Skip to content

ASM Standalone #2903

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 47 commits into from
Dec 16, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
0bae93d
Add asm standalone traces
estringana Oct 18, 2024
6444fac
Adapt sample rate to asm standalone
estringana Oct 23, 2024
ce56a29
Remove leaks
estringana Oct 23, 2024
a97936a
Send computed stats header when sidecar not used
estringana Oct 23, 2024
f7548c5
Fix adding apm enabled tag
estringana Oct 24, 2024
717e4c4
Add new meta tag _dd.p.appsec
estringana Oct 24, 2024
b156ae5
Set asm events on new tag _dd.p.appsec
estringana Oct 25, 2024
339ad5a
Mount hunter cache folder
estringana Nov 4, 2024
101b120
Wip
estringana Nov 4, 2024
c42403e
Revert "Mount hunter cache folder"
estringana Nov 4, 2024
c6e78d9
Lint
estringana Nov 4, 2024
575a117
Add appsec event as soon as detected
estringana Nov 6, 2024
21c90ec
Add asm event to propagated tags
estringana Nov 8, 2024
6706053
wip
estringana Nov 8, 2024
11ffdde
Set sampling priority as soon as changed on appsec
estringana Nov 8, 2024
5fbe354
Amend compilation error
estringana Nov 8, 2024
5e98df9
wip
estringana Nov 8, 2024
933600a
Finish fixing system tests
estringana Nov 12, 2024
ea641d2
Remove commented code
estringana Nov 12, 2024
3d76fb3
Refactor asm implementation
estringana Nov 13, 2024
6007b81
Fix error adding twice _dd.p.appsec
estringana Nov 13, 2024
fe3feb8
Fix lint
estringana Nov 13, 2024
a43565a
Amend bug finding propagated tags
estringana Nov 14, 2024
8ea6167
Create asm standalone limiter
estringana Nov 18, 2024
a0483b1
Set one minute for the asm standalone limiter
estringana Nov 18, 2024
f3d67e2
Fix standalone limiter
estringana Nov 18, 2024
757a83d
Add tests for standalone
estringana Nov 20, 2024
7ca6d72
Set traces to auto reject when standalone sampling priority not allow
estringana Nov 20, 2024
9387d96
Test auto flush
estringana Nov 21, 2024
7db1264
Test curl asm event propagation
estringana Nov 21, 2024
98481d1
Test asm standalone from appsec extension
estringana Nov 22, 2024
bad67ab
Fix tests incompatible with PHP 7 versions
estringana Nov 25, 2024
135a0e0
Change test execution order
estringana Nov 25, 2024
38b33f0
Add request replayer to appsec tests
estringana Nov 25, 2024
ecc0389
Remove httpbin tests from extension tests
estringana Nov 25, 2024
a350767
Improve standalone limiter
estringana Nov 29, 2024
324bfaf
Address comments comming from PR
estringana Nov 29, 2024
f2e5668
Add changes comng from PR
estringana Dec 2, 2024
b99049f
Amend tags values
estringana Dec 2, 2024
b5e21be
Removei unnecessary code
estringana Dec 2, 2024
a3b8609
Fix segmentation fault
estringana Dec 3, 2024
76b4871
Rename appsec module on tracer
estringana Dec 4, 2024
3952e69
Amend dm and priority on asm events
estringana Dec 4, 2024
7090812
Amend error generated by rebase
estringana Dec 12, 2024
dd051d6
Update recommended rules to latest
estringana Dec 13, 2024
a845679
Lint
estringana Dec 13, 2024
13d4688
Amend sampling rate
estringana Dec 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
512 changes: 355 additions & 157 deletions appsec/recommended.json

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions appsec/src/extension/commands_helpers.c
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,7 @@ dd_result dd_command_proc_resp_verd_span_data(

if (res == dd_should_block || res == dd_should_redirect ||
res == dd_should_record) {
dd_trace_emit_asm_event();
_set_appsec_span_data(
mpack_node_array_at(root, RESP_INDEX_APPSEC_SPAN_DATA));
}
Expand Down
1 change: 1 addition & 0 deletions appsec/src/extension/configuration.h
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ extern bool runtime_config_first_init;
CONFIG(STRING, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML, "") \
CONFIG(STRING, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, "") \
CONFIG(DOUBLE, DD_API_SECURITY_REQUEST_SAMPLE_RATE, "0.1", .ini_change = zai_config_system_ini_change) \
CONFIG(BOOL, DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED, "false") \
CONFIG(BOOL, DD_API_SECURITY_ENABLED, "true", .ini_change = zai_config_system_ini_change)
// clang-format on

Expand Down
37 changes: 37 additions & 0 deletions appsec/src/extension/ddtrace.c
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,16 @@ static void (*nullable _ddtrace_close_all_spans_and_flush)(void);
static void (*nullable _ddtrace_set_priority_sampling_on_span_zobj)(
zend_object *nonnull zobj, zend_long priority,
enum dd_sampling_mechanism mechanism);
static void (*nullable _ddtrace_add_propagated_tag_on_span_zobj)(
zend_string *nonnull key, zval *nonnull value);

static bool (*nullable _ddtrace_user_req_add_listeners)(
ddtrace_user_req_listeners *listeners);

static zend_string *(*_ddtrace_ip_extraction_find)(zval *server);

static const char *nullable (*_ddtrace_remote_config_get_path)(void);
static void *(*nullable _ddtrace_emit_asm_event)(void);

static void _test_ddtrace_metric_register_buffer(
zend_string *nonnull name, ddtrace_metric_type type, ddtrace_metric_ns ns);
Expand Down Expand Up @@ -99,6 +102,14 @@ static void dd_trace_load_symbols(void)
dlerror()); // NOLINT(concurrency-mt-unsafe)
}

_ddtrace_add_propagated_tag_on_span_zobj =
dlsym(handle, "ddtrace_add_propagated_tag_on_span_zobj");
if (_ddtrace_add_propagated_tag_on_span_zobj == NULL) {
mlog(dd_log_error,
"Failed to load ddtrace_add_propagated_tag_on_span_zobj: %s",
dlerror()); // NOLINT(concurrency-mt-unsafe)
}

_ddtrace_user_req_add_listeners =
dlsym(handle, "ddtrace_user_req_add_listeners");
if (_ddtrace_user_req_add_listeners == NULL) {
Expand Down Expand Up @@ -133,6 +144,13 @@ static void dd_trace_load_symbols(void)
dlerror()); // NOLINT(concurrency-mt-unsafe)
}

_ddtrace_emit_asm_event = dlsym(handle, "ddtrace_emit_asm_event");
if (_ddtrace_emit_asm_event == NULL) {
mlog(dd_log_error,
// NOLINTNEXTLINE(concurrency-mt-unsafe)
"Failed to load ddtrace_emit_asm_event: %s", dlerror());
}

dlclose(handle);
}

Expand Down Expand Up @@ -414,6 +432,25 @@ const char *nullable dd_trace_remote_config_get_path()
return path;
}

void dd_trace_span_add_propagated_tags(
zend_string *nonnull key, zval *nonnull value)
{
if (UNEXPECTED(_ddtrace_add_propagated_tag_on_span_zobj == NULL)) {
return;
}

_ddtrace_add_propagated_tag_on_span_zobj(key, value);
}

void dd_trace_emit_asm_event(void)
{
if (UNEXPECTED(_ddtrace_emit_asm_event == NULL)) {
return;
}

_ddtrace_emit_asm_event();
}

static PHP_FUNCTION(datadog_appsec_testing_ddtrace_rshutdown)
{
if (zend_parse_parameters_none() == FAILURE) {
Expand Down
4 changes: 4 additions & 0 deletions appsec/src/extension/ddtrace.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ enum dd_sampling_mechanism {
DD_MECHANISM_REMOTE_RATE = 2,
DD_MECHANISM_RULE = 3,
DD_MECHANISM_MANUAL = 4,
DD_MECHANISM_ASM = 5,
};

typedef zend_object root_span_t;
Expand Down Expand Up @@ -48,11 +49,14 @@ bool dd_trace_span_add_tag_str(zend_object *nonnull zobj,
// Flush the tracer spans, can be used on RINIT
void dd_trace_close_all_spans_and_flush(void);

void dd_trace_emit_asm_event(void);

// Provides the array zval representing $root_span->meta, if any.
// It is ready for modification, with refcount == 1
zval *nullable dd_trace_span_get_meta(zend_object *nonnull);
zval *nullable dd_trace_span_get_metrics(zend_object *nonnull);
zval *nullable dd_trace_span_get_meta_struct(zend_object *nonnull);
void dd_trace_span_add_propagated_tags(zend_string *nonnull key, zval *nonnull value);
zend_string *nullable dd_trace_get_formatted_runtime_id(bool persistent);

// Set sampling priority on root span
Expand Down
25 changes: 20 additions & 5 deletions appsec/src/extension/tags.c
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#endif

#define DD_TAG_DATA "_dd.appsec.json"
#define DD_TAG_P_APPSEC "_dd.p.appsec"
#define DD_TAG_EVENT "appsec.event"
#define DD_TAG_BLOCKED "appsec.blocked"
#define DD_TAG_RUNTIME_FAMILY "_dd.runtime_family"
Expand Down Expand Up @@ -65,6 +66,7 @@
static zend_string *_dd_tag_data_zstr;
static zend_string *_dd_tag_event_zstr;
static zend_string *_dd_tag_blocked_zstr;
static zend_string *_dd_tag_p_appsec_zstr;
static zend_string *_dd_tag_http_method_zstr;
static zend_string *_dd_tag_http_user_agent_zstr;
static zend_string *_dd_tag_http_status_code_zstr;
Expand Down Expand Up @@ -96,6 +98,7 @@ static zend_string *_key_server_name_zstr;
static zend_string *_key_http_user_agent_zstr;
static zend_string *_key_https_zstr;
static zend_string *_key_remote_addr_zstr;
static zend_string *_1_zstr;
static zend_string *_true_zstr;
static zend_string *_false_zstr;
static zend_string *_track_zstr;
Expand Down Expand Up @@ -130,9 +133,12 @@ void dd_tags_startup()
zend_string_init_interned(LSTRARG(DD_TAG_EVENT), 1 /* permanent */);
_dd_tag_blocked_zstr =
zend_string_init_interned(LSTRARG(DD_TAG_BLOCKED), 1 /* permanent */);
_1_zstr = zend_string_init_interned(LSTRARG("1"), 1 /* permanent */);
_true_zstr = zend_string_init_interned(LSTRARG("true"), 1 /* permanent */);
_false_zstr =
zend_string_init_interned(LSTRARG("false"), 1 /* permanent */);
_dd_tag_p_appsec_zstr =
zend_string_init_interned(LSTRARG(DD_TAG_P_APPSEC), 1 /* permanent */);

_dd_tag_http_method_zstr =
zend_string_init_interned(LSTRARG(DD_TAG_HTTP_METHOD), 1);
Expand Down Expand Up @@ -345,8 +351,11 @@ void dd_tags_add_tags(
_set_runtime_family(span);

if (_force_keep) {
dd_trace_set_priority_sampling_on_span_zobj(
span, PRIORITY_SAMPLING_USER_KEEP, DD_MECHANISM_MANUAL);
dd_trace_set_priority_sampling_on_span_zobj(span,
PRIORITY_SAMPLING_USER_KEEP,
get_DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED()
? DD_MECHANISM_ASM
: DD_MECHANISM_MANUAL);
mlog(dd_log_debug, "Updated sampling priority to user_keep");
}

Expand All @@ -361,6 +370,12 @@ void dd_tags_add_tags(
return;
}

// If we reach this point, there are asm events
zval _1_zval;
ZVAL_STR(&_1_zval, _1_zstr);

dd_trace_span_add_propagated_tags(_dd_tag_p_appsec_zstr, &_1_zval);

zend_string *tag_value = _concat_json_fragments();

zval tag_value_zv;
Expand All @@ -374,9 +389,9 @@ void dd_tags_add_tags(
}

// tag appsec.event
zval true_zv;
ZVAL_STR_COPY(&true_zv, _true_zstr);
res = dd_trace_span_add_tag(span, _dd_tag_event_zstr, &true_zv);
zval _true_zval;
ZVAL_STR(&_true_zval, _true_zstr);
res = dd_trace_span_add_tag(span, _dd_tag_event_zstr, &_true_zval);
if (!res) {
mlog(dd_log_info, "Failed adding tag " DD_TAG_EVENT " to root span");
return;
Expand Down
6 changes: 3 additions & 3 deletions appsec/tests/extension/client_init_record_span_tags.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ tags:
Array
(
[_dd.appsec.json] => {"triggers":[{"found":"attack"},{"another":"attack"},{"yet another":"attack"}]}
[_dd.p.dm] => -0
[_dd.p.appsec] => 1
[_dd.p.dm] => -5
[_dd.p.tid] => %s
[_dd.runtime_family] => php
[appsec.event] => true
Expand All @@ -106,8 +107,7 @@ Array
[metric_1] => 2
[metric_2] => 10
[_dd.appsec.enabled] => 1
[_dd.agent_psr] => 1
[_sampling_priority_v1] => 1
[_sampling_priority_v1] => 2
[php.compilation.total_time_ms] => %f
[php.memory.peak_usage_bytes] => %f
[php.memory.peak_real_usage_bytes] => %f
Expand Down
16 changes: 16 additions & 0 deletions appsec/tests/extension/inc/mock_helper.php
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,22 @@ function response_config_sync() {
return response("config_sync", []);
}

function request_without_events()
{
return [
response_list(response_request_init([[['ok', []]], [], true])),
response_list(response_request_shutdown([[['ok', []]], [], true])),
];
}

function request_with_events()
{
return [
response_list(response_request_init([[['record', []]],['{"found":"attack"}','{"another":"attack"}'],true])),
response_list(response_request_shutdown([[['record', []]], ['{"yet another":"attack"}'], true]))
];
}


// vim: set et sw=4 ts=4:
?>
66 changes: 66 additions & 0 deletions appsec/tests/extension/rinit_asm_events_propagate_tags.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
--TEST--
Asm events are added as meta tags and also as propagated tags
--INI--
extension=ddtrace.so
datadog.appsec.log_file=/tmp/php_appsec_test.log
datadog.appsec.log_level=debug
datadog.appsec.enabled=1
--ENV--
HTTP_X_DATADOG_TRACE_ID=42
HTTP_X_DATADOG_PARENT_ID=10
HTTP_X_DATADOG_ORIGIN=datadog
HTTP_X_DATADOG_TAGS=_dd.p.custom_tag=inherited,_dd.p.second_tag=bar
--FILE--
<?php
use function datadog\appsec\testing\{rinit,rshutdown,ddtrace_rshutdown,root_span_get_meta};
include __DIR__ . '/inc/ddtrace_version.php';

ddtrace_version_at_least('0.79.0');

include __DIR__ . '/inc/mock_helper.php';

$helper = Helper::createInitedRun([
response_list(response_request_init([[['record', []]], ['{"found":"attack"}','{"another":"attack"}']])),
response_list(
response_request_shutdown(
[[['record', []]], ['{"yet another":"attack"}'], false, ["rshutdown_tag" => "rshutdown_value"], ["rshutdown_metric" => 2.1]]
)
),
], ['continuous' => true]);

echo "rinit\n";
var_dump(rinit());
$helper->get_commands(); //ignore

$context = DDTrace\current_context();
echo "_dd.p.appsec on distributed propagated tags? ";
echo isset($context['distributed_tracing_propagated_tags']['_dd.p.appsec']) && $context['distributed_tracing_propagated_tags']['_dd.p.appsec'] == 1 ? "Yes": "No";
echo PHP_EOL;

echo "rshutdown\n";
var_dump(rshutdown());
$helper->get_commands(); //ignore

echo "ddtrace_rshutdown\n";
var_dump(ddtrace_rshutdown());
dd_trace_internal_fn('synchronous_flush');

$commands = $helper->get_commands();
$tags = $commands[0]['payload'][0][0]['meta'];

echo "_dd.p.appsec? ";
echo isset($tags['_dd.p.appsec']) && $tags['_dd.p.appsec'] === "1" ? "Yes": "No";
echo PHP_EOL;

$helper->finished_with_commands();

?>
--EXPECTF--
rinit
bool(true)
_dd.p.appsec on distributed propagated tags? Yes
rshutdown
bool(true)
ddtrace_rshutdown
bool(true)
_dd.p.appsec? Yes
6 changes: 3 additions & 3 deletions appsec/tests/extension/rinit_record_span_tags.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ tags:
Array
(
[_dd.appsec.json] => {"triggers":[{"found":"attack"},{"another":"attack"},{"yet another":"attack"}]}
[_dd.p.dm] => -0
[_dd.p.appsec] => 1
[_dd.p.dm] => -5
[_dd.p.tid] => %s
[_dd.runtime_family] => php
[appsec.event] => true
Expand All @@ -99,8 +100,7 @@ Array
[%s] => %d
[rshutdown_metric] => 2.1
[_dd.appsec.enabled] => 1
[_dd.agent_psr] => 1
[_sampling_priority_v1] => 1
[_sampling_priority_v1] => 2
[php.compilation.total_time_ms] => %f
[php.memory.peak_usage_bytes] => %f
[php.memory.peak_real_usage_bytes] => %f
Expand Down
2 changes: 2 additions & 0 deletions config.m4
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ if test "$PHP_DDTRACE" != "no"; then
DD_TRACE_PHP_SOURCES="$EXTRA_PHP_SOURCES \
ext/ddtrace.c \
ext/agent_info.c \
ext/asm_event.c \
ext/arrays.c \
ext/auto_flush.c \
ext/autoload_php_files.c \
Expand All @@ -188,6 +189,7 @@ if test "$PHP_DDTRACE" != "no"; then
ext/integrations/exec_integration.c \
ext/integrations/integrations.c \
ext/ip_extraction.c \
ext/standalone_limiter.c \
ext/live_debugger.c \
ext/logging.c \
ext/limiter/limiter.c \
Expand Down
2 changes: 2 additions & 0 deletions config.w32
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ if (PHP_DDTRACE != 'no') {

var DDTRACE_EXT_SOURCES = "agent_info.c";
DDTRACE_EXT_SOURCES += " arrays.c";
DDTRACE_EXT_SOURCES += " asm_event.c";
DDTRACE_EXT_SOURCES += " auto_flush.c";
DDTRACE_EXT_SOURCES += " autoload_php_files.c";
DDTRACE_EXT_SOURCES += " collect_backtrace.c";
Expand All @@ -36,6 +37,7 @@ if (PHP_DDTRACE != 'no') {
DDTRACE_EXT_SOURCES += " handlers_internal.c";
DDTRACE_EXT_SOURCES += " handlers_pcntl.c";
DDTRACE_EXT_SOURCES += " ip_extraction.c";
DDTRACE_EXT_SOURCES += " standalone_limiter.c";
DDTRACE_EXT_SOURCES += " live_debugger.c";
DDTRACE_EXT_SOURCES += " logging.c";
DDTRACE_EXT_SOURCES += " memory_limit.c";
Expand Down
2 changes: 2 additions & 0 deletions ddtrace.sym
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@ ddtrace_close_all_spans_and_flush
ddtrace_get_profiling_context
ddtrace_get_root_span
ddtrace_set_priority_sampling_on_span_zobj
ddtrace_add_propagated_tag_on_span_zobj
ddtrace_runtime_id
ddtrace_user_req_add_listeners
ddtrace_ip_extraction_find
ddtrace_set_all_thread_vm_interrupt
ddtrace_remote_config_get_path
ddtrace_metric_register_buffer
ddtrace_metric_add_point
ddtrace_emit_asm_event
ddog_remote_config_reader_for_path
ddog_remote_config_read
ddog_remote_config_reader_drop
Expand Down
Loading
Loading