Skip to content

Commit daed7ad

Browse files
authored
Enhance logging (#2490)
* Add sidecar dump * Add dyanmic log file capabilities Signed-off-by: Bob Weinand <[email protected]> * Fix compat Signed-off-by: Bob Weinand <[email protected]> * Fix PHP 8.4 / macos build Signed-off-by: Bob Weinand <[email protected]> * Remove regex (within log level computation) from the hot path Signed-off-by: Bob Weinand <[email protected]> * Add test for DD_TRACE_LOG_FILE Signed-off-by: Bob Weinand <[email protected]> * Fix json bindings on apache reload Signed-off-by: Bob Weinand <[email protected]> * Add a couple comments around log ginit Signed-off-by: Bob Weinand <[email protected]> * Ensure rust code doesn't prevent unloading of ddtrace Signed-off-by: Bob Weinand <[email protected]> --------- Signed-off-by: Bob Weinand <[email protected]>
1 parent 3b74f22 commit daed7ad

File tree

113 files changed

+1257
-642
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

113 files changed

+1257
-642
lines changed

Cargo.lock

Lines changed: 350 additions & 361 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

appsec/src/extension/ddappsec.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
#include "tags.h"
4242
#include "user_tracking.h"
4343

44+
#include <json/json.h>
45+
4446
#if ZTS
4547
static atomic_int _thread_count;
4648
#endif
@@ -171,10 +173,12 @@ static PHP_GSHUTDOWN_FUNCTION(ddappsec)
171173
if (prev == 1) {
172174
dd_log_shutdown();
173175
zai_config_mshutdown();
176+
zai_json_shutdown_bindings();
174177
}
175178
#else
176179
dd_log_shutdown();
177180
zai_config_mshutdown();
181+
zai_json_shutdown_bindings();
178182
#endif
179183

180184
memset(ddappsec_globals, '\0', sizeof(*ddappsec_globals)); // NOLINT

appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ class AppSecContainer<SELF extends AppSecContainer<SELF>> extends GenericContain
7070
withEnv 'DD_TRACE_ENABLED', '1'
7171
withEnv 'DD_SERVICE', 'appsec_int_tests'
7272
withEnv 'DD_ENV', 'integration'
73+
withEnv 'DD_TRACE_LOG_LEVEL', 'info,startup=off'
7374
withEnv 'DD_TRACE_AGENT_FLUSH_AFTER_N_REQUESTS', '0'
7475
withEnv 'DD_TRACE_AGENT_FLUSH_INTERVAL', '0'
7576
withEnv 'DD_TRACE_DEBUG', '1'

components-rs/Cargo.toml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,18 @@ spawn_worker = { path = "../libdatadog/spawn_worker" }
1818
bitflags = "2.3.3"
1919
json = "0.12.4"
2020
lazy_static = "1.4"
21+
log = "0.4.20"
22+
env_logger = "0.10.1"
2123
uuid = { version = "1.0", features = ["v4"] }
2224
paste = "1"
2325
tempfile = {version = "3.3"}
24-
26+
tracing = { version = "0.1", default-features = false, features = ["std"] }
27+
tracing-core = "0.1.32"
28+
tracing-subscriber = { version = "0.3", default-features = false, features = [
29+
"std",
30+
"fmt",
31+
"env-filter",
32+
] }
2533

2634
[build-dependencies]
2735
cbindgen = "0.24"

components-rs/ddtrace.h

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,19 @@
88
#include "telemetry.h"
99
#include "sidecar.h"
1010

11-
#define ddog_Log_None (ddog_Log){ .bits = (uint32_t)0 }
12-
#define ddog_Log_Once (ddog_Log){ .bits = (uint32_t)(1 << 0) }
13-
#define ddog_Log_Error (ddog_Log){ .bits = (uint32_t)(1 << 1) }
14-
#define ddog_Log_Warn (ddog_Log){ .bits = (uint32_t)(1 << 2) }
15-
#define ddog_Log_Info (ddog_Log){ .bits = (uint32_t)(1 << 3) }
16-
#define ddog_Log_Deprecated (ddog_Log){ .bits = (uint32_t)((1 << 4) | 1) }
17-
#define ddog_Log_Startup (ddog_Log){ .bits = (uint32_t)(1 << 5) }
11+
#define ddog_Log_Error (ddog_Log){ .bits = (uint32_t)1 }
12+
#define ddog_Log_Warn (ddog_Log){ .bits = (uint32_t)2 }
13+
#define ddog_Log_Info (ddog_Log){ .bits = (uint32_t)3 }
14+
#define ddog_Log_Debug (ddog_Log){ .bits = (uint32_t)4 }
15+
#define ddog_Log_Trace (ddog_Log){ .bits = (uint32_t)5 }
16+
#define ddog_Log_Once (ddog_Log){ .bits = (uint32_t)(1 << 3) }
17+
#define ddog_Log__Deprecated (ddog_Log){ .bits = (uint32_t)(3 | (1 << 4)) }
18+
#define ddog_Log_Deprecated (ddog_Log){ .bits = (uint32_t)((3 | (1 << 4)) | (1 << 3)) }
19+
#define ddog_Log_Startup (ddog_Log){ .bits = (uint32_t)(3 | (2 << 4)) }
20+
#define ddog_Log_Startup_Warn (ddog_Log){ .bits = (uint32_t)(1 | (2 << 4)) }
21+
#define ddog_Log_Span (ddog_Log){ .bits = (uint32_t)(4 | (3 << 4)) }
22+
#define ddog_Log_Span_Trace (ddog_Log){ .bits = (uint32_t)(5 | (3 << 4)) }
23+
#define ddog_Log_Hook_Trace (ddog_Log){ .bits = (uint32_t)(5 | (4 << 4)) }
1824

1925
typedef uint64_t ddog_QueueId;
2026

@@ -127,7 +133,7 @@ typedef ddog_Bytes ddog_Uuid;
127133

128134
extern ddog_Uuid ddtrace_runtime_id;
129135

130-
extern void (*ddog_log_callback)(struct ddog_Log, ddog_CharSlice);
136+
extern void (*ddog_log_callback)(ddog_CharSlice);
131137

132138
/**
133139
* # Safety
@@ -141,13 +147,11 @@ ddog_CharSlice ddtrace_get_container_id(void);
141147

142148
void ddtrace_set_container_cgroup_path(ddog_CharSlice path);
143149

144-
bool ddog_shall_log(struct ddog_Log level);
150+
bool ddog_shall_log(struct ddog_Log category);
145151

146-
void ddog_set_log_category(struct ddog_Log level);
152+
void ddog_set_error_log_level(bool once);
147153

148-
void ddog_parse_log_category(const ddog_CharSlice *category_names,
149-
uintptr_t num,
150-
bool startup_logs_by_default);
154+
void ddog_set_log_level(ddog_CharSlice level, bool once);
151155

152156
void ddog_log(struct ddog_Log category, ddog_CharSlice msg);
153157

@@ -158,7 +162,10 @@ bool ddtrace_detect_composer_installed_json(ddog_SidecarTransport **transport,
158162
const ddog_QueueId *queue_id,
159163
ddog_CharSlice path);
160164

161-
ddog_MaybeError ddog_sidecar_connect_php(ddog_SidecarTransport **connection, bool enable_telemetry);
165+
ddog_MaybeError ddog_sidecar_connect_php(ddog_SidecarTransport **connection,
166+
const char *error_path,
167+
ddog_CharSlice log_level,
168+
bool enable_telemetry);
162169

163170
struct ddog_TelemetryActionsBuffer *ddog_sidecar_telemetry_buffer_alloc(void);
164171

components-rs/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#![feature(allow_internal_unstable)]
2+
#![feature(local_key_cell_methods)]
23

34
pub mod log;
45
pub mod telemetry;

components-rs/log.rs

Lines changed: 147 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,114 +1,189 @@
11
use std::cell::RefCell;
22
use std::collections::BTreeSet;
33
use std::ffi::c_char;
4-
use std::slice;
4+
use std::fmt::Debug;
55
use bitflags::bitflags;
6+
use tracing::Level;
7+
use tracing_core::{Event, Field, LevelFilter, Subscriber};
8+
use tracing_subscriber::EnvFilter;
9+
use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields};
10+
use tracing_subscriber::fmt::format::Writer;
11+
use tracing_subscriber::registry::LookupSpan;
12+
use tracing_subscriber::util::SubscriberInitExt;
613
use ddcommon_ffi::CharSlice;
714
use ddcommon_ffi::slice::AsBytes;
815

916
bitflags! {
1017
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
1118
#[repr(C)]
1219
pub struct Log: u32 {
13-
const None = 0;
14-
const Once = 1 << 0; // I.e. once per request
15-
const Error = 1 << 1;
16-
const Warn = 1 << 2;
17-
const Info = 1 << 3;
18-
const Deprecated = (1 << 4) | 1 /* Once */;
19-
const Startup = 1 << 5;
20-
}
21-
}
22-
23-
#[macro_export]
24-
macro_rules! log {
25-
($source:ident, $msg:expr) => { if ($crate::ddog_shall_log($crate::Log::$source)) { $crate::log($crate::Log::$source, $msg) } }
26-
}
27-
28-
#[allow_internal_unstable(thread_local)]
29-
macro_rules! thread_local {
30-
($($tokens:tt)*) => {
31-
#[thread_local]
32-
$($tokens)*
20+
const Error = 1;
21+
const Warn = 2;
22+
const Info = 3;
23+
const Debug = 4;
24+
const Trace = 5;
25+
const Once = 1 << 3; // I.e. once per request
26+
const _Deprecated = 3 | (1 << 4);
27+
const Deprecated = 3 | (1 << 4) | (1 << 3) /* Once */;
28+
const Startup = 3 | (2 << 4);
29+
const Startup_Warn = 1 | (2 << 4);
30+
const Span = 4 | (3 << 4);
31+
const Span_Trace = 5 | (3 << 4);
32+
const Hook_Trace = 5 | (4 << 4);
3333
}
3434
}
3535

3636
#[no_mangle]
3737
#[allow(non_upper_case_globals)]
38-
pub static mut ddog_log_callback: Option<extern "C" fn(Log, CharSlice)> = None;
38+
pub static mut ddog_log_callback: Option<extern "C" fn(CharSlice)> = None;
3939

4040
// Avoid RefCell for performance
41-
thread_local! { static mut LOG_CATEGORY: Log = Log::Once.union(Log::Error); }
4241
std::thread_local! {
4342
static LOGGED_MSGS: RefCell<BTreeSet<String>> = RefCell::default();
43+
static TRACING_GUARDS: RefCell<Option<tracing_core::dispatcher::DefaultGuard>> = RefCell::default();
44+
}
45+
46+
macro_rules! with_target {
47+
($cat:ident, tracing::$p:ident!($($t:tt)*)) => {
48+
match $cat {
49+
Log::Error => tracing::$p!(target: "ddtrace", Level::ERROR, $($t)*),
50+
Log::Warn => tracing::$p!(target: "ddtrace", Level::WARN, $($t)*),
51+
Log::Info => tracing::$p!(target: "ddtrace", Level::INFO, $($t)*),
52+
Log::Debug => tracing::$p!(target: "ddtrace", Level::DEBUG, $($t)*),
53+
Log::Trace => tracing::$p!(target: "ddtrace", Level::TRACE, $($t)*),
54+
Log::_Deprecated => tracing::$p!(target: "deprecated", Level::INFO, $($t)*),
55+
Log::Startup => tracing::$p!(target: "startup", Level::INFO, $($t)*),
56+
Log::Span => tracing::$p!(target: "span", Level::DEBUG, $($t)*),
57+
Log::Span_Trace => tracing::$p!(target: "span", Level::TRACE, $($t)*),
58+
Log::Hook_Trace => tracing::$p!(target: "hook", Level::TRACE, $($t)*),
59+
_ => unreachable!()
60+
}
61+
}
4462
}
4563

4664
#[no_mangle]
47-
pub extern "C" fn ddog_shall_log(level: Log) -> bool {
48-
unsafe { LOG_CATEGORY }.contains(level & !Log::Once)
65+
pub extern "C" fn ddog_shall_log(category: Log) -> bool {
66+
let category = category & !Log::Once;
67+
with_target!(category, tracing::event_enabled!())
4968
}
5069

51-
pub fn log<S>(level: Log, msg: S) where S: AsRef<str> {
52-
ddog_log(level, CharSlice::from(msg.as_ref()))
70+
pub fn log<S>(category: Log, msg: S) where S: AsRef<str> + tracing::Value {
71+
let once = !(category & Log::Once).is_empty();
72+
let category = category & !Log::Once;
73+
if once {
74+
with_target!(category, tracing::event!(once = true, msg));
75+
} else {
76+
with_target!(category, tracing::event!(msg));
77+
}
5378
}
5479

55-
#[no_mangle]
56-
pub extern "C" fn ddog_set_log_category(level: Log) {
57-
unsafe { LOG_CATEGORY = level; }
80+
struct LogFormatter {
81+
pub once: bool,
5882
}
5983

60-
#[no_mangle]
61-
pub unsafe extern "C" fn ddog_parse_log_category(category_names: *const CharSlice, num: usize, startup_logs_by_default: bool) {
62-
let mut categories = Log::None;
63-
let category_names = slice::from_raw_parts(category_names, num);
64-
65-
if category_names.len() == 1 {
66-
let first_level = category_names[0].to_utf8_lossy();
67-
if first_level == "1" || first_level == "true" || first_level == "On" {
68-
categories = Log::Error | Log::Warn | Log::Info | Log::Deprecated;
69-
if startup_logs_by_default {
70-
categories |= Log::Startup;
71-
}
72-
}
84+
struct LogVisitor {
85+
pub msg: Option<String>,
86+
pub once: bool,
87+
}
88+
89+
impl tracing_core::field::Visit for LogVisitor {
90+
fn record_bool(&mut self, _field: &Field, value: bool) {
91+
self.once = value;
7392
}
7493

75-
for category_name in category_names {
76-
for (name, category) in Log::all().iter_names() {
77-
if name.eq_ignore_ascii_case(&category_name.to_utf8_lossy()) {
78-
categories |= category;
79-
}
80-
}
94+
fn record_str(&mut self, _field: &Field, msg: &str) {
95+
self.msg = Some(msg.to_string());
8196
}
8297

83-
// Info always implies warn
84-
if categories.contains(Log::Info) {
85-
categories |= Log::Warn;
98+
fn record_debug(&mut self, _field: &Field, value: &dyn Debug) {
99+
self.msg = Some(format!("{value:?}"));
86100
}
87-
// Warn always implies error
88-
if categories.contains(Log::Warn) {
89-
categories |= Log::Error;
101+
}
102+
103+
impl<S, N> FormatEvent<S, N> for LogFormatter
104+
where
105+
S: Subscriber + for<'a> LookupSpan<'a>,
106+
N: for<'a> FormatFields<'a> + 'static {
107+
fn format_event(
108+
&self,
109+
_ctx: &FmtContext<'_, S, N>,
110+
_writer: Writer<'_>,
111+
event: &Event<'_>
112+
) -> core::fmt::Result {
113+
let mut visitor = LogVisitor { msg: None, once: false };
114+
event.record(&mut visitor);
115+
116+
fn fmt_msg(event: &Event<'_>, msg: &str, suffix: &str) -> String {
117+
let data = event.metadata();
118+
let target = if data.target() == "ddtrace" {
119+
match *data.level() {
120+
Level::ERROR => "error",
121+
Level::WARN => "warning",
122+
Level::INFO => "info",
123+
Level::DEBUG => "debug",
124+
Level::TRACE => "trace",
125+
}
126+
} else {
127+
data.target()
128+
};
129+
format!("[ddtrace] [{}] {}{}\0", target, msg, suffix)
130+
}
131+
132+
if let Some(msg) = visitor.msg {
133+
if let Some(cb) = unsafe { ddog_log_callback } {
134+
let msg = if self.once && visitor.once {
135+
if let Some(formatted) = LOGGED_MSGS.with(|logged| {
136+
let mut logged = logged.borrow_mut();
137+
if logged.contains(msg.as_str()) {
138+
return None;
139+
}
140+
let formatted = Some(fmt_msg(event, &msg, "; This message is only displayed once. Specify DD_TRACE_ONCE_LOGS=0 to show all messages."));
141+
logged.insert(msg);
142+
formatted
143+
}) {
144+
formatted
145+
} else {
146+
return Ok(());
147+
}
148+
} else {
149+
fmt_msg(event, &msg, "")
150+
};
151+
cb(unsafe { CharSlice::new(msg.as_ptr() as *const c_char, msg.len() - 1) });
152+
}
153+
}
154+
Ok(())
90155
}
156+
}
91157

92-
ddog_set_log_category(categories);
158+
#[no_mangle]
159+
pub unsafe extern "C" fn ddog_set_error_log_level(once: bool) {
160+
let subscriber = tracing_subscriber::fmt()
161+
.with_max_level(LevelFilter::ERROR)
162+
.event_format(LogFormatter { once });
163+
set_log_subscriber(subscriber)
93164
}
94165

95166
#[no_mangle]
96-
pub extern "C" fn ddog_log(category: Log, msg: CharSlice) {
97-
if let Some(cb) = unsafe { ddog_log_callback } {
98-
if category.contains(Log::Once) && !unsafe { LOG_CATEGORY }.contains(Log::Once) {
99-
LOGGED_MSGS.with(|logged| {
100-
let mut logged = logged.borrow_mut();
101-
let msgstr = unsafe { msg.to_utf8_lossy() };
102-
if !logged.contains(msgstr.as_ref()) {
103-
let msg = msgstr.to_string();
104-
let once_msg = format!("{}; This message is only displayed once. Specify DD_TRACE_ONCE_LOGS=0 to show all messages.\0", msg);
105-
cb(category, unsafe { CharSlice::new(once_msg.as_ptr() as *const c_char, once_msg.len() - 1) });
106-
logged.insert(msg);
107-
}
108-
});
109-
} else {
110-
cb(category, msg);
111-
}
167+
pub unsafe extern "C" fn ddog_set_log_level(level: CharSlice, once: bool) {
168+
let subscriber = tracing_subscriber::fmt()
169+
.with_env_filter(EnvFilter::builder().parse_lossy(level.to_utf8_lossy()))
170+
.event_format(LogFormatter { once });
171+
set_log_subscriber(subscriber)
172+
}
173+
174+
fn set_log_subscriber<S>(subscriber: S) where S: SubscriberInitExt {
175+
TRACING_GUARDS.replace(None); // drop first to avoid a prior guard to reset the thread local subscriber it upon replace()
176+
TRACING_GUARDS.replace(Some(subscriber.set_default()));
177+
}
178+
179+
#[no_mangle]
180+
pub unsafe extern "C" fn ddog_log(category: Log, msg: CharSlice) {
181+
let once = !(category & Log::Once).is_empty();
182+
let category = category & !Log::Once;
183+
if once {
184+
with_target!(category, tracing::event!(once = true, "{}", msg.to_utf8_lossy()));
185+
} else {
186+
with_target!(category, tracing::event!("{}", msg.to_utf8_lossy()));
112187
}
113188
}
114189

0 commit comments

Comments
 (0)