Skip to content

Commit

Permalink
Merge branch 'master' into dougqh/crashtracking-adjust-timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
dougqh authored Nov 7, 2024
2 parents 1475406 + 060ab72 commit f330e67
Show file tree
Hide file tree
Showing 31 changed files with 805 additions and 70 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ private RequestBody makeTelemetryRequestBody(@Nonnull String content) throws IOE
writer.name("message").value(crashLog.toJson());
writer.name("level").value("ERROR");
writer.name("tags").value("severity:crash");
writer.name("is_sensitive").value(true);
writer.endObject();
writer.endArray();
writer.name("application");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,13 +188,8 @@ public CrashLog parse(String crashLog) {
state = State.DONE;
} else {
// Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
if (line.contains("libjvm.so") || line.contains("libjavaProfiler")) {
message.append(line).append('\n');
frames.add(parseLine(line));
} else {
message.append(line.charAt(0)).append(" [redacted]\n");
frames.add(new StackFrame(null, 0, "[redacted]"));
}
message.append(line).append('\n');
frames.add(parseLine(line));
}
break;
case DONE:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ public class CrashUploaderTest {
@BeforeEach
public void setup() throws IOException {
server.start();
System.out.println("Setting up test: " + server.getPort());
url = server.url(URL_PATH);

when(config.getEnv()).thenReturn(ENV);
Expand Down Expand Up @@ -170,6 +171,7 @@ public void testTelemetryHappyPath(String log) throws Exception {
// payload:
assertEquals("ERROR", event.get("payload").get(0).get("level").asText());

assertTrue(event.get("payload").get(0).get("is_sensitive").asBoolean());
// we need to sanitize the UIID which keeps on changing
String message = event.get("payload").get(0).get("message").asText();
CrashLog extracted = CrashLog.fromJson(message);
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"error":{"is_crash":true,"kind":"SIGSEGV","message":"\n\nJRE version: OpenJDK Runtime Environment Temurin-22.0.1+8 (22.0.1+8) (build 22.0.1+8)\nJava VM: OpenJDK 64-Bit Server VM Temurin-22.0.1+8 (22.0.1+8, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)\nProblematic frame:\nC [libpthread.so.0+0x9cd5] __pthread_clockjoin_ex+0x255\n\nNative frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)\nC [redacted]\n","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"[redacted]","line":0}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.40.0-SNAPSHOT~b4718bd887","tags":{}},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[14,6,1]}},"proc_info":{"pid":"576034"},"timestamp":"2024-09-20T13:19:06Z","uuid":"477a8d3f-d381-4352-a2a9-76eeefeef242","version_id":0}
{"error":{"is_crash":true,"kind":"SIGSEGV","message":"\n\nJRE version: OpenJDK Runtime Environment Temurin-22.0.1+8 (22.0.1+8) (build 22.0.1+8)\nJava VM: OpenJDK 64-Bit Server VM Temurin-22.0.1+8 (22.0.1+8, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)\nProblematic frame:\nC [libpthread.so.0+0x9cd5] __pthread_clockjoin_ex+0x255\n\nNative frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)\nC [libpthread.so.0+0x9cd5] __pthread_clockjoin_ex+0x255\n","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"__pthread_clockjoin_ex","line":0}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.42.0-SNAPSHOT~aa29078ace","tags":{}},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[14,7,0]}},"proc_info":{"pid":"576034"},"timestamp":"2024-09-20T13:19:06Z","uuid":"363ac4ba-c104-4c8f-9aaa-5870486a8926","version_id":0}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"error":{"is_crash":true,"kind":"INVALID","message":"\n\n fatal error: OutOfMemory encountered: Java heap space\nJRE version: OpenJDK Runtime Environment (Zulu 8.70.0.23-CA-macos-aarch64) (8.0_372-b07) (build 1.8.0_372-b07)\nJava VM: OpenJDK 64-Bit Server VM (25.372-b07 mixed mode bsd-aarch64 compressed oops)\n\nNative frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)\nV [redacted]\nV [redacted]\nV [redacted]\nV [redacted]\nV [redacted]\nV [redacted]\nj [redacted]\nv [redacted]\nV [redacted]\nV [redacted]\nV [redacted]\nC [redacted]\nC [redacted]\nC [redacted]\n","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0},{"function":"[redacted]","line":0}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.40.0-SNAPSHOT~b4718bd887","tags":{}},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[14,6,1]}},"proc_info":{"pid":"96267"},"uuid":"9b651ca7-0671-4805-bd91-c83ba131ece9","version_id":0}
{"error":{"is_crash":true,"kind":"INVALID","message":"\n\n fatal error: OutOfMemory encountered: Java heap space\nJRE version: OpenJDK Runtime Environment (Zulu 8.70.0.23-CA-macos-aarch64) (8.0_372-b07) (build 1.8.0_372-b07)\nJava VM: OpenJDK 64-Bit Server VM (25.372-b07 mixed mode bsd-aarch64 compressed oops)\n\nNative frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)\nV [libjvm.dylib+0x565d30] VMError::report_and_die()+0x468\nV [libjvm.dylib+0x1941a0] report_vm_error(char const*, int, char const*, char const*)+0x5c\nV [libjvm.dylib+0x1943d8] report_java_out_of_memory(char const*)+0xfc\nV [libjvm.dylib+0x70430] CollectedHeap::common_mem_allocate_noinit(KlassHandle, unsigned long, Thread*)+0x128\nV [libjvm.dylib+0x53eba8] TypeArrayKlass::allocate_common(int, bool, Thread*)+0xfc\nV [libjvm.dylib+0x285b6c] InterpreterRuntime::newarray(JavaThread*, BasicType, int)+0x48\nj datadog.smoketest.crashtracking.CrashtrackingTestApplication.main([Ljava/lang/String;)V+105\nv ~StubRoutines::call_stub\nV [libjvm.dylib+0x28f86c] JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)+0x840\nV [libjvm.dylib+0x2d3b44] jni_invoke_static(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)+0x294\nV [libjvm.dylib+0x2d7160] jni_CallStaticVoidMethod+0x188\nC [java+0x6404] JavaMain+0xa10\nC [libsystem_pthread.dylib+0x6f94] _pthread_start+0x88\nC [libsystem_pthread.dylib+0x1d34] thread_start+0x8\n","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"VMError::report_and_die()","line":0},{"function":"report_vm_error(char const*, int, char const*, char const*)","line":0},{"function":"report_java_out_of_memory(char const*)","line":0},{"function":"CollectedHeap::common_mem_allocate_noinit(KlassHandle, unsigned long, Thread*)","line":0},{"function":"TypeArrayKlass::allocate_common(int, bool, Thread*)","line":0},{"function":"InterpreterRuntime::newarray(JavaThread*, BasicType, int)","line":0},{"function":"datadog.smoketest.crashtracking.CrashtrackingTestApplication.main([Ljava/lang/String;)V","line":0},{"function":" ~StubRoutines::call_stub","line":0},{"function":"JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)","line":0},{"function":"jni_invoke_static(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)","line":0},{"function":"jni_CallStaticVoidMethod","line":0},{"function":"JavaMain","line":0},{"function":"_pthread_start","line":0},{"function":"thread_start","line":0}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.42.0-SNAPSHOT~aa29078ace","tags":{}},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[14,7,0]}},"proc_info":{"pid":"96267"},"uuid":"bfe4e4b4-b59f-4954-bee3-91f60e653e61","version_id":0}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"error":{"is_crash":true,"kind":"SIGSEGV","message":"\n\nJRE version: OpenJDK Runtime Environment Zulu17.42+20-SA (17.0.7+7) (build 17.0.7+7-LTS)\nJava VM: OpenJDK 64-Bit Server VM Zulu17.42+20-SA (17.0.7+7-LTS, mixed mode, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)\nProblematic frame:\nV [libjvm.so+0x6b7187] vframeStreamForte::forte_next()+0x797\n\nNative frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)\nV [libjvm.so+0x6b7187] vframeStreamForte::forte_next()+0x797\nV [libjvm.so+0x6b79ad] forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]+0x7bd\nV [libjvm.so+0x6b8123] AsyncGetCallTrace+0x193\nC [libjavaProfiler586350205236920700.so+0x146c8] Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]+0xd8\nC [libjavaProfiler586350205236920700.so+0x1c18d] Profiler::recordSample(void*, unsigned long long, int, int, Event*)+0xa2d\nC [libjavaProfiler586350205236920700.so+0x1d128] WallClock::sharedSignalHandler(int, siginfo_t*, void*)+0x148\nC [redacted]\nV [libjvm.so+0x85f9e9] JfrStackTrace::record_safe(JavaThread*, int)+0x5a9\nV [libjvm.so+0x861650] JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)+0x50\nV [libjvm.so+0xb1ecba] ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)+0x10a\nV [libjvm.so+0x81abee] JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)+0x5e\nV [libjvm.so+0x3afa20] AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)+0x30\nV [libjvm.so+0xa8f8ed] MemAllocator::allocate() const+0x13d\nV [libjvm.so+0x7e443c] InstanceKlass::allocate_objArray(int, int, JavaThread*)+0x13c\nV [libjvm.so+0xbe1b59] OptoRuntime::new_array_C(Klass*, int, JavaThread*)+0x129\n","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"vframeStreamForte::forte_next()","line":0},{"function":"forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]","line":0},{"function":"AsyncGetCallTrace","line":0},{"function":"Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]","line":0},{"function":"Profiler::recordSample(void*, unsigned long long, int, int, Event*)","line":0},{"function":"WallClock::sharedSignalHandler(int, siginfo_t*, void*)","line":0},{"function":"[redacted]","line":0},{"function":"JfrStackTrace::record_safe(JavaThread*, int)","line":0},{"function":"JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)","line":0},{"function":"ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)","line":0},{"function":"JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)","line":0},{"function":"AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)","line":0},{"function":"MemAllocator::allocate() const","line":0},{"function":"InstanceKlass::allocate_objArray(int, int, JavaThread*)","line":0},{"function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)","line":0}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.40.0-SNAPSHOT~b4718bd887","tags":{}},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[14,6,1]}},"proc_info":{"pid":"161958"},"timestamp":"2023-10-17T20:25:14+08:00","uuid":"a570abb4-60e6-4590-be2f-5f8445f664a9","version_id":0}
{"error":{"is_crash":true,"kind":"SIGSEGV","message":"\n\nJRE version: OpenJDK Runtime Environment Zulu17.42+20-SA (17.0.7+7) (build 17.0.7+7-LTS)\nJava VM: OpenJDK 64-Bit Server VM Zulu17.42+20-SA (17.0.7+7-LTS, mixed mode, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)\nProblematic frame:\nV [libjvm.so+0x6b7187] vframeStreamForte::forte_next()+0x797\n\nNative frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)\nV [libjvm.so+0x6b7187] vframeStreamForte::forte_next()+0x797\nV [libjvm.so+0x6b79ad] forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]+0x7bd\nV [libjvm.so+0x6b8123] AsyncGetCallTrace+0x193\nC [libjavaProfiler586350205236920700.so+0x146c8] Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]+0xd8\nC [libjavaProfiler586350205236920700.so+0x1c18d] Profiler::recordSample(void*, unsigned long long, int, int, Event*)+0xa2d\nC [libjavaProfiler586350205236920700.so+0x1d128] WallClock::sharedSignalHandler(int, siginfo_t*, void*)+0x148\nC [libpthread.so.0+0x12cf0]\nV [libjvm.so+0x85f9e9] JfrStackTrace::record_safe(JavaThread*, int)+0x5a9\nV [libjvm.so+0x861650] JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)+0x50\nV [libjvm.so+0xb1ecba] ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)+0x10a\nV [libjvm.so+0x81abee] JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)+0x5e\nV [libjvm.so+0x3afa20] AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)+0x30\nV [libjvm.so+0xa8f8ed] MemAllocator::allocate() const+0x13d\nV [libjvm.so+0x7e443c] InstanceKlass::allocate_objArray(int, int, JavaThread*)+0x13c\nV [libjvm.so+0xbe1b59] OptoRuntime::new_array_C(Klass*, int, JavaThread*)+0x129\n","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"vframeStreamForte::forte_next()","line":0},{"function":"forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]","line":0},{"function":"AsyncGetCallTrace","line":0},{"function":"Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]","line":0},{"function":"Profiler::recordSample(void*, unsigned long long, int, int, Event*)","line":0},{"function":"WallClock::sharedSignalHandler(int, siginfo_t*, void*)","line":0},{"function":"libpthread.so.0+0x12cf0","line":0},{"function":"JfrStackTrace::record_safe(JavaThread*, int)","line":0},{"function":"JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)","line":0},{"function":"ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)","line":0},{"function":"JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)","line":0},{"function":"AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)","line":0},{"function":"MemAllocator::allocate() const","line":0},{"function":"InstanceKlass::allocate_objArray(int, int, JavaThread*)","line":0},{"function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)","line":0}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.42.0-SNAPSHOT~aa29078ace","tags":{}},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[14,7,0]}},"proc_info":{"pid":"161958"},"timestamp":"2023-10-17T20:25:14+08:00","uuid":"0d1c4f7c-9d41-481d-909a-343ad12257b2","version_id":0}
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import datadog.trace.bootstrap.debugger.util.Redaction;
import datadog.trace.bootstrap.debugger.util.TimeoutChecker;
import datadog.trace.bootstrap.debugger.util.WellKnownClasses;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand Down Expand Up @@ -183,6 +184,10 @@ public ValueReferenceResolver withExtensions(Map<String, Object> extensions) {
return new CapturedContext(this, extensions);
}

public void removeExtension(String name) {
extensions.remove(name);
}

private void addExtension(String name, Object value) {
extensions.put(name, value);
}
Expand Down Expand Up @@ -642,7 +647,7 @@ public TimeoutChecker getTimeoutChecker() {
public static class CapturedThrowable {
private final String type;
private final String message;
private final transient Throwable throwable;
private final transient WeakReference<Throwable> throwable;

/*
* Need to exclude stacktrace from equals/hashCode computation.
Expand All @@ -663,7 +668,7 @@ public CapturedThrowable(
this.type = type;
this.message = message;
this.stacktrace = new ArrayList<>(stacktrace);
this.throwable = t;
this.throwable = new WeakReference<>(t);
}

public String getType() {
Expand All @@ -679,7 +684,7 @@ public List<CapturedStackFrame> getStacktrace() {
}

public Throwable getThrowable() {
return throwable;
return throwable.get();
}

private static List<CapturedStackFrame> captureFrames(StackTraceElement[] stackTrace) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,14 @@ boolean shouldCaptureException(String fingerprint, Clock clock) {

public void addSnapshot(Snapshot snapshot) {
Throwable throwable = snapshot.getCaptures().getReturn().getCapturedThrowable().getThrowable();
if (throwable == null) {
LOGGER.debug("Snapshot has no throwable: {}", snapshot.getId());
return;
}
throwable = ExceptionHelper.getInnerMostThrowable(throwable);
if (throwable == null) {
LOGGER.debug(
"Unable to find root cause of exception: {}",
snapshot.getCaptures().getReturn().getCapturedThrowable().getThrowable().toString());
throwable = snapshot.getCaptures().getReturn().getCapturedThrowable().getThrowable();
LOGGER.debug("Unable to find root cause of exception: {}", String.valueOf(throwable));
return;
}
ThrowableState state =
Expand All @@ -172,6 +175,10 @@ void updateLastCapture(String fingerprint, Clock clock) {
fingerprints.put(fingerprint, Instant.now(clock));
}

boolean hasExceptionStateTracked() {
return !snapshotsByThrowable.isEmpty();
}

public static class ThrowableState {
private final String exceptionId;
private List<Snapshot> snapshots;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import datadog.trace.bootstrap.debugger.ProbeId;
import datadog.trace.bootstrap.debugger.ProbeImplementation;
import datadog.trace.bootstrap.debugger.ProbeLocation;
import datadog.trace.bootstrap.debugger.el.ValueReferences;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -67,6 +68,10 @@ public void evaluate(
return;
}
Throwable throwable = context.getCapturedThrowable().getThrowable();
if (throwable == null) {
LOGGER.debug("Throwable cleared by GC");
return;
}
Throwable innerMostThrowable = getInnerMostThrowable(throwable);
String fingerprint =
Fingerprinter.fingerprint(innerMostThrowable, exceptionProbeManager.getClassNameFilter());
Expand Down Expand Up @@ -107,6 +112,9 @@ public void commit(
* - DebuggerContext.commit()
*/
snapshot.recordStackTrace(4);
// clear any strong ref to the exception before adding the snapshot to avoid leaking snapshots
// inside the stateByThrowable map
clearExceptionRefs(snapshot);
// add snapshot for later to wait for triggering point (ExceptionDebugger::handleException)
exceptionProbeManager.addSnapshot(snapshot);
LOGGER.debug(
Expand All @@ -117,6 +125,11 @@ public void commit(
}
}

private void clearExceptionRefs(Snapshot snapshot) {
snapshot.getCaptures().getReturn().getLocals().remove(ValueReferences.EXCEPTION_REF);
snapshot.getCaptures().getReturn().removeExtension(ValueReferences.EXCEPTION_EXTENSION_NAME);
}

@Override
public void buildLocation(InstrumentationResult result) {
String type = where.getTypeName();
Expand Down
Loading

0 comments on commit f330e67

Please sign in to comment.