Add support for new telemetry crash data format (#7675)

Author: jbachorik

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashLogParser.java

@@ -0,0 +1,10 @@
+package com.datadog.crashtracking;
+
+import com.datadog.crashtracking.dto.CrashLog;
+import com.datadog.crashtracking.parsers.HotspotCrashLogParser;
+
+public final class CrashLogParser {
+  public static CrashLog fromHotspotCrashLog(String logText) {
+    return new HotspotCrashLogParser().parse(logText);
+  }
+}

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploader.java

@@ -7,6 +7,7 @@
 import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT;
 import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT;
 
+import com.datadog.crashtracking.dto.CrashLog;
 import com.squareup.moshi.JsonWriter;
 import datadog.common.container.ContainerInfo;
 import datadog.common.version.VersionInfo;
@@ -16,13 +17,12 @@
 import datadog.trace.bootstrap.config.provider.ConfigProvider;
 import datadog.trace.util.PidHelper;
 import de.thetaphi.forbiddenapis.SuppressForbidden;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
+import java.io.*;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Instant;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -54,7 +54,7 @@ public final class CrashUploader {
   static final String JAVA_TRACING_LIBRARY = "dd-trace-java";
   static final String HEADER_DD_EVP_ORIGIN_VERSION = "DD-EVP-ORIGIN-VERSION";
   static final String HEADER_DD_TELEMETRY_API_VERSION = "DD-Telemetry-API-Version";
-  static final String TELEMETRY_API_VERSION = "v1";
+  static final String TELEMETRY_API_VERSION = "v2";
   static final String HEADER_DD_TELEMETRY_REQUEST_TYPE = "DD-Telemetry-Request-Type";
   static final String TELEMETRY_REQUEST_TYPE = "logs";
 
@@ -114,43 +114,44 @@ private String tagsToString(final Map<String, String> tags) {
         .collect(Collectors.joining(","));
   }
 
-  public void upload(@Nonnull List<InputStream> files) throws IOException {
-    List<String> filesContent = new ArrayList<>(files.size());
-    for (InputStream file : files) {
-      filesContent.add(readContent(file));
+  public void upload(@Nonnull List<Path> files) throws IOException {
+    for (Path file : files) {
+      uploadToLogs(file);
+      uploadToTelemetry(file);
     }
-    uploadToLogs(filesContent);
-    uploadToTelemetry(filesContent);
   }
 
-  void uploadToLogs(@Nonnull List<String> filesContent) throws IOException {
-    uploadToLogs(filesContent, System.out);
+  boolean uploadToLogs(@Nonnull Path file) {
+    try {
+      uploadToLogs(new String(Files.readAllBytes(file), StandardCharsets.UTF_8), System.out);
+    } catch (IOException e) {
+      log.error("Failed to upload crash file: {}", file, e);
+      return false;
+    }
+    return true;
   }
 
-  void uploadToLogs(@Nonnull List<String> filesContent, @Nonnull PrintStream out)
-      throws IOException {
+  void uploadToLogs(@Nonnull String message, @Nonnull PrintStream out) throws IOException {
     // print on the output, and the application/container/host log will pick it up
-    for (String message : filesContent) {
-      try (Buffer buf = new Buffer()) {
-        try (JsonWriter writer = JsonWriter.of(buf)) {
-          writer.beginObject();
-          writer.name("ddsource").value("crashtracker");
-          writer.name("ddtags").value(tags);
-          writer.name("hostname").value(config.getHostName());
-          writer.name("service").value(config.getServiceName());
-          writer.name("message").value(message);
-          writer.name("level").value("ERROR");
-          writer.name("error");
-          writer.beginObject();
-          writer.name("kind").value(extractErrorKind(message));
-          writer.name("message").value(extractErrorMessage(message));
-          writer.name("stack").value(extractErrorStackTrace(message, false));
-          writer.endObject();
-          writer.endObject();
-        }
-
-        out.println(buf.readByteString().utf8());
+    try (Buffer buf = new Buffer()) {
+      try (JsonWriter writer = JsonWriter.of(buf)) {
+        writer.beginObject();
+        writer.name("ddsource").value("crashtracker");
+        writer.name("ddtags").value(tags);
+        writer.name("hostname").value(config.getHostName());
+        writer.name("service").value(config.getServiceName());
+        writer.name("message").value(message);
+        writer.name("level").value("ERROR");
+        writer.name("error");
+        writer.beginObject();
+        writer.name("kind").value(extractErrorKind(message));
+        writer.name("message").value(extractErrorMessage(message));
+        writer.name("stack").value(extractErrorStackTrace(message, false));
+        writer.endObject();
+        writer.endObject();
       }
+
+      out.println(buf.readByteString().utf8());
     }
   }
 
@@ -235,16 +236,26 @@ private String extractErrorStackTrace(String fileContent) {
     return extractErrorStackTrace(fileContent, true);
   }
 
-  void uploadToTelemetry(@Nonnull List<String> filesContent) throws IOException {
-    handleCall(makeTelemetryRequest(filesContent));
+  boolean uploadToTelemetry(@Nonnull Path file) {
+    try {
+      String content = new String(Files.readAllBytes(file), Charset.defaultCharset());
+      handleCall(makeTelemetryRequest(content));
+    } catch (IOException e) {
+      log.error("Failed to upload crash file: {}", file, e);
+      return false;
+    }
+    return true;
   }
 
-  private Call makeTelemetryRequest(@Nonnull List<String> filesContent) throws IOException {
-    final RequestBody requestBody = makeTelemetryRequestBody(filesContent);
+  private Call makeTelemetryRequest(@Nonnull String content) throws IOException {
+    final RequestBody requestBody = makeTelemetryRequestBody(content);
 
     final Map<String, String> headers = new HashMap<>();
     // Set chunked transfer
-    headers.put("Content-Type", requestBody.contentType().toString());
+    MediaType contentType = requestBody.contentType();
+    if (contentType != null) {
+      headers.put("Content-Type", contentType.toString());
+    }
     headers.put("Content-Length", Long.toString(requestBody.contentLength()));
     headers.put("Transfer-Encoding", "chunked");
     headers.put(HEADER_DD_EVP_ORIGIN, JAVA_TRACING_LIBRARY);
@@ -258,8 +269,11 @@ private Call makeTelemetryRequest(@Nonnull List<String> filesContent) throws IOE
             .build());
   }
 
-  private RequestBody makeTelemetryRequestBody(@Nonnull List<String> filesContent)
-      throws IOException {
+  private RequestBody makeTelemetryRequestBody(@Nonnull String content) throws IOException {
+    CrashLog crashLog = CrashLogParser.fromHotspotCrashLog(content);
+    if (crashLog == null) {
+      throw new IOException("Failed to parse crash log");
+    }
     try (Buffer buf = new Buffer()) {
       try (JsonWriter writer = JsonWriter.of(buf)) {
         writer.beginObject();
@@ -275,13 +289,11 @@ private RequestBody makeTelemetryRequestBody(@Nonnull List<String> filesContent)
         writer.name("debug").value(true);
         writer.name("payload");
         writer.beginArray();
-        for (String message : filesContent) {
-          writer.beginObject();
-          writer.name("message").value(extractErrorStackTrace(message));
-          writer.name("level").value("ERROR");
-          writer.name("tags").value("severity:crash");
-          writer.endObject();
-        }
+        writer.beginObject();
+        writer.name("message").value(crashLog.toJson());
+        writer.name("level").value("ERROR");
+        writer.name("tags").value("severity:crash");
+        writer.endObject();
         writer.endArray();
         writer.name("application");
         writer.beginObject();

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/CrashLog.java

@@ -0,0 +1,97 @@
+package com.datadog.crashtracking.dto;
+
+import com.squareup.moshi.Json;
+import com.squareup.moshi.JsonAdapter;
+import com.squareup.moshi.Moshi;
+import java.io.IOException;
+import java.util.Objects;
+import java.util.UUID;
+
+public final class CrashLog {
+  private static final int VERSION = 0;
+
+  private static final JsonAdapter<CrashLog> ADAPTER;
+
+  static {
+    Moshi moshi = new Moshi.Builder().add(new SemanticVersion.SemanticVersionAdapter()).build();
+    ADAPTER = moshi.adapter(CrashLog.class);
+  }
+
+  public final String uuid = UUID.randomUUID().toString();
+  public final String timestamp;
+  public final boolean incomplete;
+  public final ErrorData error;
+  public final Metadata metadata;
+
+  @Json(name = "os_info")
+  public final OSInfo osInfo;
+
+  @Json(name = "proc_info")
+  public final ProcInfo procInfo;
+
+  @Json(name = "version_id")
+  public final int version = VERSION;
+
+  public CrashLog(
+      boolean incomplete,
+      String timestamp,
+      ErrorData error,
+      Metadata metadata,
+      OSInfo osInfo,
+      ProcInfo procInfo) {
+    this.incomplete = incomplete;
+    this.timestamp = timestamp;
+    this.error = error;
+    this.metadata = metadata;
+    this.osInfo = osInfo;
+    this.procInfo = procInfo;
+  }
+
+  public String toJson() {
+    return ADAPTER.toJson(this);
+  }
+
+  public static CrashLog fromJson(String json) throws IOException {
+    return ADAPTER.fromJson(json);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    CrashLog crashLog = (CrashLog) o;
+    return incomplete == crashLog.incomplete
+        && version == crashLog.version
+        && Objects.equals(uuid, crashLog.uuid)
+        && Objects.equals(timestamp, crashLog.timestamp)
+        && Objects.equals(error, crashLog.error)
+        && Objects.equals(metadata, crashLog.metadata)
+        && Objects.equals(osInfo, crashLog.osInfo)
+        && Objects.equals(procInfo, crashLog.procInfo);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(uuid, timestamp, incomplete, error, metadata, osInfo, procInfo, version);
+  }
+
+  public boolean equalsForTest(Object o) {
+    // for tests, we need to ignore UUID, OSInfo and Metadata part
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    CrashLog crashLog = (CrashLog) o;
+    return incomplete == crashLog.incomplete
+        && version == crashLog.version
+        && Objects.equals(timestamp, crashLog.timestamp)
+        && Objects.equals(error, crashLog.error)
+        && Objects.equals(procInfo, crashLog.procInfo);
+  }
+}

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/ErrorData.java

@@ -0,0 +1,44 @@
+package com.datadog.crashtracking.dto;
+
+import com.squareup.moshi.Json;
+import java.util.Objects;
+
+public final class ErrorData {
+  @Json(name = "is_crash")
+  public final boolean isCrash = true;
+
+  public final String kind;
+  public final String message;
+
+  @Json(name = "source_type")
+  public final String sourceType = "crashtracking";
+
+  public final StackTrace stack;
+
+  public ErrorData(String kind, String message, StackTrace stack) {
+    this.kind = kind;
+    this.message = message;
+    this.stack = stack;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    ErrorData errorData = (ErrorData) o;
+    return isCrash == errorData.isCrash
+        && Objects.equals(kind, errorData.kind)
+        && Objects.equals(message, errorData.message)
+        && Objects.equals(sourceType, errorData.sourceType)
+        && Objects.equals(stack, errorData.stack);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(isCrash, kind, message, sourceType, stack);
+  }
+}

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/Metadata.java

@@ -0,0 +1,45 @@
+package com.datadog.crashtracking.dto;
+
+import com.squareup.moshi.Json;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+
+public final class Metadata {
+  @Json(name = "library_name")
+  public final String libraryName;
+
+  @Json(name = "library_version")
+  public final String libraryVersion;
+
+  public final String family;
+  public final Map<String, String> tags;
+
+  public Metadata(
+      String libraryName, String libraryVersion, String family, Map<String, String> tags) {
+    this.libraryName = libraryName;
+    this.libraryVersion = libraryVersion;
+    this.family = family;
+    this.tags = tags != null ? Collections.unmodifiableMap(tags) : Collections.emptyMap();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    Metadata metadata = (Metadata) o;
+    return Objects.equals(libraryName, metadata.libraryName)
+        && Objects.equals(libraryVersion, metadata.libraryVersion)
+        && Objects.equals(family, metadata.family)
+        && Objects.equals(tags, metadata.tags);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(libraryName, libraryVersion, family, tags);
+  }
+}