Report JVM argument flags for crash tracking (#11013)

chore(crashtracking): Report VM argument flags

fix(crashtracking): use stringbuilder instead of concatenation

Co-authored-by: devflow.devflow-routing-intake <devflow.devflow-routing-intake@kubernetes.us1.ddbuild.io>

Author: bric3

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java

@@ -575,15 +575,27 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool
           writer.endObject();
         }
         // experimental
-        if (payload.experimental != null && payload.experimental.ucontext != null) {
+        if (payload.experimental != null
+            && (payload.experimental.ucontext != null
+                || payload.experimental.runtimeArgs != null)) {
           writer.name("experimental");
           writer.beginObject();
-          writer.name("ucontext");
-          writer.beginObject();
-          for (Map.Entry<String, String> entry : payload.experimental.ucontext.entrySet()) {
-            writer.name(entry.getKey()).value(entry.getValue());
+          if (payload.experimental.ucontext != null) {
+            writer.name("ucontext");
+            writer.beginObject();
+            for (Map.Entry<String, String> entry : payload.experimental.ucontext.entrySet()) {
+              writer.name(entry.getKey()).value(entry.getValue());
+            }
+            writer.endObject();
+          }
+          if (payload.experimental.runtimeArgs != null) {
+            writer.name("runtime_args");
+            writer.beginArray();
+            for (String arg : payload.experimental.runtimeArgs) {
+              writer.value(arg);
+            }
+            writer.endArray();
           }
-          writer.endObject();
           writer.endObject();
         }
         // files (e.g. /proc/self/maps or dynamic_libraries)

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/Experimental.java

@@ -1,24 +1,34 @@
 package datadog.crashtracking.dto;
 
+import com.squareup.moshi.Json;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
 public final class Experimental {
   public final Map<String, String> ucontext;
 
+  @Json(name = "runtime_args")
+  public final List<String> runtimeArgs;
+
   public Experimental(Map<String, String> ucontext) {
+    this(ucontext, null);
+  }
+
+  public Experimental(Map<String, String> ucontext, List<String> runtimeArgs) {
     this.ucontext = ucontext;
+    this.runtimeArgs = runtimeArgs;
   }
 
   @Override
   public boolean equals(Object o) {
     if (!(o instanceof Experimental)) return false;
     Experimental that = (Experimental) o;
-    return Objects.equals(ucontext, that.ucontext);
+    return Objects.equals(ucontext, that.ucontext) && Objects.equals(runtimeArgs, that.runtimeArgs);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(ucontext);
+    return Objects.hash(ucontext, runtimeArgs);
   }
 }

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java

@@ -43,6 +43,7 @@
  * resulting {@link datadog.crashtracking.dto.CrashLog} will be marked {@code incomplete}.
  */
 public final class HotspotCrashLogParser {
+  private static final String HOTSPOT_JVM_ARGS_PREFIX = "jvm_args:";
   private static final DateTimeFormatter ZONED_DATE_TIME_FORMATTER =
       DateTimeFormatter.ofPattern("EEE MMM ppd HH:mm:ss yyyy zzz", Locale.getDefault());
   private static final DateTimeFormatter OFFSET_DATE_TIME_FORMATTER =
@@ -62,7 +63,8 @@ enum State {
     THREAD,
     STACKTRACE,
     REGISTERS,
-    SEEK_DYNAMIC_LIBRARIES,
+    PROCESS,
+    VM_ARGUMENTS,
     DYNAMIC_LIBRARIES,
     SYSTEM,
     DONE
@@ -332,6 +334,13 @@ static String parseCurrentThreadName(String line) {
     return threadDescriptor;
   }
 
+  private static List<String> parseHotspotJvmArgs(String line) {
+    if (line == null || !line.startsWith(HOTSPOT_JVM_ARGS_PREFIX)) {
+      return null;
+    }
+    return RuntimeArgs.parseVmArgs(line.substring(HOTSPOT_JVM_ARGS_PREFIX.length()));
+  }
+
   public CrashLog parse(String uuid, String crashLog) {
     SigInfo sigInfo = null;
     String pid = null;
@@ -342,6 +351,7 @@ public CrashLog parse(String uuid, String crashLog) {
     boolean incomplete = false;
     String oomMessage = null;
     Map<String, String> registers = null;
+    List<String> runtimeArgs = null;
     List<String> dynamicLibraryLines = null;
     String dynamicLibraryKey = null;
 
@@ -420,7 +430,7 @@ public CrashLog parse(String uuid, String crashLog) {
             registers = new LinkedHashMap<>();
             state = State.REGISTERS;
           } else if (line.contains("P R O C E S S")) {
-            state = State.SEEK_DYNAMIC_LIBRARIES;
+            state = State.PROCESS;
           } else {
             // Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
             final StackFrame frame = parseLine(line);
@@ -440,18 +450,27 @@ public CrashLog parse(String uuid, String crashLog) {
             }
           }
           break;
-        case SEEK_DYNAMIC_LIBRARIES:
-          if (line.startsWith("Dynamic libraries:")) {
+        case PROCESS:
+          if (runtimeArgs == null && line.startsWith("VM Arguments:")) {
+            state = State.VM_ARGUMENTS;
+          } else if (line.startsWith("Dynamic libraries:")) {
             state = State.DYNAMIC_LIBRARIES;
           } else if (line.contains("S Y S T E M")) {
             state = State.SYSTEM;
           } else if (line.equals("END.")) {
             state = State.DONE;
           }
           break;
+        case VM_ARGUMENTS:
+          if (line.isEmpty()) {
+            state = State.PROCESS;
+          } else if (runtimeArgs == null && line.startsWith(HOTSPOT_JVM_ARGS_PREFIX)) {
+            runtimeArgs = parseHotspotJvmArgs(line);
+          }
+          break;
         case DYNAMIC_LIBRARIES:
           if (line.isEmpty()) {
-            state = State.SEEK_DYNAMIC_LIBRARIES;
+            state = State.PROCESS;
           } else {
             if (dynamicLibraryKey == null) {
               dynamicLibraryKey = detectDynamicLibrariesKey(line);
@@ -491,8 +510,8 @@ public CrashLog parse(String uuid, String crashLog) {
       }
     }
 
-    // SEEK_DYNAMIC_LIBRARIES and SYSTEM sections are late enough that all critical data is captured
-    if (state != State.DONE && state != State.SEEK_DYNAMIC_LIBRARIES && state != State.SYSTEM) {
+    // PROCESS and SYSTEM sections are late enough that all critical data is captured
+    if (state != State.DONE && state != State.PROCESS && state != State.SYSTEM) {
       // incomplete crash log
       incomplete = true;
     }
@@ -553,7 +572,10 @@ public CrashLog parse(String uuid, String crashLog) {
     Integer parsedPid = safelyParseInt(pid);
     ProcInfo procInfo = parsedPid != null ? new ProcInfo(parsedPid) : null;
     Experimental experimental =
-        (registers != null && !registers.isEmpty()) ? new Experimental(registers) : null;
+        (registers != null && !registers.isEmpty())
+                || (runtimeArgs != null && !runtimeArgs.isEmpty())
+            ? new Experimental(registers, runtimeArgs)
+            : null;
     DynamicLibs files =
         (dynamicLibraryLines != null && !dynamicLibraryLines.isEmpty())
             ? new DynamicLibs(dynamicLibraryKey, dynamicLibraryLines)

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/J9JavacoreParser.java

@@ -45,6 +45,7 @@
  * </ul>
  */
 public final class J9JavacoreParser {
+  private static final String J9_USER_ARG_PREFIX = "2CIUSERARG";
 
   private final BuildIdCollector buildIdCollector;
 
@@ -117,10 +118,17 @@ public CrashLog parse(String uuid, String javacoreContent) {
     boolean foundThreadSection = false;
 
     Map<String, String> registers = null;
+    RuntimeArgs j9UserArgs = new RuntimeArgs();
 
     String[] lines = NEWLINE_SPLITTER.split(javacoreContent);
 
     for (String line : lines) {
+      // Full command line is available under 1CICMDLINE, but it's harder to parse properly,
+      // than adding them from 2CIUSERARGS
+      if (line.startsWith(J9_USER_ARG_PREFIX)) {
+        j9UserArgs.addArg(line.substring(J9_USER_ARG_PREFIX.length()).trim());
+      }
+
       // Track section changes
       if (line.startsWith(SECTION_MARKER)) {
         currentSection = detectSection(line);
@@ -290,8 +298,12 @@ public CrashLog parse(String uuid, String javacoreContent) {
     Metadata metadata = new Metadata("dd-trace-java", VersionInfo.VERSION, "java", null);
     Integer parsedPid = safelyParseInt(pid);
     ProcInfo procInfo = parsedPid != null ? new ProcInfo(parsedPid) : null;
+    List<String> runtimeArgs = j9UserArgs.build();
     Experimental experimental =
-        (registers != null && !registers.isEmpty()) ? new Experimental(registers) : null;
+        (registers != null && !registers.isEmpty())
+                || (runtimeArgs != null && !runtimeArgs.isEmpty())
+            ? new Experimental(registers, runtimeArgs)
+            : null;
 
     return new CrashLog(
         uuid,