Merge branch 'master' into ygree/aws-payload-tagging

Author: ygree

.circleci/collect_results.sh

@@ -10,8 +10,17 @@ shopt -s globstar
 TEST_RESULTS_DIR=./results
 mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1
 
-echo "saving test results"
 mkdir -p $TEST_RESULTS_DIR
-find workspace/**/build/test-results -name \*.xml -exec sh -c '
+
+mkdir -p workspace
+mapfile -t test_result_dirs < <(find workspace -name test-results -type d)
+
+if [[ ${#test_result_dirs[@]} -eq 0 ]]; then
+  echo "No test results found"
+  exit 0
+fi
+
+echo "saving test results"
+find "${test_result_dirs[@]}" -name \*.xml -exec sh -c '
   file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
   cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \;

.circleci/config.continue.yml.j2

@@ -28,15 +28,15 @@ debugger_test_matrix: &debugger_test_matrix
 
 system_test_matrix: &system_test_matrix
   parameters:
-    weblog-variant: [ 'spring-boot', 'spring-boot-jetty', 'spring-boot-openliberty', 'spring-boot-3-native', 'jersey-grizzly2', 'resteasy-netty3','ratpack', 'vertx3' ]
+    weblog-variant: ['akka-http', 'jersey-grizzly2', 'play', 'resteasy-netty3', 'ratpack', 'spring-boot', 'spring-boot-jetty', 'spring-boot-openliberty', 'spring-boot-payara', 'spring-boot-undertow', 'spring-boot-wildfly', 'spring-boot-3-native', 'uds-spring-boot', 'vertx3', 'vertx4']
 
 agent_integration_tests_modules: &agent_integration_tests_modules "dd-trace-core|communication|internal-api|utils"
 core_modules: &core_modules "dd-java-agent|dd-trace-core|communication|internal-api|telemetry|utils|dd-java-agent/agent-bootstrap|dd-java-agent/agent-installer|dd-java-agent/agent-tooling|dd-java-agent/agent-builder|dd-java-agent/appsec|dd-java-agent/agent-crashtracking|dd-trace-api|dd-trace-ot"
-instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
+instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-iast|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 0f27f203c90bba8bce7b58e2b75ff9d7035169eb
+default_system_tests_commit: &default_system_tests_commit 39bf65e0d94278c60fdd6ff85ba668cb436467f0
 
 parameters:
   nightly:
@@ -99,6 +99,11 @@ commands:
   setup_code:
     steps:
       - checkout
+{% if use_git_changes %}
+      - run:
+          name: Fetch base branch
+          command: git fetch origin {{ pr_base_ref }}
+{% endif %}
       - run:
           name: Checkout merge commit
           command: .circleci/checkout_merge_commit.sh
@@ -259,12 +264,6 @@ commands:
       - restore_build_cache:
           cacheType: lib
 
-      - run:
-          name: Install python 3.9
-          command: |
-            sudo apt-get install python3.9-full python3.9-dev python3.9-venv
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >>"$BASH_ENV"
-
       - run:
           name: Clone system-tests
           command: |
@@ -274,6 +273,13 @@ commands:
             git fetch origin << parameters.systemTestsCommit >>
             git reset --hard FETCH_HEAD
 
+      - run:
+          name: Install python 3.12
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y python3.12-full python3.12-dev python3.12-venv
+            echo 'export PATH="$HOME/.local/bin:$PATH"' >>"$BASH_ENV"
+
 jobs:
   build:
     <<: *defaults
@@ -304,13 +310,16 @@ jobs:
           name: Build Project
           command: >-
 {% if is_nightly %}
-            ./gradlew resolveAndLockAll --write-locks
+            ./gradlew resolveAndLockAll --write-locks &&
 {% endif %}
             MAVEN_OPTS="-Xms64M -Xmx256M"
             GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
             ./gradlew clean
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             << pipeline.parameters.gradle_flags >>
             --max-workers=8
             --rerun-tasks
@@ -410,6 +419,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PrunBuildSrcTests
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             << pipeline.parameters.gradle_flags >>
@@ -488,6 +500,9 @@ jobs:
       - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}<< parameters.testJvm >>
 
     parameters:
+      environment:
+        type: string
+        default: ""
       testJvm:
         type: string
         default: ""
@@ -549,12 +564,16 @@ jobs:
             then
               PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/<< parameters.stage >>-<< parameters.testJvm >>.jfr,dumponexit=true"
             fi
-            
+
+            << parameters.environment >>
             MAVEN_OPTS="-Xms64M -Xmx512M"
             GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms<< parameters.maxDaemonHeapSize >> -Xmx<< parameters.maxDaemonHeapSize >> $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M"
             ./gradlew
             << parameters.gradleTarget >>
             << parameters.gradleParameters >>
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><</ parameters.testJvm >>
             << pipeline.parameters.gradle_flags >>
@@ -614,10 +633,10 @@ jobs:
               echo "APM Test Agent is not running and was not used for testing. No checks failed."
               exit 0
             fi
-            
+
             RESPONSE=$(curl -s -w "\n%{http_code}" -o response.txt http://localhost:8126/test/trace_check/failures)
             RESPONSE_CODE=$(echo "$RESPONSE" | awk 'END {print $NF}')
-               
+
             if [[ $RESPONSE_CODE -eq 200 ]]; then
               echo "All APM Test Agent Check Traces returned successful! (HTTP 200)"
               echo "APM Test Agent Check Traces Summary Results:"
@@ -797,8 +816,7 @@ jobs:
 
   system-tests:
     machine:
-      # https://support.circleci.com/hc/en-us/articles/360007324514-How-can-I-use-Docker-volume-mounting-on-CircleCI-
-      image: ubuntu-2004:current
+      image: ubuntu-2404:current
     resource_class: medium
     parameters:
       weblog-variant:
@@ -816,7 +834,7 @@ jobs:
           name: Build
           command: |
             cd system-tests
-            ./build.sh java --weblog-variant << parameters.weblog-variant >> 
+            ./build.sh java --weblog-variant << parameters.weblog-variant >>
 
       - run:
           name: Run
@@ -849,8 +867,7 @@ jobs:
 
   integrations-system-tests:
     machine:
-      # https://support.circleci.com/hc/en-us/articles/360007324514-How-can-I-use-Docker-volume-mounting-on-CircleCI-
-      image: ubuntu-2004:current
+      image: ubuntu-2404:current
     resource_class: medium
     steps:
       - setup_system_tests
@@ -863,10 +880,9 @@ jobs:
 
       - run:
           name: Build
-          # Build the default framework, which is springboot
           command: |
             cd system-tests
-            ./build.sh java
+            ./build.sh --library java --weblog-variant spring-boot
 
       - run:
           name: Run APM Integrations tests
@@ -884,7 +900,7 @@ jobs:
 
   debugger-system-tests:
     machine:
-      image: ubuntu-2004:current
+      image: ubuntu-2404:current
     resource_class: medium
     steps:
       - setup_system_tests
@@ -897,10 +913,9 @@ jobs:
 
       - run:
           name: Build
-          # Build the default framework, which is springboot
           command: |
             cd system-tests
-            ./build.sh java
+            ./build.sh --library java --weblog-variant spring-boot
 
       - run:
           name: Run Dynamic Instrumentation system tests
@@ -918,9 +933,8 @@ jobs:
 
   parametric-tests:
     machine:
-      # https://support.circleci.com/hc/en-us/articles/360007324514-How-can-I-use-Docker-volume-mounting-on-CircleCI-
-      image: ubuntu-2004:current
-    resource_class: large
+      image: ubuntu-2404:current
+    resource_class: xlarge
     steps:
       - setup_system_tests
 
@@ -932,29 +946,26 @@ jobs:
             cp ~/dd-trace-java/workspace/dd-trace-api/build/libs/*.jar system-tests/binaries/
             cp ~/dd-trace-java/workspace/dd-java-agent/build/libs/*.jar system-tests/binaries/
 
+
       - run:
-          name: Install requirements
+          name: Build runner
           command: |
             cd system-tests
-            pyenv local system
-            python3.9 --version
-            python3.9 -m pip install wheel
-            python3.9 -m pip install -r requirements.txt
-            sudo ln -sf /usr/bin/python3.9 /usr/bin/python
+            export TEST_LIBRARY=java
+            ./build.sh -i runner
 
       - run:
           name: Run
           command: |
             set -e
             cd system-tests
             export TEST_LIBRARY=java
-            export PYTEST_WORKER_COUNT=8
-            ./build.sh -i runner
+            export PYTEST_XDIST_AUTO_NUM_WORKERS=6
             set +e
             RUN_ATTEMPTS=1
             while [ $RUN_ATTEMPTS -le 3 ]; do
               echo "Running parametric test attempt $RUN_ATTEMPTS"
-              timeout 780s ./run.sh PARAMETRIC --log-cli-level=DEBUG --durations=30 -vv
+              timeout 12m ./run.sh PARAMETRIC --log-cli-level=DEBUG --durations=30 -vv
               status=$?
               #timneout returns 124 if it times out
               #if the return code is not 124, then we exit with the status
@@ -1065,7 +1076,7 @@ build_test_jobs: &build_test_jobs
       name: z_test_<< matrix.testJvm >>_base
       triggeredBy: *core_modules
       gradleTarget: ":baseTest"
-      gradleParameters: "-PskipFlakyTests -PskipInstTests -PskipSmokeTests -PskipProfilingTests -PskipDebuggerTests"
+      gradleParameters: "-PskipFlakyTests"
       stage: core
       cacheType: base
       parallelism: 4
@@ -1078,8 +1089,8 @@ build_test_jobs: &build_test_jobs
         - ok_to_test
       name: z_test_8_base
       triggeredBy: *core_modules
-      gradleTarget: :baseTest jacocoTestReport jacocoTestCoverageVerification
-      gradleParameters: "-PskipFlakyTests -PskipInstTests -PskipSmokeTests -PskipProfilingTests -PskipDebuggerTests"
+      gradleTarget: :baseTest
+      gradleParameters: "-PskipFlakyTests -PcheckCoverage"
       stage: core
       cacheType: base
       parallelism: 4
@@ -1237,6 +1248,20 @@ build_test_jobs: &build_test_jobs
       matrix:
         <<: *test_matrix
 
+  - tests:
+      requires:
+        - ok_to_test
+      name: z_test_<< matrix.testJvm >>_ssi_smoke
+      environment: "DD_INJECT_FORCE=true DD_INJECTION_ENABLED=tracer"
+      gradleTarget: "stageMainDist :smokeTest"
+      gradleParameters: "-PskipFlakyTests"
+      stage: smoke
+      cacheType: smoke
+      parallelism: 4
+      maxWorkers: 3
+      matrix:
+        <<: *test_matrix
+
   - tests:
       requires:
         - ok_to_test
@@ -1257,7 +1282,7 @@ build_test_jobs: &build_test_jobs
       stage: smoke
       cacheType: smoke
       testJvm: "graalvm17"
-  
+
   - tests:
       requires:
         - ok_to_test
@@ -1279,11 +1304,25 @@ build_test_jobs: &build_test_jobs
       maxWorkers: 3
       testJvm: "8"
 
+  - tests:
+      requires:
+        - ok_to_test
+      name: z_test_8_ssi_smoke
+      environment: "DD_INJECT_FORCE=true DD_INJECTION_ENABLED=tracer"
+      gradleTarget: "stageMainDist :smokeTest"
+      gradleParameters: "-PskipFlakyTests"
+      stage: smoke
+      cacheType: smoke
+      parallelism: 4
+      maxWorkers: 3
+      testJvm: "8"
+
   - fan_in:
       requires:
         - z_test_<< matrix.testJvm >>_base
         - z_test_<< matrix.testJvm >>_inst
         - z_test_<< matrix.testJvm >>_smoke
+        - z_test_<< matrix.testJvm >>_ssi_smoke
       name: test_<< matrix.testJvm >>
       stage: tracing
       matrix:
@@ -1294,6 +1333,7 @@ build_test_jobs: &build_test_jobs
         - z_test_8_base
         - z_test_8_inst
         - z_test_8_smoke
+        - z_test_8_ssi_smoke
       name: test_8
       stage: tracing
       testJvm: "8"