Merge branch 'master' into kr-igor/dsm-service-name-override

Author: kr-igor

.circleci/config.continue.yml.j2

@@ -816,6 +816,7 @@ jobs:
     parameters:
       weblog-variant:
         type: string
+    parallelism: 3
     steps:
       - setup_system_tests
 
@@ -833,32 +834,38 @@ jobs:
 
       - run:
           name: Run
-          command: |
-            cd system-tests
-            DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh
-
-      - run:
-          name: Run APM E2E default tests
           # Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
           no_output_timeout: 5m
           command: |
             cd system-tests
-            DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh APM_TRACING_E2E
+            echo "
+            DEFAULT
+            APM_TRACING_E2E
+            APM_TRACING_E2E_SINGLE_SPAN
+            " | circleci tests split > scenarios.list
+            for scenario in $(<scenarios.list); do
+              if [[ $scenario =~ .*_E2E.* ]]; then
+                export DD_SITE=datadoghq.com
+                export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
+                export DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY
+              else
+                export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
+              fi
+              echo "Running scenario $scenario"
+              ./run.sh $scenario
+            done
 
       - run:
-          name: Run APM E2E Single Span tests
-          # Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
-          no_output_timeout: 5m
+          name: Collect artifacts
           command: |
+            mkdir -p artifacts
             cd system-tests
-            DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh APM_TRACING_E2E_SINGLE_SPAN
-
-      - run:
-          name: Collect artifacts
-          command: tar -cvzf logs_java_<< parameters.weblog-variant >>_dev.tar.gz -C system-tests logs logs_apm_tracing_e2e logs_apm_tracing_e2e_single_span
+            for log_dir in logs*; do
+              tar -cvzf ../artifacts/${log_dir}_<< parameters.weblog-variant >>.tar.gz $log_dir
+            done
 
       - store_artifacts:
-          path: logs_java_<< parameters.weblog-variant >>_dev.tar.gz
+          path: artifacts
 
   integrations-system-tests:
     machine:
@@ -1363,6 +1370,14 @@ build_test_jobs: &build_test_jobs
       stage: tracing
       testJvm: "8"
 
+  - fan_in:
+      requires:
+        - test_8_inst_latest
+        - test_17_inst_latest
+        - test_21_inst_latest
+      name: test_inst_latest
+      stage: tracing
+
   - agent_integration_tests:
       requires:
         - ok_to_test
@@ -1435,6 +1450,7 @@ build_test_jobs: &build_test_jobs
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}"
 {% endfor %}
+        - test_inst_latest
         - muzzle
         - profiling
         - debugger

.circleci/render_config.py

@@ -28,7 +28,7 @@
 }
 # Version to use for all the base Docker images, see
 # https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build
-DOCKER_IMAGE_VERSION="v24.08"
+DOCKER_IMAGE_VERSION="v24.10"
 
 # Get labels from pull requests to override some defaults for jobs to run.
 # `run-tests: all` will run all tests.

.circleci/upload_ciapp.sh

@@ -1,26 +1,42 @@
 #!/usr/bin/env bash
 SERVICE_NAME="dd-trace-java"
+PIPELINE_STAGE=$1
+TEST_JVM=$2
 
 # JAVA_???_HOME are set in the base image for each used JDK https://github.com/DataDog/dd-trace-java-docker-build/blob/master/Dockerfile#L86
-java_home="JAVA_$2_HOME"
-java_bin="${!java_home}/bin/java"
-if [ ! -x $java_bin ]; then
-    java_bin=$(which java)
+JAVA_HOME="JAVA_${TEST_JVM}_HOME"
+JAVA_BIN="${!JAVA_HOME}/bin/java"
+if [ ! -x "$JAVA_BIN" ]; then
+    JAVA_BIN=$(which java)
 fi
 
-java_props=$($java_bin -XshowSettings:properties -version 2>&1)
-java_prop () {
-    echo "$(echo "$java_props" | grep $1 | head -n1 | cut -d'=' -f2 | xargs)"
+# Extract Java properties from the JVM used to run the tests
+JAVA_PROPS=$($JAVA_BIN -XshowSettings:properties -version 2>&1)
+java_prop() {
+    local PROP_NAME=$1
+    echo "$JAVA_PROPS" | grep "$PROP_NAME" | head -n1 | cut -d'=' -f2 | xargs
 }
 
-# based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
-datadog-ci junit upload --service $SERVICE_NAME \
-    --logs \
-    --tags "test.traits:{\"marker\":[\"$1\"]}" \
-    --tags "runtime.name:$(java_prop java.runtime.name)" \
-    --tags "runtime.vendor:$(java_prop java.vendor)" \
-    --tags "runtime.version:$(java_prop java.version)" \
-    --tags "os.architecture:$(java_prop os.arch)" \
-    --tags "os.platform:$(java_prop os.name)" \
-    --tags "os.version:$(java_prop os.version)" \
-./results
+# Upload test results to CI Visibility
+junit_upload() {
+    # based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
+    DD_API_KEY=$1 \
+        datadog-ci junit upload --service $SERVICE_NAME \
+        --logs \
+        --tags "test.traits:{\"marker\":[\"$PIPELINE_STAGE\"]}" \
+        --tags "runtime.name:$(java_prop java.runtime.name)" \
+        --tags "runtime.vendor:$(java_prop java.vendor)" \
+        --tags "runtime.version:$(java_prop java.version)" \
+        --tags "os.architecture:$(java_prop os.arch)" \
+        --tags "os.platform:$(java_prop os.name)" \
+        --tags "os.version:$(java_prop os.version)" \
+        ./results
+}
+
+# Make sure we do not use DATADOG_API_KEY from the environment
+unset DATADOG_API_KEY
+
+# Upload test results to production environment like all other CI jobs
+junit_upload "$DATADOG_API_KEY_PROD"
+# And also upload to staging environment to benefit from the new features not yet released
+junit_upload "$DATADOG_API_KEY_DDSTAGING"

.github/workflows/README.md

@@ -20,6 +20,14 @@ _Action:_ Append the new release to the Cloud Foundry repository.
 
 _Recovery:_ Manually edit and push the `index.yml` file from [the cloudfoundry branch](https://github.com/DataDog/dd-trace-java/tree/cloudfoundry).
 
+### check-pull-requests [🔗](check-pull-requests.yaml)
+
+_Trigger:_ When creating or updating a pull request.
+
+_Action:_ Check the pull request complies with [the contribution guidelines](https://github.com/DataDog/dd-trace-java/blob/master/CONTRIBUTING.md).
+
+_Recovery:_ Manually verify the guideline compliance.
+
 ### create-next-milestone [🔗](create-next-milestone.yaml)
 
 _Trigger:_ When closing a milestone.
@@ -46,6 +54,7 @@ _Recovery:_ Manually trigger the action again on the relevant tag.
 _Trigger:_ When creating a minor or major version tag.
 
 _Actions:_
+
 * Close the milestone related to the tag,
 * Create a new milestone by incrementing minor version.
 
@@ -68,6 +77,7 @@ _Notes:_ _Download releases_ are special GitHub releases with fixed URL and tags
 _Trigger:_ When a release is published. Releases of type `prereleased` should skip this.
 
 _Action:_
+
 * Find all issues related to the release by checking the related milestone,
 * Add a comment to let know the issue was addressed by the newly published release,
 * Close all those issues.
@@ -97,10 +107,13 @@ _Recovery:_ Manually trigger the action again.
 
 _Trigger:_ When pushing commits to `master` or any pull request targeting `master`.
 
-_Action:_ 
+_Action:_
+
 * Run [DataDog Static Analysis](https://docs.datadoghq.com/static_analysis/) and upload result to DataDog Code Analysis,
-* Run [GitHub CodeQL](https://codeql.github.com/) action, upload result to GitHub security tab and DataDog Code Analysis -- do not apply to pull request, only when pushing to `master`,
-* Run [Trivy security scanner](https://github.com/aquasecurity/trivy) on built artifacts and upload result to GitHub security tab.
+* Run [GitHub CodeQL](https://codeql.github.com/) action, upload result to GitHub security tab -- do not apply to pull request, only when pushing to `master`,
+* Run [Trivy security scanner](https://github.com/aquasecurity/trivy) on built artifacts and upload result to GitHub security tab and Datadog Code Analysis.
+
+_Notes:_ Results are sent on both production and staging environments.
 
 ### comment-on-submodule-update [🔗](comment-on-submodule-update.yaml)
 

.github/workflows/analyze-changes.yaml

@@ -16,13 +16,25 @@ jobs:
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
         with:
           submodules: 'recursive'
-      - name: Check code meets quality standards
+      - name: Check code meets quality standards (production)
         id: datadog-static-analysis
         uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1
         with:
-          dd_app_key: ${{ secrets.DD_APP_KEY }}
-          dd_api_key: ${{ secrets.DD_API_KEY }}
-          dd_site: datad0g.com
+          dd_app_key: ${{ secrets.DATADOG_APP_KEY_PROD }}
+          dd_api_key: ${{ secrets.DATADOG_API_KEY_PROD }}
+          dd_site: "datadoghq.com"
+          dd_service: "dd-trace-java"
+          dd_env: "ci"
+          cpu_count: 2
+          enable_performance_statistics: false
+      # Also run the static analysis on the staging environment to benefit from the new features not yet released
+      - name: Check code meets quality standards (staging)
+        id: datadog-static-analysis-staging
+        uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1
+        with:
+          dd_app_key: ${{ secrets.DATADOG_APP_KEY_STAGING }}
+          dd_api_key: ${{ secrets.DATADOG_API_KEY_STAGING }}
+          dd_site: "datad0g.com"
           dd_service: "dd-trace-java"
           dd_env: "ci"
           cpu_count: 2
@@ -77,11 +89,19 @@ jobs:
       # For now, CodeQL SARIF results are not supported by Datadog CI
       # - name: Upload results to Datadog CI Static Analysis
       #   run: |
-      #     wget --no-verbose https://github.com/DataDog/datadog-ci/releases/download/v2.42.0/datadog-ci_linux-x64 -O datadog-ci
+      #     wget --no-verbose https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64 -O datadog-ci
       #     chmod +x datadog-ci
       #     ./datadog-ci sarif upload /home/runner/work/dd-trace-java/results/java.sarif --service dd-trace-java --env ci
       #   env:
-      #     DD_API_KEY: ${{ secrets.DD_API_KEY }}
+      #     DD_API_KEY: ${{ secrets.DATADOG_APP_KEY_PROD }}
+      #     DD_SITE: datadoghq.com
+
+      # For now, CodeQL SARIF results are not supported by Datadog CI
+      # - name: Upload results to Datadog Staging CI Static Analysis
+      #   run: |
+      #     ./datadog-ci sarif upload /home/runner/work/dd-trace-java/results/java.sarif --service dd-trace-java --env ci
+      #   env:
+      #     DD_API_KEY: ${{ secrets.DATADOG_API_KEY_STAGING }}
       #     DD_SITE: datad0g.com
 
   trivy:
@@ -140,6 +160,9 @@ jobs:
           output: 'trivy-results.sarif'
           severity: 'CRITICAL,HIGH'
           limit-severities-for-sarif: true
+        env:
+          TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db
+          TRIVY_JAVA_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-java-db,public.ecr.aws/aquasecurity/trivy-java-db
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
@@ -149,9 +172,16 @@ jobs:
 
       - name: Upload results to Datadog CI Static Analysis
         run: |
-          wget --no-verbose https://github.com/DataDog/datadog-ci/releases/download/v2.42.0/datadog-ci_linux-x64 -O datadog-ci
+          wget --no-verbose https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64 -O datadog-ci
           chmod +x datadog-ci
           ./datadog-ci sarif upload trivy-results.sarif --service dd-trace-java --env ci
         env:
-          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          DD_API_KEY: ${{ secrets.DATADOG_API_KEY_PROD }}
+          DD_SITE: datadoghq.com
+
+      - name: Upload results to Datadog Staging CI Static Analysis
+        run: |
+          ./datadog-ci sarif upload trivy-results.sarif --service dd-trace-java --env ci
+        env:
+          DD_API_KEY: ${{ secrets.DATADOG_API_KEY_STAGING }}
           DD_SITE: datad0g.com

.github/workflows/check-pull-requests.yaml

@@ -0,0 +1,75 @@
+name: Check pull requests
+on:
+  pull_request:
+    types: [opened, edited, labeled, unlabeled]
+    branches:
+    - master
+    - release/v*
+jobs:
+  check_pull_requests:
+    name: Check pull requests
+    permissions:
+      issues: write # Required to create a comment on the pull request
+      pull-requests: write # Required to create a comment on the pull request
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check pull requests
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # 7.0.1
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            // Skip draft pull requests
+            if (context.payload.pull_request.draft) {
+              return
+            }
+            // Check at least one type and (component or instrumentation) label is set
+            const labels = context.payload.pull_request.labels.map(label => label.name)
+            const ignoreReleaseNotes = labels.filter(label => label == 'tag: no release notes').length > 0
+            const hasTypeLabel = labels.filter(label => label.startsWith('type:')).length > 0
+            const hasComponentLabel = labels.filter(label => label.startsWith('comp:')).length > 0
+            const hasInstrumentationLabel = labels.filter(label => label.startsWith('instr:')).length > 0
+            const labelsCheckFailed = !ignoreReleaseNotes && (!hasTypeLabel || (!hasComponentLabel && !hasInstrumentationLabel));
+            if (labelsCheckFailed) {
+              core.setFailed('Please add at least one type, and one component or instrumentation label to the pull request.')
+            }
+            // Check title does not contain tag
+            const title = context.payload.pull_request.title
+            const titleCheckFailed = title.match(/\[.*\]/)
+            if (titleCheckFailed) {
+              core.setFailed('Please remove the tag from the pull request title.')
+            }
+            // Add comment to the pull request
+            if (labelsCheckFailed || titleCheckFailed) {
+              // Define comment body
+              const commentMarker = '<!-- dd-trace-java-check-pull-requests-workflow -->'
+              const commentBody = 'Hi! 👋 Thanks for your pull request! 🎉\n\n' +
+                  'To help us review it, please make sure to:\n\n' +
+                  (labelsCheckFailed ? '* Add at least one type, and one component or instrumentation label to the pull request\n' : '') +
+                  (titleCheckFailed ? '* Remove the tag from the pull request title\n' : '') +
+                  '\nIf you need help, please check our [contributing guidelines](https://github.com/DataDog/dd-trace-java/blob/master/CONTRIBUTING.md).' +
+                  '\n\n' + commentMarker
+              // Look for previous comment
+              const comments = await github.rest.issues.listComments({
+                issue_number: context.payload.pull_request.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo
+              })
+              const previousComment = comments.data.find(comment => comment.body.includes(commentMarker))
+              if (previousComment) {
+                // Update previous comment
+                await github.rest.issues.updateComment({
+                  comment_id: previousComment.id,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body: commentBody
+                })
+              } else {
+                // Create new comment
+                await github.rest.issues.createComment({
+                  issue_number: context.payload.pull_request.number,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body: commentBody
+                })
+              }
+            }

.github/workflows/tests/check-pull-requests/payload-pull-request-draft.json

@@ -0,0 +1,8 @@
+{
+    "pull_request": {
+        "number": 7884,
+        "draft": true,
+        "labels": [],
+        "title": "Adding some new features"
+    }
+}

.github/workflows/tests/check-pull-requests/payload-pull-request-missing-label.json

@@ -0,0 +1,12 @@
+{
+    "pull_request": {
+        "number": 7884,
+        "draft": false,
+        "labels": [
+            {
+                "name": "comp: api"
+            }
+        ],
+        "title": "Adding some new features"
+    }
+}

.github/workflows/tests/check-pull-requests/payload-pull-request-no-release-notes.json

@@ -0,0 +1,12 @@
+{
+    "pull_request": {
+        "number": 7884,
+        "draft": false,
+        "labels": [
+            {
+                "name": "tag: no release notes"
+            }
+        ],
+        "title": "Adding some new features"
+    }
+}