Merge remote-tracking branch 'origin/main' into pawel/nix_fmt

.circleci/config.yml

@@ -4,20 +4,20 @@ jobs:
   system-tests:
     machine:
       # https://support.circleci.com/hc/en-us/articles/360007324514-How-can-I-use-Docker-volume-mounting-on-CircleCI-
-      image: ubuntu-2004:current
+      image: ubuntu-2404:current
     resource_class: large
     steps:
       - checkout
 
       - run:
-          name: Install python 3.9
-          command: sudo apt-get install python3.9-venv
+          name: Install python 3.12
+          command: sudo apt-get install python3.12-venv
 
       - run:
           name: versions
           command: |
             docker --version
-            python3.9 --version
+            python3.12 --version
             pip --version
 
       - run:

.dockerignore

@@ -8,6 +8,7 @@ logs*
 shell.nix
 __pycache__
 scenario_groups.yml
+/binaries/**/.git/config
 
 # dotnet
 **/dotnet/**/bin/
@@ -16,3 +17,4 @@ scenario_groups.yml
 **/dotnet/**/out/
 **/dotnet/**/*.user
 **/dotnet/**/*.suo
+/system-tests.sln

.gitattributes

@@ -0,0 +1,11 @@
+# default behavior for line endings in text files
+* text=auto
+
+# Set line endings in *.sh files to LF, even on Windows.
+# Otherwise, execution within Docker fails on a Windos host.
+# See https://docs.github.com/en/get-started/getting-started-with-git/configuring-git-to-handle-line-endings
+*.sh text eol=lf
+
+# Stop hiding directories called "build" in the github file search
+# https://docs.github.com/en/search-github/searching-on-github/finding-files-on-github#customizing-excluded-files
+build/** linguist-generated=false

.github/CODEOWNERS

@@ -1,22 +1,32 @@
 * @DataDog/system-tests-core
+
+/utils/build/docker/cpp/ @DataDog/dd-trace-cpp @DataDog/system-tests-core
 /utils/build/docker/dotnet*/ @DataDog/apm-dotnet @DataDog/asm-dotnet @DataDog/system-tests-core
-/utils/build/docker/golang*/ @DataDog/apm-go @DataDog/system-tests-core
+/utils/build/docker/golang*/ @DataDog/dd-trace-go-guild @DataDog/system-tests-core
 /utils/build/docker/java*/ @DataDog/apm-java @DataDog/asm-java @DataDog/system-tests-core
 /utils/build/docker/java_otel/ @DataDog/opentelemetry @DataDog/system-tests-core
 /utils/build/docker/nodejs*/ @DataDog/apm-js @DataDog/asm-js @DataDog/system-tests-core
 /utils/build/docker/php*/ @DataDog/apm-php @DataDog/system-tests-core
 /utils/build/docker/python*/ @DataDog/apm-python @DataDog/asm-python @DataDog/system-tests-core
-/utils/build/docker/ruby*/ @DataDog/apm-ruby @DataDog/asm-ruby @DataDog/system-tests-core
-/parametric/ @Kyle-Verhoog @DataDog/system-tests-core
-/tests/parametric/ @Kyle-Verhoog @DataDog/system-tests-core
+/utils/build/docker/ruby*/ @DataDog/ruby-guild @DataDog/asm-ruby @DataDog/system-tests-core
+
+/tests/parametric/ @mabdinur @DataDog/system-tests-core @DataDog/apm-sdk-api
 /tests/otel_tracing_e2e/ @DataDog/opentelemetry @DataDog/system-tests-core
-/tests/remote_config/ @DataDog/system-tests-core @DataDog/remote-config @DataDog/system-tests-core
+/tests/remote_config/ @DataDog/remote-config @DataDog/system-tests-core
 /tests/appsec/ @DataDog/asm-libraries @DataDog/system-tests-core
-/manifests/cpp.yml @DataDog/system-tests-core
-/manifests/dotnet.yml @DataDog/apm-dotnet @DataDog/asm-dotnet @DataDog/system-tests-core
-/manifests/golang.yml @DataDog/dd-trace-go-guild @DataDog/system-tests-core
-/manifests/java.yml @DataDog/asm-java @DataDog/apm-java @DataDog/system-tests-core
-/manifests/nodejs.yml @DataDog/apm-js @DataDog/asm-js @DataDog/system-tests-core
-/manifests/php.yml @DataDog/apm-php @DataDog/asm-php @DataDog/system-tests-core
-/manifests/python.yml @DataDog/apm-python @DataDog/asm-python @DataDog/system-tests-core
-/manifests/ruby.yml @DataDog/apm-ruby @DataDog/asm-ruby @DataDog/system-tests-core
+/tests/debugger/ @DataDog/debugger @DataDog/system-tests-core
+/tests/telemetry_intake/ @DataDog/apm-sdk-api @DataDog/system-tests-core
+/tests/test_telemetry.py @DataDog/libdatadog-telemetry @DataDog/apm-sdk-api @DataDog/system-tests-core
+/tests/serverless @DataDog/serverless @DataDog/system-tests-core
+
+/manifests/cpp.yml @DataDog/dd-trace-cpp
+/manifests/dotnet.yml @DataDog/apm-dotnet @DataDog/asm-dotnet
+/manifests/golang.yml @DataDog/dd-trace-go-guild
+/manifests/java.yml @DataDog/asm-java @DataDog/apm-java
+/manifests/nodejs.yml @DataDog/apm-js @DataDog/asm-js
+/manifests/php.yml @DataDog/apm-php @DataDog/asm-php
+/manifests/python.yml @DataDog/apm-python @DataDog/asm-python
+/manifests/ruby.yml @DataDog/ruby-guild @DataDog/asm-ruby
+
+# Allows everyone to easily make changes
+/tests/telemetry_intake/static/ @DataDog/apm-ecosystems

.github/actions/install_runner/action.yaml

@@ -1,11 +1,12 @@
 name: Install_system_tests_runner
 description: "Install python virtual env, and system dependencies, using github action cache if possible"
+
 runs:
   using: composite
   steps:
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.9"
+        python-version: "3.12"
     - uses: actions/cache@v4
       id: runner_cache
       with:

.github/actions/lint_code/action.yaml

@@ -26,7 +26,7 @@ runs:
         node-version: 20
     - name: 'Run nodejs lint'
       shell: bash
-      working-directory: ./utils/build/docker/nodejs/express4
+      working-directory: ./utils/build/docker/nodejs/express
       run: |
         npm install
         npm run lint

.github/actions/pull_images/action.yml

@@ -1,51 +1,62 @@
 name: "Pull docker images"
 description: "Pull docker images"
 inputs:
-  pull-all:
-    description: "True to pull all images required for all scenarios"
+  library:
+    description: "Which library will be tested (python, cpp, java, ...)"
     required: true
-    default: "false"
+  weblog:
+    description: "Which weblog will be tested"
+    required: true
+  scenarios:
+    description: "JSON array of scenarios that will be executed"
+    required: true
+  cleanup:
+    description: "Whether to cleanup the disk to free up more space. Should be disabled when a larger machine can be used instead."
+    required: false
+    default: "true"
+  dockerhub_username:
+    description: "To prevent reaching docker hub API limits, provide a username and token to login to docker hub"
+    required: false
+    default: ""
+  dockerhub_token:
+    description: "To prevent reaching docker hub API limits, provide a username and token to login to docker hub"
+    required: false
+    default: ""
+
 runs:
   using: composite
   steps:
-    - name: Let's free some place
+    - name: Free some place for scenarios known to require lot of images
+      if: ${{ (inputs.cleanup == 'true') && (contains(inputs.scenarios, '"INTEGRATIONS"') || contains(inputs.scenarios, '"CROSSED_TRACING_LIBRARIES"')) }}
       shell: bash
       run: |
-        if [[ "${{ inputs.pull-all }}" == "true" ]]; then
-          df -h
+        df -h
 
-          echo "Removing docker images, dotnet, android, ghc, and CodeQL cache to free space"
+        echo "Removing docker images, dotnet, android, ghc, and CodeQL cache to free space"
 
-          sudo rm -rf /usr/local/lib/android  # 9Gb!
-          sudo rm -rf /opt/hostedtoolcache/CodeQL  # 5Gb !
-          sudo rm -rf /usr/share/dotnet # 1Gb
+        sudo rm -rf /usr/local/lib/android  # 9Gb!
+        sudo rm -rf /opt/hostedtoolcache/CodeQL  # 5Gb !
+        sudo rm -rf /usr/share/dotnet # 1Gb
 
-          # sudo docker image prune --all --force  # if ever needed, but it's slow. Only 3Gb
+        # sudo docker image prune --all --force  # if ever needed, but it's slow. Only 3Gb
 
-          df -h
-        fi
+        df -h
 
-    - name: Pull images needed for default scenario
+    - name: Get image list
       shell: bash
       run: |
-        docker pull datadog/system-tests:proxy-v1 || true
-        docker pull postgres:latest
-    - name: Pull images needed for all scenarios
+        source venv/bin/activate
+        python utils/scripts/get-image-list.py ${{ inputs.library }} ${{ inputs.weblog }} '${{ inputs.scenarios }}' > compose.yaml
+      env:
+        PYTHONPATH: "."
+
+    - name: Login to Docker Hub
+      if: inputs.dockerhub_username != '' && inputs.dockerhub_token != ''
+      uses: docker/login-action@v3
+      with:
+        username: ${{ inputs.dockerhub_username }}
+        password: ${{ inputs.dockerhub_token }}
+
+    - name: Pull
       shell: bash
-      run: |
-        if [[ "${{ inputs.pull-all }}" == "true" ]]; then
-          docker pull mongo:latest
-          docker pull bitnami/kafka:3.1
-          docker pull bitnami/zookeeper:latest
-          docker pull cassandra:latest
-          docker pull rabbitmq:3-management-alpine
-          docker pull mysql/mysql-server:latest
-          docker pull mcr.microsoft.com/mssql/server:latest
-          docker pull mcr.microsoft.com/azure-sql-edge:latest
-          docker pull softwaremill/elasticmq:latest
-          docker pull datadog/system-tests:python_buddy-v0
-          docker pull datadog/system-tests:nodejs_buddy-v0
-          docker pull datadog/system-tests:java_buddy-v0
-          docker pull datadog/system-tests:ruby_buddy-v0
-          docker pull datadog/system-tests:golang_buddy-v0
-        fi
+      run: docker compose pull

.github/pull_request_template.md

@@ -8,7 +8,6 @@
 
 ## Workflow
 
-
 1. ⚠️ Create your PR as draft ⚠️
 2. Work on you PR until the CI passes (if something not related to your task is failing, you can ignore it)
 3. Mark it as ready for review
@@ -21,12 +20,10 @@
 
 ## Reviewer checklist
 
-* [ ] [Relevant label](https://github.com/DataDog/system-tests/blob/main/docs/CI/labels.md) (`run-parametric-scenario`, `run-profiling-scenario`...) are presents
 * [ ] If PR title starts with `[<language>]`, double-check that only `<language>` is impacted by the change
 * [ ] No system-tests internal is modified. Otherwise, I have the approval from [R&P team](https://dd.enterprise.slack.com/archives/C025TJ4RZ8X)
 * [ ] CI is green, or failing jobs are not related to this change (and you are 100% sure about this statement)
 * [ ] A docker base image is modified?
     * [ ] the relevant `build-XXX-image` label is present
 * [ ] A scenario is added (or removed)?
     * [ ] Get a review from [R&P team](https://dd.enterprise.slack.com/archives/C025TJ4RZ8X)
-

.github/workflows/ci.yml

@@ -1,8 +1,6 @@
 name: Testing the test
 on:
   workflow_dispatch: {}
-  schedule:
-  - cron: 00 02 * * 2-6
 
   pull_request:
     branches:
@@ -13,10 +11,6 @@ on:
     - labeled
     - unlabeled
 
-  push:
-    branches:
-    - main
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -47,27 +41,6 @@ jobs:
           /bin/bash run.sh ++dry APPSEC_SCENARIOS
           /bin/bash run.sh ++dry TRACER_RELEASE_SCENARIOS
 
-  # commenting, as python 3.9 is not available anymore
-  # test_the_test_platforms:
-  #   strategy:
-  #     matrix:
-  #       os:
-  #         - macos-latest
-  #   name: Test the test (${{ matrix.os }})
-  #   runs-on: ${{ matrix.os }}
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v4
-  #     - name: Install runner
-  #       uses: ./.github/actions/install_runner
-  #     # force /bin/bash in order to test against bash 3.2 on macOS
-  #     - name: Test the test (direct)
-  #       run: /bin/bash run.sh TEST_THE_TEST
-  #     - name: Test group parsing
-  #       run: |
-  #         /bin/bash run.sh ++dry APPSEC_SCENARIOS
-  #         /bin/bash run.sh ++dry TRACER_RELEASE_SCENARIOS
-
   scenarios:
     name: Get scenarios and groups
     uses: ./.github/workflows/compute-scenarios.yml
@@ -89,8 +62,10 @@ jobs:
         uses: actions/checkout@v4
       - name: Get library artifact
         run: ./utils/scripts/load-binary.sh ${{ matrix.library }}
+
       - name: Get agent artifact
         run: ./utils/scripts/load-binary.sh agent
+
       # ### appsec-event-rules is now a private repo. The GH_TOKEN provided can't read private repos.
       # ### skipping this, waiting for a proper solution
       # - name: Load WAF rules
@@ -120,6 +95,9 @@ jobs:
         - dev
       fail-fast: false
     uses: ./.github/workflows/system-tests.yml
+    permissions:
+      contents: read
+      packages: write
     secrets: inherit
     with:
       library: ${{ matrix.library }}
@@ -131,6 +109,8 @@ jobs:
       build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
       build_lib_injection_app_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-lib-injection-app-images') }}
+      _experimental_parametric_job_count: ${{ matrix.version == 'dev' && 2 || 1 }}  # test both use cases
+      skip_empty_scenarios: true
 
   system_tests_docker_mode:
     name: Ruby Docker Mode
@@ -141,6 +121,8 @@ jobs:
     - get_dev_artifacts  # non official set-up, this needs put this job in last
     if: contains(needs.impacted_libraries.outputs.impacted_libraries, 'ruby')
     uses: ./.github/workflows/run-docker-mode.yml
+    permissions:
+      packages: write
     secrets: inherit
 
   exotics:
@@ -149,6 +131,21 @@ jobs:
     uses: ./.github/workflows/run-exotics.yml
     secrets: inherit
 
+  fancy-report:
+    runs-on: ubuntu-latest
+    needs:
+    - system_tests
+    if: always()
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - run: pip install requests
+    - run: python utils/scripts/get-workflow-summary.py ${{ github.run_id }} >> $GITHUB_STEP_SUMMARY
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   update-CI-visibility:
     name: Update CI Visibility Dashboard
     runs-on: ubuntu-latest