Add parallel mode in CI #4035 (#4158)

Author: cbeauchesne

.github/workflows/ci.yml

@@ -130,6 +130,7 @@ jobs:
       build_lib_injection_app_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-lib-injection-app-images') }}
       parametric_job_count: ${{ matrix.version == 'dev' && 2 || 1 }}  # test both use cases
       skip_empty_scenarios: true
+      desired_execution_time: 3600  # one hour
 
   system_tests_docker_mode:
     name: Ruby Docker Mode

.github/workflows/compute-workflow-parameters.yml

@@ -20,6 +20,11 @@ on:
         default: 1
         required: false
         type: number
+      desired_execution_time:
+        description: "In seconds, system-tests will try to respect this time budget."
+        default: -1
+        required: false
+        type: number
       _ci_environment:
         description: "Which CI environment is running the tests, used for FPD. used internally to skip some weblogs on prod"
         default: 'custom'
@@ -49,16 +54,24 @@ on:
       parametric_job_matrix:
         description: ""
         value: ${{ jobs.main.outputs.parametric_job_matrix }}
-      endtoend_defs_weblogs:
+      endtoend_defs_parallel_enable:
+        description: ""
+        value: ${{ jobs.main.outputs.endtoend_defs_parallel_enable }}
+      endtoend_defs_parallel_weblogs:
         description: ""
-        value: ${{ jobs.main.outputs.endtoend_defs_weblogs }}
+        value: ${{ jobs.main.outputs.endtoend_defs_parallel_weblogs }}
+      endtoend_defs_parallel_jobs:
+        description: ""
+        value: ${{ jobs.main.outputs.endtoend_defs_parallel_jobs }}
 
 jobs:
   main:
     name: Get parameters
     runs-on: ubuntu-latest
     outputs:
-      endtoend_defs_weblogs: ${{ steps.main.outputs.endtoend_defs_weblogs }}
+      endtoend_defs_parallel_enable: ${{ steps.main.outputs.endtoend_defs_parallel_enable }}
+      endtoend_defs_parallel_weblogs: ${{ steps.main.outputs.endtoend_defs_parallel_weblogs }}
+      endtoend_defs_parallel_jobs: ${{ steps.main.outputs.endtoend_defs_parallel_jobs }}
       externalprocessing_scenarios: ${{ steps.main.outputs.externalprocessing_scenarios }}
       libinjection_scenarios: ${{ steps.main.outputs.libinjection_scenarios }}
       libinjection_enable: ${{ steps.main.outputs.libinjection_enable }}
@@ -81,18 +94,18 @@ jobs:
         python utils/scripts/compute-workflow-parameters.py ${{ inputs.library }} \
           -s "${{ inputs.scenarios }}" \
           -g "${{ inputs.scenarios_groups }}" \
+          -t ${{ inputs.desired_execution_time }} \
           --parametric-job-count ${{ inputs.parametric_job_count }} \
-          --ci-environment "${{ inputs._ci_environment }}" >> $GITHUB_OUTPUT
-      env:
-        PYTHONPATH: "."
+          --ci-environment "${{ inputs._ci_environment }}" \
+          --output $GITHUB_OUTPUT
     - name: log
       run: |
         source venv/bin/activate
         python utils/scripts/compute-workflow-parameters.py ${{ inputs.library }} \
           -s "${{ inputs.scenarios }}" \
           -g "${{ inputs.scenarios_groups }}" \
+          -t ${{ inputs.desired_execution_time }} \
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --ci-environment "${{ inputs._ci_environment }}" \
           --format json | jq
-      env:
-        PYTHONPATH: "."
+

.github/workflows/run-end-to-end.yml

@@ -51,6 +51,10 @@ on:
         default: false
         required: false
         type: boolean
+      logs_artifact_name:
+        description: "The name of the artifact to use for logs"
+        required: true
+        type: string
 
 env:
   REGISTRY: ghcr.io
@@ -371,7 +375,7 @@ jobs:
       uses: actions/upload-artifact@v4
       with:
         # log name convention to respect : logs_$SCENARIO-FAMILY_$LIBRARY_$WEBLOG_$CI-ENVIRONMENT
-        name: logs_endtoend_${{ inputs.library }}_${{ inputs.weblog }}_${{ inputs.ci_environment }}
+        name: ${{ inputs.logs_artifact_name }}
         path: artifact.tar.gz
 
     - name: Push results to Feature Parity Dashboard

.github/workflows/system-tests.yml

@@ -61,6 +61,11 @@ on:
         default: 1
         required: false
         type: number
+      desired_execution_time:
+        description: "In seconds, system-tests will try to respect this time budget."
+        default: -1
+        required: false
+        type: number
       _experimental_parametric_job_count:
         description: "*DEPRECATED*"
         default: 1
@@ -76,6 +81,7 @@ jobs:
       scenarios: ${{ inputs.scenarios }}
       scenarios_groups: ${{ inputs.scenarios_groups }}
       parametric_job_count: ${{ inputs.parametric_job_count }}
+      desired_execution_time: ${{ inputs.desired_execution_time }}
       _ci_environment: ${{ inputs.ci_environment }}
 
   parametric:
@@ -91,7 +97,7 @@ jobs:
       job_count: ${{ inputs.parametric_job_count }}
       job_matrix: ${{ needs.compute_parameters.outputs.parametric_job_matrix }}
 
-  lib-injection:
+  lib_injection:
     needs:
       - compute_parameters
     if: ${{ needs.compute_parameters.outputs.libinjection_enable == 'true' }}
@@ -105,29 +111,66 @@ jobs:
       library: ${{ inputs.library }}
       version: ${{ inputs.binaries_artifact == '' && 'prod' || 'dev' }}
 
-  end-to-end:
-    name: end-to-end (${{ matrix.weblogs.weblog_name }})
+  build_end_to_end:
+    name: Build end-to-end (${{ matrix.weblog }})
     needs:
       - compute_parameters
-    if: ${{ needs.compute_parameters.outputs.endtoend_defs_weblogs != '[]' }}
+    if: ${{ needs.compute_parameters.outputs.endtoend_defs_parallel_enable == 'true' }}
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        weblogs: ${{ fromJson(needs.compute_parameters.outputs.endtoend_defs_weblogs) }}
+        weblog: ${{ fromJson(needs.compute_parameters.outputs.endtoend_defs_parallel_weblogs) }}
+      fail-fast: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Get binaries artifact
+        if : ${{ inputs.binaries_artifact != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.binaries_artifact }}
+          path: binaries/
+      - name: Build python's weblog base images
+        if: inputs.library == 'python' && inputs.build_python_base_images
+        run: |
+          ./utils/build/build_python_base_images.sh
+      - name: Build agent
+        run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh -i agent -s
+      - name: Build weblog
+        id: build
+        run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog }} -s
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries_${{ inputs.ci_environment}}_${{ inputs.library }}_${{ matrix.weblog }}
+          path: binaries/
+
+  run_end_to_end:
+    name: "End-to-end #${{ matrix.job.weblog_instance }}"
+    needs:
+      - compute_parameters
+      - build_end_to_end
+    if: ${{ needs.compute_parameters.outputs.endtoend_defs_parallel_enable == 'true' }}
+    strategy:
+      matrix:
+        job: ${{ fromJson(needs.compute_parameters.outputs.endtoend_defs_parallel_jobs) }}
       fail-fast: false
     uses: ./.github/workflows/run-end-to-end.yml
     secrets: inherit
     with:
-      library: ${{ matrix.weblogs.library }}
-      weblog: ${{ matrix.weblogs.weblog_name }}
-      scenarios: ${{ toJson(matrix.weblogs.scenarios) }}
-      build_python_base_images: ${{ inputs.build_python_base_images }}
+      library: ${{ matrix.job.library }}
+      weblog: ${{ matrix.job.weblog }}
+      scenarios: ${{ toJson(matrix.job.scenarios) }}
       build_buddies_images: ${{ inputs.build_buddies_images }}
       build_proxy_image: ${{ inputs.build_proxy_image }}
-      binaries_artifact: ${{ inputs.binaries_artifact }}
+      binaries_artifact: binaries_${{ inputs.ci_environment}}_${{ inputs.library }}_${{ matrix.job.weblog }}
       ci_environment: ${{ inputs.ci_environment }}
       skip_empty_scenarios: ${{ inputs.skip_empty_scenarios }}
+      logs_artifact_name: logs_endtoend_${{ inputs.library }}_${{ matrix.job.weblog }}_${{ inputs.ci_environment }}_${{ matrix.job.weblog_instance }}
 
-  external-processing:
+  external_processing:
     needs:
       - compute_parameters
     if: ${{ needs.compute_parameters.outputs.externalprocessing_scenarios != '[]' && inputs.library == 'golang' && inputs.binaries_artifact != ''}}

utils/build/docker/python/install_ddtrace.sh

@@ -9,8 +9,8 @@ export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc)
 if [ -e "dd-trace-py" ]; then
     echo "Install from local folder /binaries/dd-trace-py"
     pip install /binaries/dd-trace-py
-elif [ "$(ls *.whl *.tar.gz | grep -v 'datadog-dotnet-apm.tar.gz' | grep -v 'dd-library-php-x86_64-linux-gnu.tar.gz' | wc -l)" = "1" ]; then
-    path=$(readlink -f $(ls *.whl *.tar.gz | grep -v 'datadog-dotnet-apm.tar.gz' | grep -v 'dd-library-php-x86_64-linux-gnu.tar.gz'))
+elif [ "$(ls *.whl | wc -l)" = "1" ]; then
+    path=$(readlink -f $(ls *.whl))
     echo "Install ddtrace from ${path}"
     pip install "ddtrace[appsec-beta] @ file://${path}"
 elif [ $(ls python-load-from-pip | wc -l) = 1 ]; then