Merge branch 'main' into conti/add-dsm-test-for-each-version-of-conte…

…xt-propagation
DataDog · Mar 7, 2024 · c311c09 · c311c09
2 parents 4543dc7 + dcc3294
commit c311c09
Show file tree

Hide file tree

Showing 33 changed files with 1,270 additions and 68 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -92,6 +92,7 @@ jobs:
       build_python_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') }}
       build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
+      build_lib_injection_app_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-lib-injection-app-images') }}
 
   update-CI-visibility:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/run-lib-injection.yml b/.github/workflows/run-lib-injection.yml
@@ -2,10 +2,17 @@ name: Lib-injection tests
 
 on:
   workflow_call:
-
+    inputs:
+      build_lib_injection_app_images:
+        description: "Shall we build python base images for tests on python tracer"
+        default: false
+        required: false
+        type: boolean 
 env:
   REGISTRY: ghcr.io
 
+
+
 jobs:
   manual-lib-injection-tests:
     runs-on: ubuntu-latest
@@ -97,4 +104,120 @@ jobs:
           docker-registry: ghcr.io
           docker-registry-username: ${{ github.actor }}
           docker-registry-password: ${{ secrets.GITHUB_TOKEN }}
-          test-script: ./lib-injection/run-auto-lib-injection.sh
+          test-script: ./lib-injection/run-auto-lib-injection.sh
+
+  #TODO rename this job after migration completed!
+  new-lib-injection-tests:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        variant:
+        - library: java
+          weblog-variant: dd-lib-java-init-test-app
+
+        - library: nodejs
+          weblog-variant: sample-app
+
+        - library: python
+          weblog-variant: dd-lib-python-init-test-django
+
+        version: 
+          - latest #Production tag
+          - latest_snapshot
+
+      fail-fast: false
+    env:
+      TEST_LIBRARY: ${{ matrix.variant.library }}
+      WEBLOG_VARIANT: ${{ matrix.variant.weblog-variant }}
+      DOCKER_REGISTRY_IMAGES_PATH: ghcr.io/datadog
+      DOCKER_IMAGE_TAG: ${{ matrix.version }}
+      BUILDX_PLATFORMS: linux/amd64
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: 'DataDog/system-tests'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3  # 3.0.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # 3.0.0
+        with:
+          install: true
+          config-inline: |
+            [worker.oci]
+              max-parallelism = 1
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # 3.0.0
+        with:
+          registry: ghcr.io/datadog
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build weblog base images (PR)
+        if: inputs.build_lib_injection_app_images
+        env:
+          DOCKER_IMAGE_WEBLOG_TAG: ${{ github.sha }}
+          APP_DOCKER_IMAGE_REPO: ghcr.io/datadog/system-tests/${{ matrix.variant.weblog-variant }}
+        run: |
+          cd lib-injection/build/docker/$TEST_LIBRARY/$WEBLOG_VARIANT 
+          LIBRARY_INJECTION_TEST_APP_IMAGE=$APP_DOCKER_IMAGE_REPO:${{ github.sha }} ./build.sh
+          cd ..
+
+      - name: Build weblog latest base images
+        if: github.ref == 'refs/heads/main'
+        env:
+          DOCKER_IMAGE_WEBLOG_TAG: latest
+          APP_DOCKER_IMAGE_REPO: ghcr.io/datadog/system-tests/${{ matrix.variant.weblog-variant }}
+        run: |
+          cd lib-injection/build/docker/$TEST_LIBRARY/$WEBLOG_VARIANT 
+          LIBRARY_INJECTION_TEST_APP_IMAGE=$APP_DOCKER_IMAGE_REPO:latest ./build.sh
+          cd ..
+
+      - name: Install runner
+        uses: ./.github/actions/install_runner       
+
+      - name: Kubernetes lib-injection tests (using custom weblog image tag)
+        if: inputs.build_lib_injection_app_images
+        id: k8s-lib-injection-tests-custom-weblog-tag
+        run: DOCKER_IMAGE_WEBLOG_TAG=${{ github.sha }} ./run.sh K8S_LIB_INJECTION 
+
+      - name: Kubernetes lib-injection tests
+        if: inputs.build_lib_injection_app_images != true
+        id: k8s-lib-injection-tests
+        run: ./run.sh K8S_LIB_INJECTION 
+
+
+
+  delete-images:
+    if: ${{ false }} #Enable when Github support fix the problem with the image versions. 
+    #if inputs.build_lib_injection_app_images 
+    needs:
+    - new-lib-injection-tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+        matrix:
+          weblog-variant: [dd-lib-java-init-test-app, sample-app, dd-lib-python-init-test-django]
+    steps:
+      - name: Log in to the Container registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # 3.0.0
+        with:
+          registry: ghcr.io/datadog
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Delete image
+        uses: bots-house/[email protected]
+        with:
+          owner: datadog
+          name: system-tests/${{ matrix.weblog-variant }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.sha }}
diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
@@ -77,6 +77,11 @@ on:
         default: false
         required: false
         type: boolean
+      build_lib_injection_app_images:
+          description: "Shall we build and push k8s lib injection weblog images"
+          default: false
+          required: false
+          type: boolean
       push_to_feature_parity_dashbaord:
         description: "Shall we push results to Feature Parity Dashbaord"
         default: false
@@ -101,6 +106,8 @@ jobs:
     if: inputs.run_libinjection || inputs.run_all
     uses: ./.github/workflows/run-lib-injection.yml
     secrets: inherit
+    with:
+      build_lib_injection_app_images: ${{ inputs.build_lib_injection_app_images }}
 
   end-to-end:
     uses: ./.github/workflows/run-end-to-end.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -13,8 +13,6 @@ stages:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:a58cc31c
   tags: ["arch:amd64"]
   before_script:
-    - apt-get update
-    - apt-get install -y jq
     # Setup AWS Credentials
     - mkdir -p ~/.aws
     - aws ssm get-parameter --region us-east-1 --name ci.system-tests.agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
@@ -183,12 +181,15 @@ onboarding_parse_results:
   script:
       - |
         for folder in reports/logs*/ ; do
-          curl -X POST ${FP_IMPORT_URL} \
-            --fail \
-            --header "Content-Type: application/json" \
-            --header "FP_API_KEY: ${FP_API_KEY}" \
-            --data "@./${folder}feature_parity.json" \
-            --include
+          if [ -e ./${folder}feature_parity.json ]
+          then
+            curl -X POST ${FP_IMPORT_URL} \
+              --fail \
+              --header "Content-Type: application/json" \
+              --header "FP_API_KEY: ${FP_API_KEY}" \
+              --data "@./${folder}feature_parity.json" \
+              --include
+          fi
         done
 
 check_merge_labels:
@@ -199,7 +200,7 @@ check_merge_labels:
     - export GH_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.gh-token --with-decryption --query "Parameter.Value" --out text) 
     - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) 
     - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) 
-  script:  
+  script:
     - ./utils/scripts/get_pr_merged_labels.sh
   rules:
     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"
diff --git a/lib-injection/build/docker/java/config.yaml b/lib-injection/build/docker/java/config.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: auto-instru
+  labels:
+    reloader: enabled
 data:
   auto-instru.json: |
     [{

diff --git a/lib-injection/build/docker/python/config.yaml b/lib-injection/build/docker/python/config.yaml
@@ -11,7 +11,7 @@ data:
       "action": "enable",
       "lib_config": {
         "library_language": "python",
-        "library_version": "v1.6.0",
+        "library_version": "latest",
         "service_name": "test-python-service",
         "env": "dev",
         "tracing_enabled": true,

diff --git a/lib-injection/common/operator-helm-values.yaml b/lib-injection/common/operator-helm-values.yaml
@@ -28,9 +28,10 @@ clusterAgent:
     failureThreshold: 12
   image:
    #comment name, tag and repository to test cluster-agent for local Mac M1
-    name: ""
-    tag: master
-    repository: datadog/cluster-agent-dev
+    #name: ""
+    #tag: master
+    #tag: liliya-belaus-7-52-0-rc2-test
+    #repository: datadog/cluster-agent-dev
     pullPolicy: Always
     doNotCheckTag: true
   admissionController:

diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml
@@ -221,9 +221,9 @@ tests/:
         Test_SNS_Propagation: missing_feature (Endpoint not implemented)
       test_sqs.py:
         Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS:
-          "*": missing_feature (Endpoint not implemented)
+          "*": missing_feature (not read in dotnet)
         Test_SQS_PROPAGATION_VIA_MESSAGE_ATTRIBUTES:
-          "*": missing_feature (Endpoint not implemented)
+          "*": v2.48.0
     test_db_integrations_sql.py:
       Test_MsSql: missing_feature
       Test_MySql: missing_feature
@@ -238,7 +238,7 @@ tests/:
       Test_DsmRabbitmq_FanoutExchange: missing_feature
       Test_DsmRabbitmq_TopicExchange: missing_feature
       Test_DsmSNS: missing_feature
-      Test_DsmSQS: missing_feature
+      Test_DsmSQS: v2.48.0
   parametric/:
     test_dynamic_configuration.py:
       TestDynamicConfigHeaderTags: missing_feature

diff --git a/manifests/python.yml b/manifests/python.yml
@@ -28,25 +28,34 @@ tests/:
         Test_API_Security_sampling: missing_feature
       test_schemas.py:
         Test_Scanners:
-          '*': missing_feature
+          '*': v2.4.0
+          fastapi: v2.5.0
         Test_Schema_Request_Cookies:
-          '*': missing_feature
+          '*': v2.1.0
+          fastapi: v2.5.0
         Test_Schema_Request_FormUrlEncoded_Body:
-          '*': missing_feature
+          '*': v2.1.0
+          fastapi: v2.5.0
         Test_Schema_Request_Headers:
-          '*': missing_feature
+          '*': v2.1.0
+          fastapi: v2.5.0
         Test_Schema_Request_Json_Body:
-          '*': missing_feature
+          '*': v2.1.0
+          fastapi: v2.5.0
         Test_Schema_Request_Path_Parameters:
-          '*': missing_feature
+          '*': v2.1.0
+          fastapi: v2.5.0
         Test_Schema_Request_Query_Parameters:
-          '*': missing_feature
+          '*': v2.1.0
+          fastapi: v2.5.0
         Test_Schema_Response_Body:
-          '*': missing_feature
+          '*': v2.1.0
+          fastapi: v2.5.0
         Test_Schema_Response_Body_env_var:
-          '*': missing_feature
+          '*': v2.6.0
         Test_Schema_Response_Headers:
-          '*': missing_feature
+          '*': v2.1.0
+          fastapi: v2.5.0
     iast/:
       sink/:
         test_command_injection.py:

diff --git a/requirements.txt b/requirements.txt
@@ -34,8 +34,10 @@ pulumi-tls==5.0.0a0
 pulumi-command==0.7.2
 pyyaml==6.0.0
 pyyaml-include==1.3
+python-vagrant==1.0.0
 
 # DSM Requirements
-kombu
+kombu==5.3.5
 
-python-vagrant==1.0.0
+#lib-injection kubernetes
+kubernetes==29.0.0
diff --git a/tests/integrations/crossed_integrations/test_sqs.py b/tests/integrations/crossed_integrations/test_sqs.py
@@ -164,6 +164,11 @@ def test_consume(self):
 
     @missing_feature(library="golang", reason="Expected to fail, Golang does not propagate context")
     @missing_feature(library="ruby", reason="Expected to fail, Ruby does not propagate context")
+    @missing_feature(
+        library="dotnet",
+        reason="Expected to fail, dotnet currently does not extract context on receive."
+        " TODO: enable after https://github.com/DataDog/dd-trace-dotnet/pull/5159",
+    )
     def test_consume_trace_equality(self):
         """This test relies on the setup for consume, it currently cannot be run on its own"""
         producer_span = self.get_span(