Merge branch 'main' into quinna.halim/add-error-extensions-graphql

Author: quinna-h

.circleci/config.templ.yml

@@ -404,25 +404,12 @@ jobs:
           paths:
             - "."
 
-  appsec_iast_packages:
-    <<: *machine_executor
-    parallelism: 5
-    steps:
-      - when:
-          condition:
-            matches: { pattern: "main", value: << pipeline.git.branch >> }
-          steps:
-            - run_test:
-                pattern: 'appsec_iast_packages'
-                snapshot: true
-      - run: echo "This test is skipped outside of main branch"
-
-  appsec_integrations:
+  appsec_integrations_pygoat:
     <<: *machine_executor
     parallelism: 13
     steps:
       - run_test:
-          pattern: 'appsec_integrations'
+          pattern: 'appsec_integrations_pygoat'
           snapshot: true
           run_agent_checks: false
           docker_services: "pygoat"

.github/CODEOWNERS

@@ -110,16 +110,17 @@ benchmarks/base/aspects_benchmarks_generate.py @DataDog/asm-python
 ddtrace/appsec/                     @DataDog/asm-python
 ddtrace/settings/asm.py             @DataDog/asm-python
 ddtrace/contrib/subprocess/         @DataDog/asm-python
+ddtrace/contrib/internal/subprocess/         @DataDog/asm-python
 ddtrace/contrib/flask_login/        @DataDog/asm-python
 ddtrace/contrib/webbrowser          @DataDog/asm-python
 ddtrace/contrib/urllib              @DataDog/asm-python
 ddtrace/internal/_exceptions.py     @DataDog/asm-python
+ddtrace/internal/appsec/            @DataDog/asm-python
+ddtrace/internal/iast/              @DataDog/asm-python
 tests/appsec/                       @DataDog/asm-python
 tests/contrib/dbapi/test_dbapi_appsec.py    @DataDog/asm-python
 tests/contrib/subprocess            @DataDog/asm-python
 tests/contrib/flask/test_flask_appsec.py    @DataDog/asm-python
-tests/contrib/django/django_app/appsec_urls.py @DataDog/asm-python
-tests/contrib/django/test_django_appsec.py  @DataDog/asm-python
 tests/snapshots/tests*appsec*.json  @DataDog/asm-python
 tests/contrib/*/test*appsec*.py     @DataDog/asm-python
 scripts/iast/*                      @DataDog/asm-python
@@ -146,6 +147,8 @@ ddtrace/contrib/internal/google_generativeai            @DataDog/ml-observabilit
 ddtrace/contrib/google_generativeai                     @DataDog/ml-observability
 ddtrace/contrib/internal/vertexai                       @DataDog/ml-observability
 ddtrace/contrib/vertexai                                @DataDog/ml-observability
+ddtrace/contrib/langgraph                               @DataDog/ml-observability
+ddtrace/contrib/internal/langgraph                      @DataDog/ml-observability
 tests/llmobs                                            @DataDog/ml-observability
 tests/contrib/openai                                    @DataDog/ml-observability
 tests/contrib/langchain                                 @DataDog/ml-observability
@@ -155,19 +158,22 @@ tests/contrib/botocore/bedrock_cassettes                @DataDog/ml-observabilit
 tests/contrib/anthropic                                 @DataDog/ml-observability
 tests/contrib/google_generativeai                       @DataDog/ml-observability
 tests/contrib/vertexai                                  @DataDog/ml-observability
+tests/contrib/langgraph                                 @DataDog/ml-observability
 .gitlab/tests/llmobs.yml                                @DataDog/ml-observability
 
 # Remote Config
 ddtrace/internal/remoteconfig       @DataDog/remote-config @DataDog/apm-core-python
 tests/internal/remoteconfig         @DataDog/remote-config @DataDog/apm-core-python
 
 # API SDK
+ddtrace/trace/                                    @DataDog/apm-sdk-api-python
 ddtrace/_trace/                                    @DataDog/apm-sdk-api-python
 ddtrace/opentelemetry/                             @DataDog/apm-sdk-api-python
 ddtrace/internal/opentelemetry                     @DataDog/apm-sdk-api-python
 ddtrace/opentracer/                                @DataDog/apm-sdk-api-python
 ddtrace/propagation/                               @DataDog/apm-sdk-api-python
 ddtrace/filters.py                                 @DataDog/apm-sdk-api-python
+ddtrace/provider.py                                @DataDog/apm-sdk-api-python
 ddtrace/pin.py                                     @DataDog/apm-sdk-api-python
 ddtrace/sampler.py                                 @DataDog/apm-sdk-api-python
 ddtrace/sampling_rule.py                           @DataDog/apm-sdk-api-python

.github/workflows/build-and-publish-image.yml

@@ -26,6 +26,9 @@ on:
 jobs:
   build_push:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
     - uses: actions/checkout@v4
       with:

.github/workflows/build_python_3.yml

@@ -35,7 +35,8 @@ jobs:
               cibuildwheel --print-build-identifiers --platform linux --arch x86_64,i686 | jq -cR '{only: ., os: "ubuntu-latest"}' \
               && cibuildwheel --print-build-identifiers --platform linux --arch aarch64  | jq -cR '{only: ., os: "arm-4core-linux"}' \
               && cibuildwheel --print-build-identifiers --platform windows --arch AMD64,x86 | grep -v 313 | jq -cR '{only: ., os: "windows-latest"}' \
-              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,universal2 | jq -cR '{only: ., os: "macos-13"}'
+              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | jq -cR '{only: ., os: "macos-13"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch arm64 | jq -cR '{only: ., os: "macos-latest"}'
             } | jq -sc
           )
           echo $MATRIX_INCLUDE
@@ -112,6 +113,7 @@ jobs:
             choco install -y 7zip &&
             7z d -r "{wheel}" *.c *.cpp *.cc *.h *.hpp *.pyx &&
             move "{wheel}" "{dest_dir}"
+          CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
           # DEV: Uncomment to debug MacOS
           # CIBW_BUILD_VERBOSITY_MACOS: 3
 
@@ -132,6 +134,10 @@ jobs:
           CIBW_BEFORE_ALL_WINDOWS: rustup target add i686-pc-windows-msvc
           CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
           CIBW_ENVIRONMENT_LINUX: PATH=$HOME/.cargo/bin:$PATH CMAKE_BUILD_PARALLEL_LEVEL=24
+          # SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a.
+          # `platform.mac_ver()` reports incorrect MacOS version at 11.0
+          # See: https://stackoverflow.com/a/65402241
+          CIBW_ENVIRONMENT_MACOS: CMAKE_BUILD_PARALLEL_LEVEL=24 SYSTEM_VERSION_COMPAT=0
           CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
             mkdir ./tempwheelhouse &&
             unzip -l {wheel} | grep '\.so' &&
@@ -148,6 +154,7 @@ jobs:
             choco install -y 7zip &&
             7z d -r "{wheel}" *.c *.cpp *.cc *.h *.hpp *.pyx &&
             move "{wheel}" "{dest_dir}"
+          CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
           # DEV: Uncomment to debug MacOS
           # CIBW_BUILD_VERBOSITY_MACOS: 3
 

.github/workflows/check_safe_main_merge.yml

@@ -0,0 +1,27 @@
+name: Check for Safe main Merge
+
+on:
+  pull_request:
+    branches:
+      - '3.x-staging'
+
+jobs:
+  check-merge:
+    runs-on: ubuntu-latest
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Step 2: Fetch the main branch
+      - name: Fetch main branch
+        run: git fetch origin main
+
+      # Step 3: Attempt to merge
+      - name: Check merge conflicts
+        run: |
+          git merge --no-commit --no-ff origin/main || exit 1
+      # Step 4: Clean up the merge (optional)
+      - name: Abort merge
+        if: failure()
+        run: git merge --abort

.github/workflows/profiling-native.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 5
+    timeout-minutes: 7
     strategy:
       fail-fast: false
       matrix:

.github/workflows/pypa_musllinux_1_2_i686.yml

@@ -12,6 +12,9 @@ on:
 jobs:
   build-and-publish:
     uses: ./.github/workflows/build-and-publish-image.yml
+    permissions:
+      contents: read
+      packages: write
     with:
       tags: 'ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:${{ github.sha }},ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest'
       platforms: 'linux/386'

.github/workflows/rust-ci.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        extension: ["src/core"]
+        extension: ["src/native"]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/system-tests.yml

@@ -47,6 +47,7 @@ jobs:
           - weblog-variant: fastapi
           # runs django-poc for 3.12
           - weblog-variant: python3.12
+          - weblog-variant: django-py3.13
       fail-fast: false
     env:
       TEST_LIBRARY: python
@@ -96,7 +97,7 @@ jobs:
     needs: [system-tests-build-agent, system-tests-build-weblog]
     strategy:
       matrix:
-        weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12]
+        weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12, django-py3.13]
         scenario: [remote-config, appsec, appsec-1, other, debugger-1, debugger-2]
 
       fail-fast: false

.github/workflows/test_frameworks.yml

@@ -111,68 +111,6 @@ jobs:
         if: needs.needs-run.outputs.outcome == 'success'
         run: cat debugger-expl.txt
 
-  sanic-testsuite:
-    strategy:
-      matrix:
-        include:
-          # TODO: profiling fails with a timeout error
-          #- suffix: Profiling
-          #  profiling: 1
-          #  iast: 0
-          #  appsec: 0
-          - suffix: IAST
-            profiling: 0
-            iast: 1
-            appsec: 0
-          - suffix: APPSEC
-            profiling: 0
-            iast: 0
-            appsec: 1
-          - suffix: Tracer only
-            profiling: 0
-            iast: 0
-            appsec: 0
-    name: Sanic 24.6 (with ${{ matrix.suffix }})
-    runs-on: ubuntu-20.04
-    needs: needs-run
-    timeout-minutes: 15
-    env:
-      DD_PROFILING_ENABLED: ${{ matrix.profiling }}
-      DD_IAST_ENABLED: ${{ matrix.iast }}
-      DD_APPSEC_ENABLED: ${{ matrix.appsec }}
-      DD_TESTING_RAISE: true
-      CMAKE_BUILD_PARALLEL_LEVEL: 12
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
-    defaults:
-      run:
-        working-directory: sanic
-    steps:
-      - uses: actions/checkout@v4
-        if: needs.needs-run.outputs.outcome == 'success'
-        with:
-          persist-credentials: false
-          path: ddtrace
-      - uses: actions/checkout@v4
-        if: needs.needs-run.outputs.outcome == 'success'
-        with:
-          persist-credentials: false
-          repository: sanic-org/sanic
-          ref: v24.6.0
-          path: sanic
-      - uses: actions/setup-python@v5
-        if: needs.needs-run.outputs.outcome == 'success'
-        with:
-          python-version: "3.11"
-      - name: Install sanic and dependencies required to run tests
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: pip3 install '.[test]' aioquic
-      - name: Install ddtrace
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: pip3 install ../ddtrace
-      - name: Run tests
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: ddtrace-run pytest -k "not test_reloader and not test_reload_listeners and not test_no_exceptions_when_cancel_pending_request and not test_add_signal and not test_ode_removes and not test_skip_touchup and not test_dispatch_signal_triggers and not test_keep_alive_connection_context and not test_redirect_with_params and not test_keep_alive_client_timeout and not test_logger_vhosts and not test_ssl_in_multiprocess_mode"
-
   django-testsuite:
     strategy:
       matrix:
@@ -963,58 +901,3 @@ jobs:
       - name: Debugger exploration results
         if: needs.needs-run.outputs.outcome == 'success'
         run: cat debugger-expl.txt
-
-  beautifulsoup-testsuite-4_12_3:
-    strategy:
-      matrix:
-        include:
-          # TODO: profiling is disabled due to a bug in the profiler paths
-          # - suffix: Profiling
-          #   profiling: 1
-          #   iast: 0
-          #   appsec: 0
-          - suffix: IAST
-            profiling: 0
-            iast: 1
-            appsec: 0
-          - suffix: APPSEC
-            profiling: 0
-            iast: 0
-            appsec: 1
-          - suffix: Tracer only
-            profiling: 0
-            iast: 0
-            appsec: 0
-    name: Beautifulsoup 4.12.3 (with ${{ matrix.suffix }})
-    runs-on: "ubuntu-latest"
-    needs: needs-run
-    env:
-      DD_TESTING_RAISE: true
-      DD_PROFILING_ENABLED: ${{ matrix.profiling }}
-      DD_IAST_ENABLED: ${{ matrix.iast }}
-      DD_APPSEC_ENABLED: ${{ matrix.appsec }}
-      CMAKE_BUILD_PARALLEL_LEVEL: 12
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
-    steps:
-      - uses: actions/setup-python@v5
-        if: needs.needs-run.outputs.outcome == 'success'
-        with:
-          python-version: '3.9'
-      - uses: actions/checkout@v4
-        if: needs.needs-run.outputs.outcome == 'success'
-        with:
-          persist-credentials: false
-          path: ddtrace
-      - name: Checkout beautifulsoup
-        if: needs.needs-run.outputs.outcome == 'success'
-        run:  |
-          git clone -b 4.12.3 https://git.launchpad.net/beautifulsoup
-      - name: Install ddtrace
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: pip3 install ./ddtrace
-      - name: Pytest fix
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: pip install pytest==8.2.1
-      - name: Run tests
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: cd beautifulsoup && ddtrace-run pytest

.gitlab-ci.yml

@@ -6,6 +6,7 @@ stages:
   - shared-pipeline
   - benchmarks
   - macrobenchmarks
+  - benchmarks-report
   - release
 
 variables:
@@ -59,18 +60,20 @@ onboarding_tests_installer:
     matrix:
       - ONBOARDING_FILTER_WEBLOG: [test-app-python,test-app-python-container,test-app-python-alpine]
 
-
 onboarding_tests_k8s_injection:
   parallel:
     matrix:
-      - WEBLOG_VARIANT:
-          - dd-lib-python-init-test-django
-          - dd-lib-python-init-test-django-gunicorn
-          - dd-lib-python-init-test-django-gunicorn-alpine
-          - dd-lib-python-init-test-django-preinstalled
-          - dd-lib-python-init-test-django-unsupported-package-force
-          - dd-lib-python-init-test-django-uvicorn
-          - dd-lib-python-init-test-protobuf-old
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django, ]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_UDS, K8S_LIB_INJECTION_NO_AC, K8S_LIB_INJECTION_NO_AC_UDS, K8S_LIB_INJECTION_PROFILING_DISABLED, K8S_LIB_INJECTION_PROFILING_ENABLED, K8S_LIB_INJECTION_PROFILING_OVERRIDE]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
+
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django-gunicorn, dd-lib-python-init-test-django-gunicorn-alpine, dd-lib-python-init-test-django-unsupported-package-force, dd-lib-python-init-test-django-uvicorn, dd-lib-python-init-test-protobuf-old ]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_PROFILING_ENABLED]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
+
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django-preinstalled]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_UDS, K8S_LIB_INJECTION_NO_AC, K8S_LIB_INJECTION_NO_AC_UDS]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
 
 deploy_to_di_backend:manual:
   stage: shared-pipeline