Merge branch 'master' into estringana/minify-json-blocking-error

estringana · web-flow · commit e1da67b7aa05 · 2025-11-28T16:05:38.000+01:00
diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml
@@ -71,19 +71,22 @@ workflow:
   - when: manual
     interruptible: true
 
+.on-master-or-nightly-benchmarks-or-release-always-and-not-interruptible: &on-master-or-nightly-benchmarks-or-release-always-and-not-interruptible
+  if: '($CI_COMMIT_REF_NAME == "master" && ($NIGHTLY_BENCHMARKS || $CI_PIPELINE_SOURCE != "schedule")) || $CI_COMMIT_REF_NAME =~ /^ddtrace-/'
+  when: always
+  interruptible: false
+
 .macrobenchmarks-rules: &macrobenchmarks-rules
-  - if: $CI_COMMIT_REF_NAME == "master" && $NIGHTLY_BENCHMARKS
-    when: always
-    interruptible: false
-  - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE != "schedule"
-    when: always
-    interruptible: false
-  - if: $CI_COMMIT_REF_NAME =~ /^ddtrace-/
-    when: always
-    interruptible: false
+  - <<: *on-master-or-nightly-benchmarks-or-release-always-and-not-interruptible
   - when: manual
     interruptible: true
 
+.pre-release-performance-quality-gates-rules: &pre-release-performance-quality-gates-rules
+  - <<: *on-master-or-nightly-benchmarks-or-release-always-and-not-interruptible
+  # Unlike on "master", on nightly benchmarks, or on release branches there's
+  # no strict requirement to run the check unconditionally with "when: always".
+  - interruptible: true
+
 .microbenchmarks:
   stage: benchmarks
   tags: ["runner:apm-k8s-tweaked-metal"]
@@ -182,13 +185,13 @@ check-big-regressions:
 
 .macrobenchmarks:
   stage: benchmarks
-  rules: *macrobenchmarks-rules
-  tags: ["runner:apm-k8s-same-cpu"]
   needs:
     - job: "package extension: [amd64, x86_64-unknown-linux-gnu]"
       artifacts: true
     - job: datadog-setup.php
       artifacts: true
+  rules: *macrobenchmarks-rules
+  tags: ["runner:apm-k8s-same-cpu"]
   timeout: 1h
   image: $MACROBENCHMARKS_CI_IMAGE
   script:
@@ -229,7 +232,7 @@ check-slo-breaches:
   extends: .check-slo-breaches
   stage: gate
   needs: ["macrobenchmarks"]
-  when: on_success
+  rules: *pre-release-performance-quality-gates-rules
   artifacts:
     name: "artifacts"
     when: always
@@ -245,6 +248,6 @@ notify-slo-breaches:
   extends: .notify-slo-breaches
   stage: notify
   needs: ["check-slo-breaches"]
-  when: always
+  rules: *pre-release-performance-quality-gates-rules
   variables:
     CHANNEL: "guild-dd-php"
diff --git a/.gitlab/check_test_agent.sh b/.gitlab/check_test_agent.sh
@@ -1,31 +1,61 @@
 #!/usr/bin/env bash
 set -eo pipefail
 
-set +e  # Disable exiting from testagent response failure
-SUMMARY_RESPONSE=$(curl -s -w "\n%{http_code}" -o summary_response.txt http://test-agent:9126/test/trace_check/summary)
-set -e
-SUMMARY_RESPONSE_CODE=$(echo "$SUMMARY_RESPONSE" | awk 'END {print $NF}')
-if [[ SUMMARY_RESPONSE_CODE -eq 200 ]]; then
+MAX_RETRIES=5
+RETRY_DELAY=3
+
+check_test_agent_with_retry() {
+  local endpoint=$1
+  local output_file=$2
+  local attempt=1
+
+  while [ $attempt -le $MAX_RETRIES ]; do
+    set +e
+    response=$(curl -s -w "\n%{http_code}" -o "$output_file" "http://test-agent:9126$endpoint")
+    response_code=$(echo "$response" | awk 'END {print $NF}')
+    set -e
+
+    if [[ $response_code -eq 200 ]] || [[ $response_code -eq 404 ]]; then
+      echo "$response_code"
+      return 0
+    fi
+
+    echo "Attempt $attempt/$MAX_RETRIES: Test agent returned $response_code, retrying in ${RETRY_DELAY}s..." >&2
+    sleep $RETRY_DELAY
+    attempt=$((attempt + 1))
+  done
+
+  echo "0"  # Failed after all retries
+  return 1
+}
+
+# Check if test agent is running
+summary_code=$(check_test_agent_with_retry "/test/trace_check/summary" "summary_response.txt")
+
+if [[ $summary_code -eq 200 ]]; then
   echo "APM Test Agent is running. (HTTP 200)"
+elif [[ $summary_code -eq 404 ]]; then
+  echo "Real APM Agent running in place of TestAgent, no checks to validate!"
+  exit 0
 else
   echo "APM Test Agent is not running and was not used for testing. No checks failed."
   exit 0
 fi
 
-RESPONSE=$(curl -s -w "\n%{http_code}" -o response.txt http://test-agent:9126/test/trace_check/failures)
-RESPONSE_CODE=$(echo "$RESPONSE" | awk 'END {print $NF}')
+# Check for failures with retry
+failures_code=$(check_test_agent_with_retry "/test/trace_check/failures" "response.txt")
 
-if [[ $RESPONSE_CODE -eq 200 ]]; then
+if [[ $failures_code -eq 200 ]]; then
   echo "All APM Test Agent Check Traces returned successful! (HTTP 200)"
   echo "APM Test Agent Check Traces Summary Results:"
   cat summary_response.txt | jq '.'
-elif [[ $RESPONSE_CODE -eq 404 ]]; then
+elif [[ $failures_code -eq 404 ]]; then
   echo "Real APM Agent running in place of TestAgent, no checks to validate!"
 else
-  echo "APM Test Agent Check Traces failed with response code: $RESPONSE_CODE"
+  echo "APM Test Agent Check Traces failed with response code: $failures_code"
   echo "Failures:"
   cat response.txt
   echo "APM Test Agent Check Traces Summary Results:"
   cat summary_response.txt | jq '.'
   exit 1
-fi  
+fi
diff --git a/.gitlab/generate-appsec.php b/.gitlab/generate-appsec.php
@@ -42,6 +42,11 @@
 
 .appsec_test:
   tags: [ "arch:${ARCH}" ]
+  interruptible: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      interruptible: false
+    - when: on_success
   before_script:
 <?php unset_dd_runner_env_vars() ?>
     - git config --global --add safe.directory "$(pwd)/appsec/third_party/libddwaf"
diff --git a/.gitlab/generate-common.php b/.gitlab/generate-common.php
@@ -50,6 +50,9 @@ function unset_dd_runner_env_vars() {
       - runner_system_failure
       - scheduler_failure
       - api_failure
+      - script_failure
+      - stuck_or_timeout_failure
+      - job_execution_timeout
 
 .all_targets: &all_minor_major_targets
 <?php
diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php
@@ -1170,7 +1170,7 @@
 
       # Install system dependencies
       apt-get update -o dir::state::lists="$APT_CACHE/lists"
-      apt-get install -y --no-install-recommends -o dir::state::lists="$APT_CACHE/lists" -o dir::cache::archives="$APT_CACHE/archives" ca-certificates curl git
+      apt-get install -y --no-install-recommends -o dir::state::lists="$APT_CACHE/lists" -o dir::cache::archives="$APT_CACHE/archives" ca-certificates curl git build-essential
       mkdir -p /etc/apt/keyrings
       curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
       echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list
diff --git a/.gitlab/generate-shared.php b/.gitlab/generate-shared.php
@@ -81,6 +81,11 @@
   tags: [ "arch:amd64" ]
   stage: test
   image: "registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-${PHP_MAJOR_MINOR}_bookworm-5"
+  interruptible: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      interruptible: false
+    - when: on_success
   after_script:
     - mkdir -p tmp/artifacts
     - cp tmp/build*/Testing/Temporary/LastTest.log tmp/artifacts/LastTest.log
diff --git a/.gitlab/generate-tracer.php b/.gitlab/generate-tracer.php
@@ -123,6 +123,7 @@ function before_script_steps() {
     GIT_CONFIG_VALUE_1: true
     CONTAINER_NAME: $CI_JOB_NAME_SLUG
     GIT_STRATEGY: clone
+    GIT_CLEAN_FLAGS: -ffdxq
     IMAGE: "registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-${PHP_MAJOR_MINOR}_windows"
   script: |
     # Make sure we actually fail if a command fails
@@ -154,13 +155,13 @@ function before_script_steps() {
     try { docker stop -t 5 ${CONTAINER_NAME} } catch { }
   after_script:
     - |
-        docker exec php cmd.exe /s /c xcopy /y /c /s /e C:\ProgramData\Microsoft\Windows\WER\ReportQueue .\app\dumps\
+        docker exec ${CONTAINER_NAME} cmd.exe /s /c xcopy /y /c /s /e C:\ProgramData\Microsoft\Windows\WER\ReportQueue .\app\dumps\
         exit 0
     - 'powershell -NoProfile -Command "try { docker logs request-replayer } catch {}"'
     - 'powershell -NoProfile -Command "try { docker logs httpbin-integration } catch {}"'
-    - 'powershell -NoProfile -Command "try { docker stop -t 5 request-replayer } catch {}"'
+    - 'powershell -NoProfile -Command "try { docker stop -t 15 request-replayer } catch {}"'
     - 'powershell -NoProfile -Command "try { docker rm -f request-replayer } catch {}"'
-    - 'powershell -NoProfile -Command "try { docker stop -t 5 httpbin-integration } catch {}"'
+    - 'powershell -NoProfile -Command "try { docker stop -t 15 httpbin-integration } catch {}"'
     - 'powershell -NoProfile -Command "try { docker rm -f httpbin-integration } catch {}"'
     - 'powershell -NoProfile -Command "try { docker network rm net } catch {}"'
   artifacts:
@@ -194,6 +195,11 @@ function before_script_steps() {
   tags: [ "arch:${ARCH}" ]
   image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-${PHP_MAJOR_MINOR}_bookworm-5
   timeout: 60m
+  interruptible: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      interruptible: false
+    - when: on_success
   variables:
     host_os: linux-gnu
     COMPOSER_MEMORY_LIMIT: "-1"
@@ -206,7 +212,7 @@ function before_script_steps() {
     HTTPBIN_PORT: 8080
   before_script:
 <?php before_script_steps() ?>
-    - for host in ${WAIT_FOR:-}; do wait-for $host --timeout=180; done
+    - .gitlab/wait-for-service-ready.sh
 
 .asan_test:
   extends: .base_test
@@ -496,7 +502,7 @@ function before_script_steps() {
     - if [[ "$MAKE_TARGET" != "test_composer" ]] || ! [[ "$PHP_MAJOR_MINOR" =~ 8.[01] ]]; then sudo composer self-update --$COMPOSER_VERSION --no-interaction; fi
     - COMPOSER_MEMORY_LIMIT=-1 composer update --no-interaction # disable composer memory limit completely
     - make composer_tests_update
-    - for host in ${WAIT_FOR:-}; do wait-for $host --timeout=180; done
+    - .gitlab/wait-for-service-ready.sh
   script:
     - DD_TRACE_AGENT_TIMEOUT=1000 make $MAKE_TARGET RUST_DEBUG_BUILD=1 PHPUNIT_OPTS="--log-junit artifacts/tests/results.xml" <?= ASSERT_NO_MEMLEAKS ?>
 <?php after_script(".", true); ?>
@@ -513,7 +519,7 @@ function before_script_steps() {
 $services["deferred_loading"] = "mysql";
 $services["deferred_loadin"] = "redis";
 $services["pdo"] = "mysql";
-$services["kafk"] = ["kafka", "zookeeper"];
+$services["kafka"] = ["kafka", "zookeeper"];  // Overwrite auto-generated entry
 
 $jobs = [];
 preg_match_all('(^TEST_(?<type>INTEGRATIONS|WEB)_(?<major>\d+)(?<minor>\d)[^\n]+(?<targets>.*?)^(?!\t))ms', file_get_contents(__DIR__ . "/../Makefile"), $matches, PREG_SET_ORDER);
diff --git a/.gitlab/wait-for-service-ready.sh b/.gitlab/wait-for-service-ready.sh
@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+detect_service_type() {
+  local host=${1}
+  case ${host} in
+    test-agent) echo "test-agent" ;;
+    mysql-integration) echo "mysql" ;;
+    elasticsearch*) echo "elasticsearch" ;;
+    kafka*) echo "kafka" ;;
+    redis*) echo "redis" ;;
+    httpbin*) echo "httpbin" ;;
+    *) echo "generic" ;;
+  esac
+}
+
+wait_for_single_service() {
+  local HOST=${1}
+  local PORT=${2}
+  local SERVICE_TYPE=${3:-generic}
+  local MAX_ATTEMPTS=${4:-30}
+  local RETRY_DELAY=${5:-5}
+
+  echo "Waiting for ${HOST}:${PORT} to be reachable..."
+  if ! wait-for "${HOST}:${PORT}" --timeout=180; then
+      echo "ERROR: Could not reach ${HOST}:${PORT}" >&2
+      return 1
+  fi
+
+  for i in $(seq 1 ${MAX_ATTEMPTS}); do
+    case ${SERVICE_TYPE} in
+      test-agent)
+        if curl -sf "http://${HOST}:${PORT}/info" > /dev/null 2>&1; then
+          echo "Test agent is ready"
+          return 0
+        fi
+        ;;
+      mysql)
+        if mysqladmin ping -h"${HOST}" --silent 2>/dev/null; then
+          echo "MySQL is ready"
+          return 0
+        fi
+        ;;
+      elasticsearch)
+        if curl -sf "http://${HOST}:${PORT}/_cluster/health?wait_for_status=yellow&timeout=1s" > /dev/null 2>&1; then
+          echo "Elasticsearch is ready"
+          return 0
+        fi
+        ;;
+      kafka)
+        # Kafka readiness via nc check + settle time
+        if timeout 5 nc -z "${HOST}" "${PORT}" 2>/dev/null; then
+          sleep 2  # Additional settle time for Kafka
+          echo "Kafka is ready"
+          return 0
+        fi
+        ;;
+      redis)
+        if redis-cli -h "${HOST}" ping 2>/dev/null | grep -q PONG; then
+          echo "Redis is ready"
+          return 0
+        fi
+        ;;
+      httpbin)
+        # httpbin-specific check
+        if curl -sf "http://${HOST}:${PORT}/status/200" > /dev/null 2>&1; then
+          echo "httpbin is ready"
+          return 0
+        fi
+        ;;
+      generic|*)
+        # For generic services, just verify port + HTTP 200/health endpoint
+        if curl -sf "http://${HOST}:${PORT}/" > /dev/null 2>&1 || curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
+          echo "${HOST}:${PORT} is ready"
+          return 0
+        fi
+        # If HTTP fails, at least verify the port is still open
+        if nc -z "${HOST}" "${PORT}" 2>/dev/null; then
+          echo "${HOST}:${PORT} port is open (non-HTTP service)"
+          return 0
+        fi
+        ;;
+    esac
+
+    echo "Attempt ${i}/${MAX_ATTEMPTS}: Service not ready yet, retrying in ${RETRY_DELAY}s..."
+    sleep ${RETRY_DELAY}
+  done
+
+  echo "ERROR: Service ${HOST}:${PORT} (${SERVICE_TYPE}) failed to become ready after ${MAX_ATTEMPTS} attempts"
+  return 1
+}
+
+if [ -n "${WAIT_FOR:-}" ]; then
+  for service in ${WAIT_FOR}; do
+    host="$(echo ${service} | cut -d: -f1)"
+    port="$(echo ${service} | cut -d: -f2)"
+    service_type="$(detect_service_type "${host}")"
+
+    if ! wait_for_single_service "${host}" "${port}" "${service_type}" 30 5; then
+      exit 1
+    fi
+  done
+fi
diff --git a/Makefile b/Makefile
@@ -1208,9 +1208,10 @@ endef
 define run_tests_debug
 	$(eval TEST_EXTRA_ENV=$(TEST_EXTRA_ENV) DD_TRACE_DEBUG=1)
 	(set -o pipefail; { $(call run_tests,$(1)) 2>&1 >&3 | \
-		tee >(grep -vE '\[ddtrace\] \[debug\]|\[ddtrace\] \[info\]' >&2) | \
-		{ ! (grep -E '\[error\]|\[warning\]|\[deprecated\]' >/dev/null && \
-		echo $$'\033[41m'"ERROR: Found debug log errors in the output."$$'\033[0m'); }; } 3>&1)
+		tee >(grep --line-buffered -vE '\[ddtrace\] \[debug\]|\[ddtrace\] \[info\]' >&2) | \
+		{ ! (grep --line-buffered -E '\[error\]|\[warning\]|\[deprecated\]' >/dev/null && \
+		echo $$'\033[41m'"ERROR: Found debug log errors in the output."$$'\033[0m'); }; } 3>&1); \
+	timeout 10 bash -c 'wait' 2>/dev/null || true
 	$(eval TEST_EXTRA_ENV=)
 endef
 
diff --git a/appsec/src/extension/request_abort.c b/appsec/src/extension/request_abort.c
@@ -515,8 +515,10 @@ zend_array *nonnull dd_request_abort_static_page_spec(
     }
 
     {
-        char buf[sizeof("18446744073709551615") - 1];
-        size_t len = sprintf(buf, "%zu", body_len);
+        // This magic number is the string representation of SIZE_MAX on 64 bit
+        // systems
+        char buf[sizeof("18446744073709551615")];
+        size_t len = snprintf(buf, sizeof(buf), "%zu", body_len);
         zend_string *s = zend_string_init(buf, len, 0);
         zval cont_len_zv;
         ZVAL_STR(&cont_len_zv, s);

Original file line number	Diff line number	Diff line change
`@@ -515,8 +515,10 @@ zend_array *nonnull dd_request_abort_static_page_spec(`
`515`	`515`	`}`
`516`	`516`
`517`	`517`	`{`
`518`		`- char buf[sizeof("18446744073709551615") - 1];`
`519`		`- size_t len = sprintf(buf, "%zu", body_len);`
	`518`	`+ // This magic number is the string representation of SIZE_MAX on 64 bit`
	`519`	`+ // systems`
	`520`	`+ char buf[sizeof("18446744073709551615")];`
	`521`	`+ size_t len = snprintf(buf, sizeof(buf), "%zu", body_len);`
`520`	`522`	`zend_string *s = zend_string_init(buf, len, 0);`
`521`	`523`	`zval cont_len_zv;`
`522`	`524`	`ZVAL_STR(&cont_len_zv, s);`