From 0cd1b98d2fb898a818e78e479f8d7a22d743c617 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 13 Dec 2024 16:59:55 +0100 Subject: [PATCH 01/43] Try to get TARGET_BRANCH from PR body --- .github/actions/get_target_branch/action.yaml | 20 +++++++++++++++++++ .github/workflows/ci.yml | 11 ++++++++-- utils/scripts/load-binary.sh | 5 ++++- 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 .github/actions/get_target_branch/action.yaml diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml new file mode 100644 index 0000000000..f39875a58c --- /dev/null +++ b/.github/actions/get_target_branch/action.yaml @@ -0,0 +1,20 @@ +name: Get_target_branch +description: "Gets target branch from the PR description" +inputs: + body: + description: "Text from which to extract the target branch" + required: true +outputs: + target-branch: + description: "Target branch" + value: ${{ steps.extract.outputs.target-branch }} + +runs: + using: composite + steps: + - name: Extract target branch + id: extract + if: ${{ contains(inputs.body, '"TARGET_BRANCH:"') }} + shell: bash + run: | + echo "target-branch=$(echo ${{ inputs.body }} | grep -oP '(?<=TARGET_BRANCH:).*' | tr -s ' ')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07698b255f..bd3360d69d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,12 +66,19 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Get target branch + uses: ./.github/actions/get_target_branch + id: get-target-branch + with: + body: ${{ github.event.pull_request.body }} - name: Get library artifact run: ./utils/scripts/load-binary.sh ${{ matrix.library }} - + env: + TARGET_BRANCH: "${{ steps.get-target-branch.outputs.target-branch }}" + - name: Get agent artifact run: ./utils/scripts/load-binary.sh agent - + # ### appsec-event-rules is now a private repo. The GH_TOKEN provided can't read private repos. # ### skipping this, waiting for a proper solution # - name: Load WAF rules diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 743f23bb54..8ea7c2dd65 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -214,8 +214,11 @@ elif [ "$TARGET" = "agent" ]; then elif [ "$TARGET" = "nodejs" ]; then assert_version_is_dev + + TARGET_BRANCH="${TARGET_BRANCH:-master}" # NPM builds the package, so we put a trigger file that tells install script to get package from github#master - echo "DataDog/dd-trace-js#master" > nodejs-load-from-npm + echo "Using \"$TARGET_BRANCH\" target branch" + echo "DataDog/dd-trace-js#$TARGET_BRANCH" > nodejs-load-from-npm elif [ "$TARGET" = "waf_rule_set_v1" ]; then exit 1 From dd53fa8fe1a18cfcef7a3c07f9a514f47ee81943 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 13 Dec 2024 17:11:33 +0100 Subject: [PATCH 02/43] lint --- .github/actions/get_target_branch/action.yaml | 2 +- .github/workflows/ci.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index f39875a58c..8c82f69821 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -17,4 +17,4 @@ runs: if: ${{ contains(inputs.body, '"TARGET_BRANCH:"') }} shell: bash run: | - echo "target-branch=$(echo ${{ inputs.body }} | grep -oP '(?<=TARGET_BRANCH:).*' | tr -s ' ')" >> $GITHUB_OUTPUT + echo "target-branch=$(echo ${{ inputs.body }} | grep -oP '(?<=TARGET_BRANCH:).*' | tr -d ' ')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd3360d69d..a59a23d457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,10 +75,8 @@ jobs: run: ./utils/scripts/load-binary.sh ${{ matrix.library }} env: TARGET_BRANCH: "${{ steps.get-target-branch.outputs.target-branch }}" - - name: Get agent artifact run: ./utils/scripts/load-binary.sh agent - # ### appsec-event-rules is now a private repo. The GH_TOKEN provided can't read private repos. # ### skipping this, waiting for a proper solution # - name: Load WAF rules From d2d0a7b43d11e568883008e596454c1db8a8dc06 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 13 Dec 2024 17:16:54 +0100 Subject: [PATCH 03/43] try --- .github/actions/get_target_branch/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index 8c82f69821..28b7982311 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -14,7 +14,7 @@ runs: steps: - name: Extract target branch id: extract - if: ${{ contains(inputs.body, '"TARGET_BRANCH:"') }} + if: ${{ contains(inputs.body, 'TARGET_BRANCH:') }} shell: bash run: | echo "target-branch=$(echo ${{ inputs.body }} | grep -oP '(?<=TARGET_BRANCH:).*' | tr -d ' ')" >> $GITHUB_OUTPUT From 2863beb1fe9abe15a7a10d48ae66db013325925b Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 13 Dec 2024 17:31:09 +0100 Subject: [PATCH 04/43] try --- .github/actions/get_target_branch/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index 28b7982311..aeb8b3c05c 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -17,4 +17,4 @@ runs: if: ${{ contains(inputs.body, 'TARGET_BRANCH:') }} shell: bash run: | - echo "target-branch=$(echo ${{ inputs.body }} | grep -oP '(?<=TARGET_BRANCH:).*' | tr -d ' ')" >> $GITHUB_OUTPUT + echo "target-branch=$(echo "${{ inputs.body }}" | grep -oP '(?<=TARGET_BRANCH:).*' | tr -d ' ')" >> $GITHUB_OUTPUT From 736fb7307e996c6c3a5769698365e40420bb0688 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 13 Dec 2024 17:52:51 +0100 Subject: [PATCH 05/43] Use PR title --- .github/actions/get_target_branch/action.yaml | 6 +++--- .github/workflows/ci.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index aeb8b3c05c..b37aa3dc79 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -1,7 +1,7 @@ name: Get_target_branch description: "Gets target branch from the PR description" inputs: - body: + text: description: "Text from which to extract the target branch" required: true outputs: @@ -14,7 +14,7 @@ runs: steps: - name: Extract target branch id: extract - if: ${{ contains(inputs.body, 'TARGET_BRANCH:') }} + if: ${{ contains(inputs.text, '[branch:') }} shell: bash run: | - echo "target-branch=$(echo "${{ inputs.body }}" | grep -oP '(?<=TARGET_BRANCH:).*' | tr -d ' ')" >> $GITHUB_OUTPUT + echo "target-branch=$(echo "${{ inputs.text }}" | grep -oP '(?<=[branch:).*' | tr -d ' ' | tr -d ']')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a59a23d457..d4ac7c604a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: uses: ./.github/actions/get_target_branch id: get-target-branch with: - body: ${{ github.event.pull_request.body }} + text: ${{ github.event.pull_request.title }} - name: Get library artifact run: ./utils/scripts/load-binary.sh ${{ matrix.library }} env: From 9d352bc6c61455419979c370c77925840efdf8f2 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 13 Dec 2024 18:07:54 +0100 Subject: [PATCH 06/43] force a new run From 027d22b03f04d28186b8f21ff7e1304fd4a82963 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 13 Dec 2024 18:21:30 +0100 Subject: [PATCH 07/43] fix regex --- .github/actions/get_target_branch/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index b37aa3dc79..c531d3a4b3 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -17,4 +17,4 @@ runs: if: ${{ contains(inputs.text, '[branch:') }} shell: bash run: | - echo "target-branch=$(echo "${{ inputs.text }}" | grep -oP '(?<=[branch:).*' | tr -d ' ' | tr -d ']')" >> $GITHUB_OUTPUT + echo "target-branch=$(echo "${{ inputs.text }}" | grep -oP '(?<=\[branch:).*\]' | tr -d ' ' | tr -d ']')" >> $GITHUB_OUTPUT From c7d4a6cb2290eedfa91b898b59e47a6a306dd78c Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 11:01:40 +0100 Subject: [PATCH 08/43] try to show branch in the UI --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4ac7c604a..dcdd88af30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,8 @@ jobs: library: ${{ fromJson(needs.impacted_libraries.outputs.impacted_libraries) }} fail-fast: false runs-on: ubuntu-latest + outputs: + target-branch: ${{ steps.get-target-branch.outputs.target-branch }} steps: - name: Checkout uses: actions/checkout@v4 @@ -91,7 +93,7 @@ jobs: path: binaries/ system_tests: - name: System Tests + name: System Tests ${{ needs.get_dev_artifacts.outputs.target-branch != '' && format('({0} branch)', needs.get_dev_artifacts.outputs.target-branch) || '' }} needs: - lint - test_the_test From 9517bc62358ffba4e41ecf6244ff6eb21c0faf75 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 11:10:53 +0100 Subject: [PATCH 09/43] force run From bec64d2c9e10c7e30a5e4c8e75c74fd6d9c1862c Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 11:18:42 +0100 Subject: [PATCH 10/43] force run not_existing_branch From f0aa746fa3266600fda144575305c1812d71484f Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 11:47:31 +0100 Subject: [PATCH 11/43] force run with existing branch From 14929181a1f91bf068ca04d64f40bf5937953d18 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 14:07:22 +0100 Subject: [PATCH 12/43] Fail if target branch job --- .github/workflows/ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcdd88af30..927f133824 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,18 @@ jobs: name: binaries_dev_${{ matrix.library }} path: binaries/ + fail-if-target-branch: + name: Check if target branch is specified + needs: + - get_dev_artifacts + if: needs.get_dev_artifacts.outputs.target-branch != '' + runs-on: ubuntu-latest + steps: + - name: Fail if PR title contains a target branch + run: | + echo "This PR can't be merged, due to the title specifying a target branch" + exit 1 + system_tests: name: System Tests ${{ needs.get_dev_artifacts.outputs.target-branch != '' && format('({0} branch)', needs.get_dev_artifacts.outputs.target-branch) || '' }} needs: From 261678322bb43f74e9ffe3860b7660f8df87e6a4 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 14:12:41 +0100 Subject: [PATCH 13/43] force run From b764f5fd938068f5831b7607c46033578c6ae41a Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 17:17:22 +0100 Subject: [PATCH 14/43] Update ruby and python load-from --- utils/scripts/load-binary.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 8ea7c2dd65..3ef7443b54 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -172,11 +172,15 @@ elif [ "$TARGET" = "dotnet" ]; then elif [ "$TARGET" = "python" ]; then assert_version_is_dev - echo "git+https://github.com/DataDog/dd-trace-py.git" > python-load-from-pip + TARGET_BRANCH="${TARGET_BRANCH:-main}" + echo "git+https://github.com/DataDog/dd-trace-py.git@$TARGET_BRANCH" > python-load-from-pip + echo "Using $(cat python-load-from-pip)" elif [ "$TARGET" = "ruby" ]; then assert_version_is_dev - echo "gem 'datadog', require: 'datadog/auto_instrument', git: 'https://github.com/Datadog/dd-trace-rb.git'" > ruby-load-from-bundle-add + + TARGET_BRANCH="${TARGET_BRANCH:-master}" + echo "gem 'datadog', require: 'datadog/auto_instrument', git: 'https://github.com/Datadog/dd-trace-rb.git', branch: '$TARGET_BRANCH'" > ruby-load-from-bundle-add echo "Using $(cat ruby-load-from-bundle-add)" elif [ "$TARGET" = "php" ]; then rm -rf *.tar.gz @@ -217,8 +221,8 @@ elif [ "$TARGET" = "nodejs" ]; then TARGET_BRANCH="${TARGET_BRANCH:-master}" # NPM builds the package, so we put a trigger file that tells install script to get package from github#master - echo "Using \"$TARGET_BRANCH\" target branch" echo "DataDog/dd-trace-js#$TARGET_BRANCH" > nodejs-load-from-npm + echo "Using $(cat nodejs-load-from-npm)" elif [ "$TARGET" = "waf_rule_set_v1" ]; then exit 1 From ceaa083fde6477b6e5eef507244311bfcb19ddef Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 17:20:17 +0100 Subject: [PATCH 15/43] force run From 70d6074b4cbf8370996e0cd25944507e344ef071 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 17:34:06 +0100 Subject: [PATCH 16/43] force run From be964ce3d53358d69de860de7830aa9516e0014d Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Mon, 16 Dec 2024 18:03:57 +0100 Subject: [PATCH 17/43] force run From a8fcdb9cdb927b21ae3f3893d5a2e90044a8b6a7 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 17 Dec 2024 09:12:34 +0100 Subject: [PATCH 18/43] use non-greedy match --- .github/actions/get_target_branch/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index c531d3a4b3..a0be7b609d 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -17,4 +17,4 @@ runs: if: ${{ contains(inputs.text, '[branch:') }} shell: bash run: | - echo "target-branch=$(echo "${{ inputs.text }}" | grep -oP '(?<=\[branch:).*\]' | tr -d ' ' | tr -d ']')" >> $GITHUB_OUTPUT + echo "target-branch=$(echo "${{ inputs.text }}" | grep -oP '(?<=\[branch:).+?\]' | tr -d ' ' | tr -d ']')" >> $GITHUB_OUTPUT From d6368d55fbb223b3f248944e4583d2e7fef425bd Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 17 Dec 2024 11:57:10 +0100 Subject: [PATCH 19/43] docs --- docs/CI/system-tests-ci.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/CI/system-tests-ci.md b/docs/CI/system-tests-ci.md index e221d06aa3..9626134594 100644 --- a/docs/CI/system-tests-ci.md +++ b/docs/CI/system-tests-ci.md @@ -14,3 +14,11 @@ The System-tests repository contains **one main workflow**: `ci.yml`. It is trig By default, after some basic test/lint jobs, this pipeline build alls weblogs (67!) in their `prod` (last release of all Datadog components) and `dev` (last commit on main of all Datadog components) versions. Then it runs the DEFAULT scenario on all of them. All of this in parallel (so 134 jobs), it takes few minutes to run. This workflow can validate any system-tests PR, as long as it modifies only the default scenario, which is the most common use case. See more details in the [docs about labels](./labels) + +### Target branch selection +`dev` version uses `main/master` branch by default but, in the case of some libraries (nodejs, python and ruby), it is possible to configure the CI to use a target branch. + +If PR's title includes `[branch:branch_name_to_test]` the workflow will use `branch_name_to_test` instead of `main/master` branch. + +As a security measure, the "Check if target branch is specified" job always fails if a target branch is selected. + From 754f83a2bc54341fb240e79ab86d402751d88dba Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 17 Dec 2024 16:00:20 +0100 Subject: [PATCH 20/43] force run From e56729f79ff32ab61024e4bb9e4a50d2bda7aafa Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 12:38:05 +0100 Subject: [PATCH 21/43] read branch from [target@branch] --- .github/actions/get_target_branch/action.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index a0be7b609d..c0ea3d9b8a 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -12,9 +12,19 @@ outputs: runs: using: composite steps: + - name: Check title + id: check-title + shell: bash + run: | + if [[ ${{ inputs.text }} =~ [java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\] ]]; then + echo "match=true" >> $GITHUB_OUTPUT + fi - name: Extract target branch id: extract - if: ${{ contains(inputs.text, '[branch:') }} + if: steps.check-title.outputs.match == 'true' shell: bash run: | - echo "target-branch=$(echo "${{ inputs.text }}" | grep -oP '(?<=\[branch:).+?\]' | tr -d ' ' | tr -d ']')" >> $GITHUB_OUTPUT + branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\]' | tr -d '[:space:]') + branch="${branch:1}" + branch="${branch::-1}" + echo "target-branch=${branch}" >> $GITHUB_OUTPUT From 0c13475c1f9ed81b816a60fa9fcf6ffec259f7a7 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 15:07:03 +0100 Subject: [PATCH 22/43] remove check-title step --- .github/actions/get_target_branch/action.yaml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index c0ea3d9b8a..13db9a7022 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -12,19 +12,13 @@ outputs: runs: using: composite steps: - - name: Check title - id: check-title - shell: bash - run: | - if [[ ${{ inputs.text }} =~ [java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\] ]]; then - echo "match=true" >> $GITHUB_OUTPUT - fi - name: Extract target branch id: extract - if: steps.check-title.outputs.match == 'true' shell: bash run: | branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\]' | tr -d '[:space:]') - branch="${branch:1}" - branch="${branch::-1}" - echo "target-branch=${branch}" >> $GITHUB_OUTPUT + if [[ ${#branch} > 0 ]]; then + branch="${branch:1}" + branch="${branch::-1}" + echo "target-branch=${branch}" >> $GITHUB_OUTPUT + if From 898d408e838d1be480bb084c72d3fd2c0151313b Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 15:49:45 +0100 Subject: [PATCH 23/43] Try to fix branch extraction and impacted libs --- .github/actions/get_target_branch/action.yaml | 2 +- .github/workflows/compute-impacted-libraries.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index 13db9a7022..f8b39735da 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -17,7 +17,7 @@ runs: shell: bash run: | branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\]' | tr -d '[:space:]') - if [[ ${#branch} > 0 ]]; then + if [ "${#branch}" -gt "0" ]; then branch="${branch:1}" branch="${branch::-1}" echo "target-branch=${branch}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/compute-impacted-libraries.yml b/.github/workflows/compute-impacted-libraries.yml index 00a8525a99..4c9c1f4150 100644 --- a/.github/workflows/compute-impacted-libraries.yml +++ b/.github/workflows/compute-impacted-libraries.yml @@ -44,7 +44,7 @@ jobs: else: pr_title = github_context["event"]["pull_request"]["title"].lower() - match = re.search(rf"^\[({libraries})\]", pr_title) + match = re.search(rf"^\[({libraries})(?:@([^\]]+)?)\]", pr_title) if match: print(f"PR title matchs => run {match[1]}") result.add(match[1]) From 0d72cebf6f90046d370195e8b27cf5c6672e3018 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 15:56:29 +0100 Subject: [PATCH 24/43] another try --- .github/actions/get_target_branch/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index f8b39735da..3b2e0ea6c1 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -17,7 +17,7 @@ runs: shell: bash run: | branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\]' | tr -d '[:space:]') - if [ "${#branch}" -gt "0" ]; then + if [ -n "$branch" ]; then branch="${branch:1}" branch="${branch::-1}" echo "target-branch=${branch}" >> $GITHUB_OUTPUT From 33fb51e356d036dfad38a905d71d08003cb388d4 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 16:00:11 +0100 Subject: [PATCH 25/43] fi --- .github/actions/get_target_branch/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index 3b2e0ea6c1..97196dd8e9 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -21,4 +21,4 @@ runs: branch="${branch:1}" branch="${branch::-1}" echo "target-branch=${branch}" >> $GITHUB_OUTPUT - if + fi From e18bca68d82cc770acb8bbfb90653918da61537a Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 16:27:39 +0100 Subject: [PATCH 26/43] rename job and update docs --- .github/workflows/ci.yml | 2 +- docs/CI/system-tests-ci.md | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 927f133824..01aeb097ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: path: binaries/ fail-if-target-branch: - name: Check if target branch is specified + name: Fail if target branch is specified needs: - get_dev_artifacts if: needs.get_dev_artifacts.outputs.target-branch != '' diff --git a/docs/CI/system-tests-ci.md b/docs/CI/system-tests-ci.md index 9626134594..d9774c36b8 100644 --- a/docs/CI/system-tests-ci.md +++ b/docs/CI/system-tests-ci.md @@ -18,7 +18,9 @@ This workflow can validate any system-tests PR, as long as it modifies only the ### Target branch selection `dev` version uses `main/master` branch by default but, in the case of some libraries (nodejs, python and ruby), it is possible to configure the CI to use a target branch. -If PR's title includes `[branch:branch_name_to_test]` the workflow will use `branch_name_to_test` instead of `main/master` branch. +If PR's title includes `[target_library@branch_name_to_test]` the workflow will use `branch_name_to_test` instead of `main/master` branch. -As a security measure, the "Check if target branch is specified" job always fails if a target branch is selected. +At the moment, it is not possible to run the CI for all libraries in a particular branch but it is limited to the target library indicated in the title. + +As a security measure, the "Fail if target branch is specified" job always fails if a target branch is selected. From 170b37d4fb56fa5522e69a3a8ebc22f682686745 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 17:02:30 +0100 Subject: [PATCH 27/43] set target-branch even when there is no branch --- .github/actions/get_target_branch/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index 97196dd8e9..fad79763aa 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -20,5 +20,5 @@ runs: if [ -n "$branch" ]; then branch="${branch:1}" branch="${branch::-1}" - echo "target-branch=${branch}" >> $GITHUB_OUTPUT fi + echo "target-branch=${branch}" >> $GITHUB_OUTPUT From d0cbf5550eb09ae9d942cc53b826bc669c2f4eff Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 17:11:49 +0100 Subject: [PATCH 28/43] fix impacted libs regex --- .github/workflows/compute-impacted-libraries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compute-impacted-libraries.yml b/.github/workflows/compute-impacted-libraries.yml index 4c9c1f4150..2de3a19db2 100644 --- a/.github/workflows/compute-impacted-libraries.yml +++ b/.github/workflows/compute-impacted-libraries.yml @@ -44,7 +44,7 @@ jobs: else: pr_title = github_context["event"]["pull_request"]["title"].lower() - match = re.search(rf"^\[({libraries})(?:@([^\]]+)?)\]", pr_title) + match = re.search(rf"^\[({libraries})(?:@([^\]]+))?\]", pr_title) if match: print(f"PR title matchs => run {match[1]}") result.add(match[1]) From 04bdec10da2bb32d366676a5bbd6608a5b21603a Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 17:30:56 +0100 Subject: [PATCH 29/43] ignore grep error when regex doesn't match --- .github/actions/get_target_branch/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index fad79763aa..d1753e22ed 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -16,7 +16,7 @@ runs: id: extract shell: bash run: | - branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\]' | tr -d '[:space:]') + branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\]' | tr -d '[:space:]' || true) if [ -n "$branch" ]; then branch="${branch:1}" branch="${branch::-1}" From 9685124d4b8287c002d6e8242d3cbfae373e9de5 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 19 Dec 2024 17:44:43 +0100 Subject: [PATCH 30/43] force run From 387195e9526823b155bab63db4bf1dc16e361e25 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 20 Dec 2024 09:06:30 +0100 Subject: [PATCH 31/43] force run From f21be766f304bbaf64184eaa67d68bb25e184344 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 20 Dec 2024 09:29:58 +0100 Subject: [PATCH 32/43] force run From 81a7000a7d8633905798601e3fa11a74f21bd73b Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 3 Jan 2025 13:20:51 +0100 Subject: [PATCH 33/43] fail if library is not supported --- .github/actions/get_target_branch/action.yaml | 12 +++++++++--- utils/scripts/load-binary.sh | 11 ++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index d1753e22ed..5c38644fc6 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -16,9 +16,15 @@ runs: id: extract shell: bash run: | - branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@.+?\]' | tr -d '[:space:]' || true) + branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@[^]]+(?=\])' | cut -c 2- | tr -d '[:space:]' || true) + if [ -n "$branch" ]; then - branch="${branch:1}" - branch="${branch::-1}" + notsupported=$(echo "$text" | grep -ioP '(java|dotnet|php|golang)') + + if [ -n "$notsupported" ]; then + echo "It is not possible to specify target branch for $notsupported yet" + exit 1 + fi fi + echo "target-branch=${branch}" >> $GITHUB_OUTPUT diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 3ef7443b54..acbb02337a 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -182,6 +182,7 @@ elif [ "$TARGET" = "ruby" ]; then TARGET_BRANCH="${TARGET_BRANCH:-master}" echo "gem 'datadog', require: 'datadog/auto_instrument', git: 'https://github.com/Datadog/dd-trace-rb.git', branch: '$TARGET_BRANCH'" > ruby-load-from-bundle-add echo "Using $(cat ruby-load-from-bundle-add)" + elif [ "$TARGET" = "php" ]; then rm -rf *.tar.gz if [ $VERSION = 'dev' ]; then @@ -192,6 +193,7 @@ elif [ "$TARGET" = "php" ]; then echo "Don't know how to load version $VERSION for $TARGET" fi mv ./temp/dd-library-php*.tar.gz . && mv ./temp/datadog-setup.php . && rm -rf ./temp + elif [ "$TARGET" = "golang" ]; then assert_version_is_dev rm -rf golang-load-from-go-get @@ -209,11 +211,14 @@ elif [ "$TARGET" = "cpp" ]; then # get_circleci_artifact "gh/DataDog/dd-opentracing-cpp" "build_test_deploy" "build" "TBD" # PROFILER: The main version is stored in s3, though we can not access this in CI # Not handled for now for system-tests. this handles artifact for parametric - echo "Using https://github.com/DataDog/dd-trace-cpp@main" - echo "https://github.com/DataDog/dd-trace-cpp@main" > cpp-load-from-git + TARGET_BRANCH="${TARGET_BRANCH:-main}" + echo "https://github.com/DataDog/dd-trace-cpp@$TARGET_BRANCH" > cpp-load-from-git + echo "Using $(cat cpp-load-from-git)" + elif [ "$TARGET" = "agent" ]; then assert_version_is_dev - echo "datadog/agent-dev:master-py3" > agent-image + TARGET_BRANCH="${TARGET_BRANCH:-master-py3}" + echo "datadog/agent-dev:$TARGET_BRANCH" > agent-image echo "Using $(cat agent-image) image" elif [ "$TARGET" = "nodejs" ]; then From 93adbfdc39111e9c0f33c926b22a3728bf18c78c Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Fri, 3 Jan 2025 13:29:59 +0100 Subject: [PATCH 34/43] linter and fix --- .github/actions/get_target_branch/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index 5c38644fc6..3ed1f51c5d 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -17,9 +17,9 @@ runs: shell: bash run: | branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@[^]]+(?=\])' | cut -c 2- | tr -d '[:space:]' || true) - + if [ -n "$branch" ]; then - notsupported=$(echo "$text" | grep -ioP '(java|dotnet|php|golang)') + notsupported=$(echo "$text" | grep -ioP '(java|dotnet|php|golang)' || true) if [ -n "$notsupported" ]; then echo "It is not possible to specify target branch for $notsupported yet" From 405c5f61fb7ff28e5a22ce23c16645004e671170 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 09:41:08 +0100 Subject: [PATCH 35/43] fail if not supported --- .github/actions/get_target_branch/action.yaml | 9 --------- utils/scripts/load-binary.sh | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index 3ed1f51c5d..ac001d6809 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -18,13 +18,4 @@ runs: run: | branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@[^]]+(?=\])' | cut -c 2- | tr -d '[:space:]' || true) - if [ -n "$branch" ]; then - notsupported=$(echo "$text" | grep -ioP '(java|dotnet|php|golang)' || true) - - if [ -n "$notsupported" ]; then - echo "It is not possible to specify target branch for $notsupported yet" - exit 1 - fi - fi - echo "target-branch=${branch}" >> $GITHUB_OUTPUT diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index acbb02337a..a0528c42ba 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -35,6 +35,17 @@ assert_version_is_dev() { exit 1 } +assert_target_branch_is_not_set() { + + if [ $TARGET_BRANCH = '' ]; then + return 0 + fi + + echo "It is not possible to specify target branch $TARGET_BRANCH for $TARGET yet" + + exit 1 +} + get_circleci_artifact() { SLUG=$1 @@ -162,10 +173,12 @@ cd binaries/ if [ "$TARGET" = "java" ]; then assert_version_is_dev + assert_target_branch_is_not_set ../utils/scripts/docker_base_image.sh ghcr.io/datadog/dd-trace-java/dd-trace-java:latest_snapshot . elif [ "$TARGET" = "dotnet" ]; then assert_version_is_dev + assert_target_branch_is_not_set rm -rf *.tar.gz ../utils/scripts/docker_base_image.sh ghcr.io/datadog/dd-trace-dotnet/dd-trace-dotnet:latest_snapshot . @@ -192,10 +205,12 @@ elif [ "$TARGET" = "php" ]; then else echo "Don't know how to load version $VERSION for $TARGET" fi + assert_target_branch_is_not_set mv ./temp/dd-library-php*.tar.gz . && mv ./temp/datadog-setup.php . && rm -rf ./temp elif [ "$TARGET" = "golang" ]; then assert_version_is_dev + assert_target_branch_is_not_set rm -rf golang-load-from-go-get # COMMIT_ID=$(curl -s 'https://api.github.com/repos/DataDog/dd-trace-go/branches/main' | jq -r .commit.sha) @@ -234,6 +249,7 @@ elif [ "$TARGET" = "waf_rule_set_v1" ]; then elif [ "$TARGET" = "waf_rule_set_v2" ]; then assert_version_is_dev + assert_target_branch_is_not_set curl --silent \ -H "Authorization: token $GH_TOKEN" \ -H "Accept: application/vnd.github.v3.raw" \ @@ -242,6 +258,7 @@ elif [ "$TARGET" = "waf_rule_set_v2" ]; then elif [ "$TARGET" = "waf_rule_set" ]; then assert_version_is_dev + assert_target_branch_is_not_set curl --fail --output "waf_rule_set.json" \ -H "Authorization: token $GH_TOKEN" \ -H "Accept: application/vnd.github.v3.raw" \ From 090faa69b0973114d30a387473b8e16b1c7616f3 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 09:44:34 +0100 Subject: [PATCH 36/43] Update docs/CI/system-tests-ci.md Co-authored-by: simon-id --- docs/CI/system-tests-ci.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CI/system-tests-ci.md b/docs/CI/system-tests-ci.md index d9774c36b8..cc743d681b 100644 --- a/docs/CI/system-tests-ci.md +++ b/docs/CI/system-tests-ci.md @@ -18,7 +18,7 @@ This workflow can validate any system-tests PR, as long as it modifies only the ### Target branch selection `dev` version uses `main/master` branch by default but, in the case of some libraries (nodejs, python and ruby), it is possible to configure the CI to use a target branch. -If PR's title includes `[target_library@branch_name_to_test]` the workflow will use `branch_name_to_test` instead of `main/master` branch. +If the PR's title includes `[target_library@branch_name_to_test]` the workflow will use `branch_name_to_test` instead of `main/master` branch. At the moment, it is not possible to run the CI for all libraries in a particular branch but it is limited to the target library indicated in the title. From f289a1d924551418fa7bafdb2f9570bc8930dc5c Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 10:12:53 +0100 Subject: [PATCH 37/43] force run From 70e6743ec27afb217a4c18a16dd6cc3ccf8e0d33 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 10:40:08 +0100 Subject: [PATCH 38/43] force run From 7a77caf478c41d03c99833658ae0274d1b5ad19f Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 10:57:11 +0100 Subject: [PATCH 39/43] force run From 21cd592cd538738d69b1305dbb309d3ebecd96b5 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 11:17:10 +0100 Subject: [PATCH 40/43] change assert_target_branch_is_not_set --- utils/scripts/load-binary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index a0528c42ba..990f6df863 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -37,7 +37,7 @@ assert_version_is_dev() { assert_target_branch_is_not_set() { - if [ $TARGET_BRANCH = '' ]; then + if [[ -z "${$TARGET_BRANCH}" ]]; then return 0 fi From bf9b6918e427a285fd3303c18d67610446eeb495 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 11:22:01 +0100 Subject: [PATCH 41/43] fix assert_target_branch_is_not_set --- utils/scripts/load-binary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 990f6df863..12a23bfc4b 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -37,7 +37,7 @@ assert_version_is_dev() { assert_target_branch_is_not_set() { - if [[ -z "${$TARGET_BRANCH}" ]]; then + if [[ -z "$TARGET_BRANCH" ]]; then return 0 fi From ae8b369ff118df98176ba0bc17fe4078250c2c9d Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 12:18:05 +0100 Subject: [PATCH 42/43] clean up messages --- docs/CI/system-tests-ci.md | 2 +- utils/scripts/load-binary.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CI/system-tests-ci.md b/docs/CI/system-tests-ci.md index cc743d681b..704ae1566c 100644 --- a/docs/CI/system-tests-ci.md +++ b/docs/CI/system-tests-ci.md @@ -16,7 +16,7 @@ By default, after some basic test/lint jobs, this pipeline build alls weblogs (6 This workflow can validate any system-tests PR, as long as it modifies only the default scenario, which is the most common use case. See more details in the [docs about labels](./labels) ### Target branch selection -`dev` version uses `main/master` branch by default but, in the case of some libraries (nodejs, python and ruby), it is possible to configure the CI to use a target branch. +`dev` version uses `main/master` branch by default but, in the case of some libraries (cpp, agent, nodejs, python and ruby), it is possible to configure the CI to use a target branch. If the PR's title includes `[target_library@branch_name_to_test]` the workflow will use `branch_name_to_test` instead of `main/master` branch. diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 12a23bfc4b..5c5724c372 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -41,7 +41,7 @@ assert_target_branch_is_not_set() { return 0 fi - echo "It is not possible to specify target branch $TARGET_BRANCH for $TARGET yet" + echo "It is not possible to specify the '$TARGET_BRANCH' target branch for $TARGET library yet" exit 1 } From fa2c84b722fda317509888c0c3d061720ca6f478 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 7 Jan 2025 14:17:37 +0100 Subject: [PATCH 43/43] fix beginning [ --- .github/actions/get_target_branch/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/get_target_branch/action.yaml b/.github/actions/get_target_branch/action.yaml index ac001d6809..884944a2bc 100644 --- a/.github/actions/get_target_branch/action.yaml +++ b/.github/actions/get_target_branch/action.yaml @@ -16,6 +16,6 @@ runs: id: extract shell: bash run: | - branch=$(echo "${{ inputs.text }}" | grep -ioP '(?<=\[java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@[^]]+(?=\])' | cut -c 2- | tr -d '[:space:]' || true) + branch=$(echo "${{ inputs.text }}" | grep -ioP '\[(?:java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@[^]]+(?=\])' | tr -d '[:space:]' || true) - echo "target-branch=${branch}" >> $GITHUB_OUTPUT + echo "target-branch=${branch#*@}" >> $GITHUB_OUTPUT