From ba158db46bf627b353857509b483a5c76280fbc3 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 25 Apr 2023 17:12:36 -0700 Subject: [PATCH 01/28] Support musllinux binary wheels on arm64 --- .../python_alpine_aarch64/Dockerfile | 20 ++++++++++ .../Dockerfile | 40 +++++++++++++++++++ tools/run_tests/artifacts/artifact_targets.py | 6 +++ .../artifacts/distribtest_targets.py | 1 + 4 files changed, 67 insertions(+) create mode 100644 tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile create mode 100644 tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile diff --git a/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile b/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile new file mode 100644 index 0000000000000..f119281d68b6f --- /dev/null +++ b/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile @@ -0,0 +1,20 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.10-alpine3.14 + +# Our test infrastructure demands bash +RUN apk update && apk add bash + +RUN pip3 install virtualenv diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile new file mode 100644 index 0000000000000..c57406d098726 --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM quay.io/pypa/musllinux_1_1_aarch64:2023-04-24-82a68e6 + +################################### +# Install Python build requirements +RUN /opt/python/cp36-cp36m/bin/pip install --upgrade cython +RUN /opt/python/cp37-cp37m/bin/pip install --upgrade cython +RUN /opt/python/cp38-cp38/bin/pip install --upgrade cython +RUN /opt/python/cp39-cp39/bin/pip install --upgrade cython +RUN /opt/python/cp310-cp310/bin/pip install --upgrade cython +RUN /opt/python/cp311-cp311/bin/pip install --upgrade cython + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.5.1 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.5.1 ccache.tar.gz + +RUN git config --global --add safe.directory /var/local/jenkins/grpc diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 50b22256a95b5..a9f3e12e7f7dc 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -452,6 +452,12 @@ def targets(): PythonArtifact( "musllinux_1_1", "x86", "cp313-cp313", presubmit=True ), + PythonArtifact("musllinux_1_1", "aarch64", "cp38-cp38", presubmit=True), + PythonArtifact("musllinux_1_1", "aarch64", "cp39-cp39"), + PythonArtifact("musllinux_1_1", "aarch64", "cp310-cp310"), + PythonArtifact("musllinux_1_1", "aarch64", "cp311-cp311"), + PythonArtifact("musllinux_1_1", "aarch64", "cp312-cp312"), + PythonArtifact("musllinux_1_1", "aarch64", "cp313-cp313", presubmit=True), PythonArtifact("macos", "x64", "python3.8", presubmit=True), PythonArtifact("macos", "x64", "python3.9"), PythonArtifact("macos", "x64", "python3.10"), diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 2750bf427866f..9d5c7f9cd09c6 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -461,6 +461,7 @@ def targets(): PythonDistribTest( "linux", "aarch64", "python38_buster", presubmit=True ), + PythonDistribTest("linux", "aarch64", "alpine", presubmit=True), PythonDistribTest( "linux", "x64", "alpine3.7", source=True, presubmit=True ), From 6ff3d8816f25310f945c8a5e4a1010fb7174c731 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 9 Aug 2023 15:10:32 -0700 Subject: [PATCH 02/28] Fixed copyright year --- tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile | 2 +- .../grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile b/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile index f119281d68b6f..a7500e39aa557 100644 --- a/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile +++ b/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2021 The gRPC Authors +# Copyright 2023 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile index c57406d098726..cc6b727bc87f2 100644 --- a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2021 The gRPC Authors +# Copyright 2023 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 72fed00513c59b8279889505dea85803acb9a671 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 16 Aug 2023 20:17:26 +0000 Subject: [PATCH 03/28] Regenerate images --- .../dockerfile/distribtest/python_alpine_aarch64.current_version | 1 + .../grpc_artifact_python_musllinux_1_1_aarch64.current_version | 1 + 2 files changed, 2 insertions(+) create mode 100644 tools/dockerfile/distribtest/python_alpine_aarch64.current_version create mode 100644 tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version diff --git a/tools/dockerfile/distribtest/python_alpine_aarch64.current_version b/tools/dockerfile/distribtest/python_alpine_aarch64.current_version new file mode 100644 index 0000000000000..a3efd8f5bcc0b --- /dev/null +++ b/tools/dockerfile/distribtest/python_alpine_aarch64.current_version @@ -0,0 +1 @@ +us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:086c6b7ba4177b5c086a4e1e36019fdd1d86a061@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version new file mode 100644 index 0000000000000..785461d641d04 --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version @@ -0,0 +1 @@ +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64:916a75b9af58419a474ed8cfafb1a1451f82434a@sha256:e23144c76abe2f7b342a2317312f1ae12ccbca6492e9ee324e2aca7a98edae8c \ No newline at end of file From f7d61e132b30f0ea1751888bb5a443571dc7bd7c Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 4 Dec 2024 16:35:23 +0100 Subject: [PATCH 04/28] Add template for dockerfile image --- .../Dockerfile.template | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template diff --git a/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template b/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template new file mode 100644 index 0000000000000..9433c7202407e --- /dev/null +++ b/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template @@ -0,0 +1,27 @@ +%YAML 1.2 +--- | + # Copyright 2024 The gRPC Authors + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64/Dockerfile.template`!!! + + FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546 + + <%include file="../python_pip_builds.include"/> + + <%include file="../ccache.include"/> + + <%include file="../git_config.include"/> + + RUN apk add openssl openssl-dev From 6c3d217af843772a5b0ef048bf4d54db5e21fa6e Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 4 Dec 2024 16:40:25 +0100 Subject: [PATCH 05/28] Refresh Dockerfile --- .../Dockerfile | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile index cc6b727bc87f2..bb392b6b39bfe 100644 --- a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2023 The gRPC Authors +# Copyright 2024 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,29 +12,55 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM quay.io/pypa/musllinux_1_1_aarch64:2023-04-24-82a68e6 +# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!! -################################### +FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546 + +#=================================== # Install Python build requirements -RUN /opt/python/cp36-cp36m/bin/pip install --upgrade cython -RUN /opt/python/cp37-cp37m/bin/pip install --upgrade cython -RUN /opt/python/cp38-cp38/bin/pip install --upgrade cython -RUN /opt/python/cp39-cp39/bin/pip install --upgrade cython -RUN /opt/python/cp310-cp310/bin/pip install --upgrade cython -RUN /opt/python/cp311-cp311/bin/pip install --upgrade cython +RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp313-cp313/bin/pip install --upgrade 'cython<4.0.0rc1' #================= # Install ccache # Install ccache from source since ccache 3.x packaged with most linux distributions # does not support Redis backend for caching. -RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.5.1/ccache-4.5.1.tar.gz \ +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.7.5/ccache-4.7.5.tar.gz \ && tar -zxf ccache.tar.gz \ - && cd ccache-4.5.1 \ + && cd ccache-4.7.5 \ && mkdir build && cd build \ && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ && make -j4 && make install \ && cd ../.. \ - && rm -rf ccache-4.5.1 ccache.tar.gz + && rm -rf ccache-4.7.5 ccache.tar.gz + +# TODO: simplify the list of third_party modules list +# NOTE: git>=2.46 allows leading paths like third_party/* to include all subdirectories +# current docker base images use git versions lower than 2.46 and hence require separate configs for each submodule RUN git config --global --add safe.directory /var/local/jenkins/grpc +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/bloaty +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/xds +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googleapis +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googletest +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opencensus-proto +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/boringssl-with-bazel +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/envoy-api +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protobuf +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/zlib +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/benchmark +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/re2 +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/abseil-cpp +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry-cpp +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protoc-gen-validate +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/cares/cares +RUN git config --global protocol.file.allow always + +RUN apk add openssl openssl-dev From 8120f83ae0d4945d89713a3a6405a6d325dbf4de Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 12 Dec 2024 23:15:41 +0100 Subject: [PATCH 06/28] Update templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template --- .../Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template b/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template index 9433c7202407e..dc34c01c34b60 100644 --- a/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template +++ b/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64/Dockerfile.template`!!! + # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!! FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546 From 63cbe6de15afb952700aaae3c2f258d36b0e973a Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 12 Dec 2024 22:46:50 +0000 Subject: [PATCH 07/28] Run push_testing_images --- tools/bazelify_tests/dockerimage_current_versions.bzl | 2 ++ .../grpc_artifact_python_musllinux_1_1_aarch64.current_version | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index 328fa4a380711..73c63276a11c5 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -36,6 +36,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:004e02902825b54c7f0d60eaed0819acf6f10c24853bf8f793001114e9969abd", "tools/dockerfile/distribtest/csharp_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu2204_x64@sha256:a439f2ccbc666f231e511a8e58eb7f66a3de4820a4d5aded1e62275cf8ac49f0", "tools/dockerfile/distribtest/php8_debian12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php8_debian12_x64@sha256:42399d061f234ee0ef79b333555db0e11ca4f106bb1d49276abf99c459c104f0", + "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e", "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:d7232ef55c81d125b66899b908668156638cd2cfc5b119083eceacb6018da7e4", "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1", "tools/dockerfile/distribtest/python_bullseye_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x64@sha256:80553398f0c59c1dc186052f4f2deaf18fea582f6d1d166eec6ea298639031fb", @@ -67,6 +68,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:263ea79d940c905233624b02751194474408338b0f5ae2fb822966b2f20f47ce", "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:0e102df67f31aeb6afe68250603288c2f1c98ccf360d1c42d751b8451da94b48", "tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:527e2e9ec4db0c52a53b50abfd59907a1b7e221168dc401686f6a48d33bddc5c", + "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27", "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:94b57e5ea31ebc29af734474bcaff3074770778e5d27557cdc06d755ee8bc7ed", "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:edf4a0c8333c9309e52f323aa7315bbc0e5643216613cab4ecd2bce3d1ec26c0", "tools/dockerfile/interoptest/grpc_interop_aspnetcore.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_aspnetcore@sha256:8e2e732e78724a8382c340dca72e7653c5f82c251a3110fa2874cc00ba538878", diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version index 785461d641d04..39e0c47d2a033 100644 --- a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64:916a75b9af58419a474ed8cfafb1a1451f82434a@sha256:e23144c76abe2f7b342a2317312f1ae12ccbca6492e9ee324e2aca7a98edae8c \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64:b171ae144fc76a473fdfad8e25c36a02b886f6a9@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27 \ No newline at end of file From 75c6914d81d6cb72fd841c3e4aea121cc11a73dc Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 13 Dec 2024 09:35:06 +0100 Subject: [PATCH 08/28] Run black over artifact_targets.py --- tools/run_tests/artifacts/artifact_targets.py | 61 +++++-------------- 1 file changed, 16 insertions(+), 45 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index a9f3e12e7f7dc..d7eb03fc8749f 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -52,8 +52,7 @@ def create_docker_jobspec( if extra_docker_args is not None: docker_env["EXTRA_DOCKER_ARGS"] = extra_docker_args jobspec = jobset.JobSpec( - cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"] - + docker_args, + cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"] + docker_args, environ=docker_env, shortname="build_artifact.%s" % (name), timeout_seconds=timeout_seconds, @@ -146,18 +145,14 @@ def build_jobspec(self, inner_jobs=None): if self.platform == "linux_extra": # Crosscompilation build for armv7 (e.g. Raspberry Pi) - environ["PYTHON"] = "/opt/python/{}/bin/python3".format( - self.py_version - ) + environ["PYTHON"] = "/opt/python/{}/bin/python3".format(self.py_version) environ["PIP"] = "/opt/python/{}/bin/pip3".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" environ["LDFLAGS"] = "-s" return create_docker_jobspec( self.name, - "tools/dockerfile/grpc_artifact_python_linux_{}".format( - self.arch - ), + "tools/dockerfile/grpc_artifact_python_linux_{}".format(self.arch), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, timeout_seconds=60 * 60 * 2, @@ -168,9 +163,7 @@ def build_jobspec(self, inner_jobs=None): environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" # Inside the manylinux container, the python installations are located in # special places... - environ["PYTHON"] = "/opt/python/{}/bin/python".format( - self.py_version - ) + environ["PYTHON"] = "/opt/python/{}/bin/python".format(self.py_version) environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" if self.arch == "aarch64": @@ -195,9 +188,7 @@ def build_jobspec(self, inner_jobs=None): timeout_seconds=60 * 60 * 2, ) elif "musllinux" in self.platform: - environ["PYTHON"] = "/opt/python/{}/bin/python".format( - self.py_version - ) + environ["PYTHON"] = "/opt/python/{}/bin/python".format(self.py_version) environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" @@ -218,9 +209,7 @@ def build_jobspec(self, inner_jobs=None): environ["EXT_COMPILER"] = "msvc" # For some reason, the batch script %random% always runs with the same # seed. We create a random temp-dir here - dir = "".join( - random.choice(string.ascii_uppercase) for _ in range(10) - ) + dir = "".join(random.choice(string.ascii_uppercase) for _ in range(10)) return create_jobspec( self.name, [ @@ -334,17 +323,13 @@ def build_jobspec(self, inner_jobs=None): environ["CXXFLAGS"] = "" environ["LDFLAGS"] = "" if self.platform == "linux": - dockerfile_dir = ( - "tools/dockerfile/grpc_artifact_centos6_{}".format( - self.arch - ) + dockerfile_dir = "tools/dockerfile/grpc_artifact_centos6_{}".format( + self.arch ) if self.arch == "aarch64": # for aarch64, use a dockcross manylinux image that will # give us both ready to use crosscompiler and sufficient backward compatibility - dockerfile_dir = ( - "tools/dockerfile/grpc_artifact_protoc_aarch64" - ) + dockerfile_dir = "tools/dockerfile/grpc_artifact_protoc_aarch64" environ["LDFLAGS"] += " -static-libgcc -static-libstdc++ -s" return create_docker_jobspec( self.name, @@ -407,51 +392,37 @@ def targets(): PythonArtifact("manylinux2014", "x64", "cp310-cp310"), PythonArtifact("manylinux2014", "x64", "cp311-cp311"), PythonArtifact("manylinux2014", "x64", "cp312-cp312"), - PythonArtifact( - "manylinux2014", "x64", "cp313-cp313", presubmit=True - ), + PythonArtifact("manylinux2014", "x64", "cp313-cp313", presubmit=True), PythonArtifact("manylinux2014", "x86", "cp38-cp38", presubmit=True), PythonArtifact("manylinux2014", "x86", "cp39-cp39", presubmit=True), PythonArtifact("manylinux2014", "x86", "cp310-cp310"), PythonArtifact("manylinux2014", "x86", "cp311-cp311"), PythonArtifact("manylinux2014", "x86", "cp312-cp312"), - PythonArtifact( - "manylinux2014", "x86", "cp313-cp313", presubmit=True - ), - PythonArtifact( - "manylinux2014", "aarch64", "cp38-cp38", presubmit=True - ), + PythonArtifact("manylinux2014", "x86", "cp313-cp313", presubmit=True), + PythonArtifact("manylinux2014", "aarch64", "cp38-cp38", presubmit=True), PythonArtifact("manylinux2014", "aarch64", "cp39-cp39"), PythonArtifact("manylinux2014", "aarch64", "cp310-cp310"), PythonArtifact("manylinux2014", "aarch64", "cp311-cp311"), PythonArtifact("manylinux2014", "aarch64", "cp312-cp312"), - PythonArtifact( - "manylinux2014", "aarch64", "cp313-cp313", presubmit=True - ), + PythonArtifact("manylinux2014", "aarch64", "cp313-cp313", presubmit=True), PythonArtifact("linux_extra", "armv7", "cp38-cp38", presubmit=True), PythonArtifact("linux_extra", "armv7", "cp39-cp39"), PythonArtifact("linux_extra", "armv7", "cp310-cp310"), PythonArtifact("linux_extra", "armv7", "cp311-cp311"), PythonArtifact("linux_extra", "armv7", "cp312-cp312"), - PythonArtifact( - "linux_extra", "armv7", "cp313-cp313", presubmit=True - ), + PythonArtifact("linux_extra", "armv7", "cp313-cp313", presubmit=True), PythonArtifact("musllinux_1_1", "x64", "cp38-cp38", presubmit=True), PythonArtifact("musllinux_1_1", "x64", "cp39-cp39"), PythonArtifact("musllinux_1_1", "x64", "cp310-cp310"), PythonArtifact("musllinux_1_1", "x64", "cp311-cp311"), PythonArtifact("musllinux_1_1", "x64", "cp312-cp312"), - PythonArtifact( - "musllinux_1_1", "x64", "cp313-cp313", presubmit=True - ), + PythonArtifact("musllinux_1_1", "x64", "cp313-cp313", presubmit=True), PythonArtifact("musllinux_1_1", "x86", "cp38-cp38", presubmit=True), PythonArtifact("musllinux_1_1", "x86", "cp39-cp39"), PythonArtifact("musllinux_1_1", "x86", "cp310-cp310"), PythonArtifact("musllinux_1_1", "x86", "cp311-cp311"), PythonArtifact("musllinux_1_1", "x86", "cp312-cp312"), - PythonArtifact( - "musllinux_1_1", "x86", "cp313-cp313", presubmit=True - ), + PythonArtifact("musllinux_1_1", "x86", "cp313-cp313", presubmit=True), PythonArtifact("musllinux_1_1", "aarch64", "cp38-cp38", presubmit=True), PythonArtifact("musllinux_1_1", "aarch64", "cp39-cp39"), PythonArtifact("musllinux_1_1", "aarch64", "cp310-cp310"), From 1e036afcff8939a2f920f6f3f8a246f7db77fcc1 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 16 Dec 2024 11:24:37 +0100 Subject: [PATCH 09/28] It looks like black does not load the config file in the current dir automatically --- tools/run_tests/artifacts/artifact_targets.py | 69 ++++++++++++++----- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index d7eb03fc8749f..45461e3e7accb 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -52,7 +52,8 @@ def create_docker_jobspec( if extra_docker_args is not None: docker_env["EXTRA_DOCKER_ARGS"] = extra_docker_args jobspec = jobset.JobSpec( - cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"] + docker_args, + cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"] + + docker_args, environ=docker_env, shortname="build_artifact.%s" % (name), timeout_seconds=timeout_seconds, @@ -145,14 +146,18 @@ def build_jobspec(self, inner_jobs=None): if self.platform == "linux_extra": # Crosscompilation build for armv7 (e.g. Raspberry Pi) - environ["PYTHON"] = "/opt/python/{}/bin/python3".format(self.py_version) + environ["PYTHON"] = "/opt/python/{}/bin/python3".format( + self.py_version + ) environ["PIP"] = "/opt/python/{}/bin/pip3".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" environ["LDFLAGS"] = "-s" return create_docker_jobspec( self.name, - "tools/dockerfile/grpc_artifact_python_linux_{}".format(self.arch), + "tools/dockerfile/grpc_artifact_python_linux_{}".format( + self.arch + ), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, timeout_seconds=60 * 60 * 2, @@ -163,7 +168,9 @@ def build_jobspec(self, inner_jobs=None): environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" # Inside the manylinux container, the python installations are located in # special places... - environ["PYTHON"] = "/opt/python/{}/bin/python".format(self.py_version) + environ["PYTHON"] = "/opt/python/{}/bin/python".format( + self.py_version + ) environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" if self.arch == "aarch64": @@ -188,7 +195,9 @@ def build_jobspec(self, inner_jobs=None): timeout_seconds=60 * 60 * 2, ) elif "musllinux" in self.platform: - environ["PYTHON"] = "/opt/python/{}/bin/python".format(self.py_version) + environ["PYTHON"] = "/opt/python/{}/bin/python".format( + self.py_version + ) environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" @@ -209,7 +218,9 @@ def build_jobspec(self, inner_jobs=None): environ["EXT_COMPILER"] = "msvc" # For some reason, the batch script %random% always runs with the same # seed. We create a random temp-dir here - dir = "".join(random.choice(string.ascii_uppercase) for _ in range(10)) + dir = "".join( + random.choice(string.ascii_uppercase) for _ in range(10) + ) return create_jobspec( self.name, [ @@ -323,13 +334,17 @@ def build_jobspec(self, inner_jobs=None): environ["CXXFLAGS"] = "" environ["LDFLAGS"] = "" if self.platform == "linux": - dockerfile_dir = "tools/dockerfile/grpc_artifact_centos6_{}".format( - self.arch + dockerfile_dir = ( + "tools/dockerfile/grpc_artifact_centos6_{}".format( + self.arch + ) ) if self.arch == "aarch64": # for aarch64, use a dockcross manylinux image that will # give us both ready to use crosscompiler and sufficient backward compatibility - dockerfile_dir = "tools/dockerfile/grpc_artifact_protoc_aarch64" + dockerfile_dir = ( + "tools/dockerfile/grpc_artifact_protoc_aarch64" + ) environ["LDFLAGS"] += " -static-libgcc -static-libstdc++ -s" return create_docker_jobspec( self.name, @@ -392,43 +407,61 @@ def targets(): PythonArtifact("manylinux2014", "x64", "cp310-cp310"), PythonArtifact("manylinux2014", "x64", "cp311-cp311"), PythonArtifact("manylinux2014", "x64", "cp312-cp312"), - PythonArtifact("manylinux2014", "x64", "cp313-cp313", presubmit=True), + PythonArtifact( + "manylinux2014", "x64", "cp313-cp313", presubmit=True + ), PythonArtifact("manylinux2014", "x86", "cp38-cp38", presubmit=True), PythonArtifact("manylinux2014", "x86", "cp39-cp39", presubmit=True), PythonArtifact("manylinux2014", "x86", "cp310-cp310"), PythonArtifact("manylinux2014", "x86", "cp311-cp311"), PythonArtifact("manylinux2014", "x86", "cp312-cp312"), - PythonArtifact("manylinux2014", "x86", "cp313-cp313", presubmit=True), - PythonArtifact("manylinux2014", "aarch64", "cp38-cp38", presubmit=True), + PythonArtifact( + "manylinux2014", "x86", "cp313-cp313", presubmit=True + ), + PythonArtifact( + "manylinux2014", "aarch64", "cp38-cp38", presubmit=True + ), PythonArtifact("manylinux2014", "aarch64", "cp39-cp39"), PythonArtifact("manylinux2014", "aarch64", "cp310-cp310"), PythonArtifact("manylinux2014", "aarch64", "cp311-cp311"), PythonArtifact("manylinux2014", "aarch64", "cp312-cp312"), - PythonArtifact("manylinux2014", "aarch64", "cp313-cp313", presubmit=True), + PythonArtifact( + "manylinux2014", "aarch64", "cp313-cp313", presubmit=True + ), PythonArtifact("linux_extra", "armv7", "cp38-cp38", presubmit=True), PythonArtifact("linux_extra", "armv7", "cp39-cp39"), PythonArtifact("linux_extra", "armv7", "cp310-cp310"), PythonArtifact("linux_extra", "armv7", "cp311-cp311"), PythonArtifact("linux_extra", "armv7", "cp312-cp312"), - PythonArtifact("linux_extra", "armv7", "cp313-cp313", presubmit=True), + PythonArtifact( + "linux_extra", "armv7", "cp313-cp313", presubmit=True + ), PythonArtifact("musllinux_1_1", "x64", "cp38-cp38", presubmit=True), PythonArtifact("musllinux_1_1", "x64", "cp39-cp39"), PythonArtifact("musllinux_1_1", "x64", "cp310-cp310"), PythonArtifact("musllinux_1_1", "x64", "cp311-cp311"), PythonArtifact("musllinux_1_1", "x64", "cp312-cp312"), - PythonArtifact("musllinux_1_1", "x64", "cp313-cp313", presubmit=True), + PythonArtifact( + "musllinux_1_1", "x64", "cp313-cp313", presubmit=True + ), PythonArtifact("musllinux_1_1", "x86", "cp38-cp38", presubmit=True), PythonArtifact("musllinux_1_1", "x86", "cp39-cp39"), PythonArtifact("musllinux_1_1", "x86", "cp310-cp310"), PythonArtifact("musllinux_1_1", "x86", "cp311-cp311"), PythonArtifact("musllinux_1_1", "x86", "cp312-cp312"), - PythonArtifact("musllinux_1_1", "x86", "cp313-cp313", presubmit=True), - PythonArtifact("musllinux_1_1", "aarch64", "cp38-cp38", presubmit=True), + PythonArtifact( + "musllinux_1_1", "x86", "cp313-cp313", presubmit=True + ), + PythonArtifact( + "musllinux_1_1", "aarch64", "cp38-cp38", presubmit=True + ), PythonArtifact("musllinux_1_1", "aarch64", "cp39-cp39"), PythonArtifact("musllinux_1_1", "aarch64", "cp310-cp310"), PythonArtifact("musllinux_1_1", "aarch64", "cp311-cp311"), PythonArtifact("musllinux_1_1", "aarch64", "cp312-cp312"), - PythonArtifact("musllinux_1_1", "aarch64", "cp313-cp313", presubmit=True), + PythonArtifact( + "musllinux_1_1", "aarch64", "cp313-cp313", presubmit=True + ), PythonArtifact("macos", "x64", "python3.8", presubmit=True), PythonArtifact("macos", "x64", "python3.9"), PythonArtifact("macos", "x64", "python3.10"), From 7e92d13d90e055e3956bc91c7287c5ec511a8f31 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Tue, 31 Dec 2024 00:07:24 +0000 Subject: [PATCH 10/28] Extend build artifact timeout from 2hr to 4hr --- tools/run_tests/artifacts/artifact_targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 45461e3e7accb..4cd999d965663 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -212,7 +212,7 @@ def build_jobspec(self, inner_jobs=None): % (self.platform, self.arch), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, - timeout_seconds=60 * 60 * 2, + timeout_seconds=60 * 60 * 4, ) elif self.platform == "windows": environ["EXT_COMPILER"] = "msvc" From 6eccb70b9812e85f4fc7d378848558cfc61a6a29 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Tue, 7 Jan 2025 02:09:27 +0000 Subject: [PATCH 11/28] Extend timeout to 8hr --- tools/run_tests/artifacts/artifact_targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 4cd999d965663..6c072ed17e443 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -212,7 +212,7 @@ def build_jobspec(self, inner_jobs=None): % (self.platform, self.arch), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, - timeout_seconds=60 * 60 * 4, + timeout_seconds=60 * 60 * 8, ) elif self.platform == "windows": environ["EXT_COMPILER"] = "msvc" From 10a9965586796df45e646f5d51e49faac7d2d15a Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Mon, 30 Dec 2024 17:25:21 +0000 Subject: [PATCH 12/28] Change timeout to 14hr --- .../internal_ci/linux/grpc_distribtests_python.cfg | 2 +- .../pull_request/grpc_distribtests_python.cfg | 2 +- .../linux/release/grpc_distribtests_python.cfg | 2 +- .../internal_ci/macos/grpc_distribtests_python.cfg | 2 +- .../pull_request/grpc_distribtests_python.cfg | 2 +- .../macos/release/grpc_distribtests_python.cfg | 2 +- tools/run_tests/artifacts/artifact_targets.py | 14 ++++++++++---- 7 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tools/internal_ci/linux/grpc_distribtests_python.cfg b/tools/internal_ci/linux/grpc_distribtests_python.cfg index 307b3b61f0abb..ee17d7a29202f 100644 --- a/tools/internal_ci/linux/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/grpc_distribtests_python.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" -timeout_mins: 240 +timeout_mins: 840 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg b/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg index de62ae1c80b99..ccc02bff0226d 100644 --- a/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" -timeout_mins: 240 +timeout_mins: 840 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg index 6a916e938710e..6ed2e7d67f603 100644 --- a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" -timeout_mins: 240 +timeout_mins: 840 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/macos/grpc_distribtests_python.cfg b/tools/internal_ci/macos/grpc_distribtests_python.cfg index e3d0b0c13d769..2f0fc7990a2cf 100644 --- a/tools/internal_ci/macos/grpc_distribtests_python.cfg +++ b/tools/internal_ci/macos/grpc_distribtests_python.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" -timeout_mins: 240 +timeout_mins: 840 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg b/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg index 4541d27ad5cdb..ee7636b30594f 100644 --- a/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" -timeout_mins: 240 +timeout_mins: 840 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/macos/release/grpc_distribtests_python.cfg b/tools/internal_ci/macos/release/grpc_distribtests_python.cfg index e3d0b0c13d769..2f0fc7990a2cf 100644 --- a/tools/internal_ci/macos/release/grpc_distribtests_python.cfg +++ b/tools/internal_ci/macos/release/grpc_distribtests_python.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" -timeout_mins: 240 +timeout_mins: 840 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 6c072ed17e443..81835d488d473 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -200,19 +200,25 @@ def build_jobspec(self, inner_jobs=None): ) environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" - environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" - environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE" - if self.arch == "x86": + if self.arch in ("x86", "aarch64"): environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" + if self.arch == "aarch64": + # As we won't strip the binary with auditwheel (see below), strip + # it at link time. + environ["LDFLAGS"] = "-s" + else: + environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" + environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE" + return create_docker_jobspec( self.name, "tools/dockerfile/grpc_artifact_python_%s_%s" % (self.platform, self.arch), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, - timeout_seconds=60 * 60 * 8, + timeout_seconds=60 * 60 * 10, ) elif self.platform == "windows": environ["EXT_COMPILER"] = "msvc" From 4173082b773917c84f4429ee50969c1fc371cb8c Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 9 Jan 2025 17:07:53 +0000 Subject: [PATCH 13/28] Keep timeout for macos --- tools/internal_ci/macos/grpc_distribtests_python.cfg | 2 +- .../internal_ci/macos/pull_request/grpc_distribtests_python.cfg | 2 +- tools/internal_ci/macos/release/grpc_distribtests_python.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/internal_ci/macos/grpc_distribtests_python.cfg b/tools/internal_ci/macos/grpc_distribtests_python.cfg index 2f0fc7990a2cf..e3d0b0c13d769 100644 --- a/tools/internal_ci/macos/grpc_distribtests_python.cfg +++ b/tools/internal_ci/macos/grpc_distribtests_python.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" -timeout_mins: 840 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg b/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg index ee7636b30594f..4541d27ad5cdb 100644 --- a/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" -timeout_mins: 840 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/macos/release/grpc_distribtests_python.cfg b/tools/internal_ci/macos/release/grpc_distribtests_python.cfg index 2f0fc7990a2cf..e3d0b0c13d769 100644 --- a/tools/internal_ci/macos/release/grpc_distribtests_python.cfg +++ b/tools/internal_ci/macos/release/grpc_distribtests_python.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" -timeout_mins: 840 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.*" From 5a3616852cd9cbe19995ba9b3ae0241c874a7b19 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Fri, 10 Jan 2025 17:30:53 +0000 Subject: [PATCH 14/28] Remove python_linux_aarch64_alpine from distribtest presubmit target --- tools/run_tests/artifacts/distribtest_targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 9d5c7f9cd09c6..93bb1b052f03b 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -461,7 +461,7 @@ def targets(): PythonDistribTest( "linux", "aarch64", "python38_buster", presubmit=True ), - PythonDistribTest("linux", "aarch64", "alpine", presubmit=True), + PythonDistribTest("linux", "aarch64", "alpine"), PythonDistribTest( "linux", "x64", "alpine3.7", source=True, presubmit=True ), From 55040ddad465115344494b7f30f9a9b5e5a90621 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Mon, 13 Jan 2025 16:56:23 +0000 Subject: [PATCH 15/28] Resolve conflict --- tools/bazelify_tests/dockerimage_current_versions.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index 73c63276a11c5..c23b1bad31343 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -37,7 +37,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/distribtest/csharp_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu2204_x64@sha256:a439f2ccbc666f231e511a8e58eb7f66a3de4820a4d5aded1e62275cf8ac49f0", "tools/dockerfile/distribtest/php8_debian12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php8_debian12_x64@sha256:42399d061f234ee0ef79b333555db0e11ca4f106bb1d49276abf99c459c104f0", "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e", - "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:d7232ef55c81d125b66899b908668156638cd2cfc5b119083eceacb6018da7e4", + "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:1bbc6fa5b4b650d3037d089e164d364e05a6daf1ed6dd1025ba07cc127f73d7d", "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1", "tools/dockerfile/distribtest/python_bullseye_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x64@sha256:80553398f0c59c1dc186052f4f2deaf18fea582f6d1d166eec6ea298639031fb", "tools/dockerfile/distribtest/python_bullseye_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x86@sha256:81300aed2bb5e6b3f9bf9f23ba751e884e0317a9cd79659d37cdef4f1cfd24ca", From 85b61c830bc587eb951e171fd067ffa01f0ce2e0 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Mon, 13 Jan 2025 23:51:54 +0000 Subject: [PATCH 16/28] Extend python run_tests_matrix timeout from 3600s to 7200s --- .../windows/pull_request/grpc_basictests_python.cfg | 2 +- tools/run_tests/run_tests_matrix.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg index ab541436b407b..fa2fc655ee7e9 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=5400" + value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=7200" } diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 6a20ec0f103f4..bb1da1d8837d0 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -33,6 +33,9 @@ # C/C++ tests can take long time _CPP_RUNTESTS_TIMEOUT = 6 * 60 * 60 +# Python tests can take long time +_PYTHON_RUNTESTS_TIMEOUT = 2 * 60 * 60 + # Set timeout high for ObjC for Cocoapods to install pods _OBJC_RUNTESTS_TIMEOUT = 4 * 60 * 60 @@ -278,6 +281,7 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): labels=["basictests", "multilang"], extra_args=extra_args + ["--report_multi_target"], inner_jobs=inner_jobs, + timeout_seconds=_PYTHON_RUNTESTS_TIMEOUT, ) # ARM64 Linux Python tests From 26bae730a1b80a88bf0dacef25819f5cfa00365a Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Wed, 22 Jan 2025 23:05:02 +0000 Subject: [PATCH 17/28] [Python dist] Build aarch64 in different Kokoro job --- .../linux/grpc_distribtests_python.cfg | 10 ++++++ .../linux/grpc_distribtests_python.sh | 8 ++++- .../linux/grpc_distribtests_python_arm64.cfg | 36 +++++++++++++++++++ .../grpc_examples_sanity_tests.cfg | 10 ++++++ .../release/grpc_distribtests_python.cfg | 10 ++++++ tools/run_tests/task_runner.py | 11 ++++++ 6 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg diff --git a/tools/internal_ci/linux/grpc_distribtests_python.cfg b/tools/internal_ci/linux/grpc_distribtests_python.cfg index ee17d7a29202f..73cc665d44b15 100644 --- a/tools/internal_ci/linux/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/grpc_distribtests_python.cfg @@ -24,3 +24,13 @@ action { regex: "github/grpc/artifacts/**" } } + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "-e aarch64" +} + +# env_vars { +# key: "TASK_RUNNER_EXCLUDE_FILTERS" +# value: "aarch64" +# } diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index dd5dba63dcf95..c6ca715d320e0 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -26,7 +26,13 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc # some distribtests use a pre-registered binfmt_misc hook # to automatically execute foreign binaries (such as aarch64) # under qemu emulator. -source tools/internal_ci/helper_scripts/prepare_qemu_rc +# TASK_RUNNER_EXTRA_FILTERS="${TASK_RUNNER_EXTRA_FILTERS:-""}" # Handle unset variable + +# if [[ "$TASK_RUNNER_EXTRA_FILTERS" == *"aarch64"* && ! ("$TASK_RUNNER_EXTRA_FILTERS" == *"x64"* || "$TASK_RUNNER_EXTRA_FILTERS" == *"x86"* || "$TASK_RUNNER_EXTRA_FILTERS" == *"armv7"* ) ]]; then +# echo "Skipping prepare_qemu_rc because TASK_RUNNER_EXTRA_FILTERS matches the condition." +# else +# source tools/internal_ci/helper_scripts/prepare_qemu_rc +# fi # configure ccache source tools/internal_ci/helper_scripts/prepare_ccache_rc diff --git a/tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg b/tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg new file mode 100644 index 0000000000000..a68238848afcb --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg @@ -0,0 +1,36 @@ +# Copyright 2025 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "aarch64 -e x64 x86 armv7" +} + +# env_vars { +# key: "TASK_RUNNER_EXCLUDE_FILTERS" +# value: "x64 x86 armv7" +# } diff --git a/tools/internal_ci/linux/pull_request/grpc_examples_sanity_tests.cfg b/tools/internal_ci/linux/pull_request/grpc_examples_sanity_tests.cfg index 6c279328a7575..7c9b23268fd4d 100644 --- a/tools/internal_ci/linux/pull_request/grpc_examples_sanity_tests.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_examples_sanity_tests.cfg @@ -23,3 +23,13 @@ action { regex: "github/grpc/reports/**" } } + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "aarch64 -e x64 x86 armv7" +} + +# env_vars { +# key: "TASK_RUNNER_EXCLUDE_FILTERS" +# value: "x64 x86 armv7" +# } diff --git a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg index 6ed2e7d67f603..7da9a597f8762 100644 --- a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg @@ -24,3 +24,13 @@ action { regex: "github/grpc/artifacts/**" } } + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "-e aarch64" +} + +# env_vars { +# key: "TASK_RUNNER_EXCLUDE_FILTERS" +# value: "aarch64" +# } diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index fb2502d2a282e..3fc9cae18df81 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -71,6 +71,14 @@ def _create_build_map(): default=[], help="Filter targets to build with AND semantics.", ) +argp.add_argument( + "-e", + "--exclude", + choices=sorted(_BUILD_MAP.keys()), + nargs="+", + default=[], + help="Target labels to exclude from building.", +) argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) argp.add_argument( "-x", @@ -106,6 +114,9 @@ def _create_build_map(): # Among targets selected by -b, filter out those that don't match the filter targets = [t for t in targets if all(f in t.labels for f in args.filter)] +# Exclude target if it has ANY of the specified exclude labels. +targets = [t for t in targets if not any(l in args.exclude for l in t.labels)] + print("Will build %d targets:" % len(targets)) for target in targets: print(" %s, labels %s" % (target.name, target.labels)) From 158dff0c1619724aa243bd74eb8c541032a86a6b Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 00:56:16 +0000 Subject: [PATCH 18/28] Add config file --- .../arm64/grpc_distribtests_python_arm64.cfg | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg diff --git a/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg b/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg new file mode 100644 index 0000000000000..a68238848afcb --- /dev/null +++ b/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg @@ -0,0 +1,36 @@ +# Copyright 2025 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "aarch64 -e x64 x86 armv7" +} + +# env_vars { +# key: "TASK_RUNNER_EXCLUDE_FILTERS" +# value: "x64 x86 armv7" +# } From c84df47a335714b177355f858425df30545d503d Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 17:16:17 +0000 Subject: [PATCH 19/28] Refactors --- .../arm64/grpc_distribtests_python_arm64.cfg | 7 +--- .../linux/grpc_distribtests_python.cfg | 7 +--- .../linux/grpc_distribtests_python.sh | 18 +++++----- .../linux/grpc_distribtests_python_arm64.cfg | 36 ------------------- .../release/grpc_distribtests_python.cfg | 7 +--- tools/run_tests/task_runner.py | 4 +-- 6 files changed, 13 insertions(+), 66 deletions(-) delete mode 100644 tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg diff --git a/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg b/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg index a68238848afcb..f166d92fd5f16 100644 --- a/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg +++ b/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg @@ -27,10 +27,5 @@ action { env_vars { key: "TASK_RUNNER_EXTRA_FILTERS" - value: "aarch64 -e x64 x86 armv7" + value: "aarch64 musllinux_1_1" } - -# env_vars { -# key: "TASK_RUNNER_EXCLUDE_FILTERS" -# value: "x64 x86 armv7" -# } diff --git a/tools/internal_ci/linux/grpc_distribtests_python.cfg b/tools/internal_ci/linux/grpc_distribtests_python.cfg index 73cc665d44b15..f7a6ea46a1f77 100644 --- a/tools/internal_ci/linux/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/grpc_distribtests_python.cfg @@ -27,10 +27,5 @@ action { env_vars { key: "TASK_RUNNER_EXTRA_FILTERS" - value: "-e aarch64" + value: "-e aarch64 musllinux_1_1" } - -# env_vars { -# key: "TASK_RUNNER_EXCLUDE_FILTERS" -# value: "aarch64" -# } diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index c6ca715d320e0..95a6f5e2a9328 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -23,16 +23,14 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -# some distribtests use a pre-registered binfmt_misc hook -# to automatically execute foreign binaries (such as aarch64) -# under qemu emulator. -# TASK_RUNNER_EXTRA_FILTERS="${TASK_RUNNER_EXTRA_FILTERS:-""}" # Handle unset variable - -# if [[ "$TASK_RUNNER_EXTRA_FILTERS" == *"aarch64"* && ! ("$TASK_RUNNER_EXTRA_FILTERS" == *"x64"* || "$TASK_RUNNER_EXTRA_FILTERS" == *"x86"* || "$TASK_RUNNER_EXTRA_FILTERS" == *"armv7"* ) ]]; then -# echo "Skipping prepare_qemu_rc because TASK_RUNNER_EXTRA_FILTERS matches the condition." -# else -# source tools/internal_ci/helper_scripts/prepare_qemu_rc -# fi +if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" ]]; then + echo "Skipping prepare_qemu_rc'" +else + # some distribtests use a pre-registered binfmt_misc hook + # to automatically execute foreign binaries (such as aarch64) + # under qemu emulator. + source tools/internal_ci/helper_scripts/prepare_qemu_rc +fi # configure ccache source tools/internal_ci/helper_scripts/prepare_ccache_rc diff --git a/tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg b/tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg deleted file mode 100644 index a68238848afcb..0000000000000 --- a/tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2025 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Config file for the internal CI (in protobuf text format) - -# Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" -timeout_mins: 240 -action { - define_artifacts { - regex: "**/*sponge_log.*" - regex: "github/grpc/reports/**" - regex: "github/grpc/artifacts/**" - } -} - -env_vars { - key: "TASK_RUNNER_EXTRA_FILTERS" - value: "aarch64 -e x64 x86 armv7" -} - -# env_vars { -# key: "TASK_RUNNER_EXCLUDE_FILTERS" -# value: "x64 x86 armv7" -# } diff --git a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg index 7da9a597f8762..bd44c7cc862e8 100644 --- a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg @@ -27,10 +27,5 @@ action { env_vars { key: "TASK_RUNNER_EXTRA_FILTERS" - value: "-e aarch64" + value: "-e aarch64 musllinux_1_1" } - -# env_vars { -# key: "TASK_RUNNER_EXCLUDE_FILTERS" -# value: "aarch64" -# } diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index 3fc9cae18df81..1ecaaa63a69c8 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -114,8 +114,8 @@ def _create_build_map(): # Among targets selected by -b, filter out those that don't match the filter targets = [t for t in targets if all(f in t.labels for f in args.filter)] -# Exclude target if it has ANY of the specified exclude labels. -targets = [t for t in targets if not any(l in args.exclude for l in t.labels)] +# Exclude target if it has ALL of the specified exclude labels. +targets = [t for t in targets if not all(l in args.exclude for l in t.labels)] print("Will build %d targets:" % len(targets)) for target in targets: From 464362abe8a9532b56b7e92722e7dc45910c3028 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 18:18:06 +0000 Subject: [PATCH 20/28] Allow use musllinux_1_1 aarch64 image to collect artifact --- .../internal_ci/linux/grpc_distribtests_python.sh | 6 +++++- tools/run_tests/artifacts/package_targets.py | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index 95a6f5e2a9328..28f9034a1799f 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -44,7 +44,11 @@ mkdir -p input_artifacts cp -r artifacts/* input_artifacts/ || true # This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/ -tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true" +if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" ]]; then + tools/run_tests/task_runner.py -f package linux python ${TASK_RUNNER_EXTRA_FILTERS} -x build_packages/sponge_log.xml || FAILED="true" +else + tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true" +fi # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. # in addition to that, preserve the contents of "artifacts" directory since we want kokoro diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index dabfb452880d9..4676b69148141 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -142,9 +142,16 @@ def build_jobspec(self, inner_jobs=None): class PythonPackage: """Collects python eggs and wheels created in the artifact phase""" - def __init__(self): + def __init__(self, platform="", arch=""): self.name = "python_package" self.labels = ["package", "python", "linux"] + self.platform = platform + self.arch = arch + if self.platform: + self.labels.append(platform) + if self.arch: + self.labels.append(arch) + def pre_build_jobspecs(self): return [] @@ -154,9 +161,12 @@ def build_jobspec(self, inner_jobs=None): # since the python package build does very little, we can use virtually # any image that has new-enough python, so reusing one of the images used # for artifact building seems natural. + dockerfile_dir = "tools/dockerfile/grpc_artifact_python_manylinux2014_x64" + if "musllinux_1_1" in self.platform and "aarch64" in self.arch: + dockerfile_dir = "tools/dockerfile/grpc_artifact_python_%s_%s" % (self.platform, self.arch), return create_docker_jobspec( self.name, - "tools/dockerfile/grpc_artifact_python_manylinux2014_x64", + dockerfile_dir, "tools/run_tests/artifacts/build_package_python.sh", environ={"PYTHON": "/opt/python/cp39-cp39/bin/python"}, ) @@ -189,5 +199,6 @@ def targets(): CSharpPackage("windows"), RubyPackage(), PythonPackage(), + PythonPackage("musllinux_1_1", "aarch64"), PHPPackage(), ] From ba944fb18e410f9ae7c3108c3b4cd32849e2cc75 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 18:29:29 +0000 Subject: [PATCH 21/28] Fix issue with PythonPackage target name --- tools/run_tests/artifacts/package_targets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 4676b69148141..d01d3f652ac03 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -149,8 +149,10 @@ def __init__(self, platform="", arch=""): self.arch = arch if self.platform: self.labels.append(platform) + self.name += "_" + platform if self.arch: self.labels.append(arch) + self.name += "_" + arch def pre_build_jobspecs(self): From 5368719521a5b9921d900bfde95a35e3886f342b Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 18:42:44 +0000 Subject: [PATCH 22/28] Add test log --- tools/run_tests/python_utils/jobset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index e475b5a82f307..630a8591bb4df 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -45,6 +45,7 @@ def strip_non_ascii_chars(s): def sanitized_environment(env): sanitized = {} for key, value in list(env.items()): + sys.stderr.write(f"strip_non_ascii_chars key: {key}, value: {value}\n"); sys.stderr.flush() sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value) return sanitized From 3c57a40ee5564efe4422892bd9a8c4a259b88004 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 18:52:59 +0000 Subject: [PATCH 23/28] Fix bug --- tools/run_tests/artifacts/package_targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index d01d3f652ac03..a350f18f1ac59 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -165,7 +165,7 @@ def build_jobspec(self, inner_jobs=None): # for artifact building seems natural. dockerfile_dir = "tools/dockerfile/grpc_artifact_python_manylinux2014_x64" if "musllinux_1_1" in self.platform and "aarch64" in self.arch: - dockerfile_dir = "tools/dockerfile/grpc_artifact_python_%s_%s" % (self.platform, self.arch), + dockerfile_dir = "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64", return create_docker_jobspec( self.name, dockerfile_dir, From 1d01acf7f25cd85c03966a24be7fd36b7ddc1e7a Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 20:19:02 +0000 Subject: [PATCH 24/28] Change distribtest tag --- tools/internal_ci/linux/grpc_distribtests_python.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index 28f9034a1799f..1b978af0f4b3a 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -60,7 +60,12 @@ cp -r artifacts/* input_artifacts/ || true # Run all python linux distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives # a better signal about which distribtest are affected by the currently broken artifact builds. -tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true" +if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" ]]; then + # We're using alpine as tag in distribtest targets. + tools/run_tests/task_runner.py -f distribtest linux python aarch64 alpine -j 12 -x distribtests/sponge_log.xml || FAILED="true" +else + tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true" +fi # This step checks if any of the artifacts exceeds a per-file size limit. tools/internal_ci/helper_scripts/check_python_artifacts_size.sh From 6e7e4f56e80ab732f02819539cb33c8cdb2356e5 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 21:33:32 +0000 Subject: [PATCH 25/28] Use actual arm64 image --- tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile | 3 ++- tools/run_tests/python_utils/jobset.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile b/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile index a7500e39aa557..27d25043fc704 100644 --- a/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile +++ b/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.10-alpine3.14 +# this an aarch64 image +FROM arm64v8/python:3.10-alpine # Our test infrastructure demands bash RUN apk update && apk add bash diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 630a8591bb4df..e475b5a82f307 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -45,7 +45,6 @@ def strip_non_ascii_chars(s): def sanitized_environment(env): sanitized = {} for key, value in list(env.items()): - sys.stderr.write(f"strip_non_ascii_chars key: {key}, value: {value}\n"); sys.stderr.flush() sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value) return sanitized From 28e764ce78404e8ac9af6356e6139c45249f3b0d Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 22:55:44 +0000 Subject: [PATCH 26/28] Push new image --- tools/bazelify_tests/dockerimage_current_versions.bzl | 2 +- .../distribtest/python_alpine_aarch64.current_version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index 6fe2efe866f53..28e037fb076c6 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -36,7 +36,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:004e02902825b54c7f0d60eaed0819acf6f10c24853bf8f793001114e9969abd", "tools/dockerfile/distribtest/csharp_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu2204_x64@sha256:a439f2ccbc666f231e511a8e58eb7f66a3de4820a4d5aded1e62275cf8ac49f0", "tools/dockerfile/distribtest/php8_debian12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php8_debian12_x64@sha256:42399d061f234ee0ef79b333555db0e11ca4f106bb1d49276abf99c459c104f0", - "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e", + "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:bcdfc865daa8c61dc6f9b67ddab63d47fcacb109ed57de5b9a7673dff907e02c", "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:1bbc6fa5b4b650d3037d089e164d364e05a6daf1ed6dd1025ba07cc127f73d7d", "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1", "tools/dockerfile/distribtest/python_bullseye_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x64@sha256:80553398f0c59c1dc186052f4f2deaf18fea582f6d1d166eec6ea298639031fb", diff --git a/tools/dockerfile/distribtest/python_alpine_aarch64.current_version b/tools/dockerfile/distribtest/python_alpine_aarch64.current_version index a3efd8f5bcc0b..5b3080154e40a 100644 --- a/tools/dockerfile/distribtest/python_alpine_aarch64.current_version +++ b/tools/dockerfile/distribtest/python_alpine_aarch64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:086c6b7ba4177b5c086a4e1e36019fdd1d86a061@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:78b5c9d459ceaedfa8dc8fa6dc52ce11666dd581@sha256:bcdfc865daa8c61dc6f9b67ddab63d47fcacb109ed57de5b9a7673dff907e02c \ No newline at end of file From 0b7fca76c1c107d794a2b43daea70d3fb2cf51b0 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 23 Jan 2025 23:19:27 +0000 Subject: [PATCH 27/28] Add back debug log --- tools/run_tests/python_utils/jobset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index e475b5a82f307..630a8591bb4df 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -45,6 +45,7 @@ def strip_non_ascii_chars(s): def sanitized_environment(env): sanitized = {} for key, value in list(env.items()): + sys.stderr.write(f"strip_non_ascii_chars key: {key}, value: {value}\n"); sys.stderr.flush() sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value) return sanitized From 45746f8db8ea463f350f8271b84c465eb17032c8 Mon Sep 17 00:00:00 2001 From: XuanWang-Amos <24593237+XuanWang-Amos@users.noreply.github.com> Date: Thu, 23 Jan 2025 23:32:10 +0000 Subject: [PATCH 28/28] Automated change: Fix sanity tests --- src/core/lib/iomgr/closure.h | 5 +---- tools/run_tests/artifacts/package_targets.py | 9 ++++++--- tools/run_tests/python_utils/jobset.py | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index 2404767cafd55..e8f7a0b7812b5 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -211,10 +211,7 @@ inline grpc_closure* grpc_closure_create(grpc_iomgr_cb_func cb, void* cb_arg) { grpc_closure_create(cb, cb_arg) #endif -#define GRPC_CLOSURE_LIST_INIT \ - { \ - nullptr, nullptr \ - } +#define GRPC_CLOSURE_LIST_INIT {nullptr, nullptr} inline void grpc_closure_list_init(grpc_closure_list* closure_list) { closure_list->head = closure_list->tail = nullptr; diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index a350f18f1ac59..d6b69f7a3472e 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -154,7 +154,6 @@ def __init__(self, platform="", arch=""): self.labels.append(arch) self.name += "_" + arch - def pre_build_jobspecs(self): return [] @@ -163,9 +162,13 @@ def build_jobspec(self, inner_jobs=None): # since the python package build does very little, we can use virtually # any image that has new-enough python, so reusing one of the images used # for artifact building seems natural. - dockerfile_dir = "tools/dockerfile/grpc_artifact_python_manylinux2014_x64" + dockerfile_dir = ( + "tools/dockerfile/grpc_artifact_python_manylinux2014_x64" + ) if "musllinux_1_1" in self.platform and "aarch64" in self.arch: - dockerfile_dir = "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64", + dockerfile_dir = ( + "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64", + ) return create_docker_jobspec( self.name, dockerfile_dir, diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 630a8591bb4df..60463f3e47c56 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -45,7 +45,8 @@ def strip_non_ascii_chars(s): def sanitized_environment(env): sanitized = {} for key, value in list(env.items()): - sys.stderr.write(f"strip_non_ascii_chars key: {key}, value: {value}\n"); sys.stderr.flush() + sys.stderr.write(f"strip_non_ascii_chars key: {key}, value: {value}\n") + sys.stderr.flush() sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value) return sanitized