From d08b0e5dc4ecf4af364121933fbe0aa61031cc66 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Tue, 17 Dec 2024 09:39:10 -0800 Subject: [PATCH] [shortfin] Use custom manylinux dockerfile in build_linux_package.sh. (#709) Package builds started failing last night when the `latest` upstream manylinux dockerfile switched to gcc 14: https://github.com/nod-ai/shark-ai/actions/runs/12371699664/job/34528374484 ``` Running command Building wheel for shortfin (pyproject.toml) -- The C compiler identification is GNU 14.2.1 -- The CXX compiler identification is GNU 14.2.1 ... [325/365] Building CXX object src/shortfin/local/CMakeFiles/shortfin_local.dylib.objects.dir/device.cc.o FAILED: src/shortfin/local/CMakeFiles/shortfin_local.dylib.objects.dir/device.cc.o /opt/rh/gcc-toolset-14/root/usr/bin/c++ -DCPUINFO_SUPPORTED_PLATFORM=1 -DSPDLOG_COMPILED_LIB -DSPDLOG_FMT_EXTERNAL -DSPDLOG_SHARED_LIB -D_SHORTFIN_BUILDING_DYLIB -Dspdlog_EXPORTS -I/home/runner/work/shark-ai/shark-ai/c/shortfin/src -I/home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/src -I/home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/spdlog-src/include -I/home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/fmt-src/include -isystem /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-src -isystem /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-build -isystem /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-src/runtime/src -isystem /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-build/runtime/src -isystem /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-src/third_party/cpuinfo/include -isystem /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-build/runtime/src/iree/base/internal/flatcc -isystem /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-src/third_party/flatcc/include -isystem /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-build/runtime/src/iree/schemas -O3 -DNDEBUG -std=gnu++20 -flto=auto -fno-fat-lto-objects -fPIC -fvisibility=hidden -fvisibility-inlines-hidden -Wall -Werror -pthread -I/home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-src/third_party/flatcc/include/ -I/home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/shortfin_iree-src/third_party/flatcc/include/flatcc/reflection/ -MD -MT src/shortfin/local/CMakeFiles/shortfin_local.dylib.objects.dir/device.cc.o -MF src/shortfin/local/CMakeFiles/shortfin_local.dylib.objects.dir/device.cc.o.d -o src/shortfin/local/CMakeFiles/shortfin_local.dylib.objects.dir/device.cc.o -c /home/runner/work/shark-ai/shark-ai/c/shortfin/src/shortfin/local/device.cc In file included from /home/runner/work/shark-ai/shark-ai/c/shortfin/src/shortfin/local/device.cc:10: /home/runner/work/shark-ai/shark-ai/c/shortfin/build/cmake/default/_deps/fmt-src/include/fmt/ranges.h:211:49: error: self-comparison always evaluates to true [-Werror=tautological-compare] 211 | integer_sequence) -> std::true_type; | ~~~^~~~~ cc1plus: all warnings being treated as errors ``` This switches to our own downstream dockerfile, defined here: https://github.com/nod-ai/base-docker-images/blob/main/dockerfiles/manylinux_x86_64.Dockerfile, which is pinned to an older version of the base image (and thus gcc). Tested successfully here: https://github.com/ScottTodd/shark-ai/actions/runs/12378199850 --- shortfin/build_tools/build_linux_package.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/shortfin/build_tools/build_linux_package.sh b/shortfin/build_tools/build_linux_package.sh index 91b944e51..db2463987 100755 --- a/shortfin/build_tools/build_linux_package.sh +++ b/shortfin/build_tools/build_linux_package.sh @@ -37,13 +37,18 @@ REPO_ROOT="$(cd "$THIS_DIR"/../../ && pwd)" SCRIPT_NAME="$(basename $0)" ARCH="$(uname -m)" -# Note: we can switch to https://github.com/nod-ai/base-docker-images as needed for extra deps. -MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-quay.io/pypa/manylinux_2_28_${ARCH}:latest}" PYTHON_VERSIONS="${OVERRIDE_PYTHON_VERSIONS:-cp311-cp311 cp312-cp312 cp313-cp313}" OUTPUT_DIR="${OUTPUT_DIR:-${THIS_DIR}/wheelhouse}" CACHE_DIR="${CACHE_DIR:-}" SHORTFIN_ENABLE_TRACING="${SHORTFIN_ENABLE_TRACING:-ON}" +if [[ "${ARCH}" == "x86_64" ]]; then + MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-ghcr.io/nod-ai/manylinux_x86_64@sha256:4acf83343706d1e37252d6001ded3c97a73bc38620580f855b4e65e35ddc5681}" +else + # TODO: publish a multi-platform manylinux image and include more deps in all platforms (rust, ccache, etc.) + MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-quay.io/pypa/manylinux_2_28_${ARCH}:latest}" +fi + function run_on_host() { echo "Running on host" echo "Launching docker image ${MANYLINUX_DOCKER_IMAGE}"