From e3d8fe183e9bceec91e2ddd6456302db5632eb97 Mon Sep 17 00:00:00 2001 From: Anirban Roychowdhury Date: Wed, 31 Aug 2022 16:19:55 -0700 Subject: [PATCH 1/6] Add preprocessor flag to unit test for importlib.metadata. Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- multipy/runtime/test_deploy.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipy/runtime/test_deploy.cpp b/multipy/runtime/test_deploy.cpp index 80a403b3..f4130389 100644 --- a/multipy/runtime/test_deploy.cpp +++ b/multipy/runtime/test_deploy.cpp @@ -459,6 +459,7 @@ result = torch.Tensor([1,2,3]) EXPECT_TRUE(w_grad0.equal(w_grad1)); } +#ifndef LEGACY_PYTHON_3_7 TEST(TorchpyTest, ImportlibMetadata) { torch::deploy::InterpreterManager m(1); m.registerModuleSource("importlib_test", R"PYTHON( @@ -470,6 +471,7 @@ result = version("torch") auto ver = I.global("importlib_test", "result").toIValue().toString(); ASSERT_EQ(ver->string(), "0.0.1+fake_multipy"); } +#endif // OSS build does not have bultin numpy support yet. Use this flag to guard the // test case. From 32253bd7cfeb341cfd085bb732f064ab5092e2af Mon Sep 17 00:00:00 2001 From: Anirban Roychowdhury Date: Wed, 31 Aug 2022 16:21:35 -0700 Subject: [PATCH 2/6] Update. Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- multipy/runtime/test_deploy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipy/runtime/test_deploy.cpp b/multipy/runtime/test_deploy.cpp index f4130389..c628f410 100644 --- a/multipy/runtime/test_deploy.cpp +++ b/multipy/runtime/test_deploy.cpp @@ -459,7 +459,7 @@ result = torch.Tensor([1,2,3]) EXPECT_TRUE(w_grad0.equal(w_grad1)); } -#ifndef LEGACY_PYTHON_3_7 +#ifndef LEGACY_PYTHON_BEFORE_3_8 TEST(TorchpyTest, ImportlibMetadata) { torch::deploy::InterpreterManager m(1); m.registerModuleSource("importlib_test", R"PYTHON( From 98efadb2d4b1566bc7484b7b9ae510f163b53592 Mon Sep 17 00:00:00 2001 From: Anirban Roychowdhury Date: Wed, 31 Aug 2022 16:56:46 -0700 Subject: [PATCH 3/6] Locals; move to another pr/commit. Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- .github/workflows/runtime_tests.yaml | 2 +- Dockerfile | 37 +++++++++++++++++----- multipy/runtime/interpreter/CMakeLists.txt | 6 ++++ multipy/runtime/test_deploy.cpp | 2 +- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/.github/workflows/runtime_tests.yaml b/.github/workflows/runtime_tests.yaml index 0e2b5632..fc5e3c9b 100644 --- a/.github/workflows/runtime_tests.yaml +++ b/.github/workflows/runtime_tests.yaml @@ -10,7 +10,7 @@ jobs: unittest: strategy: matrix: - python-version: [3.8, 3.9, '3.10'] + python-version: [3.7, 3.8, 3.9, '3.10'] platform: [ubuntu-18.04] fail-fast: false runs-on: ${{ matrix.platform }} diff --git a/Dockerfile b/Dockerfile index 9e92742c..010c34f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,20 +48,41 @@ WORKDIR /opt/multipy COPY . . RUN git submodule update --init --recursive --jobs 0 -# Install conda + neccessary python dependencies -FROM dev-base as conda +# Check python version. +# ARG is in scope of stage it is defined in. +FROM dev-base as prep ARG PYTHON_VERSION=3.8 -RUN curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ +ENV MULTIPY_BUILD_PYTHON_VERSION=${PYTHON_VERSION} +ENV MULTIPY_BUILD_PYTHON_MAJOR_VERSION=${MULTIPY_BUILD_PYTHON_VERSION%%.*} +ENV MULTIPY_BUILD_PYTHON_MINOR_VERSION=${MULTIPY_BUILD_PYTHON_VERSION##*.} +RUN if [[ $MULTIPY_BUILD_PYTHON_MAJOR_VERSION -eq 3]] && [[ $MULTIPY_BUILD_PYTHON_MINOR_VERSION -gt 7]]; then \ + export LEGACY_PYTHON_PRE_3_8=0 && \ + elif [[ $MULTIPY_BUILD_PYTHON_MAJOR_VERSION -eq 3]] && [[ $MULTIPY_BUILD_PYTHON_MINOR_VERSION -eq 7]]; then \ + export LEGACY_PYTHON_PRE_3_8=1 \ + fi + +# Install conda + neccessary python dependencies for 3.8+. +# Use pyenv for 3.7, as libpython-static is available in conda forge for 3.8+. +FROM prep as conda_pyenv +RUN if [[ $LEGACY_PYTHON_PRE_3_8 -eq 0]]; then \ + curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ chmod +x ~/miniconda.sh && \ ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ - /opt/conda/bin/conda install -y python=${PYTHON_VERSION} cmake mkl mkl-include conda-build pyyaml numpy ipython && \ - /opt/conda/bin/conda install -y -c conda-forge libpython-static=${PYTHON_VERSION} && \ + /opt/conda/bin/conda install -y python=${MULTIPY_BUILD_PYTHON_VERSION} cmake mkl mkl-include conda-build pyyaml numpy ipython && \ + /opt/conda/bin/conda install -y -c conda-forge libpython-static=${MULTIPY_BUILD_PYTHON_VERSION} && \ /opt/conda/bin/conda install -y pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch-nightly && \ - /opt/conda/bin/conda clean -ya + /opt/conda/bin/conda clean -ya \ + else \ + export CFLAGS="-fPIC -g" && \ + # install pyenv ? + pyenv install --force 3.7.10 && \ + virtualenv -p ~/.pyenv/versions/3.7.10/bin/python3 ~/venvs/multipy_3_7_10 && \ + source ~/venvs/multipy_3_7_10/bin/activate \ + fi # Build/Install pytorch with post-cxx11 ABI -FROM conda as build +FROM conda_pyenv as build WORKDIR /opt/multipy/multipy/runtime/third-party/pytorch COPY --from=conda /opt/conda /opt/conda COPY --from=submodule-update /opt/multipy /opt/multipy @@ -71,7 +92,7 @@ WORKDIR /opt/multipy # Build Multipy RUN mkdir multipy/runtime/build && \ cd multipy/runtime/build && \ - cmake .. && \ + cmake .. -DLEGACY_PYTHON_PRE_3_8 && \ cmake --build . --config Release && \ cmake --install . --prefix "." diff --git a/multipy/runtime/interpreter/CMakeLists.txt b/multipy/runtime/interpreter/CMakeLists.txt index c55e9f3d..1dbb1942 100644 --- a/multipy/runtime/interpreter/CMakeLists.txt +++ b/multipy/runtime/interpreter/CMakeLists.txt @@ -13,6 +13,8 @@ include_directories(BEFORE "${PYTORCH_ROOT}/torch/include") include_directories(BEFORE "${PYTORCH_ROOT}/torch/include/torch/csrc/api/include/") SET(MULTIPY_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../../utils") +SET(LEGACY_PYTHON_PRE_3_8 False) + find_package (Python3 COMPONENTS Interpreter Development) message(STATUS "Python3_EXECUTABLE - ${Python3_EXECUTABLE}" ) @@ -32,6 +34,10 @@ link_directories(BEFORE "${Python3_SITELIB}/torch/lib") include_directories(BEFORE "${Python3_INCLUDE_DIRS}") include_directories(BEFORE ${CMAKE_SOURCE_DIR}/../..) +if (${LEGACY_PYTHON_PRE_3_8}) + add_compile_definitions(LEGACY_PYTHON_PRE_3_8=1) +endif() + # add gtest dependency include(FetchContent) FetchContent_Declare( diff --git a/multipy/runtime/test_deploy.cpp b/multipy/runtime/test_deploy.cpp index c628f410..6bbee4dc 100644 --- a/multipy/runtime/test_deploy.cpp +++ b/multipy/runtime/test_deploy.cpp @@ -459,7 +459,7 @@ result = torch.Tensor([1,2,3]) EXPECT_TRUE(w_grad0.equal(w_grad1)); } -#ifndef LEGACY_PYTHON_BEFORE_3_8 +#ifndef LEGACY_PYTHON_PRE_3_8 TEST(TorchpyTest, ImportlibMetadata) { torch::deploy::InterpreterManager m(1); m.registerModuleSource("importlib_test", R"PYTHON( From 42f4cb0c4a987f80703727c3d09c5507d57278b6 Mon Sep 17 00:00:00 2001 From: Anirban Roychowdhury Date: Wed, 31 Aug 2022 17:31:32 -0700 Subject: [PATCH 4/6] Upd. Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- .github/workflows/runtime_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runtime_tests.yaml b/.github/workflows/runtime_tests.yaml index fc5e3c9b..0e2b5632 100644 --- a/.github/workflows/runtime_tests.yaml +++ b/.github/workflows/runtime_tests.yaml @@ -10,7 +10,7 @@ jobs: unittest: strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] + python-version: [3.8, 3.9, '3.10'] platform: [ubuntu-18.04] fail-fast: false runs-on: ${{ matrix.platform }} From 6048af3e538becb9cf194441dd40a2f9368eda20 Mon Sep 17 00:00:00 2001 From: Anirban Roychowdhury Date: Wed, 31 Aug 2022 17:35:13 -0700 Subject: [PATCH 5/6] Remove changes from downstream? branch. Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- Dockerfile | 29 +++------------------- multipy/runtime/interpreter/CMakeLists.txt | 6 ----- 2 files changed, 4 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 010c34f0..28cca480 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,22 +48,8 @@ WORKDIR /opt/multipy COPY . . RUN git submodule update --init --recursive --jobs 0 -# Check python version. -# ARG is in scope of stage it is defined in. -FROM dev-base as prep -ARG PYTHON_VERSION=3.8 -ENV MULTIPY_BUILD_PYTHON_VERSION=${PYTHON_VERSION} -ENV MULTIPY_BUILD_PYTHON_MAJOR_VERSION=${MULTIPY_BUILD_PYTHON_VERSION%%.*} -ENV MULTIPY_BUILD_PYTHON_MINOR_VERSION=${MULTIPY_BUILD_PYTHON_VERSION##*.} -RUN if [[ $MULTIPY_BUILD_PYTHON_MAJOR_VERSION -eq 3]] && [[ $MULTIPY_BUILD_PYTHON_MINOR_VERSION -gt 7]]; then \ - export LEGACY_PYTHON_PRE_3_8=0 && \ - elif [[ $MULTIPY_BUILD_PYTHON_MAJOR_VERSION -eq 3]] && [[ $MULTIPY_BUILD_PYTHON_MINOR_VERSION -eq 7]]; then \ - export LEGACY_PYTHON_PRE_3_8=1 \ - fi - # Install conda + neccessary python dependencies for 3.8+. -# Use pyenv for 3.7, as libpython-static is available in conda forge for 3.8+. -FROM prep as conda_pyenv +FROM dev-base as conda RUN if [[ $LEGACY_PYTHON_PRE_3_8 -eq 0]]; then \ curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ chmod +x ~/miniconda.sh && \ @@ -72,17 +58,10 @@ RUN if [[ $LEGACY_PYTHON_PRE_3_8 -eq 0]]; then \ /opt/conda/bin/conda install -y python=${MULTIPY_BUILD_PYTHON_VERSION} cmake mkl mkl-include conda-build pyyaml numpy ipython && \ /opt/conda/bin/conda install -y -c conda-forge libpython-static=${MULTIPY_BUILD_PYTHON_VERSION} && \ /opt/conda/bin/conda install -y pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch-nightly && \ - /opt/conda/bin/conda clean -ya \ - else \ - export CFLAGS="-fPIC -g" && \ - # install pyenv ? - pyenv install --force 3.7.10 && \ - virtualenv -p ~/.pyenv/versions/3.7.10/bin/python3 ~/venvs/multipy_3_7_10 && \ - source ~/venvs/multipy_3_7_10/bin/activate \ - fi + /opt/conda/bin/conda clean -ya # Build/Install pytorch with post-cxx11 ABI -FROM conda_pyenv as build +FROM conda as build WORKDIR /opt/multipy/multipy/runtime/third-party/pytorch COPY --from=conda /opt/conda /opt/conda COPY --from=submodule-update /opt/multipy /opt/multipy @@ -92,7 +71,7 @@ WORKDIR /opt/multipy # Build Multipy RUN mkdir multipy/runtime/build && \ cd multipy/runtime/build && \ - cmake .. -DLEGACY_PYTHON_PRE_3_8 && \ + cmake .. && \ cmake --build . --config Release && \ cmake --install . --prefix "." diff --git a/multipy/runtime/interpreter/CMakeLists.txt b/multipy/runtime/interpreter/CMakeLists.txt index 1dbb1942..c55e9f3d 100644 --- a/multipy/runtime/interpreter/CMakeLists.txt +++ b/multipy/runtime/interpreter/CMakeLists.txt @@ -13,8 +13,6 @@ include_directories(BEFORE "${PYTORCH_ROOT}/torch/include") include_directories(BEFORE "${PYTORCH_ROOT}/torch/include/torch/csrc/api/include/") SET(MULTIPY_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../../utils") -SET(LEGACY_PYTHON_PRE_3_8 False) - find_package (Python3 COMPONENTS Interpreter Development) message(STATUS "Python3_EXECUTABLE - ${Python3_EXECUTABLE}" ) @@ -34,10 +32,6 @@ link_directories(BEFORE "${Python3_SITELIB}/torch/lib") include_directories(BEFORE "${Python3_INCLUDE_DIRS}") include_directories(BEFORE ${CMAKE_SOURCE_DIR}/../..) -if (${LEGACY_PYTHON_PRE_3_8}) - add_compile_definitions(LEGACY_PYTHON_PRE_3_8=1) -endif() - # add gtest dependency include(FetchContent) FetchContent_Declare( From 8f708e1e0825e231cb266bf7d42412ae220b555b Mon Sep 17 00:00:00 2001 From: Anirban Roychowdhury Date: Wed, 31 Aug 2022 17:36:26 -0700 Subject: [PATCH 6/6] Remove downstream? changes. Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 28cca480..3ff3555c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,13 +50,13 @@ RUN git submodule update --init --recursive --jobs 0 # Install conda + neccessary python dependencies for 3.8+. FROM dev-base as conda -RUN if [[ $LEGACY_PYTHON_PRE_3_8 -eq 0]]; then \ - curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ +ARG PYTHON_VERSION=3.8 +RUN curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ chmod +x ~/miniconda.sh && \ ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ - /opt/conda/bin/conda install -y python=${MULTIPY_BUILD_PYTHON_VERSION} cmake mkl mkl-include conda-build pyyaml numpy ipython && \ - /opt/conda/bin/conda install -y -c conda-forge libpython-static=${MULTIPY_BUILD_PYTHON_VERSION} && \ + /opt/conda/bin/conda install -y python=${PYTHON_VERSION} cmake mkl mkl-include conda-build pyyaml numpy ipython && \ + /opt/conda/bin/conda install -y -c conda-forge libpython-static=${PYTHON_VERSION} && \ /opt/conda/bin/conda install -y pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch-nightly && \ /opt/conda/bin/conda clean -ya