WIP: test community repos/examples (#351)

Author: kostmo

.circleci/.gitignore

@@ -0,0 +1 @@
+.idea

.circleci/config.in.yml

@@ -0,0 +1,242 @@
+version: 2.1
+
+# This script was adapted from: https://github.com/pytorch/vision/blob/master/.circleci/config.yml.in
+
+# How to test the Linux jobs:
+#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
+#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
+#     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
+#       Job names are 'name:' key.
+
+orbs:
+  win: circleci/[email protected]
+
+binary_common: &binary_common
+  parameters:
+    # Edit these defaults to do a release`
+    build_version:
+      description: "version number of release binary; by default, build a nightly"
+      type: string
+      default: ""
+    pytorch_version:
+      description: "PyTorch version to build against; by default, use a nightly"
+      type: string
+      default: ""
+    # Don't edit these
+    python_version:
+      description: "Python version to build against (e.g., 3.7)"
+      type: string
+    cu_version:
+      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
+      type: string
+    unicode_abi:
+      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
+      type: string
+      default: ""
+    wheel_docker_image:
+      description: "Wheel only: what docker image to use"
+      type: string
+      default: "soumith/manylinux-cuda100"
+  environment:
+    PYTHON_VERSION: << parameters.python_version >>
+    BUILD_VERSION: << parameters.build_version >>
+    PYTORCH_VERSION: << parameters.pytorch_version >>
+    UNICODE_ABI: << parameters.unicode_abi >>
+    CU_VERSION: << parameters.cu_version >>
+
+jobs:
+  circleci_consistency:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - checkout
+      - run:
+          command: |
+            pip install --user --progress-bar off jinja2 pyyaml
+            python .circleci/regenerate.py
+            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
+
+  binary_linux_wheel:
+    <<: *binary_common
+    docker:
+      - image: << parameters.wheel_docker_image >>
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: test_community_repos/run_all.sh
+      - store_artifacts:
+          path: dist
+      - persist_to_workspace:
+          root: dist
+          paths:
+            - "*"
+
+  binary_linux_conda:
+    <<: *binary_common
+    docker:
+      - image: "soumith/conda-cuda"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: test_community_repos/run_all.sh
+      - store_artifacts:
+          path: /opt/conda/conda-bld/linux-64
+      - persist_to_workspace:
+          root: /opt/conda/conda-bld/linux-64
+          paths:
+            - "*"
+
+  binary_linux_conda_cuda:
+    <<: *binary_common
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: gpu.medium
+    steps:
+    - checkout
+    - run:
+        name: Setup environment
+        command: |
+          set -e
+
+          curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
+          curl -L https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
+
+          sudo apt-get update
+
+          sudo apt-get install \
+              apt-transport-https \
+              ca-certificates \
+              curl \
+              gnupg-agent \
+              software-properties-common
+
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+
+          sudo add-apt-repository \
+             "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+             $(lsb_release -cs) \
+             stable"
+
+          sudo apt-get update
+          export DOCKER_VERSION="5:19.03.2~3-0~ubuntu-xenial"
+          sudo apt-get install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io
+
+          # Add the package repositories
+          distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+          curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+          curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+
+          export NVIDIA_CONTAINER_VERSION="1.0.3-1"
+          sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION}
+          sudo systemctl restart docker
+
+          DRIVER_FN="NVIDIA-Linux-x86_64-410.104.run"
+          wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
+          sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
+          nvidia-smi
+
+    - run:
+        name: Pull docker image
+        command: |
+          set -e
+          export DOCKER_IMAGE=soumith/conda-cuda
+          echo Pulling docker image $DOCKER_IMAGE
+          docker pull $DOCKER_IMAGE >/dev/null
+
+    - run:
+        name: Build and run tests
+        command: |
+          set -e
+
+          cd ${HOME}/project/
+
+          export DOCKER_IMAGE=soumith/conda-cuda
+          export VARS_TO_PASS="-e PYTHON_VERSION -e BUILD_VERSION -e PYTORCH_VERSION -e UNICODE_ABI -e CU_VERSION"
+
+          docker run --gpus all  --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./test_community_repos/run_all.sh
+
+  binary_win_conda:
+    <<: *binary_common
+    executor:
+      name: win/vs2019
+      shell: bash.exe
+    steps:
+      - checkout
+      - run:
+          command: |
+            choco install miniconda3
+            $env:PATH = "C:\tools\miniconda3;C:\tools\miniconda3\Library\usr\bin;C:\tools\miniconda3\Scripts;C:\tools\miniconda3\bin" + $env:PATH
+            conda install -yq conda-build
+            bash test_community_repos/run_all.sh
+          shell: powershell.exe
+
+  binary_macos_wheel:
+    <<: *binary_common
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          # Cannot easily deduplicate this as source'ing activate
+          # will set environment variables which we need to propagate
+          # to build_wheel.sh
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            test_community_repos/run_all.sh
+      - store_artifacts:
+          path: dist
+      - persist_to_workspace:
+          root: dist
+          paths:
+            - "*"
+
+  binary_macos_conda:
+    <<: *binary_common
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            conda install -yq conda-build
+            test_community_repos/run_all.sh
+      - store_artifacts:
+          path: /Users/distiller/miniconda3/conda-bld/osx-64
+      - persist_to_workspace:
+          root: /Users/distiller/miniconda3/conda-bld/osx-64
+          paths:
+            - "*"
+
+
+workflows:
+  build:
+{%- if True %}
+    jobs:
+      - circleci_consistency
+      {{ workflows() }}
+      - binary_linux_conda_cuda:
+          name: torchvision_linux_py3.7_cu100
+          python_version: "3.7"
+          cu_version: "cu100"
+      - binary_win_conda:
+          name: torchvision_win_py3.6_cpu
+          python_version: "3.6"
+          cu_version: "cpu"
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 9 * * *"
+          filters:
+            branches:
+              only:
+                - master
+{%- endif %}
+    jobs:
+      - circleci_consistency
+      {{ workflows(prefix="nightly_", upload=True) }}