Merge pull request #2874 from ytausch/parallel-tests

beckermr · web-flow · commit ec3d3ca13c1b · 2024-08-08T00:32:31.000-04:00
enable parallel tests with pytest-xdist and pytest-split
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -10,14 +10,14 @@ conda_forge_tick/models appropriately and document any new fields or files. Tick
 Note that the model exists next to and independent of the actual production code.
 -->
 
-Description:
+### Description
 
 <!-- Please describe your PR here. -->
 
-Checklist:
+### Checklist
 
 - [ ] Pydantic model updated or no update needed
 
-Cross-refs, links to issues, etc:
+### Cross-refs, links to issues, etc
 
 <!-- Please cross-link your PR to any open issues, other PRs, etc. here. -->
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -19,6 +19,15 @@ jobs:
   tests:
     name: tests
     runs-on: "ubuntu-latest"
+    permissions:
+      actions: write  # for deleting cache entries
+      contents: read
+    strategy:
+      # continue running the tests even if one of the groups fails
+      fail-fast: false
+      matrix:
+        # if change the number of groups here, also change it next to --splits below
+        group: [1, 2]
     defaults:
       run:
         shell: bash -leo pipefail {0}
@@ -53,7 +62,7 @@ jobs:
 
           python -m pip install -v --no-deps --no-build-isolation -e .
 
-      - name: Start MongoDB
+      - name: start MongoDB
         uses: MongoCamp/mongodb-github-action@e76ad215d47c31a99b4b0b1fde05f6cd1185df1a # e76ad215d47c31a99b4b0b1fde05f6cd1185df1a
         with:
           mongodb-version: "latest"
@@ -68,12 +77,30 @@ jobs:
         run: |
           docker build -t ${{ env.IMAGE_NAME }}:test .
 
+      - name: restore test durations
+        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9  # v4
+        with:
+          path: .test_durations
+          key: test-durations-${{ github.ref }}
+          restore-keys: |
+            test-durations-
+
       - name: run pytest
         run: |
           export TEST_BOT_TOKEN_VAL=unpassword
           export BOT_TOKEN=${TEST_BOT_TOKEN_VAL}
+          # note: we do not use pytest-xdist (-n auto) here for now because they interfere with hiding the
+          # MONGODB_CONNECTION_STRING sensitive environment variable
+          if [[ -f .test_durations ]]; then
+            cp .test_durations .test_durations.${{ matrix.group }}
+          fi
           pytest \
             -v \
+            --splits 2 --group ${{ matrix.group }} \
+            --store-durations \
+            --durations-path=.test_durations.${{ matrix.group }} \
+            --randomly-seed=${{ github.run_id }} \
+            --splitting-algorithm least_duration \
             --cov=conda_forge_tick \
             --cov=tests \
             --cov-config=.coveragerc \
@@ -86,7 +113,62 @@ jobs:
           MONGODB_CONNECTION_STRING: "mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000"
           RUN_URL: ""
 
+      - name: upload test durations
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4
+        with:
+          name: test-durations-${{ matrix.group }}
+          path: .test_durations.${{ matrix.group }}
+
+      - name: upload coverage
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4
+        with:
+          name: coverage-${{ matrix.group }}
+          path: .coverage
+
+  test-coverage-and-durations:
+    name: test-coverage-and-durations
+    needs: tests
+    runs-on: "ubuntu-latest"
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 # v1
+        with:
+          environment-file: conda-lock.yml
+          environment-name: cf-scripts
+          condarc-file: autotick-bot/condarc
+
+      - name: download coverage artifacts
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        with:
+          pattern: coverage-*
+
+      - name: combine coverage
+        run: |
+          coverage combine coverage-*/.coverage*
+          coverage xml
+
       - name: upload codecov
         uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: cache test durations
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9  # v4
+        with:
+          path: .test_durations
+          key: test-durations-${{ github.ref }}
+          restore-keys: |
+            test-durations-
+
+      - name: download test duration artifacts
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        with:
+          pattern: test-durations-*
+
+      - name: combine test durations
+        run: |
+          jq '. + input' test-durations-*/.test_durations.* > .test_durations
diff --git a/.gitignore b/.gitignore
@@ -57,3 +57,4 @@ versions/*
 .pixi
 pixi.lock
 pixi.toml
+.ruff_cache/
diff --git a/README.md b/README.md
@@ -285,6 +285,8 @@ As `conda-forge` grew, the "single job + global data access" model became increa
 
 #### Current Bot Jobs and Structure
 
+[**Current GitHub Runner Allocation**](docs/runner_allocation.md)
+
 In this section, we list the collection of jobs that comprise the bot. Each job touches a distinct part of the bot's data structure and is run in parallel with the other jobs. We have also specified the GitHub Actions workflow that runs each job. See those files for further details on which commands are run.
 
 **bot** / `bot-bot.yml`: The main job that runs the bot, making PRs to feedstocks, etc. This job writes data on the PRs it makes to `cf-graph-countyfair/pr_info` and `cf-graph-countyfair/version_pr_info`. It also writes new PR JSON blobs to `cf-graph-countyfair/pr_json` for each PR to track their statuses on GitHub.
diff --git a/conda-lock.yml b/conda-lock.yml
@@ -3,9 +3,9 @@ metadata:
     - url: conda-forge
       used_env_vars: []
   content_hash:
-    linux-64: 2b3c0976e57cb73f4954a13b0019b2082ff53a96d28c7dc99af6330878412da3
-    osx-64: a823286e2d460d09bcee256454c86ab7a53966fa2b7539cf573af939e8711fff
-    osx-arm64: c3255180ca12c8077951328d65d1f846ebe057235066735d459a689c605ae02d
+    linux-64: 74b8ba2455a1c26564be8d99f414d006c68b4448b50a09a2682b6cd487deb834
+    osx-64: d4f9ea8b9b5e8c6656029f5f2c36281636e03170c29250e41541975a8d5c78b6
+    osx-arm64: 0df568ade8cdfaea2654d2331a61e925288d6d56e0072d96c53941b5e8221271
   platforms:
     - osx-arm64
     - linux-64
@@ -10447,6 +10447,93 @@ package:
     url:
       https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.1.3-pyhd8ed1ab_0.conda
     version: 1.1.3
+  - category: main
+    dependencies:
+      importlib-metadata: '>=3.6.0'
+      pytest: ''
+      python: '>=3.6'
+    hash:
+      md5: 203b5d3f85a47940f7ec6b6e1747786e
+      sha256: bd1953e4bc20ffd52cfee41b27b3a781ca6e281004d0dd59e2dd60b0192c7a86
+    manager: conda
+    name: pytest-randomly
+    optional: false
+    platform: linux-64
+    url:
+      https://conda.anaconda.org/conda-forge/noarch/pytest-randomly-3.15.0-pyhd8ed1ab_0.conda
+    version: 3.15.0
+  - category: main
+    dependencies:
+      importlib-metadata: '>=3.6.0'
+      pytest: ''
+      python: '>=3.6'
+    hash:
+      md5: 203b5d3f85a47940f7ec6b6e1747786e
+      sha256: bd1953e4bc20ffd52cfee41b27b3a781ca6e281004d0dd59e2dd60b0192c7a86
+    manager: conda
+    name: pytest-randomly
+    optional: false
+    platform: osx-64
+    url:
+      https://conda.anaconda.org/conda-forge/noarch/pytest-randomly-3.15.0-pyhd8ed1ab_0.conda
+    version: 3.15.0
+  - category: main
+    dependencies:
+      importlib-metadata: '>=3.6.0'
+      pytest: ''
+      python: '>=3.6'
+    hash:
+      md5: 203b5d3f85a47940f7ec6b6e1747786e
+      sha256: bd1953e4bc20ffd52cfee41b27b3a781ca6e281004d0dd59e2dd60b0192c7a86
+    manager: conda
+    name: pytest-randomly
+    optional: false
+    platform: osx-arm64
+    url:
+      https://conda.anaconda.org/conda-forge/noarch/pytest-randomly-3.15.0-pyhd8ed1ab_0.conda
+    version: 3.15.0
+  - category: main
+    dependencies:
+      pytest: '>=5.4'
+      python: '>=3.6,<4'
+    hash:
+      md5: d15a8b5c3a050fc3cf7d4509c70441f4
+      sha256: 34b2849d08d1cd7672b873c72df37d071d10402df89c6f8ac82d43cbeaa33442
+    manager: conda
+    name: pytest-split
+    optional: false
+    platform: linux-64
+    url:
+      https://conda.anaconda.org/conda-forge/noarch/pytest-split-0.8.2-pyhd8ed1ab_0.conda
+    version: 0.8.2
+  - category: main
+    dependencies:
+      pytest: '>=5.4'
+      python: '>=3.6,<4'
+    hash:
+      md5: d15a8b5c3a050fc3cf7d4509c70441f4
+      sha256: 34b2849d08d1cd7672b873c72df37d071d10402df89c6f8ac82d43cbeaa33442
+    manager: conda
+    name: pytest-split
+    optional: false
+    platform: osx-64
+    url:
+      https://conda.anaconda.org/conda-forge/noarch/pytest-split-0.8.2-pyhd8ed1ab_0.conda
+    version: 0.8.2
+  - category: main
+    dependencies:
+      pytest: '>=5.4'
+      python: '>=3.6,<4'
+    hash:
+      md5: d15a8b5c3a050fc3cf7d4509c70441f4
+      sha256: 34b2849d08d1cd7672b873c72df37d071d10402df89c6f8ac82d43cbeaa33442
+    manager: conda
+    name: pytest-split
+    optional: false
+    platform: osx-arm64
+    url:
+      https://conda.anaconda.org/conda-forge/noarch/pytest-split-0.8.2-pyhd8ed1ab_0.conda
+    version: 0.8.2
   - category: main
     dependencies:
       execnet: '>=2.1'
diff --git a/docs/runner_allocation.md b/docs/runner_allocation.md
@@ -0,0 +1,24 @@
+# GitHub Runner Allocation
+
+Last Updated: 2024-07-29
+
+We have a
+[concurrency of 20 runners available](https://docs.github.com/en/actions/administering-github-actions/usage-limits-billing-and-administration#usage-limits) in the entire `regro` account.
+
+These are split across our workflows as follows:
+- `bot-bot` - 1 runner
+- `bot-cache` - currently disabled
+- `bot-feedstocks` (daily for ~30 minutes) - 1 runner
+- `bot-make-graph` - 3 runners
+- `bot-make-migrators` - 1 runner
+- `bot-prs` - 4 runners
+- `bot-pypi-mapping` (hourly for ~5 minutes) - 1 runner
+- `bot-update-status-page` - 1 runner
+- `bot-versions` - 6 runners
+- `docker` (on demand) - 1 runner
+- `keepalive` (hourly for ~5 minutes) - 1 runner
+- `relock` (every 3 hours for ~1 minute) - 1 runner
+- `test-model` (daily for ~4 minutes, on demand) - 1 runner
+- `tests` (on demand) - 1 runner
+
+Total: 16 runners used permanently, 4 runners can be used on demand.
diff --git a/environment.yml b/environment.yml
@@ -68,5 +68,7 @@ dependencies:
   - pytest-xdist
   - pytest-cov
   - pytest-env
+  - pytest-randomly
+  - pytest-split
   - setuptools_scm>=7
   - python-build
diff --git a/tests/test_lazy_json_backends.py b/tests/test_lazy_json_backends.py
@@ -502,6 +502,9 @@ def test_lazy_json(tmpdir, backend):
             assert len(lj) == 0
             assert not lj
         finally:
+            be = LAZY_JSON_BACKENDS[backend]()
+            be.hdel("lazy_json", ["hi"])
+
             conda_forge_tick.lazy_json_backends.CF_TICK_GRAPH_DATA_BACKENDS = (
                 old_backend
             )
