Update dependencies to Pandas v2.0 (#1112)

* Update requirements files * Add Pandas v2.1.0 tox environment * Add Pandas v2.1 comment triggered tests * Fix indentation in workflow file * Rollback version pins to maintain Python 3.8 compatibility
UCL · Sep 28, 2023 · a9b2776 · a9b2776
1 parent cd39e2c
commit a9b2776
Show file tree

Hide file tree

Showing 6 changed files with 267 additions and 174 deletions.
diff --git a/.github/workflows/comment-triggered-ci.yml b/.github/workflows/comment-triggered-ci.yml
@@ -90,6 +90,34 @@ jobs:
       application-id: ${{ secrets.COMMENT_BOT_APP_ID }}
       application-private-key: ${{ secrets.COMMENT_BOT_APP_PRIVATE_KEY }}
 
+  python311_pandas21_fast_tests:
+    uses: ./.github/workflows/run-on-comment.yml
+    with:
+      runs-on: self-hosted
+      keyword: python3.11-pandas2.1-fast-tests
+      commands: |
+        tox -v -e py311-pandas21,report -- pytest --skip-slow --cov --cov-report=term-missing -vv tests
+      description: Run of fast tests on Python v3.11 with Pandas v2.1
+      timeout-minutes: 8640
+      application-organization: UCL
+    secrets:
+      application-id: ${{ secrets.COMMENT_BOT_APP_ID }}
+      application-private-key: ${{ secrets.COMMENT_BOT_APP_PRIVATE_KEY }}
+
+  python311_pandas21_all_tests:
+    uses: ./.github/workflows/run-on-comment.yml
+    with:
+      runs-on: self-hosted
+      keyword: python3.11-pandas2.1-all-tests
+      commands: |
+        tox -v -e py311-pandas21,report -- pytest --cov --cov-report=term-missing -vv tests
+      description: Run of all tests on Python v3.11 with Pandas v2.1
+      timeout-minutes: 8640
+      application-organization: UCL
+    secrets:
+      application-id: ${{ secrets.COMMENT_BOT_APP_ID }}
+      application-private-key: ${{ secrets.COMMENT_BOT_APP_PRIVATE_KEY }}
+
   dummy_job:
     uses: ./.github/workflows/run-on-comment.yml
     with:

diff --git a/requirements/base.in b/requirements/base.in
@@ -6,9 +6,9 @@ matplotlib
 pyshp
 squarify
 numpy
-pandas
+pandas~=2.0.0
 scipy
-openpyxl
+openpyxl==3.1.0  # avoid https://foss.heptapod.net/openpyxl/openpyxl/-/issues/1963
 
 # To submit jobs to Azure Batch
 azure-batch

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -2,144 +2,160 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=requirements/base.txt requirements/base.in
+#    pip-compile --output-file=base.txt base.in
 #
-adal==1.2.6
+adal==1.2.7
     # via msrestazure
-azure-batch==10.0.0
-    # via -r requirements/base.in
-azure-common==1.1.26
+azure-batch==14.0.0
+    # via -r base.in
+azure-common==1.1.28
     # via
     #   azure-batch
     #   azure-keyvault-certificates
     #   azure-keyvault-keys
     #   azure-keyvault-secrets
-azure-core==1.11.0
+azure-core==1.29.4
     # via
     #   azure-identity
     #   azure-keyvault-certificates
     #   azure-keyvault-keys
     #   azure-keyvault-secrets
     #   azure-storage-file-share
-azure-identity==1.5.0
-    # via -r requirements/base.in
-azure-keyvault-certificates==4.2.1
+    #   msrest
+azure-identity==1.14.0
+    # via -r base.in
+azure-keyvault-certificates==4.7.0
     # via azure-keyvault
-azure-keyvault-keys==4.3.1
+azure-keyvault-keys==4.8.0
     # via azure-keyvault
-azure-keyvault-secrets==4.2.0
+azure-keyvault-secrets==4.7.0
     # via azure-keyvault
-azure-keyvault==4.1.0
-    # via -r requirements/base.in
-azure-storage-file-share==12.4.1
-    # via -r requirements/base.in
-certifi==2020.12.5
+azure-keyvault==4.2.0
+    # via -r base.in
+azure-storage-file-share==12.14.1
+    # via -r base.in
+certifi==2023.7.22
     # via
     #   msrest
     #   requests
-cffi==1.14.5
+cffi==1.15.1
     # via cryptography
-chardet==4.0.0
+charset-normalizer==3.2.0
     # via requests
-click==7.1.2
-    # via -r requirements/base.in
-cryptography==3.4.6
+click==8.1.7
+    # via -r base.in
+contourpy==1.1.1
+    # via matplotlib
+cryptography==41.0.3
     # via
     #   adal
     #   azure-identity
     #   azure-keyvault-keys
     #   azure-storage-file-share
     #   msal
     #   pyjwt
-cycler==0.10.0
+cycler==0.11.0
     # via matplotlib
-et-xmlfile==1.0.1
+et-xmlfile==1.1.0
     # via openpyxl
-gitdb==4.0.5
+fonttools==4.42.1
+    # via matplotlib
+gitdb==4.0.10
     # via gitpython
-gitpython==3.1.13
-    # via -r requirements/base.in
-idna==2.10
+gitpython==3.1.36
+    # via -r base.in
+idna==3.4
     # via requests
-isodate==0.6.0
-    # via msrest
-jdcal==1.4.1
-    # via openpyxl
-kiwisolver==1.3.1
+importlib-resources==6.0.1
     # via matplotlib
-matplotlib==3.3.4
-    # via -r requirements/base.in
-msal-extensions==0.3.0
-    # via azure-identity
-msal==1.9.0
-    # via
-    #   azure-identity
-    #   msal-extensions
-msrest==0.6.21
+isodate==0.6.1
     # via
-    #   azure-batch
     #   azure-keyvault-certificates
     #   azure-keyvault-keys
     #   azure-keyvault-secrets
     #   azure-storage-file-share
-    #   msrestazure
+    #   msrest
+kiwisolver==1.4.5
+    # via matplotlib
+matplotlib==3.7.3
+    # via -r base.in
+msal-extensions==1.0.0
+    # via azure-identity
+msal==1.24.0
+    # via
+    #   azure-identity
+    #   msal-extensions
+msrest==0.7.1
+    # via msrestazure
 msrestazure==0.6.4
     # via azure-batch
-numpy==1.20.1
+numpy==1.24.4
     # via
-    #   -r requirements/base.in
+    #   -r base.in
+    #   contourpy
     #   matplotlib
     #   pandas
     #   scipy
-oauthlib==3.1.0
+oauthlib==3.2.2
     # via requests-oauthlib
-openpyxl==3.0.6
-    # via -r requirements/base.in
-pandas==1.2.2
-    # via -r requirements/base.in
-pillow==8.1.0
+openpyxl==3.1.0
+    # via -r base.in
+packaging==23.1
     # via matplotlib
-portalocker==1.7.1
+pandas==2.0.3
+    # via -r base.in
+pillow==10.0.1
+    # via matplotlib
+portalocker==2.8.2
     # via msal-extensions
-pycparser==2.20
+pycparser==2.21
     # via cffi
-pyjwt[crypto]==2.0.1
+pyjwt[crypto]==2.8.0
     # via
     #   adal
     #   msal
-pyparsing==2.4.7
+pyparsing==3.1.1
     # via matplotlib
-pyshp==2.3.0
-    # via -r requirements/base.in
-python-dateutil==2.8.1
+pyshp==2.3.1
+    # via -r base.in
+python-dateutil==2.8.2
     # via
     #   adal
     #   matplotlib
     #   pandas
-pytz==2021.1
+pytz==2023.3.post1
     # via pandas
-requests-oauthlib==1.3.0
+requests-oauthlib==1.3.1
     # via msrest
-requests==2.25.1
+requests==2.31.0
     # via
     #   adal
     #   azure-core
     #   msal
     #   msrest
     #   requests-oauthlib
-scipy==1.6.1
-    # via -r requirements/base.in
-six==1.15.0
+scipy==1.10.1
+    # via -r base.in
+six==1.16.0
     # via
     #   azure-core
-    #   azure-identity
-    #   cycler
     #   isodate
     #   msrestazure
     #   python-dateutil
-smmap==3.0.5
+smmap==5.0.1
     # via gitdb
 squarify==0.4.3
-    # via -r requirements/base.in
-urllib3==1.26.3
+    # via -r base.in
+typing-extensions==4.8.0
+    # via
+    #   azure-core
+    #   azure-keyvault-certificates
+    #   azure-keyvault-keys
+    #   azure-keyvault-secrets
+    #   azure-storage-file-share
+tzdata==2023.3
+    # via pandas
+urllib3==2.0.4
     # via requests
+zipp==3.16.2
+    # via importlib-resources
diff --git a/requirements/dev.in b/requirements/dev.in
@@ -2,11 +2,13 @@
 
 # Running tests
 pytest
+pytest-cov
+pytest-xdist
 virtualenv
 tox
 
 # Profiling
-pyinstrument
+pyinstrument>=4.3
 
 # Building requirements files
 pip-tools