Backport PR #58181 on branch 2.2.x (CI: correct error msg in test_view_index)

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 jobs:
   test-arm:
     machine:
-      image: ubuntu-2004:2022.04.1
+      image: default
     resource_class: arm.large
     environment:
       ENV_FILE: ci/deps/circle-310-arm64.yaml
@@ -46,7 +46,7 @@ jobs:
       cibw-build:
         type: string
     machine:
-      image: ubuntu-2004:2022.04.1
+      image: default
     resource_class: arm.large
     environment:
       TRIGGER_SOURCE: << pipeline.trigger_source >>

.github/actions/run-tests/action.yml

@@ -1,16 +1,9 @@
 name: Run tests and report results
-inputs:
-  preload:
-    description: Preload arguments for sanitizer
-    required: false
-  asan_options:
-    description: Arguments for Address Sanitizer (ASAN)
-    required: false
 runs:
   using: composite
   steps:
     - name: Test
-      run: ${{ inputs.asan_options }} ${{ inputs.preload }} ci/run_tests.sh
+      run: ci/run_tests.sh
       shell: bash -el {0}
 
     - name: Publish test results

.github/workflows/code-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
 
 env:
   ENV_FILE: environment.yml

.github/workflows/docbuild-and-upload.yml

@@ -4,13 +4,13 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
 
 env:
   ENV_FILE: environment.yml

.github/workflows/package-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "consortium-standard", "all"]
+        extra: ["test", "pyarrow", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "consortium-standard", "all"]
       fail-fast: false
     name: Install Extras - ${{ matrix.extra }}
     concurrency:

.github/workflows/unit-tests.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     paths-ignore:
       - "doc/**"
       - "web/**"
@@ -92,18 +92,10 @@ jobs:
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
-            test_args: "-W error::FutureWarning"
+            test_args: "-W error::DeprecationWarning -W error::FutureWarning"
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
-          - name: "ASAN / UBSAN"
-            env_file: actions-311-sanitizers.yaml
-            pattern: "not slow and not network and not single_cpu and not skip_ubsan"
-            asan_options: "ASAN_OPTIONS=detect_leaks=0"
-            preload: LD_PRELOAD=$(gcc -print-file-name=libasan.so)
-            meson_args: --config-settings=setup-args="-Db_sanitize=address,undefined"
-            cflags_adds: -fno-sanitize-recover=all
-            pytest_workers: -1  # disable pytest-xdist as it swallows stderr from ASAN
       fail-fast: false
     name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
     env:
@@ -190,18 +182,12 @@ jobs:
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
       if: ${{ matrix.name != 'Pypy' }}
-      with:
-        preload: ${{ matrix.preload }}
-        asan_options: ${{ matrix.asan_options }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}
 
     - name: Test (single_cpu)
       uses: ./.github/actions/run-tests
-      with:
-        preload: ${{ matrix.preload }}
-        asan_options: ${{ matrix.asan_options }}
       env:
         PATTERN: 'single_cpu'
         PYTEST_WORKERS: 0
@@ -211,7 +197,8 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        os: [macos-latest, windows-latest]
+        # Note: Don't use macOS latest since macos 14 appears to be arm64 only
+        os: [macos-13, macos-14, windows-latest]
         env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
@@ -224,8 +211,7 @@ jobs:
       PANDAS_CI: 1
       PYTEST_TARGET: pandas
       PATTERN: "not slow and not db and not network and not single_cpu"
-      # GH 47443: PYTEST_WORKERS > 0 crashes Windows builds with memory related errors
-      PYTEST_WORKERS: ${{ matrix.os == 'macos-latest' && 'auto' || '0' }}
+      PYTEST_WORKERS: 'auto'
 
     steps:
       - name: Checkout
@@ -351,7 +337,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macOS-latest, windows-latest]
+        # Separate out macOS 13 and 14, since macOS 14 is arm64 only
+        os: [ubuntu-22.04, macOS-13, macOS-14, windows-latest]
 
     timeout-minutes: 90
 

.github/workflows/wheels.yml

@@ -94,7 +94,9 @@ jobs:
         buildplat:
         - [ubuntu-22.04, manylinux_x86_64]
         - [ubuntu-22.04, musllinux_x86_64]
-        - [macos-12, macosx_*]
+        - [macos-12, macosx_x86_64]
+        # Note: M1 images on Github Actions start from macOS 14
+        - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
         python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
@@ -128,7 +130,7 @@ jobs:
       # Python version used to build sdist doesn't matter
       # wheel will be built from sdist with the correct version
       - name: Unzip sdist (macOS)
-        if: ${{ matrix.buildplat[1] == 'macosx_*' }}
+        if: ${{ startsWith(matrix.buildplat[1], 'macosx') }}
         run: |
           tar -xzf ./dist/${{ env.sdist_name }} -C ./dist
 
@@ -137,20 +139,19 @@ jobs:
         shell: bash -el {0}
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
-      - name: Build normal wheels
-        if: ${{ (env.IS_SCHEDULE_DISPATCH != 'true' || env.IS_PUSH == 'true') }}
-        uses: pypa/[email protected]
+      - name: Build wheels
+        uses: pypa/[email protected]
         with:
-         package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+         package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
       - name: Build nightly wheels (with NumPy pre-release)
         if: ${{ (env.IS_SCHEDULE_DISPATCH == 'true' && env.IS_PUSH != 'true') }}
-        uses: pypa/cibuildwheel@v2.16.2
+        uses: pypa/cibuildwheel@v2.17.0
         with:
-         package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+         package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           # The nightly wheels should be build witht he NumPy 2.0 pre-releases
           # which requires the additional URL.
@@ -183,15 +184,15 @@ jobs:
           $TST_CMD = @"
           python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
-          python -c `'import pandas as pd; pd.test(extra_args=[\"`\"--no-strict-data-files`\"\", \"`\"-m not clipboard and not single_cpu and not slow and not network and not db`\"\"])`';
+          python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`';
           "@
           # add rc to the end of the image name if the Python version is unreleased
           docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
           docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
 
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
+          name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
           path: ./wheelhouse/*.whl
 
       - name: Upload wheels & sdist

.pre-commit-config.yaml

@@ -32,6 +32,8 @@ repos:
         # TODO: remove autofixe-only rules when they are checked by ruff
         name: ruff-selected-autofixes
         alias: ruff-selected-autofixes
+        files: ^pandas
+        exclude: ^pandas/tests
         args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
 -   repo: https://github.com/jendrikseipp/vulture
     rev: 'v2.10'
@@ -356,18 +358,6 @@ repos:
         files: ^pandas/
         exclude: ^(pandas/_libs/|pandas/tests/|pandas/errors/__init__.py$|pandas/_version.py)
         types: [python]
-    -   id: future-annotations
-        name: import annotations from __future__
-        entry: 'from __future__ import annotations'
-        language: pygrep
-        args: [--negate]
-        files: ^pandas/
-        types: [python]
-        exclude: |
-            (?x)
-            /(__init__\.py)|(api\.py)|(_version\.py)|(testing\.py)|(conftest\.py)$
-            |/tests/
-            |/_testing/
     -   id: check-test-naming
         name: check that test names start with 'test'
         entry: python -m scripts.check_test_naming

asv_bench/benchmarks/io/csv.py

@@ -408,6 +408,9 @@ def time_read_stringcsv(self, engine):
     def time_read_bytescsv(self, engine):
         read_csv(self.data(self.BytesIO_input), engine=engine)
 
+    def peakmem_read_csv(self, engine):
+        read_csv(self.data(self.BytesIO_input), engine=engine)
+
 
 class ReadCSVCategorical(BaseIO):
     fname = "__test__.csv"

ci/deps/actions-310.yaml

@@ -14,13 +14,12 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-localserver>=0.7.1
   - pytest-qt>=4.2.0
   - boto3
 
   # required dependencies
   - python-dateutil
-  - numpy<2
+  - numpy
   - pytz
 
   # optional dependencies
@@ -61,3 +60,4 @@ dependencies:
     - adbc-driver-postgresql>=0.8.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7
+    - pytest-localserver>=0.7.1

ci/deps/actions-311-downstream_compat.yaml

@@ -21,7 +21,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy<2
+  - numpy
   - pytz
 
   # optional dependencies

ci/deps/actions-311-pyarrownightly.yaml

@@ -18,7 +18,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy<2
+  - numpy
   - pytz
   - pip
 

ci/deps/actions-311.yaml

@@ -14,13 +14,12 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-localserver>=0.7.1
   - pytest-qt>=4.2.0
   - boto3
 
   # required dependencies
   - python-dateutil
-  - numpy<2
+  - numpy
   - pytz
 
   # optional dependencies
@@ -60,4 +59,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.8.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - pytest-localserver>=0.7.1

ci/deps/actions-312.yaml

@@ -14,13 +14,12 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-localserver>=0.7.1
   - pytest-qt>=4.2.0
   - boto3
 
   # required dependencies
   - python-dateutil
-  - numpy<2
+  - numpy
   - pytz
 
   # optional dependencies
@@ -61,3 +60,4 @@ dependencies:
     - adbc-driver-postgresql>=0.8.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7
+    - pytest-localserver>=0.7.1

ci/deps/actions-39-minimum_versions.yaml

@@ -22,7 +22,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.2
-  - numpy=1.22.4, <2
+  - numpy=1.22.4
   - pytz=2020.1
 
   # optional dependencies