Merge branch 'master' into datetime64-indexing-fix

Author: keewis

.pre-commit-config.yaml

@@ -2,24 +2,24 @@
 repos:
   # isort should run before black as black sometimes tweaks the isort output
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.1.0
+    rev: 5.4.2
     hooks:
       - id: isort
   # https://github.com/python/black#version-control-integration
-  - repo: https://github.com/python/black
-    rev: stable
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
     hooks:
       - id: black
   - repo: https://github.com/keewis/blackdoc
-    rev: stable
+    rev: v0.1.2
     hooks:
       - id: blackdoc
   - repo: https://gitlab.com/pycqa/flake8
     rev: 3.8.3
     hooks:
       - id: flake8
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.780  # Must match ci/requirements/*.yml
+    rev: v0.782  # Must match ci/requirements/*.yml
     hooks:
       - id: mypy
   # run this occasionally, ref discussion https://github.com/pydata/xarray/pull/3194

HOW_TO_RELEASE.md

@@ -1,79 +1,93 @@
-# How to issue an xarray release in 17 easy steps
+# How to issue an xarray release in 20 easy steps
 
 Time required: about an hour.
 
+These instructions assume that `upstream` refers to the main repository:
+```
+$ git remote -v
+{...}
+upstream        https://github.com/pydata/xarray (fetch)
+upstream        https://github.com/pydata/xarray (push)
+```
+
  1. Ensure your master branch is synced to upstream:
-      ```
-      git pull upstream master
-      ```
+     ```sh
+     git pull upstream master
+     ```
  2. Get a list of contributors with:
-    ```
+    ```sh
     git log "$(git tag --sort="v:refname" | sed -n 'x;$p').." --format=%aN | sort -u | perl -pe 's/\n/$1, /'
     ```
-    or by substituting the _previous_ release in:
-    ```
-    git log v0.X.Y-1.. --format=%aN | sort -u | perl -pe 's/\n/$1, /'
+    or by substituting the _previous_ release in {0.X.Y-1}:
+    ```sh
+    git log v{0.X.Y-1}.. --format=%aN | sort -u | perl -pe 's/\n/$1, /'
     ```
     Add these into `whats-new.rst` somewhere :)
- 3. Look over whats-new.rst and the docs. Make sure "What's New" is complete
-    (check the date!) and consider adding a brief summary note describing the
-    release at the top.
+ 3. Write a release summary: ~50 words describing the high level features. This
+    will be used in the release emails, tweets, GitHub release notes, etc. 
+ 4. Look over whats-new.rst and the docs. Make sure "What's New" is complete
+    (check the date!) and add the release summary at the top.
     Things to watch out for:
     - Important new features should be highlighted towards the top.
     - Function/method references should include links to the API docs.
     - Sometimes notes get added in the wrong section of whats-new, typically
       due to a bad merge. Check for these before a release by using git diff,
-      e.g., `git diff v0.X.Y whats-new.rst` where 0.X.Y is the previous
+      e.g., `git diff v{0.X.Y-1} whats-new.rst` where {0.X.Y-1} is the previous
       release.
- 4. If you have any doubts, run the full test suite one final time!
-      ```
+ 5. If possible, open a PR with the release summary and whatsnew changes.
+ 6. After merging, again ensure your master branch is synced to upstream:
+     ```sh
+     git pull upstream master
+     ```
+ 7. If you have any doubts, run the full test suite one final time!
+      ```sh
       pytest
       ```
- 5. Check that the ReadTheDocs build is passing.
- 6. On the master branch, commit the release in git:
+ 8. Check that the ReadTheDocs build is passing.
+ 9. On the master branch, commit the release in git:
+      ```s
+      git commit -am 'Release v{0.X.Y}'
       ```
-      git commit -am 'Release v0.X.Y'
+10. Tag the release:
+      ```sh
+      git tag -a v{0.X.Y} -m 'v{0.X.Y}'
       ```
- 7. Tag the release:
-      ```
-      git tag -a v0.X.Y -m 'v0.X.Y'
-      ```
- 8. Build source and binary wheels for pypi:
-      ```
-      git clean -xdf  # this deletes all uncommited changes!
+11. Build source and binary wheels for PyPI:
+      ```sh
+      git clean -xdf  # this deletes all uncommitted changes!
       python setup.py bdist_wheel sdist
       ```
- 9. Use twine to check the package build:
-      ```
-      twine check dist/xarray-0.X.Y*
+12. Use twine to check the package build:
+      ```sh
+      twine check dist/xarray-{0.X.Y}*
       ```
-10. Use twine to register and upload the release on pypi. Be careful, you can't
+13. Use twine to register and upload the release on PyPI. Be careful, you can't
     take this back!
-      ```
-      twine upload dist/xarray-0.X.Y*
+      ```sh
+      twine upload dist/xarray-{0.X.Y}*
       ```
     You will need to be listed as a package owner at
     https://pypi.python.org/pypi/xarray for this to work.
-11. Push your changes to master:
-      ```
+14. Push your changes to master:
+      ```sh
       git push upstream master
       git push upstream --tags
       ```
-12. Update the stable branch (used by ReadTheDocs) and switch back to master:
-     ```
+15. Update the stable branch (used by ReadTheDocs) and switch back to master:
+     ```sh
       git checkout stable
       git rebase master
-      git push upstream stable
+      git push --force upstream stable
       git checkout master
      ```
     It's OK to force push to 'stable' if necessary. (We also update the stable 
-    branch with `git cherrypick` for documentation only fixes that apply the 
+    branch with `git cherry-pick` for documentation only fixes that apply the 
     current released version.)
-13. Add a section for the next release (v.X.Y+1) to doc/whats-new.rst:
+16. Add a section for the next release {0.X.Y+1} to doc/whats-new.rst:
      ```
-     .. _whats-new.0.X.Y+1:
+     .. _whats-new.{0.X.Y+1}:
 
-     v0.X.Y+1 (unreleased)
+     v{0.X.Y+1} (unreleased)
      ---------------------
 
      Breaking changes
@@ -95,20 +109,20 @@ Time required: about an hour.
      Internal Changes
      ~~~~~~~~~~~~~~~~
      ```
-14. Commit your changes and push to master again:
-      ```
+17. Commit your changes and push to master again:
+      ```sh
       git commit -am 'New whatsnew section'
       git push upstream master
       ```
     You're done pushing to master!
-15. Issue the release on GitHub. Click on "Draft a new release" at
-    https://github.com/pydata/xarray/releases. Type in the version number, but
-    don't bother to describe it -- we maintain that on the docs instead.
-16. Update the docs. Login to https://readthedocs.org/projects/xray/versions/
+18. Issue the release on GitHub. Click on "Draft a new release" at
+    https://github.com/pydata/xarray/releases. Type in the version number
+    and paste the release summary in the notes.
+19. Update the docs. Login to https://readthedocs.org/projects/xray/versions/
     and switch your new release tag (at the bottom) from "Inactive" to "Active".
     It should now build automatically.
-17. Issue the release announcement! For bug fix releases, I usually only email
-    [email protected]. For major/feature releases, I will email a broader
+20. Issue the release announcement to mailing lists & Twitter. For bug fix releases, I 
+    usually only email [email protected]. For major/feature releases, I will email a broader
     list (no more than once every 3-6 months):
       - [email protected]
       - [email protected]

ci/min_deps_check.py

@@ -23,9 +23,21 @@
     "pytest-env",
 }
 
-POLICY_MONTHS = {"python": 42, "numpy": 24, "pandas": 12, "scipy": 12}
-POLICY_MONTHS_DEFAULT = 6
-
+POLICY_MONTHS = {"python": 42, "numpy": 24, "setuptools": 42}
+POLICY_MONTHS_DEFAULT = 12
+POLICY_OVERRIDE = {
+    # dask < 2.9 has trouble with nan-reductions
+    # TODO remove this special case and the matching note in installing.rst
+    #      after January 2021.
+    "dask": (2, 9),
+    "distributed": (2, 9),
+    # setuptools-scm doesn't work with setuptools < 36.7 (Nov 2017).
+    # The conda metadata is malformed for setuptools < 38.4 (Jan 2018)
+    # (it's missing a timestamp which prevents this tool from working).
+    # TODO remove this special case and the matching note in installing.rst
+    #      after July 2021.
+    "setuptools": (38, 4),
+}
 has_errors = False
 
 
@@ -151,6 +163,11 @@ def process_pkg(
         policy_minor = minor
         policy_published_actual = published
 
+    try:
+        policy_major, policy_minor = POLICY_OVERRIDE[pkg]
+    except KeyError:
+        pass
+
     if (req_major, req_minor) < (policy_major, policy_minor):
         status = "<"
     elif (req_major, req_minor) > (policy_major, policy_minor):

ci/requirements/doc.yml

@@ -13,14 +13,22 @@ dependencies:
   - ipython
   - iris>=2.3
   - jupyter_client
+  - matplotlib-base=3.3.0
   - nbsphinx
   - netcdf4>=1.5
   - numba
   - numpy>=1.17
-  - pandas>=1.0
+  # FIXME https://github.com/pydata/xarray/issues/4287
+  # - pandas>=1.0
+  - pandas=1.0
   - rasterio>=1.1
   - seaborn
   - setuptools
-  - sphinx>=2.3
+  - sphinx=3.2
   - sphinx_rtd_theme>=0.4
+  - sphinx-autosummary-accessors
   - zarr>=2.4
+  - pip
+  - pip:
+      # relative to this file. Needs to be editable to be accepted.
+      - -e ../..

ci/requirements/py36-bare-minimum.yml

@@ -10,4 +10,4 @@ dependencies:
   - pytest-env
   - numpy=1.15
   - pandas=0.25
-  - setuptools=41.2
+  - setuptools=38.4

ci/requirements/py36-min-all-deps.yml

@@ -26,7 +26,7 @@ dependencies:
   - isort
   - lxml=4.4  # Optional dep of pydap
   - matplotlib=3.1
-  - mypy=0.761  # Must match .pre-commit-config.yaml
+  - mypy=0.782  # Must match .pre-commit-config.yaml
   - nc-time-axis=1.2
   - netcdf4=1.4
   - numba=0.46
@@ -43,7 +43,7 @@ dependencies:
   - rasterio=1.0
   - scipy=1.3
   - seaborn=0.9
-  - setuptools=41.2
+  - setuptools=38.4
   # - sparse  # See py36-min-nep18.yml
   - toolz=0.10
   - zarr=2.3

ci/requirements/py36-min-nep18.yml

@@ -15,6 +15,6 @@ dependencies:
   - pytest
   - pytest-cov
   - pytest-env
-  - scipy=1.2
-  - setuptools=41.2
+  - scipy=1.3
+  - setuptools=38.4
   - sparse=0.8

ci/requirements/py36.yml

@@ -22,7 +22,7 @@ dependencies:
   - isort
   - lxml    # Optional dep of pydap
   - matplotlib
-  - mypy=0.761  # Must match .pre-commit-config.yaml
+  - mypy=0.782  # Must match .pre-commit-config.yaml
   - nc-time-axis
   - netcdf4
   - numba

ci/requirements/py37-windows.yml

@@ -22,7 +22,7 @@ dependencies:
   - isort
   - lxml    # Optional dep of pydap
   - matplotlib
-  - mypy=0.761  # Must match .pre-commit-config.yaml
+  - mypy=0.782  # Must match .pre-commit-config.yaml
   - nc-time-axis
   - netcdf4
   - numba

ci/requirements/py37.yml

@@ -22,7 +22,7 @@ dependencies:
   - isort
   - lxml    # Optional dep of pydap
   - matplotlib
-  - mypy=0.761  # Must match .pre-commit-config.yaml
+  - mypy=0.782  # Must match .pre-commit-config.yaml
   - nc-time-axis
   - netcdf4
   - numba

ci/requirements/py38-all-but-dask.yml

@@ -19,7 +19,7 @@ dependencies:
   - isort
   - lxml    # Optional dep of pydap
   - matplotlib
-  - mypy=0.761  # Must match .pre-commit-config.yaml
+  - mypy=0.782  # Must match .pre-commit-config.yaml
   - nc-time-axis
   - netcdf4
   - numba

conftest.py

@@ -24,7 +24,7 @@ def pytest_runtest_setup(item):
 
 
 @pytest.fixture(autouse=True)
-def add_standard_imports(doctest_namespace):
+def add_standard_imports(doctest_namespace, tmpdir):
     import numpy as np
     import pandas as pd
 
@@ -33,3 +33,9 @@ def add_standard_imports(doctest_namespace):
     doctest_namespace["np"] = np
     doctest_namespace["pd"] = pd
     doctest_namespace["xr"] = xr
+
+    # always seed numpy.random to make the examples deterministic
+    np.random.seed(0)
+
+    # always switch to the temporary directory, so files get written there
+    tmpdir.chdir()