merge develop to main for 0.5.9

.github/workflows/ci.yml

@@ -42,13 +42,20 @@ jobs:
     with:
       anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
+      skip_matrix_jobs: |
+        gnu@debian-11
+        [email protected]
+        gnu@debian-11
+        [email protected]
+        [email protected]
+        gnu@fedora-37
     secrets: inherit
 
-   # Build downstream packages on HPC
-  downstream-ci-hpc:
-    name: downstream-ci-hpc
-    if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
-    with:
-      anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
-    secrets: inherit
+#   # Build downstream packages on HPC
+#  downstream-ci-hpc:
+#    name: downstream-ci-hpc
+#    if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
+#    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+#    with:
+#      anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
+#    secrets: inherit

.github/workflows/python-pull-request.yml

@@ -1,12 +1,17 @@
 # This workflow will upload a Python Package using Twine when a release is created
 # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
 
-name: Code Quality checks for PRs
+name: Test PR
 
 on:
-  push:
   pull_request:
     types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - develop
+  schedule:
+    - cron: "9 2 * * 0" # at 9:02 on sunday
+
 
 jobs:
   quality:
@@ -17,7 +22,7 @@ jobs:
   checks:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.11"]
     uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
     with:
       python-version: ${{ matrix.python-version }}

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
     -   id: python-check-blanket-noqa # Check for # noqa: all
     -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.8.0
+  rev: 24.10.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -40,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.9
+  rev: v0.7.2
   hooks:
   - id: ruff
     args:
@@ -59,13 +59,8 @@ repos:
   hooks:
   - id: rstfmt
     exclude: 'cli/.*' # Because we use argparse
-- repo: https://github.com/b8raoult/pre-commit-docconvert
-  rev: "0.1.5"
-  hooks:
-  - id: docconvert
-    args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.2.4"
+  rev: "v2.5.0"
   hooks:
   - id: pyproject-fmt
 

CHANGELOG.md

@@ -10,6 +10,13 @@ Keep it human-readable, your future self will thank you!
 
 ## [Unreleased](https://github.com/ecmwf/anemoi-datasets/compare/0.5.8...HEAD)
 
+
+### Added
+
+- Call filters from anemoi-transform
+- make test optional when adls is not installed Pull request #110
+
+
 ## [0.5.8](https://github.com/ecmwf/anemoi-datasets/compare/0.5.7...0.5.8) - 2024-10-26
 
 ### Changed
@@ -25,9 +32,13 @@ Keep it human-readable, your future self will thank you!
 - Various bug fixes
 - Control compatibility check in xy/zip
 - Add `merge` feature
+- Add support for storing `supporting_arrays` in checkpoint files
+- Allow naming of datasets components
+- Contributors file (#105)
 
 ### Changed
 
+- Upload with ssh (experimental)
 - Remove upstream dependencies from downstream-ci workflow (temporary) (#83)
 - ci: pin python versions to 3.9 ... 3.12 for checks (#93)
 - Fix `__version__` import in init

CONTRIBUTORS.md

@@ -0,0 +1,13 @@
+## How to Contribute
+
+Please see the [read the docs](https://anemoi-training.readthedocs.io/en/latest/dev/contributing.html).
+
+
+## Contributors
+
+Thank you to all the wonderful people who have contributed to Anemoi. Contributions can come in many forms, including code, documentation, bug reports, feature suggestions, design, and more. A list of code-based contributors can be found [here](https://github.com/ecmwf/anemoi-datasets/graphs/contributors).
+
+
+## Contributing Organisations
+
+Significant contributions have been made by the following organisations: [DWD](https://www.dwd.de/), [MET Norway](https://www.met.no/), [MeteoSwiss](https://www.meteoswiss.admin.ch/), [RMI](https://www.meteo.be/) & [ECMWF](https://www.ecmwf.int/)

docs/building/sources/xarray-zarr.rst

@@ -2,5 +2,17 @@
  xarray-zarr
 #############
 
+Here is an example recipe that builds a dataset using one of the many
+regridded versions of ERA5 hosted by Google in Analysis-Ready, Cloud
+Optimized format. See `here
+<https://cloud.google.com/storage/docs/public-datasets/era5>`_ for more
+information.
+
 .. literalinclude:: yaml/xarray-zarr.yaml
    :language: yaml
+
+Note that unlike the ``mars`` examples, there is no need to include a
+``grid`` specification. Also, in order to subselect the vertical levels,
+it is necessary to use the :ref:`join <building-join>` operation to join
+separate lists containing 2D variables and 3D variables. If all vertical
+levels are desired, then it is OK to specify a single source.

docs/building/sources/yaml/xarray-zarr.yaml

@@ -1,3 +1,22 @@
+dates:
+  start: 2021-01-01T00:00:00
+  end: 2021-01-10T12:00:00
+  frequency: 6h
+
 input:
-  xarray-zarr:
-    url: https://...
+  join:
+      - xarray-zarr:
+          url: "gs://gcp-public-data-arco-era5/ar/1959-2022-6h-128x64_equiangular_conservative.zarr"
+          param:
+              - surface_pressure
+              - 2m_temperature
+              - 10m_u_component_of_wind
+              - 10m_v_component_of_wind
+
+      - xarray-zarr:
+          url: "gs://gcp-public-data-arco-era5/ar/1959-2022-6h-128x64_equiangular_conservative.zarr"
+          param:
+              - temperature
+          level:
+              - 1000
+              - 500

docs/using/missing.rst

@@ -4,6 +4,16 @@
  Managing missing dates
 ########################
 
+*********************************************
+ Managing missing dates with anemoi-training
+*********************************************
+
+Anemoi-training has internal handling of missing dates, and will
+calculate the valid date indices used during training using the
+``missing`` property. Consequenctly, when training a model with
+anemoi-training, you should `not` specify a method to deal with missing
+dates in the dataloader configuration file.
+
 **************************************************
  Filling the missing dates with artificial values
 **************************************************

pyproject.toml

@@ -1,14 +1,12 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
-
 [build-system]
 requires = [
   "setuptools>=60",
@@ -42,6 +40,7 @@ classifiers = [
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
@@ -97,6 +96,9 @@ optional-dependencies.tests = [
 optional-dependencies.xarray = [
   "gcsfs",
   "kerchunk",
+  "pandas",
+  "planetary-computer",
+  "pystac-client",
 ]
 
 urls.Documentation = "https://anemoi-datasets.readthedocs.io/"

src/anemoi/datasets/__init__.py

@@ -1,6 +1,8 @@
-# (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.

src/anemoi/datasets/__main__.py

@@ -1,12 +1,11 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
-#
 
 from anemoi.utils.cli import cli_main
 from anemoi.utils.cli import make_parser

src/anemoi/datasets/commands/__init__.py

@@ -1,12 +1,11 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
-#
 
 import os