feat: new data sources (#258)

Co-authored-by: Florian Pinault <[email protected]>

Author: b8raoult

.gitignore

@@ -135,3 +135,8 @@ _version.py
 *.to_upload
 tempCodeRunnerFile.python
 Untitled-*.py
+*.zip
+*.json
+*.db
+*.tgz
+_api/

.readthedocs.yaml

@@ -4,16 +4,16 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.11"
-  jobs:
-    pre_build:
-      - bash docs/scripts/api_build.sh
+#  jobs:
+#    pre_build:
+#    - bash docs/scripts/api_build.sh
 
 sphinx:
   configuration: docs/conf.py
 
 python:
-   install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - docs
+  install:
+  - method: pip
+    path: .
+    extra_requirements:
+    - docs

03-constant-fields.rst

@@ -0,0 +1,3 @@
+########################
+ Adding constant fields
+########################

docs/cli/grib-index.rst

@@ -0,0 +1,24 @@
+.. _grib-index_command:
+
+Grib-index Command
+============
+
+The `grib-index` command is used to create an index file for GRIB files. The index file is then used
+by the `grib-index` :ref:`source <grib-index_source>`.
+
+The command will recursively scan the directories provided and open all the GRIB files found. It will
+then create an index file for each GRIB file, which will be used to read the data.
+
+.. code:: bash
+
+    anemoi-datasets grib-index --index index.db /path1/to/grib/files /path2/to/grib/files
+
+
+See :ref:`grib_flavour` for more information about GRIB flavours.
+
+
+.. argparse::
+    :module: anemoi.datasets.__main__
+    :func: create_parser
+    :prog: anemoi-datasets
+    :path: grib-index

docs/datasets/building/filters/yaml/orog_to_z.yaml

@@ -1,10 +1,11 @@
 input:
   pipe:
-    - source: # mars, grib, netcdf, etc.
+  - source:
+      # mars, grib, netcdf, etc.
       # source attributes here
       # ...
       # Must load an orography variable
 
-    - orog_to_z:
-        orog: orog  # Name of orography (input) variable
-        z: z        # Name of z (output) variable
+  - orog_to_z:
+      orog: orog # Name of orography (input) variable
+      z: z # Name of z (output) variable

docs/datasets/building/filters/yaml/regrid1.yaml

@@ -1,10 +1,11 @@
 input:
   pipe:
-    - source: # mars, grib, netcdf, etc.
+  - source:
+      # mars, grib, netcdf, etc.
       # source attributes here
       # ...
 
-    - regrid:
-        method: nearest
-        in_grid: o32
-        out_grid: o48
+  - regrid:
+      method: nearest
+      in_grid: o32
+      out_grid: o48

docs/datasets/building/filters/yaml/regrid2.yaml

@@ -1,8 +1,9 @@
 input:
   pipe:
-    - source: # mars, grib, netcdf, etc.
+  - source:
+      # mars, grib, netcdf, etc.
       # source attributes here
       # ...
 
-    - regrid:
-        matrix: /path/to/regrid/matrix.npz
+  - regrid:
+      matrix: /path/to/regrid/matrix.npz

docs/datasets/building/filters/yaml/rename.yaml

@@ -1,11 +1,13 @@
 input:
   pipe:
-    - source: # mars, grib, netcdf, etc.
+  - source:
+      # mars, grib, netcdf, etc.
       # source attributes here
       # ...
 
-    - rename:
-        param: # Map old `param` names to new ones
-          temperature_2m: 2t
-          temperature_850hPa: t_850
-          # ...
+  - rename:
+      param:
+        # Map old `param` names to new ones
+        temperature_2m: 2t
+        temperature_850hPa: t_850
+        # ...

docs/datasets/building/filters/yaml/sum.yaml

@@ -1,13 +1,15 @@
 input:
   pipe:
-    - source: # mars, grib, netcdf, etc.
+  - source:
+      # mars, grib, netcdf, etc.
       # source attributes here
       # ...
       # Must load the variables to be summed
 
-    - sum:
-        params:  # List of input variables
-          variable1
-          variable2
-          variable3
-        output: variable_total    # Name of output variable
+  - sum:
+      params:
+      # List of input variables
+      - variable1
+      - variable2
+      - variable3
+      output: variable_total # Name of output variable

docs/datasets/building/filters/yaml/wz_to_w.yaml

@@ -1,10 +1,11 @@
 input:
   pipe:
-    - source: # mars, grib, netcdf, etc.
+  - source:
+      # mars, grib, netcdf, etc.
       # source attributes here
       # ...
       # Must load geometric vertical velocity
 
-    - wz_to_w:
-        wz: wz  # Name of geometric vertical velocity (input) variable
-        x: z    # Name of pressure vertical velocity (output) variable
+  - wz_to_w:
+      wz: wz # Name of geometric vertical velocity (input) variable
+      x: z # Name of pressure vertical velocity (output) variable

docs/datasets/building/sources.rst

@@ -21,17 +21,19 @@ The following `sources` are currently available:
    :maxdepth: 1
 
    sources/accumulations
-   sources/eccc_fstd
+   sources/anemoi-dataset
+   sources/cds
+   sources/eccc-fstd
    sources/forcings
    sources/grib
+   sources/grib-index
    sources/hindcasts
    sources/mars
-   sources/cds
    sources/netcdf
    sources/opendap
    sources/recentre
-   sources/repeated_dates
+   sources/repeated-dates
+   sources/xarray-based
    sources/xarray-kerchunk
    sources/xarray-zarr
-   sources/xarray-based
    sources/zenodo

docs/datasets/building/sources/anemoi-dataset.rst

@@ -0,0 +1,19 @@
+.. _anemoi-dataset_source:
+
+################
+ anemoi-dataset
+################
+
+.. admonition:: Experimental
+   :class: important
+
+      This source is experimental and may change in the future.
+
+An anemoi-dataset can be a source for a dataset:
+
+.. literalinclude:: yaml/anemoi-dataset.yaml
+   :language: yaml
+
+The parameters are the same as the ones used in the ``open_dataset``
+function, which allows you to subset and combine datasets. See
+:ref:`opening-datasets` for more information.

docs/datasets/building/sources/eccc_fstd.rst renamed to docs/datasets/building/sources/eccc-fstd.rst

@@ -5,12 +5,14 @@
 To read files in the standard format used at Environment and Climate
 Change Canada (ECCC), the following source can be used:
 
-.. literalinclude:: yaml/eccc_fstd.yaml
+.. literalinclude:: yaml/eccc-fstd.yaml
    :language: yaml
 
 The recipe will build a dataset from a standard file using the
 ``fstd2nc`` xarray plugin.
 
-The ``fstd2nc`` dependencie is not part of the default anemoi-datasets
+The ``fstd2nc`` dependency is not part of the default anemoi-datasets
 installation and has to be install following the `fstd2nc project
 description <https://github.com/neishm/fstd2nc>`_.
+
+See :ref:`create-cf-data` for more information.

docs/datasets/building/sources/grib-index.rst

@@ -0,0 +1,11 @@
+.. _grib-index_source:
+
+############
+ grib-index
+############
+
+The `grib-index` source is used to read GRIB files with the help of an
+index file created with the `grib-index` :ref:`command
+<grib-index_command>`.
+
+See :ref:`create-grib-data` for more information.

docs/datasets/building/sources/grib.rst

@@ -1,3 +1,5 @@
+.. _grib_source:
+
 ######
  grib
 ######
@@ -32,6 +34,8 @@ hour, you can use the following configuration:
 The patterns in between the curly brackets are replaced by the values of
 the `date` and formatted according to the Python strftime_ method.
 
+See :ref:`create-grib-data` for more information.
+
 .. note::
 
    You can combine all the above options when selecting GRIB messages

docs/datasets/building/sources/netcdf.rst

@@ -24,3 +24,5 @@ files, using Unix’ shell `wildcards
    information, but in some cases it is not possible. If you encounter
    this or similar issues, please open an issue in the anemoi-datasets
    repository.
+
+See :ref:`create-cf-data` for more information.

docs/datasets/building/sources/opendap.rst

@@ -4,3 +4,5 @@
 
 .. literalinclude:: yaml/opendap.yaml
    :language: yaml
+
+See :ref:`create-cf-data` for more information.

docs/datasets/building/sources/repeated_dates.rst renamed to docs/datasets/building/sources/repeated-dates.rst

@@ -1,14 +1,14 @@
 ################
- repeated_dates
+ repeated-dates
 ################
 
-The `repeated_dates` source is used to repeat a single source multiple
+The `repeated-dates` source is used to repeat a single source multiple
 times, so that its data is present at multiple dates. A simple example
-of this is when you have source that contains a constant field, such as
-orography or bathymetry, that you want to have repeated at all the dates
-of the dataset.
+of this is when you have a source that contains a constant field, such
+as orography or bathymetry, that you want to have repeated at all the
+dates of the dataset.
 
-The generale format of the `repeated_dates` source is:
+The general format of the `repeated-dates` source is:
 
 .. literalinclude:: yaml/repeated_dates1.yaml
    :language: yaml
@@ -21,19 +21,19 @@ where ``source`` is any of the :ref:`operations <operations>` or
  constant
 **********
 
-.. literalinclude:: yaml/repeated_dates2.yaml
+.. literalinclude:: yaml/repeated-dates2.yaml
    :language: yaml
 
 *************
  climatology
 *************
 
-.. literalinclude:: yaml/repeated_dates3.yaml
+.. literalinclude:: yaml/repeated-dates3.yaml
    :language: yaml
 
 *********
  closest
 *********
 
-.. literalinclude:: yaml/repeated_dates4.yaml
+.. literalinclude:: yaml/repeated-dates4.yaml
    :language: yaml

docs/datasets/building/sources/xarray-based.rst

@@ -2,14 +2,16 @@
  xarray-based-sources
 ######################
 
-More in general, you can specify any valid xarray.open_dataset_
-arguments as the source and anemoi-dataset will try to build a dataset
-from it. Examples of valid xarray.open_dataset_ arguments are: netCDF,
-zarr, opendap, etc.
+More generally, you can specify any valid xarray.open_dataset_ arguments
+as the source, and anemoi-dataset will try to build a dataset from it.
+Examples of valid xarray.open_dataset_ arguments include: netCDF, Zarr,
+OpenDAP, etc.
 
 .. literalinclude:: yaml/xarray-based.yaml
    :language: yaml
 
+See :ref:`create-cf-data` for more information.
+
 .. _cf conventions: http://cfconventions.org/
 
 .. _wildcards: https://en.wikipedia.org/wiki/Glob_(programming)

docs/datasets/building/sources/xarray-kerchunk.rst

@@ -12,6 +12,8 @@ install the relevant packages before running the code below.
 .. literalinclude:: xarray-kerchunk.py
    :language: python
 
+See :ref:`create-cf-data` for more information.
+
 .. _era5 dataset available on aws: https://registry.opendata.aws/ecmwf-era5/
 
 .. _kerchunk tutorial: https://fsspec.github.io/kerchunk/tutorial.html

docs/datasets/building/sources/xarray-zarr.rst

@@ -3,16 +3,18 @@
 #############
 
 Here is an example recipe that builds a dataset using one of the many
-regridded versions of ERA5 hosted by Google in Analysis-Ready, Cloud
-Optimized format. See `here
+regridded versions of ERA5 hosted by Google in an Analysis-Ready,
+Cloud-Optimised format. See `here
 <https://cloud.google.com/storage/docs/public-datasets/era5>`_ for more
 information.
 
 .. literalinclude:: yaml/xarray-zarr.yaml
    :language: yaml
 
-Note that unlike the ``mars`` examples, there is no need to include a
-``grid`` specification. Also, in order to subselect the vertical levels,
+Note that, unlike the ``mars`` examples, there is no need to include a
+``grid`` specification. Additionally, to subselect the vertical levels,
 it is necessary to use the :ref:`join <building-join>` operation to join
 separate lists containing 2D variables and 3D variables. If all vertical
-levels are desired, then it is OK to specify a single source.
+levels are desired, then it is acceptable to specify a single source.
+
+See :ref:`create-cf-data` for more information.

docs/datasets/building/sources/yaml/accumulations1.yaml

@@ -2,5 +2,5 @@ input:
   accumulations:
     accumulation_period: 6
     class: ea
-    param: [tp, cp, sf]
+    param: [ tp, cp, sf ]
     levtype: sfc

docs/datasets/building/sources/yaml/accumulations2.yaml

@@ -1,6 +1,6 @@
 input:
   accumulations:
-    accumulation_period: [6, 12]
+    accumulation_period: [ 6, 12 ]
     class: od
-    param: [tp, cp, sf]
+    param: [ tp, cp, sf ]
     levtype: sfc

docs/datasets/building/sources/yaml/anemoi-dataset.yaml

@@ -0,0 +1,11 @@
+input:
+  anemoi-dataset:
+    join:
+    - dataset: dataset1
+      select: [ z_500, t_500, u_500, v_500 ]
+      frequency: 6h
+    - dataset: dataset2
+      select: [ msl, 2t, 10u, 10v ]
+      frequency: 6h
+    start: 2000
+    end: 2001