Adding support to download and reshuffle SMOS L2

.github/workflows/build.yml

@@ -14,44 +14,67 @@ on:
 
 jobs:
   build:
-    name: Build py${{ matrix.python-version }} @ ${{ matrix.os }} 🐍
+    name: py${{ matrix.python-version }} @ ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9']
-        os: ["ubuntu-latest", "windows-latest"]
-
+        include:
+          - os: "ubuntu-latest"
+            python-version: '3.8'  # first supported
+#          - os: "windows-latest"
+#            python-version: '3.8'  # first supported
+          - os: "ubuntu-latest"
+            python-version: '3.12'  # latest supported
+          - os: "windows-latest"
+            python-version: '3.12'  # latest supported
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 0
-      - uses: conda-incubator/[email protected]
+          lfs: true
+      - name: Checkout LFS objects
+        run: git lfs checkout
+#      - uses: mamba-org/setup-micromamba@v2
+#        with:
+#          micromamba-version: "latest"
+#          environment-file: environment.yml
+#          create-args: python=${{ matrix.python-version }}
+#          post-cleanup: "all"
+#          init-shell: >-
+#            bash
+#            powershell
+      - uses: conda-incubator/setup-miniconda@v3
         with:
-          miniconda-version: "latest"
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
           environment-file: environment.yml
+          channel-priority: flexible
           activate-environment: smos
           auto-activate-base: false
-      - name: Print environment infos
+      - name: Print Infos
         shell: bash -l {0}
         run: |
+          ls -R tests/smos-test-data
           conda info -a
-          conda list
-          pip list
           which pip
           which python
+          conda list
       - name: Export Environment
         shell: bash -l {0}
         run: |
-          mkdir -p .artifacts
+          mkdir -p artifacts
           filename=env_py${{ matrix.python-version }}_${{ matrix.os }}.yml
-          conda env export --no-builds | grep -v "prefix" > .artifacts/$filename
-      - name: Install package and test
+          conda env export --no-builds | grep -v "prefix" > artifacts/$filename
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: Artifacts-py${{ matrix.python-version }}-${{ matrix.os }}
+          path: artifacts/*
+      - name: Install base package and run tests
         shell: bash -l {0}
         run: |
-          pip install .
+          pip install -e .[testing]
           pytest
       - name: Upload Coverage
         shell: bash -l {0}
@@ -69,17 +92,12 @@ jobs:
           then
             # build whls on windows
             pip install wheel
-            python setup.py bdist_wheel --dist-dir .artifacts/dist
+            python setup.py bdist_wheel --dist-dir artifacts/dist
           else
             # build dist on linux
-            python setup.py sdist --dist-dir .artifacts/dist
+            python setup.py sdist --dist-dir artifacts/dist
           fi
-          ls .artifacts/dist
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: Artifacts
-          path: .artifacts/*
+          ls artifacts/dist
   coveralls:
     name: Submit Coveralls 👚
     needs: build
@@ -102,7 +120,11 @@ jobs:
           echo "GITHUB_REF = $GITHUB_REF"
           echo "GITHUB_REPOSITORY = $GITHUB_REPOSITORY"
       - name: Download Artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
+        with:
+          path: Artifacts
+          pattern: Artifacts-*
+          merge-multiple: true
       - name: Display downloaded files
         run: ls -aR
       - name: Upload to PyPI
@@ -113,4 +135,4 @@ jobs:
           verify_metadata: true
           packages_dir: Artifacts/dist/
           user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }} # this needs to be uploaded to github actions secrets
+          password: ${{ secrets.PYPI_API_TOKEN }} # this needs to be uploaded to github actions secrets

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "tests/smos-test-data"]
 	path = tests/smos-test-data
-	url = https://www.geo.tuwien.ac.at/downloads/gittd/smos-test-data.git/
+	url = https://git.geo.tuwien.ac.at/public_projects/rs/rs_testdata/smos-test-data.git

docker/Dockerfile

@@ -0,0 +1,35 @@
+FROM mambaorg/micromamba:1.3.1-alpine
+MAINTAINER Wolfgang Preimesberger <[email protected]>
+
+USER root
+
+ARG GIT_BRANCH_TAG_COMMIT
+ARG GIT_URL
+
+RUN apk update && \
+    apk upgrade && \
+    apk add git && \
+    apk add build-base && \
+    apk add g++ && \
+    apk add bsd-compat-headers && \
+    apk add tiff
+
+RUN apk add lftp
+
+# Check out the SMECV code at the chose tag using your credentials
+RUN git clone --recursive $GIT_URL && \
+    cd smos && \
+    git checkout $GIT_BRANCH_TAG_COMMIT
+
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+
+RUN micromamba install -y -n base -c conda-forge python=3.12
+RUN cd smos &&  \
+    pip install .
+
+RUN micromamba clean --all --yes
+
+# Clean up the src code, as it is installed now
+RUN rm -rf smos
+
+ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"]

docker/build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Before running this, make sure that
+# -1) The machine you install the docker container from can download and
+# install all packages listed in the conda_ci_env.yml file.
+
+if [ `id -u` -ne 0 ]; then
+  echo "ERROR: Please run the script $(basename "$0") with sudo!"
+  exit 1
+fi
+
+# Tag/branch/commit in https://git.eodc.eu/cci-sm-work/cci_sm_ecvps_py_src
+GIT_BRANCH_TAG_COMMIT=$1
+
+GIT_URL="https://github.com/TUW-GEO/smos.git"
+
+echo "Calling Dockerfile at $this_dir/docker/Dockerfile"
+echo "Checking out source tag $GIT_BRANCH_TAG_COMMIT"
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+echo "This DIR: $SCRIPTPATH"
+
+sudo docker build -t smos:$GIT_BRANCH_TAG_COMMIT \
+    --build-arg GIT_BRANCH_TAG_COMMIT=$GIT_BRANCH_TAG_COMMIT \
+    --build-arg GIT_URL=$GIT_URL \
+    .

environment.yml

@@ -1,24 +1,25 @@
 name: smos
 channels:
+  - defaults
   - conda-forge
 dependencies:
-- numpy>=1.13.0
+- numpy>=1.13.0,<2
 - pandas
-- netcdf4
+- netCDF4!=1.6.2
+- hdf5
 - scipy
+- dask[distributed]
+- xarray
 - pyresample
-- ipykernel
 - pip
 - pip:
     - pygeobase
     - pygeogrids
     - pynetcf
-    - repurpose
+    - pyproj
+    - git+https://github.com/TUW-GEO/repurpose@master
     - trollsift
     - ease_grid
     - more_itertools
-    - sphinx==4.0.3  # https://github.com/spatialaudio/nbsphinx/issues/584, when this is resolved this line can be deleted to use the latest version.
-    - nbsphinx
-    - sphinx_rtd_theme
-    - pytest
-    - pytest-cov
+    - cf-xarray==0.8.4
+    - git+https://github.com/awst-austria/qa4sm-preprocessing@smos-package

setup.cfg

@@ -31,18 +31,23 @@ package_dir =
 # Add here dependencies of your project (semicolon/line-separated), e.g.
 install_requires =
     importlib-metadata; python_version<"3.8"
-    numpy>=1.13.0
+    numpy>=1.13.0,<2
     scipy
     pandas
-    netcdf4
+    dask[distributed]
+    xarray
+    netCDF4
     repurpose
     pyresample
     pygeogrids>=0.3.2
-    pynetcf
+    pynetcf>=0.5.1
     pygeobase
     ease_grid
     trollsift
+    h5py
     more_itertools
+    cf-xarray==0.8.4
+    # qa4sm_preprocessing>=0.2
 # The usage of test_requires is discouraged, see `Dependency Management` docs
 #tests_require = pytest; pytest-cov; coverage
 # Require a specific Python version, e.g. Python 2.7 or >= 3.4
@@ -73,7 +78,7 @@ testing =
 # pyscaffold.cli =
 #     awesome = pyscaffoldext.awesome.extension:AwesomeExtension
 console_scripts =
-    smos_repurpose = smos.smos_ic.reshuffle:run
+    smos_l2 = smos.smos_l2.cli:smos_l2
 
 [test]
 # py.test options when running `python setup.py test`

src/smos/misc.py

@@ -0,0 +1,86 @@
+import pandas as pd
+import os
+from datetime import date
+import typing as t
+
+def _get_first_and_last_file(path: str):
+    # Get list of all years (folders) in the path
+    years = sorted([folder for folder in os.listdir(path) if folder.isdigit()], key=int)
+
+    if not years:
+        return None, None
+
+    # Get the first year and last year
+    first_year = years[0]
+    last_year = years[-1]
+
+    # Handle the first year
+    first_year_path = os.path.join(path, first_year)
+    first_months = sorted([folder for folder in os.listdir(first_year_path) if folder.isdigit()], key=int)
+
+    if first_months:
+        first_month = first_months[0]
+        first_month_path = os.path.join(first_year_path, first_month)
+        first_days = sorted([folder for folder in os.listdir(first_month_path) if folder.isdigit()], key=int)
+
+        if first_days:
+            first_day = first_days[0]
+            first_day_path = os.path.join(first_month_path, first_day)
+            first_files = sorted(os.listdir(first_day_path))
+            first_file = first_files[0] if first_files else None
+        else:
+            first_day_path = first_month_path
+            first_files = sorted(os.listdir(first_day_path))
+            first_file = first_files[0] if first_files else None
+    else:
+        first_month_path = first_year_path
+        first_files = sorted(os.listdir(first_month_path))
+        first_file = first_files[0] if first_files else None
+
+    # Handle the last year
+    last_year_path = os.path.join(path, last_year)
+    last_months = sorted([folder for folder in os.listdir(last_year_path) if folder.isdigit()], key=int, reverse=True)
+
+    if last_months:
+        last_month = last_months[0]
+        last_month_path = os.path.join(last_year_path, last_month)
+        last_days = sorted([folder for folder in os.listdir(last_month_path) if folder.isdigit()], key=int, reverse=True)
+
+        if last_days:
+            last_day = last_days[0]
+            last_day_path = os.path.join(last_month_path, last_day)
+            last_files = sorted(os.listdir(last_day_path))
+        else:
+            last_day_path = last_month_path
+            last_files = sorted(os.listdir(last_day_path))
+    else:
+        last_month_path = last_year_path
+        last_files = sorted(os.listdir(last_month_path))
+
+    return first_file, last_files[-1] if last_files else None
+
+
+def _get_date(f: str) -> t.Union[date, None]:
+    for e in f.split('_'):
+        try:
+            dt = pd.to_datetime(e).to_pydatetime().date()
+            return dt
+        except Exception:
+            continue
+    return None
+
+
+def get_first_last_day_images(img_path: str) -> (date, date):
+
+    f, l = _get_first_and_last_file(img_path)
+    first_date = _get_date(f)
+    last_date = _get_date(l)
+
+    return first_date, last_date
+
+
+if __name__ == '__main__':
+
+    f, l = get_first_last_day_images("/home/wpreimes/shares/climers/Projects/FRM4SM/07_data/SMOSL2/MIR_SMUDP2_nc")
+    print(f, l)
+

src/smos/smos_ic/download.py

@@ -0,0 +1 @@
+# https://data.catds.fr/cecsm/Land_products/L3_SMOS_IC_Soil_Moisture/