Revert "pypromicev1.4.0 (#291)"

This reverts commit 735a8ba.

Author: BaptisteVandecrux

.github/workflows/process_test.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.8"        
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
@@ -31,26 +31,14 @@ jobs:
         run: | 
           cd $GITHUB_WORKSPACE
           git clone --depth 1 https://oauth2:${{ env.GITLAB_TOKEN }}@geusgitlab.geus.dk/glaciology-and-climate/promice/aws-l0.git
-      - name: Run L0 to L2 processing
+      - name: Run data processing
         env:
-          TEST_STATION: KAN_U HUM
+          TEST_STATION: KPC_U CEN2 JAR
         shell: bash
         run: |
           mkdir $GITHUB_WORKSPACE/out/
-          mkdir $GITHUB_WORKSPACE/out/L0toL2/
-          mkdir $GITHUB_WORKSPACE/data_issues
           for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do
-            python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx --issues $GITHUB_WORKSPACE/data_issues -o $GITHUB_WORKSPACE/out/L0toL2/ --data_issues_path $GITHUB_WORKSPACE/data_issues
-          done
-      - name: Run L2 to L3 processing
-        env:
-          TEST_STATION: KAN_U HUM
-        shell: bash
-        run: |
-          mkdir $GITHUB_WORKSPACE/out/L2toL3/
-          for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do
-            echo ${i}_hour.nc
-            python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2tol3.py -c $GITHUB_WORKSPACE/aws-l0/metadata/station_configurations/ -i $GITHUB_WORKSPACE/out/L0toL2/${i}/${i}_hour.nc -o $GITHUB_WORKSPACE/out/L2toL3/ --data_issues_path $GITHUB_WORKSPACE/data_issues
+            python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/
           done
       - name: Upload test output
         uses: actions/upload-artifact@v3

.github/workflows/unit_test.yml

@@ -4,12 +4,12 @@ on:
   workflow_dispatch:
 
 jobs:
-  test:
+  build:
     name: unit_test
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python_version: ['3.10', '3.11']
+        python_version: ['3.8','3.9','3.10']
     steps:
       - name: Install Python
         uses: actions/setup-python@v4
@@ -19,9 +19,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install eccodes
-        run : |
-          sudo apt-get install -y libeccodes-dev
       - name: Install dependencies
         shell: bash
         run: |
@@ -33,4 +30,4 @@ jobs:
       - name: Run unit tests
         shell: bash
         run: |
-          python3 -m unittest discover tests
+          python3 -m unittest discover pypromice

MANIFEST.in

@@ -1,2 +1 @@
 include src/pypromice/test/*
-include src/pypromice/resources/*

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pypromice",
-    version="1.4.0",
+    version="1.3.6",
     author="GEUS Glaciology and Climate",
     description="PROMICE/GC-Net data processing toolbox",
     long_description=long_description,
@@ -31,25 +31,21 @@
     packages=setuptools.find_packages(where="src"),
     python_requires=">=3.8",
     package_data={
+    	"pypromice.process": ["metadata.csv", "variables.csv"],
     	"pypromice.tx": ["payload_formats.csv", "payload_types.csv"],
         "pypromice.qc.percentiles": ["thresholds.csv"],
-        "pypromice.postprocess": ["positions_seed.csv"],
+        "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"],
     },
-    install_requires=['numpy~=1.23', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'],
+    install_requires=['numpy>=1.23.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse', 'eccodes','scikit-learn>=1.1.0'],
 #    extras_require={'postprocess': ['eccodes','scikit-learn>=1.1.0']},
     entry_points={
     'console_scripts': [
         'get_promice_data = pypromice.get.get_promice_data:get_promice_data',
         'get_l0tx = pypromice.tx.get_l0tx:get_l0tx',
-        'join_l2 = pypromice.process.join_l2:main',
-        'join_l3 = pypromice.process.join_l3:main',
-        'get_l2 = pypromice.process.get_l2:main',
-        'get_l2tol3 = pypromice.process.get_l2tol3:main',
-        'make_metadata_csv = pypromice.postprocess.make_metadata_csv:main',
+        'get_l3 = pypromice.process.get_l3:get_l3',
+        'join_l3 = pypromice.process.join_l3:join_l3',
         'get_watsontx = pypromice.tx.get_watsontx:get_watsontx',
         'get_bufr = pypromice.postprocess.get_bufr:main',
-        'create_bufr_files = pypromice.postprocess.create_bufr_files:main',
-        'bufr_to_csv = pypromice.postprocess.bufr_to_csv:main',
         'get_msg = pypromice.tx.get_msg:get_msg'
     ],
 },

src/pypromice/postprocess/bufr_to_csv.py

@@ -3,14 +3,9 @@
 
 from pypromice.postprocess.bufr_utilities import read_bufr_file
 
-
-def main():
+if __name__ == "__main__":
     parser = argparse.ArgumentParser("BUFR to CSV converter")
     parser.add_argument("path", type=Path)
     args = parser.parse_args()
 
     print(read_bufr_file(args.path).to_csv())
-
-
-if __name__ == "__main__":
-    main()

src/pypromice/postprocess/bufr_utilities.py

@@ -45,7 +45,6 @@ def round(value: float):
 
     return round
 
-
 # Enforce precision
 # Note the sensor accuracies listed here:
 # https://essd.copernicus.org/articles/13/3819/2021/#section8
@@ -65,82 +64,28 @@ class BUFRVariables:
     * heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformWSPD: Corresponds to "#7#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform" which is height if anemometer relative to ground or deck of marine platform.
 
     """
-
-    # Station type: "mobile" or "land"
-    # ===============================
-    # Fixed land station schema: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307080
-    # Mobile station schema: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307090
-
-    station_type: str
-
-    # WMO station identifier
-    # Land stations: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/301090
-    # Mobile stations: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/301092
-    # ======================================================================================================
     wmo_id: str
+    station_type: str
     timestamp: datetime.datetime
-
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/005001
-    # Scale: 5, unit: degrees
-    # TODO: Test if eccodes does the rounding as well. The rounding is was 6 which is larger that the scale.
-    latitude: float = attrs.field(converter=round_converter(5))
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/006001
-    # Scale: 5, unit: degrees
-    longitude: float = attrs.field(converter=round_converter(5))
-
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007030
-    # Scale: 1, unit: m
+    relativeHumidity: float = attrs.field(converter=round_converter(0))
+    airTemperature: float = attrs.field(converter=round_converter(1))
+    pressure: float = attrs.field(converter=round_converter(1))
+    windDirection: float = attrs.field(converter=round_converter(0))
+    windSpeed: float = attrs.field(converter=round_converter(1))
+    latitude: float = attrs.field(converter=round_converter(6))
+    longitude: float = attrs.field(converter=round_converter(6))
     heightOfStationGroundAboveMeanSeaLevel: float = attrs.field(
-        converter=round_converter(1)
+        converter=round_converter(2)
     )
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007031
-    # Scale: 1, unit: m
+    #
     heightOfBarometerAboveMeanSeaLevel: float = attrs.field(
-        converter=round_converter(1),
+        converter=round_converter(2),
     )
-
-    # Pressure information
-    # ====================
-    # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302031
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/010004
-    # Scale: -1, unit: Pa
-    nonCoordinatePressure: float = attrs.field(converter=round_converter(-1))
-    # There are two other pressure variables in the template: 007004 - pressure and 010062 24-hour pressure change
-
-    # Basic synoptic "instantaneous" data
-    # ===================================
-    # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302035
-    # This section only include the temperature and humidity data (302032).
-    # Precipitation and cloud data are currently ignored.
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007032
-    # Scale: 2, unit: m
-    # This is the first appearance of this variable id.
     heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformTempRH: float = attrs.field(
-        converter=round_converter(2),
+        converter=round_converter(4),
     )
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012101
-    # Scale: 2, unit: K
-    airTemperature: float = attrs.field(converter=round_converter(2))
-    # There is also a Dewpoint temperature in this template: 012103 which is currently unused.
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012103
-    # Scale: 0, unit: %
-    relativeHumidity: float = attrs.field(converter=round_converter(0))
-
-    # Basic synoptic "period" data
-    # ============================
-    # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302043
-    # Wind data: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302042
-    # Wind direction: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/011001
-    # Scale: 0, unit: degrees
-    windDirection: float = attrs.field(converter=round_converter(0))
-    # Wind speed: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/011002
-    # Scale: 1, unit: m/s
-    windSpeed: float = attrs.field(converter=round_converter(1))
-    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007032
-    # Scale: 2, unit: m
-    # This is the 7th appearance of this variable id.
     heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformWSPD: float = attrs.field(
-        converter=round_converter(2)
+        converter=round_converter(4)
     )
 
     def as_series(self) -> pd.Series:
@@ -184,7 +129,6 @@ def __eq__(self, other: "BUFRVariables"):
 
 BUFR_TEMPLATES = {
     "mobile": {
-        # Template definition: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307090
         "unexpandedDescriptors": (307090),  # message template, "synopMobil"
         "edition": 4,  # latest edition
         "masterTableNumber": 0,
@@ -200,7 +144,6 @@ def __eq__(self, other: "BUFRVariables"):
         "compressedData": 0,
     },
     "land": {
-        # Template definition: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307080
         "unexpandedDescriptors": (307080),  # message template, "synopLand"
         "edition": 4,  # latest edition
         "masterTableNumber": 0,
@@ -303,11 +246,6 @@ def set_station(ibufr, station_type: str, wmo_id: str):
     elif station_type == "land":
         # StationNumber for land stations are integeres
         wmo_id_int = int(wmo_id)
-        if wmo_id_int >= 1024:
-            raise ValueError(
-                f"Invalid WMO ID {wmo_id}. Land station number must be less than 1024."
-                "See https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/001002"
-            )
         station_config = dict(stationNumber=wmo_id_int)
     else:
         raise Exception(f"Unsupported station station type {station_type}")
@@ -342,7 +280,7 @@ def set_AWS_variables(
 
     set_bufr_value(ibufr, "relativeHumidity", variables.relativeHumidity)
     set_bufr_value(ibufr, "airTemperature", variables.airTemperature)
-    set_bufr_value(ibufr, "nonCoordinatePressure", variables.nonCoordinatePressure)
+    set_bufr_value(ibufr, "pressure", variables.pressure)
     set_bufr_value(ibufr, "windDirection", variables.windDirection)
     set_bufr_value(ibufr, "windSpeed", variables.windSpeed)
 
@@ -434,7 +372,7 @@ def get_bufr_value(msgid: int, key: str) -> float:
         raise ValueError(f"Unsupported BUFR value type {type(value)} for key {key}")
 
 
-def read_bufr_message(fp: BinaryIO, backwards_compatible: bool = False) -> Optional[BUFRVariables]:
+def read_bufr_message(fp: BinaryIO) -> Optional[BUFRVariables]:
     """
     Read and parse BUFR message from binary IO stream.
 
@@ -445,8 +383,6 @@ def read_bufr_message(fp: BinaryIO, backwards_compatible: bool = False) -> Optio
     ----------
     fp
         Readable binary io stream
-    backwards_compatible
-        Use legacy pressure if nonCoordinatePressure is nan
 
     Returns
     -------
@@ -499,19 +435,11 @@ def read_bufr_message(fp: BinaryIO, backwards_compatible: bool = False) -> Optio
             f"Unknown BUFR template unexpandedDescriptors: {unexpanded_descriptors}"
         )
 
-    nonCoordinatePressure = get_bufr_value(ibufr, "nonCoordinatePressure")
-    if math.isnan(nonCoordinatePressure) and backwards_compatible:
-        nonCoordinatePressure = get_bufr_value(ibufr, "pressure")
-        if not math.isnan(nonCoordinatePressure):
-            logger.warning(
-                f"nonCoordinatePressure is nan, using legacy pressure instead"
-            )
-
     variables = BUFRVariables(
         timestamp=timestamp,
         relativeHumidity=get_bufr_value(ibufr, "relativeHumidity"),
         airTemperature=get_bufr_value(ibufr, "airTemperature"),
-        nonCoordinatePressure=nonCoordinatePressure,
+        pressure=get_bufr_value(ibufr, "pressure"),
         windDirection=get_bufr_value(ibufr, "windDirection"),
         windSpeed=get_bufr_value(ibufr, "windSpeed"),
         latitude=get_bufr_value(ibufr, "latitude"),
@@ -557,6 +485,5 @@ def read_bufr_file(path: PathLike) -> pd.DataFrame:
             message_vars = read_bufr_message(fp)
             if message_vars is None:
                 break
-            lines.append(message_vars.as_series())
-    data_frame = pd.DataFrame(lines).set_index("wmo_id")
-    return data_frame
+            lines.append(message_vars)
+    return pd.DataFrame(lines).rename_axis("message_index")