Develop

.github/workflows/process_test.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"        
+          python-version: "3.10"
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
@@ -31,14 +31,26 @@ jobs:
         run: | 
           cd $GITHUB_WORKSPACE
           git clone --depth 1 https://oauth2:${{ env.GITLAB_TOKEN }}@geusgitlab.geus.dk/glaciology-and-climate/promice/aws-l0.git
-      - name: Run data processing
+      - name: Run L0 to L2 processing
         env:
-          TEST_STATION: KPC_U CEN2 JAR
+          TEST_STATION: KAN_U HUM
         shell: bash
         run: |
           mkdir $GITHUB_WORKSPACE/out/
+          mkdir $GITHUB_WORKSPACE/out/L0toL2/
+          mkdir $GITHUB_WORKSPACE/data_issues
           for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do
-            python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/
+            python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx --issues $GITHUB_WORKSPACE/data_issues -o $GITHUB_WORKSPACE/out/L0toL2/ --data_issues_path $GITHUB_WORKSPACE/data_issues
+          done
+      - name: Run L2 to L3 processing
+        env:
+          TEST_STATION: KAN_U HUM
+        shell: bash
+        run: |
+          mkdir $GITHUB_WORKSPACE/out/L2toL3/
+          for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do
+            echo ${i}_hour.nc
+            python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2tol3.py -c $GITHUB_WORKSPACE/aws-l0/metadata/station_configurations/ -i $GITHUB_WORKSPACE/out/L0toL2/${i}/${i}_hour.nc -o $GITHUB_WORKSPACE/out/L2toL3/ --data_issues_path $GITHUB_WORKSPACE/data_issues
           done
       - name: Upload test output
         uses: actions/upload-artifact@v3

.github/workflows/unit_test.yml

@@ -4,12 +4,12 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  test:
     name: unit_test
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python_version: ['3.8','3.9','3.10']
+        python_version: ['3.10', '3.11']
     steps:
       - name: Install Python
         uses: actions/setup-python@v4
@@ -19,6 +19,9 @@ jobs:
         uses: actions/checkout@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install eccodes
+        run : |
+          sudo apt-get install -y libeccodes-dev
       - name: Install dependencies
         shell: bash
         run: |
@@ -30,4 +33,4 @@ jobs:
       - name: Run unit tests
         shell: bash
         run: |
-          python3 -m unittest discover pypromice
+          python3 -m unittest discover tests

MANIFEST.in

@@ -1 +1,2 @@
 include src/pypromice/test/*
+include src/pypromice/resources/*

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pypromice",
-    version="1.3.6",
+    version="1.4.0",
     author="GEUS Glaciology and Climate",
     description="PROMICE/GC-Net data processing toolbox",
     long_description=long_description,
@@ -31,21 +31,25 @@
     packages=setuptools.find_packages(where="src"),
     python_requires=">=3.8",
     package_data={
-    	"pypromice.process": ["metadata.csv", "variables.csv"],
     	"pypromice.tx": ["payload_formats.csv", "payload_types.csv"],
         "pypromice.qc.percentiles": ["thresholds.csv"],
-        "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"],
+        "pypromice.postprocess": ["positions_seed.csv"],
     },
-    install_requires=['numpy>=1.23.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse', 'eccodes','scikit-learn>=1.1.0'],
+    install_requires=['numpy~=1.23', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'],
 #    extras_require={'postprocess': ['eccodes','scikit-learn>=1.1.0']},
     entry_points={
     'console_scripts': [
         'get_promice_data = pypromice.get.get_promice_data:get_promice_data',
         'get_l0tx = pypromice.tx.get_l0tx:get_l0tx',
-        'get_l3 = pypromice.process.get_l3:get_l3',
-        'join_l3 = pypromice.process.join_l3:join_l3',
+        'join_l2 = pypromice.process.join_l2:main',
+        'join_l3 = pypromice.process.join_l3:main',
+        'get_l2 = pypromice.process.get_l2:main',
+        'get_l2tol3 = pypromice.process.get_l2tol3:main',
+        'make_metadata_csv = pypromice.postprocess.make_metadata_csv:main',
         'get_watsontx = pypromice.tx.get_watsontx:get_watsontx',
         'get_bufr = pypromice.postprocess.get_bufr:main',
+        'create_bufr_files = pypromice.postprocess.create_bufr_files:main',
+        'bufr_to_csv = pypromice.postprocess.bufr_to_csv:main',
         'get_msg = pypromice.tx.get_msg:get_msg'
     ],
 },

src/pypromice/postprocess/bufr_to_csv.py

@@ -3,9 +3,14 @@
 
 from pypromice.postprocess.bufr_utilities import read_bufr_file
 
-if __name__ == "__main__":
+
+def main():
     parser = argparse.ArgumentParser("BUFR to CSV converter")
     parser.add_argument("path", type=Path)
     args = parser.parse_args()
 
     print(read_bufr_file(args.path).to_csv())
+
+
+if __name__ == "__main__":
+    main()

src/pypromice/postprocess/bufr_utilities.py

@@ -45,6 +45,7 @@ def round(value: float):
 
     return round
 
+
 # Enforce precision
 # Note the sensor accuracies listed here:
 # https://essd.copernicus.org/articles/13/3819/2021/#section8
@@ -64,28 +65,82 @@ class BUFRVariables:
     * heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformWSPD: Corresponds to "#7#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform" which is height if anemometer relative to ground or deck of marine platform.
 
     """
-    wmo_id: str
+
+    # Station type: "mobile" or "land"
+    # ===============================
+    # Fixed land station schema: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307080
+    # Mobile station schema: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307090
+
     station_type: str
+
+    # WMO station identifier
+    # Land stations: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/301090
+    # Mobile stations: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/301092
+    # ======================================================================================================
+    wmo_id: str
     timestamp: datetime.datetime
-    relativeHumidity: float = attrs.field(converter=round_converter(0))
-    airTemperature: float = attrs.field(converter=round_converter(1))
-    pressure: float = attrs.field(converter=round_converter(1))
-    windDirection: float = attrs.field(converter=round_converter(0))
-    windSpeed: float = attrs.field(converter=round_converter(1))
-    latitude: float = attrs.field(converter=round_converter(6))
-    longitude: float = attrs.field(converter=round_converter(6))
+
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/005001
+    # Scale: 5, unit: degrees
+    # TODO: Test if eccodes does the rounding as well. The rounding is was 6 which is larger that the scale.
+    latitude: float = attrs.field(converter=round_converter(5))
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/006001
+    # Scale: 5, unit: degrees
+    longitude: float = attrs.field(converter=round_converter(5))
+
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007030
+    # Scale: 1, unit: m
     heightOfStationGroundAboveMeanSeaLevel: float = attrs.field(
-        converter=round_converter(2)
+        converter=round_converter(1)
     )
-    #
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007031
+    # Scale: 1, unit: m
     heightOfBarometerAboveMeanSeaLevel: float = attrs.field(
-        converter=round_converter(2),
+        converter=round_converter(1),
     )
+
+    # Pressure information
+    # ====================
+    # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302031
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/010004
+    # Scale: -1, unit: Pa
+    nonCoordinatePressure: float = attrs.field(converter=round_converter(-1))
+    # There are two other pressure variables in the template: 007004 - pressure and 010062 24-hour pressure change
+
+    # Basic synoptic "instantaneous" data
+    # ===================================
+    # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302035
+    # This section only include the temperature and humidity data (302032).
+    # Precipitation and cloud data are currently ignored.
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007032
+    # Scale: 2, unit: m
+    # This is the first appearance of this variable id.
     heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformTempRH: float = attrs.field(
-        converter=round_converter(4),
+        converter=round_converter(2),
     )
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012101
+    # Scale: 2, unit: K
+    airTemperature: float = attrs.field(converter=round_converter(2))
+    # There is also a Dewpoint temperature in this template: 012103 which is currently unused.
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012103
+    # Scale: 0, unit: %
+    relativeHumidity: float = attrs.field(converter=round_converter(0))
+
+    # Basic synoptic "period" data
+    # ============================
+    # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302043
+    # Wind data: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302042
+    # Wind direction: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/011001
+    # Scale: 0, unit: degrees
+    windDirection: float = attrs.field(converter=round_converter(0))
+    # Wind speed: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/011002
+    # Scale: 1, unit: m/s
+    windSpeed: float = attrs.field(converter=round_converter(1))
+    # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007032
+    # Scale: 2, unit: m
+    # This is the 7th appearance of this variable id.
     heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformWSPD: float = attrs.field(
-        converter=round_converter(4)
+        converter=round_converter(2)
     )
 
     def as_series(self) -> pd.Series:
@@ -129,6 +184,7 @@ def __eq__(self, other: "BUFRVariables"):
 
 BUFR_TEMPLATES = {
     "mobile": {
+        # Template definition: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307090
         "unexpandedDescriptors": (307090),  # message template, "synopMobil"
         "edition": 4,  # latest edition
         "masterTableNumber": 0,
@@ -144,6 +200,7 @@ def __eq__(self, other: "BUFRVariables"):
         "compressedData": 0,
     },
     "land": {
+        # Template definition: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307080
         "unexpandedDescriptors": (307080),  # message template, "synopLand"
         "edition": 4,  # latest edition
         "masterTableNumber": 0,
@@ -246,6 +303,11 @@ def set_station(ibufr, station_type: str, wmo_id: str):
     elif station_type == "land":
         # StationNumber for land stations are integeres
         wmo_id_int = int(wmo_id)
+        if wmo_id_int >= 1024:
+            raise ValueError(
+                f"Invalid WMO ID {wmo_id}. Land station number must be less than 1024."
+                "See https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/001002"
+            )
         station_config = dict(stationNumber=wmo_id_int)
     else:
         raise Exception(f"Unsupported station station type {station_type}")
@@ -280,7 +342,7 @@ def set_AWS_variables(
 
     set_bufr_value(ibufr, "relativeHumidity", variables.relativeHumidity)
     set_bufr_value(ibufr, "airTemperature", variables.airTemperature)
-    set_bufr_value(ibufr, "pressure", variables.pressure)
+    set_bufr_value(ibufr, "nonCoordinatePressure", variables.nonCoordinatePressure)
     set_bufr_value(ibufr, "windDirection", variables.windDirection)
     set_bufr_value(ibufr, "windSpeed", variables.windSpeed)
 
@@ -372,7 +434,7 @@ def get_bufr_value(msgid: int, key: str) -> float:
         raise ValueError(f"Unsupported BUFR value type {type(value)} for key {key}")
 
 
-def read_bufr_message(fp: BinaryIO) -> Optional[BUFRVariables]:
+def read_bufr_message(fp: BinaryIO, backwards_compatible: bool = False) -> Optional[BUFRVariables]:
     """
     Read and parse BUFR message from binary IO stream.
 
@@ -383,6 +445,8 @@ def read_bufr_message(fp: BinaryIO) -> Optional[BUFRVariables]:
     ----------
     fp
         Readable binary io stream
+    backwards_compatible
+        Use legacy pressure if nonCoordinatePressure is nan
 
     Returns
     -------
@@ -435,11 +499,19 @@ def read_bufr_message(fp: BinaryIO) -> Optional[BUFRVariables]:
             f"Unknown BUFR template unexpandedDescriptors: {unexpanded_descriptors}"
         )
 
+    nonCoordinatePressure = get_bufr_value(ibufr, "nonCoordinatePressure")
+    if math.isnan(nonCoordinatePressure) and backwards_compatible:
+        nonCoordinatePressure = get_bufr_value(ibufr, "pressure")
+        if not math.isnan(nonCoordinatePressure):
+            logger.warning(
+                f"nonCoordinatePressure is nan, using legacy pressure instead"
+            )
+
     variables = BUFRVariables(
         timestamp=timestamp,
         relativeHumidity=get_bufr_value(ibufr, "relativeHumidity"),
         airTemperature=get_bufr_value(ibufr, "airTemperature"),
-        pressure=get_bufr_value(ibufr, "pressure"),
+        nonCoordinatePressure=nonCoordinatePressure,
         windDirection=get_bufr_value(ibufr, "windDirection"),
         windSpeed=get_bufr_value(ibufr, "windSpeed"),
         latitude=get_bufr_value(ibufr, "latitude"),
@@ -485,5 +557,6 @@ def read_bufr_file(path: PathLike) -> pd.DataFrame:
             message_vars = read_bufr_message(fp)
             if message_vars is None:
                 break
-            lines.append(message_vars)
-    return pd.DataFrame(lines).rename_axis("message_index")
+            lines.append(message_vars.as_series())
+    data_frame = pd.DataFrame(lines).set_index("wmo_id")
+    return data_frame