Skip to content

Commit f2ee9da

Browse files
Revert "pypromicev1.4.0 (#291)"
This reverts commit 735a8ba.
1 parent 735a8ba commit f2ee9da

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+35916
-38615
lines changed

.github/workflows/process_test.yml

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
- name: Install Python
1212
uses: actions/setup-python@v4
1313
with:
14-
python-version: "3.10"
14+
python-version: "3.8"
1515
- name: Checkout repo
1616
uses: actions/checkout@v3
1717
with:
@@ -31,26 +31,14 @@ jobs:
3131
run: |
3232
cd $GITHUB_WORKSPACE
3333
git clone --depth 1 https://oauth2:${{ env.GITLAB_TOKEN }}@geusgitlab.geus.dk/glaciology-and-climate/promice/aws-l0.git
34-
- name: Run L0 to L2 processing
34+
- name: Run data processing
3535
env:
36-
TEST_STATION: KAN_U HUM
36+
TEST_STATION: KPC_U CEN2 JAR
3737
shell: bash
3838
run: |
3939
mkdir $GITHUB_WORKSPACE/out/
40-
mkdir $GITHUB_WORKSPACE/out/L0toL2/
41-
mkdir $GITHUB_WORKSPACE/data_issues
4240
for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do
43-
python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx --issues $GITHUB_WORKSPACE/data_issues -o $GITHUB_WORKSPACE/out/L0toL2/ --data_issues_path $GITHUB_WORKSPACE/data_issues
44-
done
45-
- name: Run L2 to L3 processing
46-
env:
47-
TEST_STATION: KAN_U HUM
48-
shell: bash
49-
run: |
50-
mkdir $GITHUB_WORKSPACE/out/L2toL3/
51-
for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do
52-
echo ${i}_hour.nc
53-
python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2tol3.py -c $GITHUB_WORKSPACE/aws-l0/metadata/station_configurations/ -i $GITHUB_WORKSPACE/out/L0toL2/${i}/${i}_hour.nc -o $GITHUB_WORKSPACE/out/L2toL3/ --data_issues_path $GITHUB_WORKSPACE/data_issues
41+
python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/
5442
done
5543
- name: Upload test output
5644
uses: actions/upload-artifact@v3

.github/workflows/unit_test.yml

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@ on:
44
workflow_dispatch:
55

66
jobs:
7-
test:
7+
build:
88
name: unit_test
99
runs-on: ubuntu-latest
1010
strategy:
1111
matrix:
12-
python_version: ['3.10', '3.11']
12+
python_version: ['3.8','3.9','3.10']
1313
steps:
1414
- name: Install Python
1515
uses: actions/setup-python@v4
@@ -19,9 +19,6 @@ jobs:
1919
uses: actions/checkout@v3
2020
with:
2121
token: ${{ secrets.GITHUB_TOKEN }}
22-
- name: Install eccodes
23-
run : |
24-
sudo apt-get install -y libeccodes-dev
2522
- name: Install dependencies
2623
shell: bash
2724
run: |
@@ -33,4 +30,4 @@ jobs:
3330
- name: Run unit tests
3431
shell: bash
3532
run: |
36-
python3 -m unittest discover tests
33+
python3 -m unittest discover pypromice

MANIFEST.in

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
11
include src/pypromice/test/*
2-
include src/pypromice/resources/*

setup.py

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
setuptools.setup(
77
name="pypromice",
8-
version="1.4.0",
8+
version="1.3.6",
99
author="GEUS Glaciology and Climate",
1010
description="PROMICE/GC-Net data processing toolbox",
1111
long_description=long_description,
@@ -31,25 +31,21 @@
3131
packages=setuptools.find_packages(where="src"),
3232
python_requires=">=3.8",
3333
package_data={
34+
"pypromice.process": ["metadata.csv", "variables.csv"],
3435
"pypromice.tx": ["payload_formats.csv", "payload_types.csv"],
3536
"pypromice.qc.percentiles": ["thresholds.csv"],
36-
"pypromice.postprocess": ["positions_seed.csv"],
37+
"pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"],
3738
},
38-
install_requires=['numpy~=1.23', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'],
39+
install_requires=['numpy>=1.23.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse', 'eccodes','scikit-learn>=1.1.0'],
3940
# extras_require={'postprocess': ['eccodes','scikit-learn>=1.1.0']},
4041
entry_points={
4142
'console_scripts': [
4243
'get_promice_data = pypromice.get.get_promice_data:get_promice_data',
4344
'get_l0tx = pypromice.tx.get_l0tx:get_l0tx',
44-
'join_l2 = pypromice.process.join_l2:main',
45-
'join_l3 = pypromice.process.join_l3:main',
46-
'get_l2 = pypromice.process.get_l2:main',
47-
'get_l2tol3 = pypromice.process.get_l2tol3:main',
48-
'make_metadata_csv = pypromice.postprocess.make_metadata_csv:main',
45+
'get_l3 = pypromice.process.get_l3:get_l3',
46+
'join_l3 = pypromice.process.join_l3:join_l3',
4947
'get_watsontx = pypromice.tx.get_watsontx:get_watsontx',
5048
'get_bufr = pypromice.postprocess.get_bufr:main',
51-
'create_bufr_files = pypromice.postprocess.create_bufr_files:main',
52-
'bufr_to_csv = pypromice.postprocess.bufr_to_csv:main',
5349
'get_msg = pypromice.tx.get_msg:get_msg'
5450
],
5551
},

src/pypromice/postprocess/bufr_to_csv.py

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,9 @@
33

44
from pypromice.postprocess.bufr_utilities import read_bufr_file
55

6-
7-
def main():
6+
if __name__ == "__main__":
87
parser = argparse.ArgumentParser("BUFR to CSV converter")
98
parser.add_argument("path", type=Path)
109
args = parser.parse_args()
1110

1211
print(read_bufr_file(args.path).to_csv())
13-
14-
15-
if __name__ == "__main__":
16-
main()

src/pypromice/postprocess/bufr_utilities.py

Lines changed: 18 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ def round(value: float):
4545

4646
return round
4747

48-
4948
# Enforce precision
5049
# Note the sensor accuracies listed here:
5150
# https://essd.copernicus.org/articles/13/3819/2021/#section8
@@ -65,82 +64,28 @@ class BUFRVariables:
6564
* heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformWSPD: Corresponds to "#7#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform" which is height if anemometer relative to ground or deck of marine platform.
6665
6766
"""
68-
69-
# Station type: "mobile" or "land"
70-
# ===============================
71-
# Fixed land station schema: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307080
72-
# Mobile station schema: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307090
73-
74-
station_type: str
75-
76-
# WMO station identifier
77-
# Land stations: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/301090
78-
# Mobile stations: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/301092
79-
# ======================================================================================================
8067
wmo_id: str
68+
station_type: str
8169
timestamp: datetime.datetime
82-
83-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/005001
84-
# Scale: 5, unit: degrees
85-
# TODO: Test if eccodes does the rounding as well. The rounding is was 6 which is larger that the scale.
86-
latitude: float = attrs.field(converter=round_converter(5))
87-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/006001
88-
# Scale: 5, unit: degrees
89-
longitude: float = attrs.field(converter=round_converter(5))
90-
91-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007030
92-
# Scale: 1, unit: m
70+
relativeHumidity: float = attrs.field(converter=round_converter(0))
71+
airTemperature: float = attrs.field(converter=round_converter(1))
72+
pressure: float = attrs.field(converter=round_converter(1))
73+
windDirection: float = attrs.field(converter=round_converter(0))
74+
windSpeed: float = attrs.field(converter=round_converter(1))
75+
latitude: float = attrs.field(converter=round_converter(6))
76+
longitude: float = attrs.field(converter=round_converter(6))
9377
heightOfStationGroundAboveMeanSeaLevel: float = attrs.field(
94-
converter=round_converter(1)
78+
converter=round_converter(2)
9579
)
96-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007031
97-
# Scale: 1, unit: m
80+
#
9881
heightOfBarometerAboveMeanSeaLevel: float = attrs.field(
99-
converter=round_converter(1),
82+
converter=round_converter(2),
10083
)
101-
102-
# Pressure information
103-
# ====================
104-
# Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302031
105-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/010004
106-
# Scale: -1, unit: Pa
107-
nonCoordinatePressure: float = attrs.field(converter=round_converter(-1))
108-
# There are two other pressure variables in the template: 007004 - pressure and 010062 24-hour pressure change
109-
110-
# Basic synoptic "instantaneous" data
111-
# ===================================
112-
# Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302035
113-
# This section only include the temperature and humidity data (302032).
114-
# Precipitation and cloud data are currently ignored.
115-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007032
116-
# Scale: 2, unit: m
117-
# This is the first appearance of this variable id.
11884
heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformTempRH: float = attrs.field(
119-
converter=round_converter(2),
85+
converter=round_converter(4),
12086
)
121-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012101
122-
# Scale: 2, unit: K
123-
airTemperature: float = attrs.field(converter=round_converter(2))
124-
# There is also a Dewpoint temperature in this template: 012103 which is currently unused.
125-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012103
126-
# Scale: 0, unit: %
127-
relativeHumidity: float = attrs.field(converter=round_converter(0))
128-
129-
# Basic synoptic "period" data
130-
# ============================
131-
# Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302043
132-
# Wind data: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302042
133-
# Wind direction: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/011001
134-
# Scale: 0, unit: degrees
135-
windDirection: float = attrs.field(converter=round_converter(0))
136-
# Wind speed: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/011002
137-
# Scale: 1, unit: m/s
138-
windSpeed: float = attrs.field(converter=round_converter(1))
139-
# https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007032
140-
# Scale: 2, unit: m
141-
# This is the 7th appearance of this variable id.
14287
heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformWSPD: float = attrs.field(
143-
converter=round_converter(2)
88+
converter=round_converter(4)
14489
)
14590

14691
def as_series(self) -> pd.Series:
@@ -184,7 +129,6 @@ def __eq__(self, other: "BUFRVariables"):
184129

185130
BUFR_TEMPLATES = {
186131
"mobile": {
187-
# Template definition: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307090
188132
"unexpandedDescriptors": (307090), # message template, "synopMobil"
189133
"edition": 4, # latest edition
190134
"masterTableNumber": 0,
@@ -200,7 +144,6 @@ def __eq__(self, other: "BUFRVariables"):
200144
"compressedData": 0,
201145
},
202146
"land": {
203-
# Template definition: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307080
204147
"unexpandedDescriptors": (307080), # message template, "synopLand"
205148
"edition": 4, # latest edition
206149
"masterTableNumber": 0,
@@ -303,11 +246,6 @@ def set_station(ibufr, station_type: str, wmo_id: str):
303246
elif station_type == "land":
304247
# StationNumber for land stations are integeres
305248
wmo_id_int = int(wmo_id)
306-
if wmo_id_int >= 1024:
307-
raise ValueError(
308-
f"Invalid WMO ID {wmo_id}. Land station number must be less than 1024."
309-
"See https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/001002"
310-
)
311249
station_config = dict(stationNumber=wmo_id_int)
312250
else:
313251
raise Exception(f"Unsupported station station type {station_type}")
@@ -342,7 +280,7 @@ def set_AWS_variables(
342280

343281
set_bufr_value(ibufr, "relativeHumidity", variables.relativeHumidity)
344282
set_bufr_value(ibufr, "airTemperature", variables.airTemperature)
345-
set_bufr_value(ibufr, "nonCoordinatePressure", variables.nonCoordinatePressure)
283+
set_bufr_value(ibufr, "pressure", variables.pressure)
346284
set_bufr_value(ibufr, "windDirection", variables.windDirection)
347285
set_bufr_value(ibufr, "windSpeed", variables.windSpeed)
348286

@@ -434,7 +372,7 @@ def get_bufr_value(msgid: int, key: str) -> float:
434372
raise ValueError(f"Unsupported BUFR value type {type(value)} for key {key}")
435373

436374

437-
def read_bufr_message(fp: BinaryIO, backwards_compatible: bool = False) -> Optional[BUFRVariables]:
375+
def read_bufr_message(fp: BinaryIO) -> Optional[BUFRVariables]:
438376
"""
439377
Read and parse BUFR message from binary IO stream.
440378
@@ -445,8 +383,6 @@ def read_bufr_message(fp: BinaryIO, backwards_compatible: bool = False) -> Optio
445383
----------
446384
fp
447385
Readable binary io stream
448-
backwards_compatible
449-
Use legacy pressure if nonCoordinatePressure is nan
450386
451387
Returns
452388
-------
@@ -499,19 +435,11 @@ def read_bufr_message(fp: BinaryIO, backwards_compatible: bool = False) -> Optio
499435
f"Unknown BUFR template unexpandedDescriptors: {unexpanded_descriptors}"
500436
)
501437

502-
nonCoordinatePressure = get_bufr_value(ibufr, "nonCoordinatePressure")
503-
if math.isnan(nonCoordinatePressure) and backwards_compatible:
504-
nonCoordinatePressure = get_bufr_value(ibufr, "pressure")
505-
if not math.isnan(nonCoordinatePressure):
506-
logger.warning(
507-
f"nonCoordinatePressure is nan, using legacy pressure instead"
508-
)
509-
510438
variables = BUFRVariables(
511439
timestamp=timestamp,
512440
relativeHumidity=get_bufr_value(ibufr, "relativeHumidity"),
513441
airTemperature=get_bufr_value(ibufr, "airTemperature"),
514-
nonCoordinatePressure=nonCoordinatePressure,
442+
pressure=get_bufr_value(ibufr, "pressure"),
515443
windDirection=get_bufr_value(ibufr, "windDirection"),
516444
windSpeed=get_bufr_value(ibufr, "windSpeed"),
517445
latitude=get_bufr_value(ibufr, "latitude"),
@@ -557,6 +485,5 @@ def read_bufr_file(path: PathLike) -> pd.DataFrame:
557485
message_vars = read_bufr_message(fp)
558486
if message_vars is None:
559487
break
560-
lines.append(message_vars.as_series())
561-
data_frame = pd.DataFrame(lines).set_index("wmo_id")
562-
return data_frame
488+
lines.append(message_vars)
489+
return pd.DataFrame(lines).rename_axis("message_index")

0 commit comments

Comments
 (0)