diff --git a/.github/workflows/process_test.yml b/.github/workflows/process_test.yml index 5ba3d51a..e7180f6f 100644 --- a/.github/workflows/process_test.yml +++ b/.github/workflows/process_test.yml @@ -38,7 +38,7 @@ jobs: run: | mkdir $GITHUB_WORKSPACE/out/ for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do - python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/ + python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/ done - name: Upload test output uses: actions/upload-artifact@v3 diff --git a/.gitignore b/.gitignore index 065beda1..cebe88d2 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ src/pypromice/postprocess/positions.csv # sqlite db files *.db +*.bak diff --git a/setup.py b/setup.py index 2a4638d3..1755a7a3 100644 --- a/setup.py +++ b/setup.py @@ -42,8 +42,9 @@ 'console_scripts': [ 'get_promice_data = pypromice.get.get_promice_data:get_promice_data', 'get_l0tx = pypromice.tx.get_l0tx:get_l0tx', + 'get_l2 = pypromice.process.get_l2:get_l2', + 'join_l2 = pypromice.process.join_l2:join_l2', 'get_l3 = pypromice.process.get_l3:get_l3', - 'join_l3 = pypromice.process.join_l3:join_l3', 'get_watsontx = pypromice.tx.get_watsontx:get_watsontx', 'get_bufr = pypromice.postprocess.get_bufr:main', 'get_msg = pypromice.tx.get_msg:get_msg' diff --git a/src/pypromice/process/L1toL2.py b/src/pypromice/process/L1toL2.py index 73ca1a7d..69bf5c48 100644 --- a/src/pypromice/process/L1toL2.py +++ b/src/pypromice/process/L1toL2.py @@ -30,7 +30,18 @@ def toL2( eps_clear=9.36508e-6, emissivity=0.97, ) -> xr.Dataset: - '''Process one Level 1 (L1) product to Level 2 + '''Process one Level 1 (L1) product to Level 2. + In this step we do: + - manual flagging and adjustments + - automated QC: persistence, percentile + - custom filter: gps_alt filter, NaN t_rad removed from dlr & ulr + - smoothing of tilt and rot + - calculation of rh with regards to ice in subfreezin conditions + - caluclation of cloud coverage + - correction of dsr and usr for tilt + - filtering of dsr based on a theoritical TOA irradiance and grazing light + - calculation of albedo + - caluclation of directional wind speed Parameters ---------- @@ -85,10 +96,20 @@ def toL2( ds['dlr'] = ds.dlr.where(ds.t_rad.notnull()) ds['ulr'] = ds.ulr.where(ds.t_rad.notnull()) + # calculating realtive humidity with regard to ice T_100 = _getTempK(T_0) ds['rh_u_cor'] = correctHumidity(ds['rh_u'], ds['t_u'], T_0, T_100, ews, ei0) + if ds.attrs['number_of_booms']==2: + ds['rh_l_cor'] = correctHumidity(ds['rh_l'], ds['t_l'], + T_0, T_100, ews, ei0) + + if hasattr(ds,'t_i'): + if ~ds['t_i'].isnull().all(): + ds['rh_i_cor'] = correctHumidity(ds['rh_i'], ds['t_i'], + T_0, T_100, ews, ei0) + # Determiune cloud cover for on-ice stations cc = calcCloudCoverage(ds['t_u'], T_0, eps_overcast, eps_clear, # Calculate cloud coverage ds['dlr'], ds.attrs['station_id']) @@ -176,22 +197,52 @@ def toL2( ds['precip_u_cor'], ds['precip_u_rate'] = correctPrecip(ds['precip_u'], ds['wspd_u']) if ds.attrs['number_of_booms']==2: - ds['rh_l_cor'] = correctHumidity(ds['rh_l'], ds['t_l'], # Correct relative humidity - T_0, T_100, ews, ei0) - if ~ds['precip_l'].isnull().all() and precip_flag: # Correct precipitation ds['precip_l_cor'], ds['precip_l_rate']= correctPrecip(ds['precip_l'], ds['wspd_l']) - if hasattr(ds,'t_i'): - if ~ds['t_i'].isnull().all(): # Instantaneous msg processing - ds['rh_i_cor'] = correctHumidity(ds['rh_i'], ds['t_i'], # Correct relative humidity - T_0, T_100, ews, ei0) + # Get directional wind speed + ds['wdir_u'] = ds['wdir_u'].where(ds['wspd_u'] != 0) + ds['wspd_x_u'], ds['wspd_y_u'] = calcDirWindSpeeds(ds['wspd_u'], ds['wdir_u']) + + if ds.attrs['number_of_booms']==2: + ds['wdir_l'] = ds['wdir_l'].where(ds['wspd_l'] != 0) + ds['wspd_x_l'], ds['wspd_y_l'] = calcDirWindSpeeds(ds['wspd_l'], ds['wdir_l']) + + if hasattr(ds, 'wdir_i'): + if ~ds['wdir_i'].isnull().all() and ~ds['wspd_i'].isnull().all(): + ds['wdir_i'] = ds['wdir_i'].where(ds['wspd_i'] != 0) + ds['wspd_x_i'], ds['wspd_y_i'] = calcDirWindSpeeds(ds['wspd_i'], ds['wdir_i']) + ds = clip_values(ds, vars_df) return ds +def calcDirWindSpeeds(wspd, wdir, deg2rad=np.pi/180): + '''Calculate directional wind speed from wind speed and direction + + Parameters + ---------- + wspd : xr.Dataarray + Wind speed data array + wdir : xr.Dataarray + Wind direction data array + deg2rad : float + Degree to radians coefficient. The default is np.pi/180 + + Returns + ------- + wspd_x : xr.Dataarray + Wind speed in X direction + wspd_y : xr.Datarray + Wind speed in Y direction + ''' + wspd_x = wspd * np.sin(wdir * deg2rad) + wspd_y = wspd * np.cos(wdir * deg2rad) + return wspd_x, wspd_y + + def calcCloudCoverage(T, T_0, eps_overcast, eps_clear, dlr, station_id): '''Calculate cloud cover from T and T_0 diff --git a/src/pypromice/process/L2toL3.py b/src/pypromice/process/L2toL3.py index a71a4028..9751122e 100755 --- a/src/pypromice/process/L2toL3.py +++ b/src/pypromice/process/L2toL3.py @@ -7,7 +7,10 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071, es_100=1013.246): - '''Process one Level 2 (L2) product to Level 3 (L3) + '''Process one Level 2 (L2) product to Level 3 (L3) meaning calculating all + derived variables: + - Sensible fluxes + Parameters ---------- @@ -32,9 +35,6 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071, T_100 = _getTempK(T_0) # Get steam point temperature as K - ds['wdir_u'] = ds['wdir_u'].where(ds['wspd_u'] != 0) # Get directional wind speed - ds['wspd_x_u'], ds['wspd_y_u'] = calcDirWindSpeeds(ds['wspd_u'], ds['wdir_u']) - # Upper boom bulk calculation T_h_u = ds['t_u'].copy() # Copy for processing p_h_u = ds['p_u'].copy() @@ -85,41 +85,9 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071, q_h_l = cleanSpHumid(q_h_l, T_h_l, Tsurf_h, p_h_l, RH_cor_h_l) # Clean sp.humid values ds['qh_l'] = (('time'), q_h_l.data) - ds['wdir_l'] = ds['wdir_l'].where(ds['wspd_l'] != 0) # Get directional wind speed - ds['wspd_x_l'], ds['wspd_y_l'] = calcDirWindSpeeds(ds['wspd_l'], ds['wdir_l']) - - if hasattr(ds, 'wdir_i'): - if ~ds['wdir_i'].isnull().all() and ~ds['wspd_i'].isnull().all(): # Instantaneous msg processing - ds['wdir_i'] = ds['wdir_i'].where(ds['wspd_i'] != 0) # Get directional wind speed - ds['wspd_x_i'], ds['wspd_y_i'] = calcDirWindSpeeds(ds['wspd_i'], ds['wdir_i']) - return ds -def calcDirWindSpeeds(wspd, wdir, deg2rad=np.pi/180): - '''Calculate directional wind speed from wind speed and direction - - Parameters - ---------- - wspd : xr.Dataarray - Wind speed data array - wdir : xr.Dataarray - Wind direction data array - deg2rad : float - Degree to radians coefficient. The default is np.pi/180 - - Returns - ------- - wspd_x : xr.Dataarray - Wind speed in X direction - wspd_y : xr.Datarray - Wind speed in Y direction - ''' - wspd_x = wspd * np.sin(wdir * deg2rad) - wspd_y = wspd * np.cos(wdir * deg2rad) - return wspd_x, wspd_y - - def calcHeatFlux(T_0, T_h, Tsurf_h, rho_atm, WS_h, z_WS, z_T, nu, q_h, p_h, kappa=0.4, WS_lim=1., z_0=0.001, g=9.82, es_0=6.1071, eps=0.622, gamma=16., L_sub=2.83e6, L_dif_max=0.01, c_pd=1005., aa=0.7, diff --git a/src/pypromice/process/aws.py b/src/pypromice/process/aws.py index 58440595..2ce64de2 100644 --- a/src/pypromice/process/aws.py +++ b/src/pypromice/process/aws.py @@ -91,12 +91,30 @@ def getL1(self): logger.info('Level 1 processing...') self.L0 = [addBasicMeta(item, self.vars) for item in self.L0] self.L1 = [toL1(item, self.vars) for item in self.L0] - self.L1A = reduce(xr.Dataset.combine_first, self.L1) + + if self.merge_flag: + self.L1A = self.hard_merge(self.L1) + else: + self.L1A = reduce(xr.Dataset.combine_first, self.L1) def getL2(self): '''Perform L1 to L2 data processing''' logger.info('Level 2 processing...') self.L2 = toL2(self.L1A, vars_df=self.vars) + self.L2 = self.resample(self.L2) + self.L2 = reformat_time(self.L2) + + # Switch gps_lon to negative (degrees_east) + # Do this here, and NOT in addMeta, otherwise we switch back to positive + # when calling getMeta in joinL2! PJW + if self.L2.attrs['station_id'] not in ['UWN', 'Roof_GEUS', 'Roof_PROMICE']: + self.L2['gps_lon'] = self.L2['gps_lon'] * -1 + + # Add variable attributes and metadata + self.L2 = self.addAttributes(self.L2) + + # Round all values to specified decimals places + self.L2 = roundValues(self.L2, self.vars) def getL3(self): '''Perform L2 to L3 data processing, including resampling and metadata @@ -104,31 +122,48 @@ def getL3(self): logger.info('Level 3 processing...') self.L3 = toL3(self.L2) - # Resample L3 product + def resample(self, dataset): + '''Resample dataset to specific temporal resolution (based on input + data type)''' f = [l.attrs['format'] for l in self.L0] if 'raw' in f or 'STM' in f: logger.info('Resampling to 10 minute') - self.L3 = resampleL3(self.L3, '10min') + resampled = resampleL2(dataset, '10min') else: - self.L3 = resampleL3(self.L3, '60min') + resampled = resampleL2(dataset, '60min') logger.info('Resampling to hour') - - # Re-format time - t = self.L3['time'].values - self.L3['time'] = list(t) - - # Switch gps_lon to negative (degrees_east) - # Do this here, and NOT in addMeta, otherwise we switch back to positive - # when calling getMeta in joinL3! PJW - if self.L3.attrs['station_id'] not in ['UWN', 'Roof_GEUS', 'Roof_PROMICE']: - self.L3['gps_lon'] = self.L3['gps_lon'] * -1 - - # Add variable attributes and metadata - self.L3 = self.addAttributes(self.L3) - - # Round all values to specified decimals places - self.L3 = roundValues(self.L3, self.vars) - + return resampled + + def merge_flag(self): + '''Determine if hard merging is needed, based on whether a hard + merge_type flag is defined in any of the configs''' + f = [l.attrs['merge_type'] for l in self.L0] + if 'hard' in f: + return True + else: + return False + + def hard_merge(self, dataset_list): + '''Determine positions where hard merging should occur, combine + data and append to list of combined data chunks, then hard merge all + combined data chunks. This should be called in instances where there + needs to be a clear break between input datasets, such as when a station + is moved (and we do not want the GPS position jumping)''' + # Define positions where hard merging should occur + m=[] + f = [l.attrs['merge_type'] for l in self.L0] + [m.append(i) for i, item in enumerate(f) if item=='hard'] + + # Perform combine between hard merge breaks and append to list of combined data + combined=[] + for i in range(len(m[:-1])): + combined.append(reduce(xr.Dataset.combine_first, dataset_list[m[i]:m[i+1]])) + combined.append(reduce(xr.Dataset.combine_first, dataset_list[m[-1]:])) + + # Hard merge all combined datasets together + return reduce(xr.Dataset.update, combined) + + def addAttributes(self, L3): '''Add variable and attribute metadata @@ -365,6 +400,12 @@ def getL0(infile, nodata, cols, skiprows, file_version, ds = xr.Dataset.from_dataframe(df) return ds +def reformat_time(dataset): + '''Re-format time''' + t = dataset['time'].values + dataset['time'] = list(t) + return dataset + def addBasicMeta(ds, vars_df): ''' Use a variable lookup table DataFrame to add the basic metadata to the xarray dataset. This is later amended to finalise L3 @@ -712,8 +753,8 @@ def getMeta(m_file=None, delimiter=','): pass return meta -def resampleL3(ds_h, t): - '''Resample L3 AWS data, e.g. hourly to daily average. This uses pandas +def resampleL2(ds_h, t): + '''Resample L2 AWS data, e.g. hourly to daily average. This uses pandas DataFrame resampling at the moment as a work-around to the xarray Dataset resampling. As stated, xarray resampling is a lengthy process that takes ~2-3 minutes per operation: ds_d = ds_h.resample({'time':"1D"}).mean() @@ -881,7 +922,7 @@ def testAddAll(self): self.assertTrue(d.attrs['station_id']=='TEST') self.assertIsInstance(d.attrs['references'], str) - def testL0toL3(self): + def testL0toL2(self): '''Test L0 to L3 processing''' try: import pypromice @@ -890,19 +931,23 @@ def testL0toL3(self): except: pAWS = AWS('../test/test_config1.toml', '../test/') pAWS.process() - self.assertIsInstance(pAWS.L3, xr.Dataset) - self.assertTrue(pAWS.L3.attrs['station_id']=='TEST1') + self.assertIsInstance(pAWS.L2, xr.Dataset) + self.assertTrue(pAWS.L2.attrs['station_id']=='TEST1') - def testCLIgetl3(self): - '''Test get_l3 CLI''' - exit_status = os.system('get_l3 -h') + def testCLIgetl2(self): + '''Test get_l2 CLI''' + exit_status = os.system('get_l2 -h') self.assertEqual(exit_status, 0) - def testCLIjoinl3(self): - '''Test join_l3 CLI''' - exit_status = os.system('join_l3 -h') + def testCLIjoinl2(self): + '''Test join_l2 CLI''' + exit_status = os.system('join_l2 -h') self.assertEqual(exit_status, 0) + def testCLIgetl3(self): + '''Test get_l3 CLI''' + exit_status = os.system('get_l3 -h') + self.assertEqual(exit_status, 0) #------------------------------------------------------------------------------ if __name__ == "__main__": diff --git a/src/pypromice/process/get_l2.py b/src/pypromice/process/get_l2.py new file mode 100644 index 00000000..7e28be7c --- /dev/null +++ b/src/pypromice/process/get_l2.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +import logging, os, sys, unittest +from argparse import ArgumentParser +from pypromice.process.aws import AWS + +def parse_arguments_l2(): + parser = ArgumentParser(description="AWS L2 processor") + + parser.add_argument('-c', '--config_file', type=str, required=True, + help='Path to config (TOML) file') + parser.add_argument('-i', '--inpath', default='data', type=str, required=True, + help='Path to input data') + parser.add_argument('-o', '--outpath', default=None, type=str, required=False, + help='Path where to write output') + parser.add_argument('-v', '--variables', default=None, type=str, + required=False, help='File path to variables look-up table') + parser.add_argument('-m', '--metadata', default=None, type=str, + required=False, help='File path to metadata') + args = parser.parse_args() + return args + +def get_l2(): + args = parse_arguments_l2() + + logging.basicConfig( + format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", + level=logging.INFO, + stream=sys.stdout, + ) + + station_name = args.config_file.split('/')[-1].split('.')[0] + station_path = os.path.join(args.inpath, station_name) + + if os.path.exists(station_path): + aws = AWS(args.config_file, station_path, args.variables, args.metadata) + else: + aws = AWS(args.config_file, args.inpath, args.variables, args.metadata) + + aws.process() + + if args.outpath is not None: + aws.write(args.outpath) + +if __name__ == "__main__": + get_l2() + diff --git a/src/pypromice/process/get_l3.py b/src/pypromice/process/get_l3.py old mode 100755 new mode 100644 index 4cc18436..ac1cbad8 --- a/src/pypromice/process/get_l3.py +++ b/src/pypromice/process/get_l3.py @@ -1,46 +1,108 @@ #!/usr/bin/env python -import logging, os, sys, unittest +import os, unittest, pkg_resources +import pandas as pd +import numpy as np +import xarray as xr from argparse import ArgumentParser -from pypromice.process.aws import AWS +from pypromice.process import getVars, getMeta, addMeta, getColNames, \ + roundValues, resampleL2, writeAll +from pypromice.process.L1toL2 import correctPrecip +from pypromice.process.L2toL3 import toL3 +from sys import exit -def parse_arguments_l3(): - parser = ArgumentParser(description="AWS L3 processor") - - parser.add_argument('-c', '--config_file', type=str, required=True, - help='Path to config (TOML) file') - parser.add_argument('-i', '--inpath', default='data', type=str, required=True, - help='Path to input data') - parser.add_argument('-o', '--outpath', default=None, type=str, required=False, - help='Path where to write output') - parser.add_argument('-v', '--variables', default=None, type=str, - required=False, help='File path to variables look-up table') - parser.add_argument('-m', '--metadata', default=None, type=str, - required=False, help='File path to metadata') - args = parser.parse_args() +def parse_arguments_getl3(debug_args=None): + parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") + parser.add_argument('-s', '--file1', type=str, required=True, nargs='+', + help='Path to source L2 file') + # here will come additional arguments for the merging with historical stations + parser.add_argument('-v', '--variables', default=None, type=str, required=False, + help='Path to variables look-up table .csv file for variable name retained'''), + parser.add_argument('-m', '--metadata', default=None, type=str, required=False, + help='Path to metadata table .csv file for metadata information'''), + parser.add_argument('-d', '--datatype', default='raw', type=str, required=False, + help='Data type to output, raw or tx') + args = parser.parse_args(args=debug_args) + args.file1 = ' '.join(args.file1) + args.folder_gcnet = ' '.join(args.folder_gcnet) + args.folder_promice = ' '.join(args.folder_promice) return args -def get_l3(): - args = parse_arguments_l3() - logging.basicConfig( - format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", - level=logging.INFO, - stream=sys.stdout, - ) +def loadArr(infile): + if infile.split('.')[-1].lower() in 'csv': + df = pd.read_csv(infile) + df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None) + df = df.set_index('time') + ds = xr.Dataset.from_dataframe(df) - station_name = args.config_file.split('/')[-1].split('.')[0] - station_path = os.path.join(args.inpath, station_name) + elif infile.split('.')[-1].lower() in 'nc': + ds = xr.open_dataset(infile) + + try: + name = ds.attrs['station_name'] + except: + name = infile.split('/')[-1].split('.')[0].split('_hour')[0].split('_10min')[0] + + print(f'{name} array loaded from {infile}') + return ds, name - if os.path.exists(station_path): - aws = AWS(args.config_file, station_path, args.variables, args.metadata) +def get_l3(): + args = parse_arguments_getl3() + + # Check files + if os.path.isfile(args.file1): + # Load L2 data arrays + ds1, n1 = loadArr(args.file1) + + # converts to L3: + # - derives sensible heat fluxes + # - more to come + ds1 = toL3(ds1) + + # here will come the merging with historical data else: - aws = AWS(args.config_file, args.inpath, args.variables, args.metadata) + print(f'Invalid file {args.file1}') + exit() - aws.process() - - if args.outpath is not None: - aws.write(args.outpath) + # Get hourly, daily and monthly datasets + print('Resampling L3 data to hourly, daily and monthly resolutions...') + l3_h = resampleL2(ds1, '60min') + l3_d = resampleL2(ds1, '1D') + l3_m = resampleL2(ds1, 'M') + + print(f'Adding variable information from {args.variables}...') + + # Load variables look-up table + var = getVars(args.variables) + + # Round all values to specified decimals places + l3_h = roundValues(l3_h, var) + l3_d = roundValues(l3_d, var) + l3_m = roundValues(l3_m, var) + # Get columns to keep + if hasattr(ds1, 'p_l'): + col_names = getColNames(var, 2, args.datatype.lower()) + else: + col_names = getColNames(var, 1, args.datatype.lower()) + + # Assign station id + for l in [l3_h, l3_d, l3_m]: + l.attrs['station_id'] = n1 + + # Assign metadata + print(f'Adding metadata from {args.metadata}...') + m = getMeta(args.metadata) + l3_h = addMeta(l3_h, m) + l3_d = addMeta(l3_d, m) + l3_m = addMeta(l3_m, m) + + # Set up output path + out = os.path.join(args.outpath, site_id) + + # Write to files + writeAll(out, site_id, l3_h, l3_d, l3_m, col_names) + print(f'Files saved to {os.path.join(out, site_id)}...') +# %% if __name__ == "__main__": get_l3() - diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l2.py similarity index 83% rename from src/pypromice/process/join_l3.py rename to src/pypromice/process/join_l2.py index 2e0d3fe3..943d9afa 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l2.py @@ -4,15 +4,15 @@ import xarray as xr from argparse import ArgumentParser from pypromice.process import getVars, getMeta, addMeta, getColNames, \ - roundValues, resampleL3, writeAll + roundValues, resampleL2, writeAll from pypromice.process.L1toL2 import correctPrecip def parse_arguments_join(): - parser = ArgumentParser(description="AWS L3 joiner for merging together two L3 products, for example an L3 RAW and L3 TX data product. An hourly, daily and monthly L3 data product is outputted to the defined output path") + parser = ArgumentParser(description="AWS L2 joiner for merging together two L2 products, for example an L2 RAW and L2 TX data product. An hourly, daily and monthly L2 data product is outputted to the defined output path") parser.add_argument('-s', '--file1', type=str, required=True, - help='Path to source L3 file, which will be preferenced in merge process') + help='Path to source L2 file, which will be preferenced in merge process') parser.add_argument('-t', '--file2', type=str, required=True, - help='Path to target L3 file, which will be used to fill gaps in merge process') + help='Path to target L2 file, which will be used to fill gaps in merge process') parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=True, help='Path where to write output') parser.add_argument('-v', '--variables', default=None, type=str, required=False, @@ -41,7 +41,7 @@ def loadArr(infile): return ds, name -def join_l3(): +def join_l2(): args = parse_arguments_join() # Check files @@ -87,10 +87,10 @@ def join_l3(): exit() # Get hourly, daily and monthly datasets - print('Resampling L3 data to hourly, daily and monthly resolutions...') - l3_h = resampleL3(all_ds, '60min') - l3_d = resampleL3(all_ds, '1D') - l3_m = resampleL3(all_ds, 'M') + print('Resampling L2 data to hourly, daily and monthly resolutions...') + l2_h = resampleL2(all_ds, '60min') + l2_d = resampleL2(all_ds, '1D') + l2_m = resampleL2(all_ds, 'M') print(f'Adding variable information from {args.variables}...') @@ -98,9 +98,9 @@ def join_l3(): var = getVars(args.variables) # Round all values to specified decimals places - l3_h = roundValues(l3_h, var) - l3_d = roundValues(l3_d, var) - l3_m = roundValues(l3_m, var) + l2_h = roundValues(l2_h, var) + l2_d = roundValues(l2_d, var) + l2_m = roundValues(l2_m, var) # Get columns to keep if hasattr(all_ds, 'p_l'): @@ -109,22 +109,22 @@ def join_l3(): col_names = getColNames(var, 1, args.datatype.lower()) # Assign station id - for l in [l3_h, l3_d, l3_m]: + for l in [l2_h, l2_d, l2_m]: l.attrs['station_id'] = name # Assign metadata print(f'Adding metadata from {args.metadata}...') m = getMeta(args.metadata) - l3_h = addMeta(l3_h, m) - l3_d = addMeta(l3_d, m) - l3_m = addMeta(l3_m, m) + l2_h = addMeta(l2_h, m) + l2_d = addMeta(l2_d, m) + l2_m = addMeta(l2_m, m) # Set up output path out = os.path.join(args.outpath, name) # Write to files - writeAll(out, name, l3_h, l3_d, l3_m, col_names) + writeAll(out, name, l2_h, l2_d, l2_m, col_names) print(f'Files saved to {os.path.join(out, name)}...') if __name__ == "__main__": - join_l3() + join_l2()