diff --git a/setup.py b/setup.py index bb6604a1..402502f2 100644 --- a/setup.py +++ b/setup.py @@ -43,8 +43,9 @@ 'get_promice_data = pypromice.get.get_promice_data:get_promice_data', 'get_l0tx = pypromice.tx.get_l0tx:get_l0tx', 'join_l2 = pypromice.process.join_l2:join_l2', - 'join_l3 = pypromice.process.join_l2:join_l3', + 'join_l3 = pypromice.process.join_l3:join_l3', 'get_l3 = pypromice.process.get_l3:get_l3', + 'l2_to_l3 = pypromice.process.l2_to_l3:l2_to_l3', 'get_watsontx = pypromice.tx.get_watsontx:get_watsontx', 'get_bufr = pypromice.postprocess.get_bufr:main', 'get_msg = pypromice.tx.get_msg:get_msg' diff --git a/src/pypromice/process/aws.py b/src/pypromice/process/aws.py index 6efed9f8..ed634707 100644 --- a/src/pypromice/process/aws.py +++ b/src/pypromice/process/aws.py @@ -509,6 +509,32 @@ def writeNC(outfile, Lx, col_names=None): names = list(Lx.keys()) Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True) +def writeAll(outpath, station_id, l3_h, l3_d, l3_m, csv_order=None): + '''Write L3 hourly, daily and monthly datasets to .nc and .csv + files + + outpath : str + Output file path + station_id : str + Station name + l3_h : xr.Dataset + L3 hourly data + l3_d : xr.Dataset + L3 daily data + l3_m : xr.Dataset + L3 monthly data + csv_order : list, optional + List order of variables + ''' + if not os.path.isdir(outpath): + os.mkdir(outpath) + outfile_h = os.path.join(outpath, station_id + '_hour') + outfile_d = os.path.join(outpath, station_id + '_day') + outfile_m = os.path.join(outpath, station_id + '_month') + for o,l in zip([outfile_h, outfile_d, outfile_m], [l3_h ,l3_d, l3_m]): + writeCSV(o+'.csv',l, csv_order) + writeNC(o+'.nc',l) + def popCols(ds, names): '''Populate dataset with all given variable names diff --git a/src/pypromice/process/get_l3.py b/src/pypromice/process/get_l3.py index b8395865..0e72c9ec 100644 --- a/src/pypromice/process/get_l3.py +++ b/src/pypromice/process/get_l3.py @@ -5,11 +5,11 @@ from pypromice.process.aws import AWS def parse_arguments_level(): - parser = ArgumentParser(description="AWS L2 processor") + parser = ArgumentParser(description="AWS L2/L3 processor") parser.add_argument('-c', '--config_file', type=str, required=True, help='Path to config (TOML) file') - parser.add_argument('-i', '--inpath', default='data', type=str, required=True, + parser.add_argument('-i', '--inpath', type=str, required=True, help='Path to input data') parser.add_argument('-o', '--outpath', default=None, type=str, required=False, help='Path where to write output') @@ -22,7 +22,7 @@ def parse_arguments_level(): args = parser.parse_args() return args -def get_level(): +def get_l3(): args = parse_arguments_level() logging.basicConfig( @@ -43,7 +43,7 @@ def get_level(): else: m = args.metadata - # Define output path + # Define input path station_name = args.config_file.split('/')[-1].split('.')[0] station_path = os.path.join(args.inpath, station_name) if os.path.exists(station_path): diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index 943d9afa..4d4ed60d 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -4,7 +4,7 @@ import xarray as xr from argparse import ArgumentParser from pypromice.process import getVars, getMeta, addMeta, getColNames, \ - roundValues, resampleL2, writeAll + roundValues, resample_dataset, writeAll from pypromice.process.L1toL2 import correctPrecip def parse_arguments_join(): @@ -88,9 +88,9 @@ def join_l2(): # Get hourly, daily and monthly datasets print('Resampling L2 data to hourly, daily and monthly resolutions...') - l2_h = resampleL2(all_ds, '60min') - l2_d = resampleL2(all_ds, '1D') - l2_m = resampleL2(all_ds, 'M') + l2_h = resample_dataset(all_ds, '60min') + l2_d = resample_dataset(all_ds, '1D') + l2_m = resample_dataset(all_ds, 'M') print(f'Adding variable information from {args.variables}...') diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 2e0d3fe3..933eae95 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -4,7 +4,7 @@ import xarray as xr from argparse import ArgumentParser from pypromice.process import getVars, getMeta, addMeta, getColNames, \ - roundValues, resampleL3, writeAll + roundValues, resample_dataset, writeAll from pypromice.process.L1toL2 import correctPrecip def parse_arguments_join(): @@ -88,9 +88,9 @@ def join_l3(): # Get hourly, daily and monthly datasets print('Resampling L3 data to hourly, daily and monthly resolutions...') - l3_h = resampleL3(all_ds, '60min') - l3_d = resampleL3(all_ds, '1D') - l3_m = resampleL3(all_ds, 'M') + l3_h = resample_dataset(all_ds, '60min') + l3_d = resample_dataset(all_ds, '1D') + l3_m = resample_dataset(all_ds, 'M') print(f'Adding variable information from {args.variables}...') diff --git a/src/pypromice/process/l2_to_l3.py b/src/pypromice/process/l2_to_l3.py index f58aaf5d..4acf6cb6 100644 --- a/src/pypromice/process/l2_to_l3.py +++ b/src/pypromice/process/l2_to_l3.py @@ -1,109 +1,72 @@ #!/usr/bin/env python -import os, unittest, pkg_resources -import pandas as pd -import numpy as np +import os, logging, sys import xarray as xr from argparse import ArgumentParser -from pypromice.process import getVars, getMeta, addMeta, getColNames, \ - roundValues, resampleL2, writeAll -from pypromice.process.L1toL2 import correctPrecip -from pypromice.process.L2toL3 import toL3 -from sys import exit +import pypromice +from pypromice.process.aws import AWS def parse_arguments_l2_to_l3(debug_args=None): - parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") - parser.add_argument('-s', '--file1', type=str, required=True, nargs='+', - help='Path to source L2 file') + parser = ArgumentParser(description="AWS L3 script for the processing L3 "+ + "data from L2 and merging the L3 data with its "+ + "historical site. An hourly, daily and monthly L3 "+ + "data product is outputted to the defined output path") + parser.add_argument('-c', '--config_file', type=str, required=True, + help='Path to config (TOML) file') + parser.add_argument('-i', '--inpath', type=str, required=True, + help='Path to input data') + parser.add_argument('-l', '--level_2', type=str, required=True, + help='Path to Level 2 .nc data file') + parser.add_argument('-o', '--outpath', default=None, type=str, required=False, + help='Path where to write output') + parser.add_argument('-v', '--variables', default=None, type=str, + required=False, help='File path to variables look-up table') + parser.add_argument('-m', '--metadata', default=None, type=str, + required=False, help='File path to metadata') + parser.add_argument('-g', '--gcnet_historical', default=None, type=str, + required=False, help='File path to historical GC-Net data file') + # here will come additional arguments for the merging with historical stations - parser.add_argument('-v', '--variables', default=None, type=str, required=False, - help='Path to variables look-up table .csv file for variable name retained'''), - parser.add_argument('-m', '--metadata', default=None, type=str, required=False, - help='Path to metadata table .csv file for metadata information'''), - parser.add_argument('-d', '--datatype', default='raw', type=str, required=False, - help='Data type to output, raw or tx') args = parser.parse_args(args=debug_args) - args.file1 = ' '.join(args.file1) - args.folder_gcnet = ' '.join(args.folder_gcnet) - args.folder_promice = ' '.join(args.folder_promice) return args - -def loadArr(infile): - if infile.split('.')[-1].lower() in 'csv': - df = pd.read_csv(infile) - df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None) - df = df.set_index('time') - ds = xr.Dataset.from_dataframe(df) - - elif infile.split('.')[-1].lower() in 'nc': - ds = xr.open_dataset(infile) - - try: - name = ds.attrs['station_name'] - except: - name = infile.split('/')[-1].split('.')[0].split('_hour')[0].split('_10min')[0] - - print(f'{name} array loaded from {infile}') - return ds, name - -def get_l3(): +def l2_to_l3(): args = parse_arguments_l2_to_l3() - - # Check files - if os.path.isfile(args.file1): - # Load L2 data arrays - ds1, n1 = loadArr(args.file1) - - # converts to L3: - # - derives sensible heat fluxes - # - more to come - ds1 = toL3(ds1) + logging.basicConfig( + format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", + level=logging.INFO, + stream=sys.stdout, + ) + + # Define variables (either from file or pypromice package defaults) + if args.variables is None: + v = os.path.join(os.path.dirname(pypromice.__file__),'process/variables.csv') + else: + v = args.variables - # here will come the merging with historical data + # Define metadata (either from file or pypromice package defaults) + if args.variables is None: + m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv') else: - print(f'Invalid file {args.file1}') - exit() + m = args.metadata - # Get hourly, daily and monthly datasets - print('Resampling L3 data to hourly, daily and monthly resolutions...') - l3_h = resampleL2(ds1, '60min') - l3_d = resampleL2(ds1, '1D') - l3_m = resampleL2(ds1, 'M') - - print(f'Adding variable information from {args.variables}...') - - # Load variables look-up table - var = getVars(args.variables) - - # Round all values to specified decimals places - l3_h = roundValues(l3_h, var) - l3_d = roundValues(l3_d, var) - l3_m = roundValues(l3_m, var) - - # Get columns to keep - if hasattr(ds1, 'p_l'): - col_names = getColNames(var, 2, args.datatype.lower()) + # Define input path + station_name = args.config_file.split('/')[-1].split('.')[0] + station_path = os.path.join(args.inpath, station_name) + if os.path.exists(station_path): + aws = AWS(args.config_file, station_path, v, m) else: - col_names = getColNames(var, 1, args.datatype.lower()) + aws = AWS(args.config_file, args.inpath, v, m) - # Assign station id - for l in [l3_h, l3_d, l3_m]: - l.attrs['station_id'] = n1 - # Assign metadata - print(f'Adding metadata from {args.metadata}...') - m = getMeta(args.metadata) - l3_h = addMeta(l3_h, m) - l3_d = addMeta(l3_d, m) - l3_m = addMeta(l3_m, m) - - # Set up output path - out = os.path.join(args.outpath, site_id) + # Define Level 2 dataset from file + aws.L2 = xr.open_dataset(args.level_2) - # Write to files - writeAll(out, site_id, l3_h, l3_d, l3_m, col_names) - print(f'Files saved to {os.path.join(out, site_id)}...') -# %% + # Perform Level 3 processing + aws.getL3() + + # Write Level 3 dataset to file if output directory given + if args.outpath is not None: + aws.writeL3(args.outpath) + if __name__ == "__main__": l2_to_l3() -