l2_to_l3 functionality from pypromice

Author: PennyHow

setup.py

@@ -43,8 +43,9 @@
         'get_promice_data = pypromice.get.get_promice_data:get_promice_data',
         'get_l0tx = pypromice.tx.get_l0tx:get_l0tx',
         'join_l2 = pypromice.process.join_l2:join_l2',
-        'join_l3 = pypromice.process.join_l2:join_l3',
+        'join_l3 = pypromice.process.join_l3:join_l3',
         'get_l3 = pypromice.process.get_l3:get_l3',
+        'l2_to_l3 = pypromice.process.l2_to_l3:l2_to_l3',
         'get_watsontx = pypromice.tx.get_watsontx:get_watsontx',
         'get_bufr = pypromice.postprocess.get_bufr:main',
         'get_msg = pypromice.tx.get_msg:get_msg'

src/pypromice/process/aws.py

@@ -509,6 +509,32 @@ def writeNC(outfile, Lx, col_names=None):
         names = list(Lx.keys())
     Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True)
 
+def writeAll(outpath, station_id, l3_h, l3_d, l3_m, csv_order=None):
+    '''Write L3 hourly, daily and monthly datasets to .nc and .csv
+    files
+
+    outpath : str
+        Output file path
+    station_id : str
+        Station name
+    l3_h : xr.Dataset
+        L3 hourly data
+    l3_d : xr.Dataset
+        L3 daily data
+    l3_m : xr.Dataset
+        L3 monthly data
+    csv_order : list, optional
+        List order of variables
+    '''
+    if not os.path.isdir(outpath):
+        os.mkdir(outpath)
+    outfile_h = os.path.join(outpath, station_id + '_hour')
+    outfile_d = os.path.join(outpath, station_id + '_day')
+    outfile_m = os.path.join(outpath, station_id + '_month')
+    for o,l in zip([outfile_h, outfile_d, outfile_m], [l3_h ,l3_d, l3_m]):
+        writeCSV(o+'.csv',l, csv_order)
+        writeNC(o+'.nc',l)
+        
 def popCols(ds, names):
     '''Populate dataset with all given variable names
 

src/pypromice/process/get_l3.py

@@ -5,11 +5,11 @@
 from pypromice.process.aws import AWS
 
 def parse_arguments_level():
-    parser = ArgumentParser(description="AWS L2 processor")
+    parser = ArgumentParser(description="AWS L2/L3 processor")
 
     parser.add_argument('-c', '--config_file', type=str, required=True,
                         help='Path to config (TOML) file')
-    parser.add_argument('-i', '--inpath', default='data', type=str, required=True, 
+    parser.add_argument('-i', '--inpath', type=str, required=True, 
                         help='Path to input data')
     parser.add_argument('-o', '--outpath', default=None, type=str, required=False, 
                         help='Path where to write output')
@@ -22,7 +22,7 @@ def parse_arguments_level():
     args = parser.parse_args()
     return args
 
-def get_level():
+def get_l3():
     args = parse_arguments_level()
 
     logging.basicConfig(
@@ -43,7 +43,7 @@ def get_level():
     else:
         m = args.metadata
 
-    # Define output path
+    # Define input path
     station_name = args.config_file.split('/')[-1].split('.')[0] 
     station_path = os.path.join(args.inpath, station_name)
     if os.path.exists(station_path):

src/pypromice/process/join_l2.py

@@ -4,7 +4,7 @@
 import xarray as xr
 from argparse import ArgumentParser
 from pypromice.process import getVars, getMeta, addMeta, getColNames, \
-    roundValues, resampleL2, writeAll
+    roundValues, resample_dataset, writeAll
 from pypromice.process.L1toL2 import correctPrecip
 
 def parse_arguments_join():
@@ -88,9 +88,9 @@ def join_l2():
 
     # Get hourly, daily and monthly datasets
     print('Resampling L2 data to hourly, daily and monthly resolutions...')
-    l2_h = resampleL2(all_ds, '60min')
-    l2_d = resampleL2(all_ds, '1D')
-    l2_m = resampleL2(all_ds, 'M')
+    l2_h = resample_dataset(all_ds, '60min')
+    l2_d = resample_dataset(all_ds, '1D')
+    l2_m = resample_dataset(all_ds, 'M')
 
     print(f'Adding variable information from {args.variables}...')
 

src/pypromice/process/join_l3.py

@@ -4,7 +4,7 @@
 import xarray as xr
 from argparse import ArgumentParser
 from pypromice.process import getVars, getMeta, addMeta, getColNames, \
-    roundValues, resampleL3, writeAll
+    roundValues, resample_dataset, writeAll
 from pypromice.process.L1toL2 import correctPrecip
 
 def parse_arguments_join():
@@ -88,9 +88,9 @@ def join_l3():
 
     # Get hourly, daily and monthly datasets
     print('Resampling L3 data to hourly, daily and monthly resolutions...')
-    l3_h = resampleL3(all_ds, '60min')
-    l3_d = resampleL3(all_ds, '1D')
-    l3_m = resampleL3(all_ds, 'M')
+    l3_h = resample_dataset(all_ds, '60min')
+    l3_d = resample_dataset(all_ds, '1D')
+    l3_m = resample_dataset(all_ds, 'M')
 
     print(f'Adding variable information from {args.variables}...')
 

src/pypromice/process/l2_to_l3.py

@@ -1,109 +1,72 @@
 #!/usr/bin/env python
-import os, unittest, pkg_resources
-import pandas as pd
-import numpy as np
+import os, logging, sys
 import xarray as xr
 from argparse import ArgumentParser
-from pypromice.process import getVars, getMeta, addMeta, getColNames, \
-    roundValues, resampleL2, writeAll
-from pypromice.process.L1toL2 import correctPrecip
-from pypromice.process.L2toL3 import toL3
-from sys import exit
+import pypromice
+from pypromice.process.aws import AWS
 
 def parse_arguments_l2_to_l3(debug_args=None):
-    parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path")
-    parser.add_argument('-s', '--file1', type=str, required=True, nargs='+',
-                        help='Path to source L2 file')
+    parser = ArgumentParser(description="AWS L3 script for the processing L3 "+
+                            "data from L2 and merging the L3 data with its "+
+                            "historical site. An hourly, daily and monthly L3 "+
+                            "data product is outputted to the defined output path")
+    parser.add_argument('-c', '--config_file', type=str, required=True,
+                        help='Path to config (TOML) file')
+    parser.add_argument('-i', '--inpath', type=str, required=True, 
+                        help='Path to input data')
+    parser.add_argument('-l', '--level_2', type=str, required=True,
+                        help='Path to Level 2 .nc data file')
+    parser.add_argument('-o', '--outpath', default=None, type=str, required=False, 
+                        help='Path where to write output')
+    parser.add_argument('-v', '--variables', default=None, type=str, 
+                        required=False, help='File path to variables look-up table')
+    parser.add_argument('-m', '--metadata', default=None, type=str, 
+                        required=False, help='File path to metadata')
+    parser.add_argument('-g', '--gcnet_historical', default=None, type=str, 
+                        required=False, help='File path to historical GC-Net data file')
+
     # here will come additional arguments for the merging with historical stations
-    parser.add_argument('-v', '--variables', default=None, type=str, required=False, 
-    			 help='Path to variables look-up table .csv file for variable name retained'''),
-    parser.add_argument('-m', '--metadata', default=None, type=str, required=False, 
-    			 help='Path to metadata table .csv file for metadata information'''),
-    parser.add_argument('-d', '--datatype', default='raw', type=str, required=False, 
-    			 help='Data type to output, raw or tx')
     args = parser.parse_args(args=debug_args)
-    args.file1 = ' '.join(args.file1)
-    args.folder_gcnet = ' '.join(args.folder_gcnet)
-    args.folder_promice = ' '.join(args.folder_promice)
     return args
 
-
-def loadArr(infile):
-    if infile.split('.')[-1].lower() in 'csv':
-        df = pd.read_csv(infile)
-        df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None)
-        df = df.set_index('time')
-        ds = xr.Dataset.from_dataframe(df)
-
-    elif infile.split('.')[-1].lower() in 'nc':
-        ds = xr.open_dataset(infile)
-    
-    try:
-        name = ds.attrs['station_name'] 
-    except:
-        name = infile.split('/')[-1].split('.')[0].split('_hour')[0].split('_10min')[0]
-        
-    print(f'{name} array loaded from {infile}')
-    return ds, name
-
-def get_l3():
+def l2_to_l3():
     args = parse_arguments_l2_to_l3()
-                    
-    # Check files    
-    if os.path.isfile(args.file1): 
-        # Load L2 data arrays
-        ds1, n1 = loadArr(args.file1)
-        
-        # converts to L3:
-        # - derives sensible heat fluxes
-        # - more to come
-        ds1 = toL3(ds1)
+    logging.basicConfig(
+        format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
+        level=logging.INFO,
+        stream=sys.stdout,
+    )
+
+    # Define variables (either from file or pypromice package defaults)
+    if args.variables is None:
+        v = os.path.join(os.path.dirname(pypromice.__file__),'process/variables.csv')
+    else:
+        v = args.variables
 
-        # here will come the merging with historical data    
+    # Define metadata (either from file or pypromice package defaults)
+    if args.variables is None:
+        m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv')
     else:
-        print(f'Invalid file {args.file1}')
-        exit()
+        m = args.metadata
 
-    # Get hourly, daily and monthly datasets
-    print('Resampling L3 data to hourly, daily and monthly resolutions...')
-    l3_h = resampleL2(ds1, '60min')
-    l3_d = resampleL2(ds1, '1D')
-    l3_m = resampleL2(ds1, 'M')
-    
-    print(f'Adding variable information from {args.variables}...')
-        
-    # Load variables look-up table
-    var = getVars(args.variables)
-        	
-    # Round all values to specified decimals places
-    l3_h = roundValues(l3_h, var)
-    l3_d = roundValues(l3_d, var)
-    l3_m = roundValues(l3_m, var)
-        
-    # Get columns to keep
-    if hasattr(ds1, 'p_l'):
-        col_names = getColNames(var, 2, args.datatype.lower())  
+    # Define input path
+    station_name = args.config_file.split('/')[-1].split('.')[0] 
+    station_path = os.path.join(args.inpath, station_name)
+    if os.path.exists(station_path):
+        aws = AWS(args.config_file, station_path, v, m)
     else:
-        col_names = getColNames(var, 1, args.datatype.lower())    
+        aws = AWS(args.config_file, args.inpath, v, m)
 
-    # Assign station id
-    for l in [l3_h, l3_d, l3_m]:
-        l.attrs['station_id'] = n1
 
-    # Assign metadata
-    print(f'Adding metadata from {args.metadata}...')
-    m = getMeta(args.metadata)
-    l3_h = addMeta(l3_h, m)
-    l3_d = addMeta(l3_d, m)
-    l3_m = addMeta(l3_m, m)
-      
-    # Set up output path
-    out = os.path.join(args.outpath, site_id)
+    # Define Level 2 dataset from file
+    aws.L2 = xr.open_dataset(args.level_2)
 
-    # Write to files
-    writeAll(out, site_id, l3_h, l3_d, l3_m, col_names)
-    print(f'Files saved to {os.path.join(out, site_id)}...')
-# %%
+    # Perform Level 3 processing
+    aws.getL3()
+    
+    # Write Level 3 dataset to file if output directory given
+    if args.outpath is not None:
+        aws.writeL3(args.outpath)
+
 if __name__ == "__main__":  
     l2_to_l3()
-