diff --git a/pvlib/iotools/bsrn.py b/pvlib/iotools/bsrn.py index 43fdbe919f..ffe09191d5 100644 --- a/pvlib/iotools/bsrn.py +++ b/pvlib/iotools/bsrn.py @@ -159,8 +159,7 @@ def get_bsrn(station, start, end, username, password, end = pd.to_datetime(end) # Generate list files to download based on start/end (SSSMMYY.dat.gz) - filenames = pd.date_range( - start, end.replace(day=1) + pd.DateOffset(months=1), freq='1M')\ + filenames = pd.date_range(start, end, freq='1MS')\ .strftime(f"{station}%m%y.dat.gz").tolist() # Create FTP connection diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index b9922af4b8..13fd92e610 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -32,7 +32,7 @@ # Dictionary mapping time steps to CAMS time step format TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', - '1d': 'P01D', '1M': 'P01M'} + '1d': 'P01D', '1MS': 'P01M'} TIME_STEPS_IN_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} @@ -40,12 +40,12 @@ '0 year 0 month 0 day 0 h 15 min 0 s': '15min', # noqa '0 year 0 month 0 day 1 h 0 min 0 s': '1h', '0 year 0 month 1 day 0 h 0 min 0 s': '1d', - '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} + '0 year 1 month 0 day 0 h 0 min 0 s': '1MS'} def get_cams(latitude, longitude, start, end, email, identifier='mcclear', altitude=None, time_step='1h', time_ref='UT', verbose=False, - integrated=False, label=None, map_variables=True, + integrated=False, map_variables=True, server=URL, timeout=30): """Retrieve irradiance and clear-sky time series from CAMS. @@ -79,7 +79,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', altitude: float, optional Altitude in meters. If not specified, then the altitude is determined from the NASA SRTM database - time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' + time_step: str, {'1min', '15min', '1h', '1d', '1MS'}, default: '1h' Time step of the time series, either 1 minute, 15 minute, hourly, daily, or monthly. time_ref: str, {'UT', 'TST'}, default: 'UT' @@ -90,9 +90,6 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label : {'right', 'left'}, optional - Which bin edge label to label time-step with. The default is 'left' for - all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the DataFrame to pvlib variable names where applicable. See variable :const:`VARIABLE_MAP`. @@ -120,7 +117,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', ======================== ====== ========================================= **Mapped field names are returned when the map_variables argument is True** --------------------------------------------------------------------------- - Observation period str Beginning/end of time period + Observation period str Start of the time period (left labeled) TOA, ghi_extra float Horizontal radiation at top of atmosphere Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal @@ -231,12 +228,12 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', # Successful requests returns a csv data file else: fbuf = io.StringIO(res.content.decode('utf-8')) - data, metadata = parse_cams(fbuf, integrated=integrated, label=label, + data, metadata = parse_cams(fbuf, integrated=integrated, map_variables=map_variables) return data, metadata -def parse_cams(fbuf, integrated=False, label=None, map_variables=True): +def parse_cams(fbuf, integrated=False, map_variables=True): """ Parse a file-like buffer with data in the format of a CAMS Radiation or McClear file. The CAMS solar radiation services are described in [1]_. @@ -248,9 +245,6 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label : {'right', 'left'}, optional - Which bin edge label to label time-step with. The default is 'left' for - all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names where applicable. See variable :const:`VARIABLE_MAP`. @@ -262,6 +256,10 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): metadata: dict Metadata available in the file. + Notes + ----- + The index timestamps correspond to the start/left of the interval. + See Also -------- pvlib.iotools.read_cams, pvlib.iotools.get_cams @@ -301,26 +299,13 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): obs_period = data['Observation period'].str.split('/') # Set index as the start observation time (left) and localize to UTC - if (label == 'left') | ((label is None) & (time_step != '1M')): - data.index = pd.to_datetime(obs_period.str[0], utc=True) - # Set index as the stop observation time (right) and localize to UTC - # default label for monthly data is 'right' following Pandas' convention - elif (label == 'right') | ((label is None) & (time_step == '1M')): - data.index = pd.to_datetime(obs_period.str[1], utc=True) - - # For time_steps '1d' and '1M', drop timezone and round to nearest midnight - if (time_step == '1d') | (time_step == '1M'): - data.index = pd.DatetimeIndex(data.index.date) - # For monthly data with 'right' label, the index should be the last - # date of the month and not the first date of the following month - if (time_step == '1M') & (label != 'left'): - data.index = data.index - pd.Timedelta(days=1) + data.index = pd.to_datetime(obs_period.str[0], utc=True) if not integrated: # Convert radiation values from Wh/m2 to W/m2 integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS if c in data.columns] - if time_step == '1M': + if time_step.endswith('MS'): time_delta = (pd.to_datetime(obs_period.str[1]) - pd.to_datetime(obs_period.str[0])) hours = time_delta.dt.total_seconds()/60/60 @@ -336,7 +321,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): return data, metadata -def read_cams(filename, integrated=False, label=None, map_variables=True): +def read_cams(filename, integrated=False, map_variables=True): """ Read a CAMS Radiation or McClear file into a pandas DataFrame. @@ -349,9 +334,6 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label : {'right', 'left}, optional - Which bin edge label to label time-step with. The default is 'left' for - all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names where applicable. See variable :const:`VARIABLE_MAP`. @@ -368,11 +350,15 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): -------- pvlib.iotools.parse_cams, pvlib.iotools.get_cams + Notes + ----- + The index timestamps correspond to the start/left of the interval. + References ---------- .. [1] `CAMS solar radiation documentation `_ """ with open(str(filename), 'r') as fbuf: - content = parse_cams(fbuf, integrated, label, map_variables) + content = parse_cams(fbuf, integrated, map_variables) return content diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py index 728c3a7093..3bcb1f09e9 100644 --- a/pvlib/iotools/srml.py +++ b/pvlib/iotools/srml.py @@ -236,8 +236,7 @@ def get_srml(station, start, end, filetype='PO', map_variables=True, end = pd.to_datetime(end) # Generate list of months - months = pd.date_range( - start, end.replace(day=1) + pd.DateOffset(months=1), freq='1M') + months = pd.date_range(start, end, freq='1MS') months_str = months.strftime('%y%m') # Generate list of filenames diff --git a/tests/iotools/test_sodapro.py b/tests/iotools/test_sodapro.py index 93e3c39cff..df5bd8fe40 100644 --- a/tests/iotools/test_sodapro.py +++ b/tests/iotools/test_sodapro.py @@ -19,7 +19,7 @@ index_verbose = pd.date_range('2020-06-01 12', periods=4, freq='1min', tz='UTC') -index_monthly = pd.date_range('2020-01-01', periods=4, freq='1M') +index_monthly = pd.date_range('2020-01-01', periods=4, freq='1MS') dtypes_mcclear_verbose = [ @@ -172,19 +172,6 @@ def test_read_cams(testfile, index, columns, values, dtypes): assert_frame_equal(out, expected, check_less_precise=True) -def test_read_cams_integrated_unmapped_label(): - # Default label is 'left' for 1 minute time resolution, hence 1 minute is - # added for label='right' - expected = generate_expected_dataframe( - values_radiation_verbose_integrated, - columns_radiation_verbose_unmapped, - index_verbose+pd.Timedelta(minutes=1), dtypes=dtypes_radiation_verbose) - out, metadata = sodapro.read_cams(testfile_radiation_verbose, - integrated=True, label='right', - map_variables=False) - assert_frame_equal(out, expected, check_less_precise=True) - - def test_read_cams_metadata(): _, metadata = sodapro.read_cams(testfile_mcclear_monthly, integrated=False) assert metadata['Time reference'] == 'Universal time (UT)' @@ -193,7 +180,7 @@ def test_read_cams_metadata(): assert metadata['longitude'] == 12.5251 assert metadata['altitude'] == 39.0 assert metadata['radiation_unit'] == 'W/m^2' - assert metadata['time_step'] == '1M' + assert metadata['time_step'] == '1MS' @pytest.mark.parametrize('testfile,index,columns,values,dtypes,identifier', [ @@ -224,7 +211,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, email='pvlib-admin@googlegroups.com', identifier=identifier, altitude=80, - time_step='1M', + time_step='1MS', verbose=False, integrated=False) expected = generate_expected_dataframe(values, columns, index, dtypes) @@ -240,7 +227,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, email='pvlib-admin@googlegroups.com', identifier=identifier, altitude=80, - time_step='1M', + time_step='1MS', verbose=True) diff --git a/tests/test_clearsky.py b/tests/test_clearsky.py index ef943ac39d..0ef5dfeacd 100644 --- a/tests/test_clearsky.py +++ b/tests/test_clearsky.py @@ -224,8 +224,9 @@ def test_lookup_linke_turbidity_nointerp(): def test_lookup_linke_turbidity_months(): - times = pd.date_range(start='2014-04-01', end='2014-07-01', - freq='1M', tz='America/Phoenix') + times = pd.date_range(start='2014-05-01', end='2014-07-01', + freq='1MS', tz='America/Phoenix', + ) - pd.Timedelta(days=1) expected = pd.Series( np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times ) @@ -234,8 +235,9 @@ def test_lookup_linke_turbidity_months(): def test_lookup_linke_turbidity_months_leapyear(): - times = pd.date_range(start='2016-04-01', end='2016-07-01', - freq='1M', tz='America/Phoenix') + times = pd.date_range(start='2016-05-01', end='2016-07-01', + freq='1MS', tz='America/Phoenix', + ) - pd.Timedelta(days=1) expected = pd.Series( np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times ) @@ -245,14 +247,16 @@ def test_lookup_linke_turbidity_months_leapyear(): def test_lookup_linke_turbidity_nointerp_months(): times = pd.date_range(start='2014-04-10', end='2014-07-10', - freq='1M', tz='America/Phoenix') + freq='1MS', tz='America/Phoenix', + ) - pd.Timedelta(days=1) expected = pd.Series(np.array([2.85, 2.95, 3.]), index=times) out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875, interp_turbidity=False) assert_series_equal(expected, out) # changing the dates shouldn't matter if interp=False times = pd.date_range(start='2014-04-05', end='2014-07-05', - freq='1M', tz='America/Phoenix') + freq='1MS', tz='America/Phoenix', + ) - pd.Timedelta(days=1) out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875, interp_turbidity=False) assert_series_equal(expected, out)