diff --git a/tests/conftest.py b/tests/conftest.py index 70bca191..cb01f6b5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -167,7 +167,7 @@ def sample_site_batch(): prefetch_factor=None, train_period=[None, None], val_period=[None, None], - sample_dir="tests/test_data/presaved_site_samples", + sample_dir="tests/test_data/presaved_samples_site", ) batch = next(iter(dm.train_dataloader())) return batch diff --git a/tests/test_data/presaved_samples/data_configuration.yaml b/tests/test_data/presaved_samples/data_configuration.yaml deleted file mode 100644 index c65402eb..00000000 --- a/tests/test_data/presaved_samples/data_configuration.yaml +++ /dev/null @@ -1,130 +0,0 @@ -general: - description: Config for producing batches on GCP - name: gcp_pvnet - -input_data: - gsp: - gsp_zarr_path: /mnt/disks/nwp_rechunk/pv_gsp/pvlive_gsp.zarr - interval_start_minutes: -120 - interval_end_minutes: 480 - time_resolution_minutes: 30 - # A random value from the list below will be chosen as the delay when dropout is used - # If set to null no dropout is applied. Only values before t0 are dropped out for GSP. - # Values after t0 are assumed as targets and cannot be dropped. - dropout_timedeltas_minutes: null - dropout_fraction: 0 # Fraction of samples with dropout - - nwp: - ukv: - nwp_provider: ukv - nwp_zarr_path: - - /mnt/disks/nwp_rechunk/nwp/ukv/UKV_intermediate_version_7.1.zarr - - /mnt/disks/nwp_rechunk/nwp/ukv/UKV_2021_missing.zarr - - /mnt/disks/nwp_rechunk/nwp/ukv/UKV_2022.zarr - - /mnt/disks/nwp_rechunk/nwp/ukv/UKV_2023.zarr - interval_start_minutes: -120 - interval_end_minutes: 480 - time_resolution_minutes: 60 - nwp_channels: - # These variables exist in the CEDA training set and in the live MetOffice live service - - t # 2-metre temperature - - dswrf # downwards short-wave radiation flux - - dlwrf # downwards long-wave radiation flux - - hcc # high cloud cover - - mcc # medium cloud cover - - lcc # low cloud cover - - sde # snow depth water equivalent - - r # relative humidty - - vis # visibility - - si10 # 10-metre wind speed - - prate # precipitation rate - nwp_image_size_pixels_height: 24 - nwp_image_size_pixels_width: 24 - dropout_timedeltas_minutes: [-180] - dropout_fraction: 1.0 - max_staleness_minutes: null - - ecmwf: - nwp_provider: ecmwf - nwp_zarr_path: /mnt/disks/nwp_rechunk/nwp/ecmwf/UK_v2.zarr - interval_start_minutes: -120 - interval_end_minutes: 480 - time_resolution_minutes: 60 - - nwp_channels: - - t2m # 2-metre temperature - - dswrf # downwards short-wave radiation flux - - dlwrf # downwards long-wave radiation flux - - hcc # high cloud cover - - mcc # medium cloud cover - - lcc # low cloud cover - - tcc # total cloud cover - - sde # snow depth water equivalent - - sr # direct solar radiation - - duvrs # downwards UV radiation at surface - - u10 # 10-metre U component of wind speed - - v10 # 10-metre V component of wind speed - - # The following channels are accumulated and need to be diffed - nwp_accum_channels: - - dswrf - - dlwrf - - sr - - duvrs - - nwp_image_size_pixels_height: 12 # roughly equivalent to ukv 48 - nwp_image_size_pixels_width: 12 # roughly equivalent to ukv 48 - dropout_timedeltas_minutes: [-360] - dropout_fraction: 1.0 - max_staleness_minutes: null - - sat_pred: - nwp_provider: sat_pred - nwp_zarr_path: /mnt/disks/sat_preds/simvp_preds/*.zarr - interval_start_minutes: 15 - interval_end_minutes: 180 - time_resolution_minutes: 15 - nwp_channels: - - IR_016 - - IR_039 - - IR_087 - - IR_097 - - IR_108 - - IR_120 - - IR_134 - - VIS006 - - VIS008 - - WV_062 - - WV_073 - nwp_image_size_pixels_height: 24 - nwp_image_size_pixels_width: 24 - dropout_timedeltas_minutes: null - dropout_fraction: 0 - max_staleness_minutes: null - - satellite: - satellite_zarr_path: - - /mnt/disks/nwp_rechunk/sat/2019_nonhrv.zarr - - /mnt/disks/nwp_rechunk/sat/2020_nonhrv.zarr - - /mnt/disks/nwp_rechunk/sat/2021_nonhrv.zarr - - /mnt/disks/nwp_rechunk/sat/2022_nonhrv.zarr - - /mnt/disks/nwp_rechunk/sat/2023_nonhrv.zarr - interval_start_minutes: -30 - interval_end_minutes: 0 - time_resolution_minutes: 5 - satellite_channels: - - IR_016 - - IR_039 - - IR_087 - - IR_097 - - IR_108 - - IR_120 - - IR_134 - - VIS006 - - VIS008 - - WV_062 - - WV_073 - satellite_image_size_pixels_height: 24 - satellite_image_size_pixels_width: 24 - dropout_timedeltas_minutes: null - dropout_fraction: 0. diff --git a/tests/test_data/presaved_samples/datamodule.yaml b/tests/test_data/presaved_samples/datamodule.yaml deleted file mode 100644 index 4b661250..00000000 --- a/tests/test_data/presaved_samples/datamodule.yaml +++ /dev/null @@ -1,18 +0,0 @@ -_target_: pvnet.data.datamodule.DataModule -configuration: /home/jamesfulton/repos/PVNet/configs/datamodule/configuration/gcp_configuration.yaml -num_workers: 160 -prefetch_factor: 1 -batch_size: 8 -train_period: - - null - - "2022-05-07" -val_period: - - "2023-01-01" - - "2023-05-08" -test_period: - - "2022-05-08" - - "2023-05-08" -seed: ${seed} -sample_output_dir: /mnt/disks/extra_batches/samples_with_simvp_sat_pred -num_train_samples: 1600000 -num_val_samples: 64000 diff --git a/tests/test_data/presaved_samples/train/00000000.pt b/tests/test_data/presaved_samples/train/00000000.pt deleted file mode 100644 index 292e1a65..00000000 Binary files a/tests/test_data/presaved_samples/train/00000000.pt and /dev/null differ diff --git a/tests/test_data/presaved_samples/train/00000001.pt b/tests/test_data/presaved_samples/train/00000001.pt deleted file mode 100644 index 4a46e05b..00000000 Binary files a/tests/test_data/presaved_samples/train/00000001.pt and /dev/null differ diff --git a/tests/test_data/presaved_samples/train/00000002.pt b/tests/test_data/presaved_samples/train/00000002.pt deleted file mode 100644 index f7fa3986..00000000 Binary files a/tests/test_data/presaved_samples/train/00000002.pt and /dev/null differ diff --git a/tests/test_data/presaved_samples/train/00000003.pt b/tests/test_data/presaved_samples/train/00000003.pt deleted file mode 100644 index 6c4a4666..00000000 Binary files a/tests/test_data/presaved_samples/train/00000003.pt and /dev/null differ diff --git a/tests/test_data/presaved_samples/train/00000004.pt b/tests/test_data/presaved_samples/train/00000004.pt deleted file mode 100644 index 13169acb..00000000 Binary files a/tests/test_data/presaved_samples/train/00000004.pt and /dev/null differ diff --git a/tests/test_data/presaved_samples/train/00000005.pt b/tests/test_data/presaved_samples/train/00000005.pt deleted file mode 100644 index 670f5978..00000000 Binary files a/tests/test_data/presaved_samples/train/00000005.pt and /dev/null differ diff --git a/tests/test_data/presaved_samples/train/00000006.pt b/tests/test_data/presaved_samples/train/00000006.pt deleted file mode 100644 index 8e238b6b..00000000 Binary files a/tests/test_data/presaved_samples/train/00000006.pt and /dev/null differ diff --git a/tests/test_data/presaved_samples/train/00000007.pt b/tests/test_data/presaved_samples/train/00000007.pt deleted file mode 100644 index c62cd85c..00000000 Binary files a/tests/test_data/presaved_samples/train/00000007.pt and /dev/null differ diff --git a/tests/test_data/presaved_site_samples/data_configuration.yaml b/tests/test_data/presaved_samples_site/data_configuration.yaml similarity index 100% rename from tests/test_data/presaved_site_samples/data_configuration.yaml rename to tests/test_data/presaved_samples_site/data_configuration.yaml diff --git a/tests/test_data/presaved_site_samples/datamodule.yaml b/tests/test_data/presaved_samples_site/datamodule.yaml similarity index 100% rename from tests/test_data/presaved_site_samples/datamodule.yaml rename to tests/test_data/presaved_samples_site/datamodule.yaml diff --git a/tests/test_data/presaved_site_samples/train/00000000.nc b/tests/test_data/presaved_samples_site/train/00000000.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000000.nc rename to tests/test_data/presaved_samples_site/train/00000000.nc diff --git a/tests/test_data/presaved_site_samples/train/00000001.nc b/tests/test_data/presaved_samples_site/train/00000001.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000001.nc rename to tests/test_data/presaved_samples_site/train/00000001.nc diff --git a/tests/test_data/presaved_site_samples/train/00000002.nc b/tests/test_data/presaved_samples_site/train/00000002.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000002.nc rename to tests/test_data/presaved_samples_site/train/00000002.nc diff --git a/tests/test_data/presaved_site_samples/train/00000003.nc b/tests/test_data/presaved_samples_site/train/00000003.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000003.nc rename to tests/test_data/presaved_samples_site/train/00000003.nc diff --git a/tests/test_data/presaved_site_samples/train/00000004.nc b/tests/test_data/presaved_samples_site/train/00000004.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000004.nc rename to tests/test_data/presaved_samples_site/train/00000004.nc diff --git a/tests/test_data/presaved_site_samples/train/00000005.nc b/tests/test_data/presaved_samples_site/train/00000005.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000005.nc rename to tests/test_data/presaved_samples_site/train/00000005.nc diff --git a/tests/test_data/presaved_site_samples/train/00000006.nc b/tests/test_data/presaved_samples_site/train/00000006.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000006.nc rename to tests/test_data/presaved_samples_site/train/00000006.nc diff --git a/tests/test_data/presaved_site_samples/train/00000007.nc b/tests/test_data/presaved_samples_site/train/00000007.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000007.nc rename to tests/test_data/presaved_samples_site/train/00000007.nc diff --git a/tests/test_data/presaved_site_samples/train/00000008.nc b/tests/test_data/presaved_samples_site/train/00000008.nc similarity index 100% rename from tests/test_data/presaved_site_samples/train/00000008.nc rename to tests/test_data/presaved_samples_site/train/00000008.nc diff --git a/tests/test_data/presaved_site_samples/val/00000000.nc b/tests/test_data/presaved_samples_site/val/00000000.nc similarity index 100% rename from tests/test_data/presaved_site_samples/val/00000000.nc rename to tests/test_data/presaved_samples_site/val/00000000.nc diff --git a/tests/test_data/presaved_site_samples/val/00000001.nc b/tests/test_data/presaved_samples_site/val/00000001.nc similarity index 100% rename from tests/test_data/presaved_site_samples/val/00000001.nc rename to tests/test_data/presaved_samples_site/val/00000001.nc diff --git a/tests/test_data/presaved_site_samples/val/00000002.nc b/tests/test_data/presaved_samples_site/val/00000002.nc similarity index 100% rename from tests/test_data/presaved_site_samples/val/00000002.nc rename to tests/test_data/presaved_samples_site/val/00000002.nc diff --git a/tests/test_data/presaved_site_samples/val/00000003.nc b/tests/test_data/presaved_samples_site/val/00000003.nc similarity index 100% rename from tests/test_data/presaved_site_samples/val/00000003.nc rename to tests/test_data/presaved_samples_site/val/00000003.nc diff --git a/tests/test_data/presaved_site_samples/val/00000004.nc b/tests/test_data/presaved_samples_site/val/00000004.nc similarity index 100% rename from tests/test_data/presaved_site_samples/val/00000004.nc rename to tests/test_data/presaved_samples_site/val/00000004.nc diff --git a/tests/test_data/sample_site_batches/data_configuration.yaml b/tests/test_data/sample_site_batches/data_configuration.yaml deleted file mode 100644 index cf603063..00000000 --- a/tests/test_data/sample_site_batches/data_configuration.yaml +++ /dev/null @@ -1,93 +0,0 @@ -general: - description: WindNet configuration for Leonardo - name: windnet_india - -input_data: - default_forecast_minutes: 2880 - default_history_minutes: 60 - nwp: - ecmwf: - # Path to ECMWF NWP data in zarr format - # n.b. It is not necessary to use multiple or any NWP data. These entries can be removed - nwp_zarr_path: "/mnt/storage_b/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/NWP/ECMWF/nw-india/zarr/20*.zarr.zip" - history_minutes: 60 - forecast_minutes: 2880 - time_resolution_minutes: 60 - nwp_channels: - #- hcc - #- lcc - #- mcc - #- prate - #- sde - #- sr - - t2m - #- tcc - - u10 - - u100 - - u200 - - v10 - - v100 - - v200 - nwp_image_size_pixels_height: 168 # roughtly equivalent to ukv 24 pixels - nwp_image_size_pixels_width: 168 - x_dim_name: "longitude" - y_dim_name: "latitude" - nwp_provider: "ecmwf" - dropout_timedeltas_minutes: [-360] # 6 hours - # Dropout applied with this probability - dropout_fraction: 1.0 - #start_datetime: "2021-01-01 00:00:00" - #end_datetime: "2024-01-01 00:00:00" - # excarta: - # nwp_zarr_path: "/mnt/storage_b/nwp/excarta/hindcast.zarr" - # history_minutes: 60 - # forecast_minutes: 2160 # 48 hours won't work much, as its only midnight ones, maybe 24 hours to ensure more coverage - # time_resolution_minutes: 60 - # nwp_channels: - # - 10u - # - 100u - # - 10v - # - 100v - # - surface_pressure - # #- mean_sea_level_pressure - # nwp_image_size_pixels_height: 64 # roughtly equivalent to ukv 24 pixels - # nwp_image_size_pixels_width: 64 - # nwp_provider: "excarta" - # x_dim_name: "longitude" - # y_dim_name: "latitude" - # dropout_timedeltas_minutes: [ -360 ] # 6 hours - # # Dropout applied with this probability - # dropout_fraction: 1.0 - wind: - wind_files_groups: - - label: india - wind_filename: /mnt/storage_ssd_4tb/india_wind_data.nc - wind_metadata_filename: /mnt/storage_ssd_4tb/india_wind_metadata.csv - n_wind_systems_per_example: 1 - #start_datetime: "2021-01-01 00:00:00" - #end_datetime: "2024-01-01 00:00:00" - sensor: - #sensor_files_groups: - # - label: meteomatics - sensor_filename: "/mnt/storage_b/nwp/meteomatics/nw_india/wind*.zarr.zip" - history_minutes: 60 - forecast_minutes: 2880 - #n_sensor_systems_per_example: 26 - time_resolution_minutes: 15 - #x_dim_name: "lon" - #y_dim_name: "lat" - sensor_variables: - - 100u - - 100v - - 10u - - 10v - - 200u - - 200v - - cape:Jkg - - air_density_25m:kgm3 - - air_density_10m:kgm3 - - air_density_100m:kgm3 - - air_density_200m:kgm3 - - wind_gusts_200m:ms - - wind_gusts_100m:ms - - wind_gusts_10m:ms diff --git a/tests/test_data/sample_site_batches/datamodule.yaml b/tests/test_data/sample_site_batches/datamodule.yaml deleted file mode 100644 index 92615f74..00000000 --- a/tests/test_data/sample_site_batches/datamodule.yaml +++ /dev/null @@ -1,15 +0,0 @@ -_target_: pvnet.data.wind_datamodule.WindDataModule -configuration: /home/jacob/PVNet/configs/datamodule/configuration/leonardo_wind_configuration.yaml -num_workers: 1 -prefetch_factor: 2 -batch_size: 8 -batch_dir: /mnt/storage_a/windnet_india_batches_large_meteomatics -train_period: - - "2019-01-01" - - "2022-11-29" -val_period: - - "2022-12-01" - - "2023-12-31" -test_period: - - "2023-09-01" - - "2023-12-31" diff --git a/tests/test_data/sample_site_batches/train/000000.nc b/tests/test_data/sample_site_batches/train/000000.nc deleted file mode 100644 index cc485da7..00000000 Binary files a/tests/test_data/sample_site_batches/train/000000.nc and /dev/null differ diff --git a/tests/test_data/sample_site_batches/train/000001.nc b/tests/test_data/sample_site_batches/train/000001.nc deleted file mode 100644 index cc485da7..00000000 Binary files a/tests/test_data/sample_site_batches/train/000001.nc and /dev/null differ