Merge pull request #88 from openclimatefix/issue/85-perciever-nwp

Issue/85 perciever nwp

Author: peterdudfield

configs/experiment/perceiver_conv3d_sat_nwp.yaml

@@ -0,0 +1,24 @@
+# @package _global_
+
+# to execute this experiment run:
+# python run.py experiment=example_simple.yaml
+
+defaults:
+  - override /trainer: default.yaml # choose trainer from 'configs/trainer/'
+  - override /model: perceiver_conv3d_sat_nwp.yaml
+  - override /datamodule: netcdf_datamodule_gcp.yaml
+  - override /callbacks: default.yaml
+  - override /logger: neptune.yaml
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+
+seed: 518
+
+trainer:
+  min_epochs: 1
+  max_epochs: 50
+
+datamodule:
+  n_train_data: 10000
+  n_val_data: 1000

configs/experiment/perceiver_sat_nwp.yaml

@@ -0,0 +1,24 @@
+# @package _global_
+
+# to execute this experiment run:
+# python run.py experiment=example_simple.yaml
+
+defaults:
+  - override /trainer: default.yaml # choose trainer from 'configs/trainer/'
+  - override /model: perceiver_sat_nwp.yaml
+  - override /datamodule: netcdf_datamodule_gcp.yaml
+  - override /callbacks: default.yaml
+  - override /logger: neptune.yaml
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+
+seed: 518
+
+trainer:
+  min_epochs: 1
+  max_epochs: 10
+
+datamodule:
+  n_train_data: 10000
+  n_val_data: 1000

configs/model/perceiver.yaml

@@ -5,4 +5,5 @@ history_minutes: 60
 batch_size: 8
 num_latents: 128
 latent_dim: 64
-embedding_dem: 16
+embedding_dem: 16
+output_variable: gsp_yield

configs/model/perceiver_conv3d_sat_nwp.yaml

@@ -0,0 +1,11 @@
+_target_: predict_pv_yield.models.perceiver.perceiver_conv3d_nwp_sat.Model
+
+forecast_minutes: 30
+history_minutes: 60
+batch_size: 32
+num_latents: 24
+latent_dim: 24
+embedding_dem: 0
+output_variable: gsp_yield
+conv3d_channels: 8
+use_future_satellite_images: 0

configs/model/perceiver_sat_nwp.yaml

@@ -0,0 +1,9 @@
+_target_: predict_pv_yield.models.perceiver.perceiver_nwp_sat.Model
+
+forecast_minutes: 30
+history_minutes: 60
+batch_size: 8
+num_latents: 128
+latent_dim: 64
+embedding_dem: 0
+output_variable: gsp_yield

experiments/2021-09/2021-09-03/experiments.txt

@@ -0,0 +1,14 @@
+Ran perciever RNN model
+
+https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-245/charts
+
+Includes validation images, so we can see how the model is perforaming after in epoch
+
+due memory of gpu had to go
+
+forecast_len: 12
+history_len: 6
+batch_size: 8
+num_latents: 32
+latent_dim: 32
+embedding_dem: 10

experiments/2021-09/2021-09-27/experiments.txt

@@ -22,4 +22,4 @@ Using both sat and nwp into two separate convolution nets.
 
 https://app.neptune.ai/OpenClimateFix/predict-pv-yield/e/PRED-320
 
-# TODO Currently running
+MAE = 0.0376 - this was after 10 epochs, and I think it was still going down.

experiments/2021-09/2021-09-28/experiments.txt

@@ -0,0 +1,38 @@
+1. Perceiver NWP SAT
+
+https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-331/monitoring
+
+Ran with
+- batch_size of 6, as GPU was out of memory
+- num_latents: int = 64,
+- latent_dim: int = 64,
+- embedding_dem: int = 0,
+
+Each epoch takes about 3 hours
+
+Decided to stop it earlier
+
+1. Perceiver Conv3d NWP SAT
+
+Idea is to have 1 conv3d + max pool later before the perceiver model
+https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-331/monitoring
+
+Conv3d did not make much memory difference, the biggest being, changing the
+- num_latents
+- latent_dim
+
+To get batch 32, set
+- num_latents = 16
+- latent_dim = 16
+- PERCEIVER_OUTPUT_SIZE = 512
+OR
+To get batch 32, set
+- num_latents = 24
+- latent_dim = 24
+- PERCEIVER_OUTPUT_SIZE = 128
+
+https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-349/monitoring
+
+~ 4 hours per epoch
+
+MAE = 0.0308 (after 10 epochs)

experiments/2021-10/2021-10-01/experiment.txt

@@ -0,0 +1,9 @@
+1. Perceiver Conv3d NWP SAT
+
+No future satellite images
+
+https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-378/charts
+
+~ 4 hours per epoch
+
+MAE = 0.0365 (after 22 epochs), compared to MAE 0.0304 when future satellite images were included

predict_pv_yield/data/dataloader.py

@@ -1,13 +1,14 @@
 import os
 from nowcasting_dataset.dataset.datasets import NetCDFDataset, worker_init_fn
 from nowcasting_dataset.dataset.validate import FakeDataset
-from nowcasting_dataset.config.model import Configuration
+from nowcasting_dataset.config.load import load_yaml_configuration
 from typing import Tuple
 import logging
 import torch
 from pytorch_lightning import LightningDataModule
 
 
+
 _LOG = logging.getLogger(__name__)
 _LOG.setLevel(logging.DEBUG)
 
@@ -20,6 +21,8 @@ def get_dataloaders(
     data_path="prepared_ML_training_data/v4/",
 ) -> Tuple:
 
+    configuration = load_yaml_configuration(filename=f'{data_path}/configuration.yaml')
+
     data_module = NetCDFDataModule(
         temp_path=temp_path, data_path=data_path, cloud=cloud, n_train_data=n_train_data, n_val_data=n_validation_data
     )
@@ -70,6 +73,10 @@ def __init__(
         self.pin_memory = pin_memory
         self.fake_data = fake_data
 
+        filename = os.path.join(data_path, 'configuration.yaml')
+        _LOG.debug(f'Will be loading the configuration file {filename}')
+        self.configuration = load_yaml_configuration(filename=filename)
+
         self.dataloader_config = dict(
             pin_memory=self.pin_memory,
             num_workers=self.num_workers,
@@ -83,40 +90,43 @@ def __init__(
 
     def train_dataloader(self):
         if self.fake_data:
-            train_dataset = FakeDataset(configuration=Configuration())
+            train_dataset = FakeDataset(configuration=self.configuration)
         else:
             train_dataset = NetCDFDataset(
                 self.n_train_data,
                 os.path.join(self.data_path, "train"),
                 os.path.join(self.temp_path, "train"),
                 cloud=self.cloud,
+                configuration=self.configuration
             )
 
         return torch.utils.data.DataLoader(train_dataset, **self.dataloader_config)
 
     def val_dataloader(self):
         if self.fake_data:
-            val_dataset = FakeDataset(configuration=Configuration())
+            val_dataset = FakeDataset(configuration=self.configuration)
         else:
             val_dataset = NetCDFDataset(
                 self.n_val_data,
                 os.path.join(self.data_path, "validation"),
                 os.path.join(self.temp_path, "validation"),
                 cloud=self.cloud,
+                configuration=self.configuration
             )
 
         return torch.utils.data.DataLoader(val_dataset, **self.dataloader_config)
 
     def test_dataloader(self):
         if self.fake_data:
-            test_dataset = FakeDataset(configuration=Configuration())
+            test_dataset = FakeDataset(configuration=self.configuration)
         else:
             # TODO need to change this to a test folder
             test_dataset = NetCDFDataset(
                 self.n_val_data,
                 os.path.join(self.data_path, "validation"),
                 os.path.join(self.temp_path, "validation"),
                 cloud=self.cloud,
+                configuration=self.configuration
             )
 
         return torch.utils.data.DataLoader(test_dataset, **self.dataloader_config)

predict_pv_yield/models/base_model.py

@@ -114,8 +114,8 @@ def training_step(self, batch, batch_idx):
 
     def validation_step(self, batch, batch_idx):
         INTERESTING_EXAMPLES = (1, 5, 6, 7, 9, 11, 17, 19)
-        name = f"validation/plot/epoch{self.current_epoch}"
-        if batch_idx == 0:
+        name = f"validation/plot/epoch_{self.current_epoch}_{batch_idx}"
+        if batch_idx in [0, 1, 2, 3, 4]:
 
             # get model outputs
             model_output = self(batch)

predict_pv_yield/models/conv3d/model_sat_nwp.py

@@ -80,7 +80,6 @@ def __init__(
             * ((image_size_pixels - 2 * self.number_of_conv3d_layers) ** 2)
             * (self.forecast_len_5 + self.history_len_5 + 1 - 2 * self.number_of_conv3d_layers)
         )
-        print(self.cnn_output_size)
 
         # conv0
         self.sat_conv0 = nn.Conv3d(

predict_pv_yield/models/perceiver/perceiver.py

@@ -178,9 +178,14 @@ def forward(self, x):
             dim=2,
         )
 
-        # take the history of the pv yield of this system,
-        pv_yield_history = x["pv_yield"][0 : self.batch_size][:, : self.history_len_5 + 1, 0].unsqueeze(-1)
-        encoder_input = torch.cat((rnn_input[:, : self.history_len_5 + 1], pv_yield_history), dim=2)
+        if self.output_variable == 'pv_yield':
+            # take the history of the pv yield of this system,
+            pv_yield_history = x["pv_yield"][0: self.batch_size][:, : self.history_len_5 + 1, 0].unsqueeze(-1)
+            encoder_input = torch.cat((rnn_input[:, : self.history_len_5 + 1], pv_yield_history), dim=2)
+        elif self.output_variable == 'gsp_yield':
+            # take the history of the gsp yield of this system,
+            gsp_history = x[self.output_variable][0: self.batch_size][:, : self.history_len_30 + 1, 0].unsqueeze(-1)
+            encoder_input = torch.cat((rnn_input[:, : self.history_len_30 + 1], gsp_history), dim=2)
 
         encoder_output, encoder_hidden = self.encoder_rnn(encoder_input)
         decoder_output, _ = self.decoder_rnn(rnn_input[:, -self.forecast_len :], encoder_hidden)