[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Author: pre-commit-ci[bot]

LICENSE

@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

README.md

@@ -16,4 +16,3 @@ pip install -e .
 ```
 
 Also download `nowcasting_dataset` and install in the `predict_pv_yield` conda environment using `pip install -e .` from within the `nowcasting_dataset` directory.
-

configs/experiment/conv3d.yaml

@@ -24,4 +24,4 @@ datamodule:
   n_val_data: 1000
 
 model:
-  conv3d_channels: 32
+  conv3d_channels: 32

configs/experiment/conv3d_sat_nwp.yaml

@@ -24,4 +24,4 @@ datamodule:
   n_val_data: 1000
 
 model:
-  conv3d_channels: 32
+  conv3d_channels: 32

configs/experiment/example_simple.yaml

@@ -24,4 +24,4 @@ datamodule:
   n_val_data: 2
   fake_data: 1
 
-validate_only: '1'  # by putting this key in the config file, the model does not get trained.
+validate_only: '1'  # by putting this key in the config file, the model does not get trained.

configs/hparams_search/conv3d_optuna.yaml

@@ -53,4 +53,3 @@ hydra:
       model.fc3_output_features:
         type: categorical
         choices: [8, 16, 32, 64, 128 ]
-

configs/model/conv3d.yaml

@@ -12,4 +12,4 @@ conv3d_channels: 32
 fc1_output_features: 128
 fc2_output_features: 128
 fc3_output_features: 64
-output_variable: gsp_yield
+output_variable: gsp_yield

configs/model/conv3d_sat_nwp.yaml

@@ -12,4 +12,4 @@ conv3d_channels: 32
 fc1_output_features: 128
 fc2_output_features: 128
 fc3_output_features: 64
-output_variable: gsp_yield
+output_variable: gsp_yield

configs/model/perceiver.yaml

@@ -6,4 +6,4 @@ batch_size: 8
 num_latents: 128
 latent_dim: 64
 embedding_dem: 16
-output_variable: gsp_yield
+output_variable: gsp_yield

configs/model/perceiver_conv3d_sat_nwp.yaml

@@ -8,4 +8,4 @@ latent_dim: 24
 embedding_dem: 0
 output_variable: gsp_yield
 conv3d_channels: 8
-use_future_satellite_images: 0
+use_future_satellite_images: 0

configs/model/perceiver_sat_nwp.yaml

@@ -6,4 +6,4 @@ batch_size: 8
 num_latents: 128
 latent_dim: 64
 embedding_dem: 0
-output_variable: gsp_yield
+output_variable: gsp_yield

configs/readme.md

@@ -1,8 +1,7 @@
 The following folders how the configuration files
 
-This idea is copied from 
+This idea is copied from
 https://github.com/ashleve/lightning-hydra-template/blob/main/configs/experiment/example_simple.yaml
 
 run experiments by:
 `python run.py experiment=example_simple `
-

environment.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - python>=3.9
   - pip
-  
+
   # Scientific Python
   - numpy
   - pandas
@@ -14,10 +14,10 @@ dependencies:
   - ipykernel
   - pyproj
   - h5netcdf
-  
+
   # Cloud & distributed compute
   - gcsfs
-  
+
   # Machine learning
   - pytorch::pytorch  # explicitly specify pytorch channel to prevent conda from using conda-forge for pytorch, and hence installing the CPU-only version.
   - pytorch-lightning
@@ -27,7 +27,7 @@ dependencies:
   - flake8
   - jedi
   - mypy
-  
+
   - pip:
     - neptune-client[pytorch-lightning]
     - tilemapbase  # For plotting human-readable geographical maps.

experiments/001_CNN_concat_all_timesteps_as_channels.py

@@ -111,21 +111,21 @@ def plot_example(batch, model_output, example_i: int=0, border: int=0):
     fig = plt.figure(figsize=(20, 20))
     ncols=4
     nrows=2
-    
+
     # Satellite data
     extent = (
-        float(batch['sat_x_coords'][example_i, 0].cpu().numpy()), 
-        float(batch['sat_x_coords'][example_i, -1].cpu().numpy()), 
-        float(batch['sat_y_coords'][example_i, -1].cpu().numpy()), 
+        float(batch['sat_x_coords'][example_i, 0].cpu().numpy()),
+        float(batch['sat_x_coords'][example_i, -1].cpu().numpy()),
+        float(batch['sat_y_coords'][example_i, -1].cpu().numpy()),
         float(batch['sat_y_coords'][example_i, 0].cpu().numpy()))  # left, right, bottom, top
-    
+
     def _format_ax(ax):
         #ax.set_xlim(extent[0]-border, extent[1]+border)
         #ax.set_ylim(extent[2]-border, extent[3]+border)
         # ax.coastlines(color='black')
         ax.scatter(
-            batch['x_meters_center'][example_i].cpu(), 
-            batch['y_meters_center'][example_i].cpu(), 
+            batch['x_meters_center'][example_i].cpu(),
+            batch['y_meters_center'][example_i].cpu(),
             s=500, color='white', marker='x')
 
     ax = fig.add_subplot(nrows, ncols, 1) #, projection=ccrs.OSGB(approx=False))
@@ -140,12 +140,12 @@ def _format_ax(ax):
     ax.imshow(sat_data[params['history_len']+1], extent=extent, interpolation='none', vmin=sat_min, vmax=sat_max)
     ax.set_title('t = 0')
     _format_ax(ax)
-    
+
     ax = fig.add_subplot(nrows, ncols, 3)
     ax.imshow(sat_data[-1], extent=extent, interpolation='none', vmin=sat_min, vmax=sat_max)
     ax.set_title('t = {}'.format(params['forecast_len']))
     _format_ax(ax)
-    
+
     ax = fig.add_subplot(nrows, ncols, 4)
     lat_lon_bottom_left = osgb_to_lat_lon(extent[0], extent[2])
     lat_lon_top_right = osgb_to_lat_lon(extent[1], extent[3])
@@ -163,7 +163,7 @@ def _format_ax(ax):
     ax = fig.add_subplot(nrows, ncols, 5)
     nwp_dt_index = pd.to_datetime(batch['nwp_target_time'][example_i].cpu().numpy(), unit='s')
     pd.DataFrame(
-        batch['nwp'][example_i, :, :, 0, 0].T.cpu().numpy(), 
+        batch['nwp'][example_i, :, :, 0, 0].T.cpu().numpy(),
         index=nwp_dt_index,
         columns=params['nwp_channels']).plot(ax=ax)
     ax.set_title('NWP')
@@ -194,14 +194,14 @@ def _format_ax(ax):
     ax.legend()
 
     # fig.tight_layout()
-    
+
     return fig
 
 
 # In[11]:
 
 
-# plot_example(batch, model_output, example_i=20);  
+# plot_example(batch, model_output, example_i=20);
 
 
 # In[12]:
@@ -234,89 +234,89 @@ def __init__(
         self,
         history_len = params['history_len'],
         forecast_len = params['forecast_len'],
-        
+
     ):
         super().__init__()
         self.history_len = history_len
         self.forecast_len = forecast_len
-        
+
         self.sat_conv1 = nn.Conv2d(in_channels=history_len+6, out_channels=CHANNELS, kernel_size=KERNEL)#, groups=history_len+1)
         self.sat_conv2 = nn.Conv2d(in_channels=CHANNELS, out_channels=CHANNELS, kernel_size=KERNEL) #, groups=CHANNELS//2)
         self.sat_conv3 = nn.Conv2d(in_channels=CHANNELS, out_channels=CHANNELS, kernel_size=KERNEL) #, groups=CHANNELS)
 
         self.maxpool = nn.MaxPool2d(kernel_size=KERNEL)
-        
+
         self.fc1 = nn.Linear(
-            in_features=CHANNELS * 11 * 11, 
+            in_features=CHANNELS * 11 * 11,
             out_features=256)
-        
+
         self.fc2 = nn.Linear(in_features=256 + EMBEDDING_DIM + NWP_SIZE + N_DATETIME_FEATURES + history_len+1, out_features=128)
         #self.fc2 = nn.Linear(in_features=EMBEDDING_DIM + N_DATETIME_FEATURES, out_features=128)
         self.fc3 = nn.Linear(in_features=128, out_features=128)
         self.fc4 = nn.Linear(in_features=128, out_features=128)
         self.fc5 = nn.Linear(in_features=128, out_features=params['forecast_len'])
-        
+
         if EMBEDDING_DIM:
             self.pv_system_id_embedding = nn.Embedding(
                 num_embeddings=len(data_module.pv_data_source.pv_metadata),
                 embedding_dim=EMBEDDING_DIM)
-        
+
     def forward(self, x):
         # ******************* Satellite imagery *************************
         # Shape: batch_size, seq_length, width, height, channel
         sat_data = x['sat_data'][:, :self.history_len+1]
         batch_size, seq_len, width, height, n_chans = sat_data.shape
-        
+
         # Move seq_length to be the last dim, ready for changing the shape
         sat_data = sat_data.permute(0, 2, 3, 4, 1)
-        
+
         # Stack timesteps into the channel dimension
         sat_data = sat_data.view(batch_size, width, height, seq_len * n_chans)
-        
+
         sat_data = sat_data.permute(0, 3, 1, 2)  # Conv2d expects channels to be the 2nd dim!
-        
+
         ### EXTRA CHANNELS
         # Center marker
         center_marker = torch.zeros((batch_size, 1, width, height), dtype=torch.float32, device=self.device)
         half_width = width // 2
         center_marker[..., half_width-2:half_width+2, half_width-2:half_width+2] = 1
-        
+
         # geo-spatial x
         x_coords = x['sat_x_coords'] - SAT_X_MEAN
         x_coords /= SAT_X_STD
         x_coords = x_coords.unsqueeze(1).expand(-1, width, -1).unsqueeze(1)
-        
+
         # geo-spatial y
         y_coords = x['sat_y_coords'] - SAT_Y_MEAN
         y_coords /= SAT_Y_STD
         y_coords = y_coords.unsqueeze(-1).expand(-1, -1, height).unsqueeze(1)
-        
+
         # pixel x & y
         pixel_range = (torch.arange(width, device=self.device) - 64) / 37
         pixel_range = pixel_range.unsqueeze(0).unsqueeze(0)
         pixel_x = pixel_range.unsqueeze(-2).expand(batch_size, 1, width, -1)
         pixel_y = pixel_range.unsqueeze(-1).expand(batch_size, 1, -1, height)
-        
+
         # Concat
         sat_data = torch.cat((sat_data, center_marker, x_coords, y_coords, pixel_x, pixel_y), dim=1)
-        
+
         del center_marker, x_coords, y_coords, pixel_x, pixel_y
-        
+
         # Pass data through the network :)
         out = F.relu(self.sat_conv1(sat_data))
         out = self.maxpool(out)
         out = F.relu(self.sat_conv2(out))
         out = self.maxpool(out)
         out = F.relu(self.sat_conv3(out))
-        
+
         out = out.view(-1, CHANNELS * 11 * 11)
         out = F.relu(self.fc1(out))
-        
+
         # *********************** NWP Data **************************************
         nwp_data = x['nwp'].float() # Shape: batch_size, channel, seq_length, width, height
         batch_size, n_nwp_chans, nwp_seq_len, nwp_width, nwp_height = nwp_data.shape
         nwp_data = nwp_data.reshape(batch_size, n_nwp_chans * nwp_seq_len * nwp_width * nwp_height)
-        
+
         # Concat
         out = torch.cat(
             (
@@ -330,15 +330,15 @@ def forward(self, x):
             ),
             dim=1)
         del nwp_data
-        
+
         # Embedding of PV system ID
         if EMBEDDING_DIM:
             pv_embedding = self.pv_system_id_embedding(x['pv_system_row_number'])
             out = torch.cat(
                 (
                     out,
                     pv_embedding
-                ), 
+                ),
                 dim=1)
 
         # Fully connected layers.
@@ -348,7 +348,7 @@ def forward(self, x):
         out = F.relu(self.fc5(out)) # PV yield is in range [0, 1].  ReLU should train more cleanly than sigmoid.
 
         return out
-    
+
     def _training_or_validation_step(self, batch, is_train_step):
         y_hat = self(batch)
         y = batch['pv_yield'][:, -self.forecast_len:]
@@ -360,19 +360,19 @@ def _training_or_validation_step(self, batch, is_train_step):
         tag = "Train" if is_train_step else "Validation"
         self.log_dict({f'MSE/{tag}': mse_loss}, on_step=is_train_step, on_epoch=True)
         self.log_dict({f'NMAE/{tag}': nmae_loss}, on_step=is_train_step, on_epoch=True)
-        
+
         return nmae_loss
 
     def training_step(self, batch, batch_idx):
         return self._training_or_validation_step(batch, is_train_step=True)
-    
+
     def validation_step(self, batch, batch_idx):
         if batch_idx == 0:
             # Plot example
             model_output = self(batch)
             fig = plot_example(batch, model_output)
             self.logger.experiment['validation/plot'].log(File.as_image(fig))
-            
+
         return self._training_or_validation_step(batch, is_train_step=False)
 
     def configure_optimizers(self):
@@ -436,4 +436,4 @@ def configure_optimizers(self):
 trainer.fit(model, data_module)
 
 
-# In[ ]:
+# In[ ]:

experiments/2021-08/2021-08-17/run_cnn3d.py

@@ -29,4 +29,4 @@ def main():
     trainer.fit(model, train_dataloader)
 
 if __name__ == '__main__':
-    main()
+    main()

experiments/2021-08/2021-08-18/run_cnn3d.py

@@ -34,4 +34,4 @@ def main():
 # 1. Large training set, and one epoch took a day, so should use GPU for this model. I was a bit suprised as I didnt
 # think the model was so big.
 # 2. Need to work on validationm general validation method. Good to base line against a really simple model. For
-# validation might need to think carefully about metrics that will be used.
+# validation might need to think carefully about metrics that will be used.

experiments/2021-08/2021-08-24/run_cnn3d_n_layers.py

@@ -41,4 +41,4 @@ def main():
 # https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-133/monitoring
 
 # https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-132/monitoring
-# https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-131/monitoring
+# https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-131/monitoring

experiments/2021-08/2021-08-26/run_cnn3d_n_layers.py

@@ -43,4 +43,3 @@ def main():
 
 # https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-138/charts
 # ran with 10000 in train data
-

experiments/2021-08/2021-08-27/experiments.md

@@ -1,8 +1,8 @@
 # Daily Experiments
 
-Ran hydra for the first time, for hyper parameters optermization. 
+Ran hydra for the first time, for hyper parameters optermization.
 It did 2 full runs, then I think ran out of memory caused a funny error.
-Now have install 'psutil' so that cpu and memory is logged to neptune. 
+Now have install 'psutil' so that cpu and memory is logged to neptune.
 
 https://app.neptune.ai/o/OpenClimateFix/org/predict-pv-yield/e/PRED-160/monitoring
 Validation error after 10 epochs - 0.073
@@ -26,4 +26,4 @@ Validation error after 2 epochs - 0.076 (then error happened in 3rd epoch)
 conv3d_channels = 32
 fc1_output_features = 64
 fc2_output_features = 16
-fc3_output_features = 8
+fc3_output_features = 8