diff --git a/experiments/india/006_da_only/bad.png b/experiments/india/006_da_only/bad.png new file mode 100644 index 00000000..61571ea5 Binary files /dev/null and b/experiments/india/006_da_only/bad.png differ diff --git a/experiments/india/006_da_only/da_only.md b/experiments/india/006_da_only/da_only.md new file mode 100644 index 00000000..ec25244a --- /dev/null +++ b/experiments/india/006_da_only/da_only.md @@ -0,0 +1,37 @@ +## DA forecasts only + +The idea was to create a forecast for DA (day-ahead) only for Windnet. +We hope this would bring down the DA MAE values. + +We do this by not forecasting the first X hours. + +Unfortunately, it doesnt not look like ignore X hours, make the DA forecast better. + +## Experiments + +1. Baseline - [here](https://wandb.ai/openclimatefix/india/runs/miszfep5) +2. Ignore first 6 hours - [here](https://wandb.ai/openclimatefix/india/runs/uosk0qug) +3. Ignore first 12 hours - [here](https://wandb.ai/openclimatefix/india/runs/s9cnn4ei) + +## Results + +| Timestep | all MAE % | 6 MAE % | 12 MAE % | +| --- | --- |---------|---------| +| 0-0 minutes | nan | nan | nan | +| 15-15 minutes | nan | nan | nan | +| 30-45 minutes | 0.065 | nan | nan | +| 45-60 minutes | 0.066 | nan | nan | +| 60-120 minutes | 0.063 | nan | nan | +| 120-240 minutes | 0.063 | nan | nan | +| 240-360 minutes | 0.064 | nan | nan | +| 360-480 minutes | 0.065 | 0.068 | nan | +| 480-720 minutes | 0.067 | 0.065 | nan | +| 720-1440 minutes | 0.068 | 0.065 | 0.065 | +| 1440-2880 minutes | 0.071 | 0.071 | 0.071 | + +![](mae_steps.png "mae_steps") + +Here's two examples from the 6 hour ignore model, one that forecated it well, one that didnt + +![](bad.png "bad") +![](good.png "good") diff --git a/experiments/india/006_da_only/good.png b/experiments/india/006_da_only/good.png new file mode 100644 index 00000000..e4ed228b Binary files /dev/null and b/experiments/india/006_da_only/good.png differ diff --git a/experiments/india/006_da_only/mae_steps.png b/experiments/india/006_da_only/mae_steps.png new file mode 100644 index 00000000..8c54c0b7 Binary files /dev/null and b/experiments/india/006_da_only/mae_steps.png differ diff --git a/pvnet/models/base_model.py b/pvnet/models/base_model.py index 46706c00..a375216e 100644 --- a/pvnet/models/base_model.py +++ b/pvnet/models/base_model.py @@ -245,7 +245,7 @@ def save_pretrained( class BaseModel(pl.LightningModule, PVNetModelHubMixin): - """Abtstract base class for PVNet submodels""" + """Abstract base class for PVNet submodels""" def __init__( self, @@ -257,6 +257,7 @@ def __init__( interval_minutes: int = 30, timestep_intervals_to_plot: Optional[list[int]] = None, use_weighted_loss: bool = False, + forecast_minutes_ignore: Optional[int] = 0, ): """Abtstract base class for PVNet submodels. @@ -270,6 +271,8 @@ def __init__( interval_minutes: The interval in minutes between each timestep in the data timestep_intervals_to_plot: Intervals, in timesteps, to plot during training use_weighted_loss: Whether to use a weighted loss function + forecast_minutes_ignore: Number of forecast minutes to ignore when calculating losses. + For example if set to 60, the model doesnt predict the first 60 minutes """ super().__init__() @@ -292,10 +295,12 @@ def __init__( self.forecast_minutes = forecast_minutes self.output_quantiles = output_quantiles self.interval_minutes = interval_minutes + self.forecast_minutes_ignore = forecast_minutes_ignore # Number of timestemps for 30 minutely data self.history_len = history_minutes // interval_minutes - self.forecast_len = forecast_minutes // interval_minutes + self.forecast_len = (forecast_minutes - forecast_minutes_ignore) // interval_minutes + self.forecast_len_ignore = forecast_minutes_ignore // interval_minutes self.weighted_losses = WeightedLosses(forecast_length=self.forecast_len) @@ -334,7 +339,7 @@ def _quantiles_to_prediction(self, y_quantiles): y_median = y_quantiles[..., idx] return y_median - def _calculate_qauntile_loss(self, y_quantiles, y): + def _calculate_quantile_loss(self, y_quantiles, y): """Calculate quantile loss. Note: @@ -366,7 +371,7 @@ def _calculate_common_losses(self, y, y_hat): losses = {} if self.use_quantile_regression: - losses["quantile_loss"] = self._calculate_qauntile_loss(y_hat, y) + losses["quantile_loss"] = self._calculate_quantile_loss(y_hat, y) y_hat = self._quantiles_to_prediction(y_hat) # calculate mse, mae diff --git a/pvnet/models/multimodal/multimodal.py b/pvnet/models/multimodal/multimodal.py index ad4c11d6..806ce6d9 100644 --- a/pvnet/models/multimodal/multimodal.py +++ b/pvnet/models/multimodal/multimodal.py @@ -71,6 +71,7 @@ def __init__( timestep_intervals_to_plot: Optional[list[int]] = None, adapt_batches: Optional[bool] = False, use_weighted_loss: Optional[bool] = False, + forecast_minutes_ignore: Optional[int] = 0, ): """Neural network which combines information from different sources. @@ -131,6 +132,8 @@ def __init__( the model to use. This allows us to overprepare batches and slice from them for the data we need for a model run. use_weighted_loss: Whether to use a weighted loss function + forecast_minutes_ignore: Number of forecast minutes to ignore when calculating losses. + For example if set to 60, the model doesnt predict the first 60 minutes """ self.include_gsp_yield_history = include_gsp_yield_history @@ -154,6 +157,7 @@ def __init__( interval_minutes=interval_minutes, timestep_intervals_to_plot=timestep_intervals_to_plot, use_weighted_loss=use_weighted_loss, + forecast_minutes_ignore=forecast_minutes_ignore, ) # Number of features expected by the output_network @@ -271,7 +275,8 @@ def __init__( if self.include_sun: self.sun_fc1 = nn.Linear( - in_features=2 * (self.forecast_len + self.history_len + 1), + in_features=2 + * (self.forecast_len + self.forecast_len_ignore + self.history_len + 1), out_features=16, ) diff --git a/tests/conftest.py b/tests/conftest.py index 4ea9e3f1..8d2ab630 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -303,3 +303,12 @@ def multimodal_weighted_quantile_model(multimodal_model_kwargs): output_quantiles=[0.1, 0.5, 0.9], **multimodal_model_kwargs, use_weighted_loss=True ) return model + + +@pytest.fixture() +def multimodal_quantile_model_ignore_minutes(multimodal_model_kwargs): + """Only forecsat second half of the 8 hours""" + model = Model( + output_quantiles=[0.1, 0.5, 0.9], **multimodal_model_kwargs, forecast_minutes_ignore=240 + ) + return model diff --git a/tests/models/multimodal/test_multimodal.py b/tests/models/multimodal/test_multimodal.py index 0cc9a7df..3e82e171 100644 --- a/tests/models/multimodal/test_multimodal.py +++ b/tests/models/multimodal/test_multimodal.py @@ -51,3 +51,14 @@ def test_weighted_quantile_model_backward(multimodal_weighted_quantile_model, sa # Backwards on sum drives sum to zero y_quantiles.sum().backward() + + +def test_weighted_quantile_model_forward(multimodal_quantile_model_ignore_minutes, sample_batch): + y_quantiles = multimodal_quantile_model_ignore_minutes(sample_batch) + + # check output is the correct shape + # batch size=2, forecast_len=8, num_quantiles=3 + assert tuple(y_quantiles.shape) == (2, 8, 3), y_quantiles.shape + + # Backwards on sum drives sum to zero + y_quantiles.sum().backward()