Remove datapipe references

Author: Sukhil Patel

pvnet/data/site_datamodule.py

@@ -25,7 +25,7 @@ def __init__(
         Can also be used with pre-made batches if `sample_dir` is set.
 
         Args:
-            configuration: Path to datapipe configuration file.
+            configuration: Path to configuration file.
             sample_dir: Path to the directory of pre-saved samples. Cannot be used together with
                 `configuration` or '[train/val]_period'.
             batch_size: Batch size.

pvnet/data/uk_regional_datamodule.py

@@ -8,7 +8,7 @@
 
 
 class DataModule(BaseDataModule):
-    """Datamodule for training pvnet and using pvnet pipeline in `ocf_datapipes`."""
+    """Datamodule for training pvnet and using pvnet pipeline in `ocf-data-sampler`."""
 
     def __init__(
         self,
@@ -25,7 +25,7 @@ def __init__(
         Can also be used with pre-made batches if `sample_dir` is set.
 
         Args:
-            configuration: Path to datapipe configuration file.
+            configuration: Path to configuration file.
             sample_dir: Path to the directory of pre-saved samples. Cannot be used together with
                 `configuration` or '[train/val]_period'.
             batch_size: Batch size.

pvnet/models/base_model.py

@@ -42,7 +42,7 @@ def make_clean_data_config(input_path, output_path, placeholder="PLACEHOLDER"):
     """Resave the data config and replace the filepaths with a placeholder.
 
     Args:
-        input_path: Path to input datapipes configuration file
+        input_path: Path to input configuration file
         output_path: Location to save the output configuration file
         placeholder: String placeholder for data sources
     """
@@ -81,7 +81,7 @@ def minimize_data_config(input_path, output_path, model):
     """Strip out parts of the data config which aren't used by the model
 
     Args:
-        input_path: Path to input datapipes configuration file
+        input_path: Path to input configuration file
         output_path: Location to save the output configuration file
         model: The PVNet model object
     """
@@ -286,7 +286,7 @@ def save_pretrained(
         if isinstance(config, dict):
             (save_directory / CONFIG_NAME).write_text(json.dumps(config, indent=4))
 
-        # Save cleaned datapipes configuration file
+        # Save cleaned configuration file
         if data_config is not None:
             new_data_config_path = save_directory / DATA_CONFIG_NAME
 

pvnet/models/model_cards/pv_india_model_card_template.md

@@ -32,7 +32,7 @@ The model is trained on data from 2019-2022 and validated on data from 2022-2023
 
 ### Preprocessing
 
-Data is prepared with the `ocf_datapipes.training.pvnet_site` datapipe [2].
+Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/site` Dataset [2].
 
 
 ## Results
@@ -48,4 +48,4 @@ Trained on a single NVIDIA Tesla T4
 ### Software
 
 - [1] https://github.com/openclimatefix/PVNet
-- [2] https://github.com/openclimatefix/ocf_datapipes
+- [2] https://github.com/openclimatefix/ocf-data-sampler

pvnet/models/model_cards/pv_uk_regional_model_card_template.md

@@ -31,7 +31,7 @@ The model is trained on data from 2019-2022 and validated on data from 2022-2023
 
 ### Preprocessing
 
-Data is prepared with the `ocf_datapipes.training.pvnet` datapipe [2].
+Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Dataset [2].
 
 
 ## Results
@@ -51,4 +51,4 @@ Trained on a single NVIDIA Tesla T4
 ### Software
 
 - [1] https://github.com/openclimatefix/PVNet
-- [2] https://github.com/openclimatefix/ocf_datapipes
+- [2] https://github.com/openclimatefix/ocf-data-sampler

pvnet/models/model_cards/wind_india_model_card_template.md

@@ -32,7 +32,7 @@ The model is trained on data from 2019-2022 and validated on data from 2022-2023
 
 ### Preprocessing
 
-Data is prepared with the `ocf_datapipes.training.windnet` datapipe [2].
+Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/site` Dataset [2].
 
 
 ## Results
@@ -48,4 +48,4 @@ Trained on a single NVIDIA Tesla T4
 ### Software
 
 - [1] https://github.com/openclimatefix/PVNet
-- [2] https://github.com/openclimatefix/ocf_datapipes
+- [2] https://github.com/openclimatefix/ocf-data-sampler

pvnet/models/multimodal/site_encoders/encoders.py

@@ -4,7 +4,6 @@
 
 import einops
 import torch
-from ocf_datapipes.batch import BatchKey
 from torch import nn
 
 from pvnet.models.multimodal.linear_networks.networks import ResFCNet2
@@ -75,13 +74,13 @@ def __init__(
         )
 
     def _calculate_attention(self, x):
-        gsp_ids = x[BatchKey.gsp_id].squeeze().int()
+        gsp_ids = x["gsp_id"].squeeze().int()
         attention = self._attention_network(gsp_ids)
         return attention
 
     def _encode_value(self, x):
         # Shape: [batch size, sequence length, PV site]
-        pv_site_seqs = x[BatchKey.pv].float()
+        pv_site_seqs = x["pv"].float()
         batch_size = pv_site_seqs.shape[0]
 
         pv_site_seqs = pv_site_seqs.swapaxes(1, 2).flatten(0, 1)
@@ -170,7 +169,6 @@ def __init__(
         self.input_key_to_use = input_key_to_use
         self.num_channels = num_channels
         self.num_sites_in_inference = num_sites_in_inference
-        print(self.sequence_length, "SEQUENCE LENGTH")
 
         if use_id_in_value:
             self.value_id_embedding = nn.Embedding(num_sites, id_embed_dim)

scripts/load_batches.py

@@ -1,4 +1,4 @@
-from ocf_datapipes.utils.location import Location
+from ocf_data_sampler.select.location import Location
 from pvnet.utils import SiteLocationLookup
 import xarray as xr
 import pytest