test failure

pytorch_forecasting/tests/_conftest.py

@@ -0,0 +1,262 @@
+import numpy as np
+import pytest
+import torch
+
+from pytorch_forecasting import TimeSeriesDataSet
+from pytorch_forecasting.data import EncoderNormalizer, GroupNormalizer, NaNLabelEncoder
+from pytorch_forecasting.data.examples import generate_ar_data, get_stallion_data
+
+torch.manual_seed(23)
+
+
+@pytest.fixture(scope="session")
+def gpus():
+    if torch.cuda.is_available():
+        return [0]
+    else:
+        return 0
+
+
+@pytest.fixture(scope="session")
+def data_with_covariates():
+    data = get_stallion_data()
+    data["month"] = data.date.dt.month.astype(str)
+    data["log_volume"] = np.log1p(data.volume)
+    data["weight"] = 1 + np.sqrt(data.volume)
+
+    data["time_idx"] = data["date"].dt.year * 12 + data["date"].dt.month
+    data["time_idx"] -= data["time_idx"].min()
+
+    # convert special days into strings
+    special_days = [
+        "easter_day",
+        "good_friday",
+        "new_year",
+        "christmas",
+        "labor_day",
+        "independence_day",
+        "revolution_day_memorial",
+        "regional_games",
+        "fifa_u_17_world_cup",
+        "football_gold_cup",
+        "beer_capital",
+        "music_fest",
+    ]
+    data[special_days] = (
+        data[special_days].apply(lambda x: x.map({0: "", 1: x.name})).astype("category")
+    )
+    data = data.astype(dict(industry_volume=float))
+
+    # select data subset
+    data = data[lambda x: x.sku.isin(data.sku.unique()[:2])][
+        lambda x: x.agency.isin(data.agency.unique()[:2])
+    ]
+
+    # default target
+    data["target"] = data["volume"].clip(1e-3, 1.0)
+
+    return data
+
+
+def make_dataloaders(data_with_covariates, **kwargs):
+    training_cutoff = "2016-09-01"
+    max_encoder_length = 4
+    max_prediction_length = 3
+
+    kwargs.setdefault("target", "volume")
+    kwargs.setdefault("group_ids", ["agency", "sku"])
+    kwargs.setdefault("add_relative_time_idx", True)
+    kwargs.setdefault("time_varying_unknown_reals", ["volume"])
+
+    training = TimeSeriesDataSet(
+        data_with_covariates[lambda x: x.date < training_cutoff].copy(),
+        time_idx="time_idx",
+        max_encoder_length=max_encoder_length,
+        max_prediction_length=max_prediction_length,
+        **kwargs,  # fixture parametrization
+    )
+
+    validation = TimeSeriesDataSet.from_dataset(
+        training,
+        data_with_covariates.copy(),
+        min_prediction_idx=training.index.time.max() + 1,
+    )
+    train_dataloader = training.to_dataloader(train=True, batch_size=2, num_workers=0)
+    val_dataloader = validation.to_dataloader(train=False, batch_size=2, num_workers=0)
+    test_dataloader = validation.to_dataloader(train=False, batch_size=1, num_workers=0)
+
+    return dict(train=train_dataloader, val=val_dataloader, test=test_dataloader)
+
+
+@pytest.fixture(
+    params=[
+        dict(),
+        dict(
+            static_categoricals=["agency", "sku"],
+            static_reals=["avg_population_2017", "avg_yearly_household_income_2017"],
+            time_varying_known_categoricals=["special_days", "month"],
+            variable_groups=dict(
+                special_days=[
+                    "easter_day",
+                    "good_friday",
+                    "new_year",
+                    "christmas",
+                    "labor_day",
+                    "independence_day",
+                    "revolution_day_memorial",
+                    "regional_games",
+                    "fifa_u_17_world_cup",
+                    "football_gold_cup",
+                    "beer_capital",
+                    "music_fest",
+                ]
+            ),
+            time_varying_known_reals=[
+                "time_idx",
+                "price_regular",
+                "price_actual",
+                "discount",
+                "discount_in_percent",
+            ],
+            time_varying_unknown_categoricals=[],
+            time_varying_unknown_reals=[
+                "volume",
+                "log_volume",
+                "industry_volume",
+                "soda_volume",
+                "avg_max_temp",
+            ],
+            constant_fill_strategy={"volume": 0},
+            categorical_encoders={"sku": NaNLabelEncoder(add_nan=True)},
+        ),
+        dict(static_categoricals=["agency", "sku"]),
+        dict(randomize_length=True, min_encoder_length=2),
+        dict(target_normalizer=EncoderNormalizer(), min_encoder_length=2),
+        dict(target_normalizer=GroupNormalizer(transformation="log1p")),
+        dict(
+            target_normalizer=GroupNormalizer(
+                groups=["agency", "sku"], transformation="softplus", center=False
+            )
+        ),
+        dict(target="agency"),
+        # test multiple targets
+        dict(target=["industry_volume", "volume"]),
+        dict(target=["agency", "volume"]),
+        dict(
+            target=["agency", "volume"], min_encoder_length=1, min_prediction_length=1
+        ),
+        dict(target=["agency", "volume"], weight="volume"),
+        # test weights
+        dict(target="volume", weight="volume"),
+    ],
+    scope="session",
+)
+def multiple_dataloaders_with_covariates(data_with_covariates, request):
+    return make_dataloaders(data_with_covariates, **request.param)
+
+
+@pytest.fixture(scope="session")
+def dataloaders_with_different_encoder_decoder_length(data_with_covariates):
+    return make_dataloaders(
+        data_with_covariates.copy(),
+        target="target",
+        time_varying_known_categoricals=["special_days", "month"],
+        variable_groups=dict(
+            special_days=[
+                "easter_day",
+                "good_friday",
+                "new_year",
+                "christmas",
+                "labor_day",
+                "independence_day",
+                "revolution_day_memorial",
+                "regional_games",
+                "fifa_u_17_world_cup",
+                "football_gold_cup",
+                "beer_capital",
+                "music_fest",
+            ]
+        ),
+        time_varying_known_reals=[
+            "time_idx",
+            "price_regular",
+            "price_actual",
+            "discount",
+            "discount_in_percent",
+        ],
+        time_varying_unknown_categoricals=[],
+        time_varying_unknown_reals=[
+            "target",
+            "volume",
+            "log_volume",
+            "industry_volume",
+            "soda_volume",
+            "avg_max_temp",
+        ],
+        static_categoricals=["agency"],
+        add_relative_time_idx=False,
+        target_normalizer=GroupNormalizer(groups=["agency", "sku"], center=False),
+    )
+
+
+@pytest.fixture(scope="session")
+def dataloaders_with_covariates(data_with_covariates):
+    return make_dataloaders(
+        data_with_covariates.copy(),
+        target="target",
+        time_varying_known_reals=["discount"],
+        time_varying_unknown_reals=["target"],
+        static_categoricals=["agency"],
+        add_relative_time_idx=False,
+        target_normalizer=GroupNormalizer(groups=["agency", "sku"], center=False),
+    )
+
+
+@pytest.fixture(scope="session")
+def dataloaders_multi_target(data_with_covariates):
+    return make_dataloaders(
+        data_with_covariates.copy(),
+        time_varying_unknown_reals=["target", "discount"],
+        target=["target", "discount"],
+        add_relative_time_idx=False,
+    )
+
+
+@pytest.fixture(scope="session")
+def dataloaders_fixed_window_without_covariates():
+    data = generate_ar_data(seasonality=10.0, timesteps=50, n_series=2)
+    validation = data.series.iloc[:2]
+
+    max_encoder_length = 30
+    max_prediction_length = 10
+
+    training = TimeSeriesDataSet(
+        data[lambda x: ~x.series.isin(validation)],
+        time_idx="time_idx",
+        target="value",
+        categorical_encoders={"series": NaNLabelEncoder().fit(data.series)},
+        group_ids=["series"],
+        static_categoricals=[],
+        max_encoder_length=max_encoder_length,
+        max_prediction_length=max_prediction_length,
+        time_varying_unknown_reals=["value"],
+        target_normalizer=EncoderNormalizer(),
+    )
+
+    validation = TimeSeriesDataSet.from_dataset(
+        training,
+        data[lambda x: x.series.isin(validation)],
+        stop_randomization=True,
+    )
+    batch_size = 2
+    train_dataloader = training.to_dataloader(
+        train=True, batch_size=batch_size, num_workers=0
+    )
+    val_dataloader = validation.to_dataloader(
+        train=False, batch_size=batch_size, num_workers=0
+    )
+    test_dataloader = validation.to_dataloader(
+        train=False, batch_size=batch_size, num_workers=0
+    )
+
+    return dict(train=train_dataloader, val=val_dataloader, test=test_dataloader)

pytorch_forecasting/tests/test_all_estimators.py

@@ -1,14 +1,19 @@
 """Automated tests based on the skbase test suite template."""
 
 from inspect import isclass
+import shutil
 
+import lightning.pytorch as pl
+from lightning.pytorch.callbacks import EarlyStopping
+from lightning.pytorch.loggers import TensorBoardLogger
 from skbase.testing import (
     BaseFixtureGenerator as _BaseFixtureGenerator,
     TestAllObjects as _TestAllObjects,
 )
 
 from pytorch_forecasting._registry import all_objects
 from pytorch_forecasting.tests._config import EXCLUDE_ESTIMATORS, EXCLUDED_TESTS
+from pytorch_forecasting.tests._conftest import make_dataloaders
 
 # whether to test only estimators from modules that are changed w.r.t. main
 # default is False, can be set to True by pytest --only_changed_modules True flag
@@ -110,6 +115,98 @@ def _all_objects(self):
     ]
 
 
+def _integration(
+    data_with_covariates,
+    tmp_path,
+    cell_type="LSTM",
+    data_loader_kwargs={},
+    clip_target: bool = False,
+    trainer_kwargs=None,
+    **kwargs,
+):
+    data_with_covariates = data_with_covariates.copy()
+    if clip_target:
+        data_with_covariates["target"] = data_with_covariates["volume"].clip(1e-3, 1.0)
+    else:
+        data_with_covariates["target"] = data_with_covariates["volume"]
+    data_loader_default_kwargs = dict(
+        target="target",
+        time_varying_known_reals=["price_actual"],
+        time_varying_unknown_reals=["target"],
+        static_categoricals=["agency"],
+        add_relative_time_idx=True,
+    )
+    data_loader_default_kwargs.update(data_loader_kwargs)
+    dataloaders_with_covariates = make_dataloaders(
+        data_with_covariates, **data_loader_default_kwargs
+    )
+
+    train_dataloader = dataloaders_with_covariates["train"]
+    val_dataloader = dataloaders_with_covariates["val"]
+    test_dataloader = dataloaders_with_covariates["test"]
+
+    early_stop_callback = EarlyStopping(
+        monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min"
+    )
+
+    logger = TensorBoardLogger(tmp_path)
+    if trainer_kwargs is None:
+        trainer_kwargs = {}
+    trainer = pl.Trainer(
+        max_epochs=3,
+        gradient_clip_val=0.1,
+        callbacks=[early_stop_callback],
+        enable_checkpointing=True,
+        default_root_dir=tmp_path,
+        limit_train_batches=2,
+        limit_val_batches=2,
+        limit_test_batches=2,
+        logger=logger,
+        **trainer_kwargs,
+    )
+
+    net = DeepAR.from_dataset(
+        train_dataloader.dataset,
+        hidden_size=5,
+        cell_type=cell_type,
+        learning_rate=0.01,
+        log_gradient_flow=True,
+        log_interval=1000,
+        n_plotting_samples=100,
+        **kwargs,
+    )
+    net.size()
+    try:
+        trainer.fit(
+            net,
+            train_dataloaders=train_dataloader,
+            val_dataloaders=val_dataloader,
+        )
+        test_outputs = trainer.test(net, dataloaders=test_dataloader)
+        assert len(test_outputs) > 0
+        # check loading
+        net = DeepAR.load_from_checkpoint(trainer.checkpoint_callback.best_model_path)
+
+        # check prediction
+        net.predict(
+            val_dataloader,
+            fast_dev_run=True,
+            return_index=True,
+            return_decoder_lengths=True,
+            trainer_kwargs=trainer_kwargs,
+        )
+    finally:
+        shutil.rmtree(tmp_path, ignore_errors=True)
+
+    net.predict(
+        val_dataloader,
+        fast_dev_run=True,
+        return_index=True,
+        return_decoder_lengths=True,
+        trainer_kwargs=trainer_kwargs,
+    )
+
+
 class TestAllPtForecasters(PackageConfig, BaseFixtureGenerator, _TestAllObjects):
     """Generic tests for all objects in the mini package."""
 
@@ -118,3 +215,7 @@ def test_doctest_examples(self, object_class):
         import doctest
 
         doctest.run_docstring_examples(object_class, globals())
+
+    def certain_failure(self, object_class):
+        """Fails for certain, for testing."""
+        assert False