different frequency

[moghadas76]

Exception has occurred: RuntimeError (note: full exception trace is shown but execution is paused at: _run_module_as_main)
cuFFT error: CUFFT_INVALID_SIZE
File "/home/seyed/PycharmProjects/step/lag-llama/data/augmentations/freq_mask.py", line 30, in freq_mask
xy_f = torch.fft.rfft(xy, dim=dim)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/PycharmProjects/step/lag-llama/lag_llama/gluon/lightning_module.py", line 777, in training_step
batch["past_target"], batch["future_target"] = freq_mask(
^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/strategies/strategy.py", line 391, in training_step
return self.lightning_module.training_step(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/trainer/call.py", line 309, in _call_strategy_hook
output = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 319, in _training_step
training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 129, in closure
step_output = self._step_fn()
^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 144, in call
self._result = self.closure(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/plugins/precision/precision.py", line 108, in _wrap_closure
closure_result = closure()
^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/torch/optim/adam.py", line 148, in step
loss = closure()
^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/torch/optim/optimizer.py", line 76, in _use_grad
ret = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/torch/optim/optimizer.py", line 391, in wrapper
out = func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/plugins/precision/precision.py", line 122, in optimizer_step
return optimizer.step(closure=closure, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/plugins/precision/amp.py", line 77, in optimizer_step
return super().optimizer_step(optimizer, model=model, closure=closure, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/strategies/strategy.py", line 239, in optimizer_step
return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/core/optimizer.py", line 152, in step
step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/core/module.py", line 1303, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/trainer/call.py", line 157, in _call_lightning_module_hook
output = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 269, in _optimizer_step
call._call_lightning_module_hook(
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 191, in run
self._optimizer_step(batch_idx, closure)
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 242, in advance
batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 138, in run
self.advance(data_fetcher)
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/fit_loop.py", line 363, in advance
self.epoch_loop.run(self._data_fetcher)
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/loops/fit_loop.py", line 205, in run
self.advance()
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1033, in _run_stage
self.fit_loop.run()
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 987, in _run
results = self._run_stage()
^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 580, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/trainer/call.py", line 44, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 543, in fit
call._call_and_handle_interrupt(
File "/home/seyed/PycharmProjects/step/lag-llama/lag_llama/gluon/estimator.py", line 1004, in train_model
trainer.fit(
File "/home/seyed/PycharmProjects/step/lag-llama/lag_llama/gluon/estimator.py", line 620, in train_with_module
return self.train_model(
^^^^^^^^^^^^^^^^^
File "/home/seyed/PycharmProjects/step/lag-llama/test_knn.py", line 1205, in
predictor = estimator.train_with_module(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/seyed/miniconda3/envs/env/lib/python3.11/runpy.py", line 88, in _run_code
exec(code, run_globals)
File "/home/seyed/miniconda3/envs/env/lib/python3.11/runpy.py", line 198, in _run_module_as_main (Current frame)
return _run_code(code, main_globals, None,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: cuFFT error: CUFFT_INVALID_SIZE