for predict noise objective, clipping still helps a lot

Author: lucidrains

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rectified-flow-pytorch"
-version = "0.1.2"
+version = "0.1.4"
 description = "Rectified Flow in Pytorch"
 authors = [
     { name = "Phil Wang", email = "[email protected]" }

rectified_flow_pytorch/rectified_flow.py

@@ -152,7 +152,9 @@ def __init__(
         consistency_delta_time = 1e-3,
         consistency_loss_weight = 1.,
         data_normalize_fn = normalize_to_neg_one_to_one,
-        data_unnormalize_fn = unnormalize_to_zero_to_one
+        data_unnormalize_fn = unnormalize_to_zero_to_one,
+        clip_during_sampling = True,
+        clip_values: Tuple[float, float] = (-1., 1.)
     ):
         super().__init__()
 
@@ -202,6 +204,9 @@ def __init__(
         self.odeint_kwargs = odeint_kwargs
         self.data_shape = data_shape
 
+        self.clip_during_sampling = clip_during_sampling
+        self.clip_values = clip_values
+
         # consistency flow matching
 
         self.use_consistency = use_consistency
@@ -291,7 +296,16 @@ def sample(
         data_shape = default(data_shape, self.data_shape)
         assert exists(data_shape), 'you need to either pass in a `data_shape` or have trained at least with one forward'
 
+        # clipping still helps for predict noise objective
+        # much like original ddpm paper trick
+
+        maybe_clip = (lambda t: t.clamp_(*self.clip_values)) if self.clip_during_sampling else identity
+
+        # ode step function
+
         def ode_fn(t, x):
+            x = maybe_clip(x)
+
             _, flow = self.predict_flow(model, x, times = t, **model_kwargs)
             return flow