Add type conversion for ROP

PiperOrigin-RevId: 644480423

Author: lingvo-bot

lingvo/core/layers.py

@@ -3514,7 +3514,8 @@ def FProp(self, theta, inputs, position=None):
       position = tf.range(seq_length, dtype=tf.float32)[tf.newaxis, :]
     position = position[:, :, tf.newaxis, tf.newaxis]
     timescale = timescale[tf.newaxis, tf.newaxis, tf.newaxis, :]
-    sinusoid_inp = position / timescale
+
+    sinusoid_inp = position / tf.cast(timescale, position.dtype)
     sin = tf.sin(sinusoid_inp)
     cos = tf.cos(sinusoid_inp)
     first_half, second_half = tf.split(inputs, 2, axis=-1)