diff --git a/modules.py b/modules.py index 16df087..a839231 100644 --- a/modules.py +++ b/modules.py @@ -283,7 +283,7 @@ def positional_encoding(inputs, # First part of the PE function: sin and cos argument position_enc = np.array([ - [pos / np.power(10000, (i-i%2)/E) for i in range(E)] + [pos / np.power(10000, 1.0*(i-i%2)/E) for i in range(E)] for pos in range(maxlen)]) # Second part, apply the cosine to even columns and sin to odds. @@ -308,4 +308,4 @@ def noam_scheme(init_lr, global_step, warmup_steps=4000.): until it reaches init_lr. ''' step = tf.cast(global_step + 1, dtype=tf.float32) - return init_lr * warmup_steps ** 0.5 * tf.minimum(step * warmup_steps ** -1.5, step ** -0.5) \ No newline at end of file + return init_lr * warmup_steps ** 0.5 * tf.minimum(step * warmup_steps ** -1.5, step ** -0.5)