Added pure python implementation of lish (#1138)

Author: gabrieldemarmiesse

tensorflow_addons/activations/lisht.py

@@ -42,3 +42,7 @@ def lisht(x: types.TensorLike) -> tf.Tensor:
 @tf.RegisterGradient("Addons>Lisht")
 def _lisht_grad(op, grad):
     return _activation_so.ops.addons_lisht_grad(grad, op.inputs[0])
+
+
+def _lisht_py(x):
+    return x * tf.math.tanh(x)

tensorflow_addons/activations/lisht_test.py

@@ -18,6 +18,7 @@
 import numpy as np
 import tensorflow as tf
 from tensorflow_addons.activations import lisht
+from tensorflow_addons.activations.lisht import _lisht_py
 from tensorflow_addons.utils import test_utils
 
 
@@ -42,6 +43,30 @@ def test_theoretical_gradients(self, dtype):
         theoretical, numerical = tf.test.compute_gradient(lisht, [x])
         self.assertAllCloseAccordingToType(theoretical, numerical, rtol=5e-4, atol=5e-4)
 
+    @parameterized.named_parameters(
+        ("float16", np.float16), ("float32", np.float32), ("float64", np.float64)
+    )
+    def test_same_as_py_func(self, dtype):
+        np.random.seed(1234)
+        for _ in range(20):
+            self.verify_funcs_are_equivalent(dtype)
+
+    def verify_funcs_are_equivalent(self, dtype):
+        x_np = np.random.uniform(-10, 10, size=(4, 4)).astype(dtype)
+        x = tf.convert_to_tensor(x_np)
+
+        with tf.GradientTape(persistent=True) as t:
+            t.watch(x)
+            y_native = lisht(x)
+            y_py = _lisht_py(x)
+
+        self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4)
+
+        grad_native = t.gradient(y_native, x)
+        grad_py = t.gradient(y_py, x)
+
+        self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4)
+
 
 if __name__ == "__main__":
     tf.test.main()