[CIBUILD]

Author: fuhailin

deepray/activations/__init__.py

@@ -14,6 +14,7 @@
 # ==============================================================================
 """Additional activation functions."""
 
+from deepray.activations.gelu import gelu
 from deepray.activations.hardshrink import hardshrink
 from deepray.activations.lisht import lisht
 from deepray.activations.mish import mish

deepray/activations/gelu.py

@@ -0,0 +1,74 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import tensorflow as tf
+import warnings
+
+from deepray.utils.types import TensorLike
+
+
+@tf.keras.utils.register_keras_serializable(package="Deepray")
+def gelu(x: TensorLike, approximate: bool = True) -> tf.Tensor:
+  r"""Gaussian Error Linear Unit.
+
+    Computes gaussian error linear:
+
+    $$
+    \mathrm{gelu}(x) = x \Phi(x),
+    $$
+
+    where
+
+    $$
+    \Phi(x) = \frac{1}{2} \left[ 1 + \mathrm{erf}(\frac{x}{\sqrt{2}}) \right]$
+    $$
+
+    when `approximate` is `False`; or
+
+    $$
+    \Phi(x) = \frac{x}{2} \left[ 1 + \tanh(\sqrt{\frac{2}{\pi}} \cdot (x + 0.044715 \cdot x^3)) \right]
+    $$
+
+    when `approximate` is `True`.
+
+    See [Gaussian Error Linear Units (GELUs)](https://arxiv.org/abs/1606.08415)
+    and [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805).
+
+    Consider using `tf.nn.gelu` instead.
+    Note that the default of `approximate` changed to `False` in `tf.nn.gelu`.
+
+    Usage:
+
+    >>> x = tf.constant([-1.0, 0.0, 1.0])
+    >>> dp.activations.gelu(x, approximate=False)
+    <tf.Tensor: shape=(3,), dtype=float32, numpy=array([-0.15865529,  0.        ,  0.8413447 ], dtype=float32)>
+    >>> dp.activations.gelu(x, approximate=True)
+    <tf.Tensor: shape=(3,), dtype=float32, numpy=array([-0.15880796,  0.        ,  0.841192  ], dtype=float32)>
+
+    Args:
+        x: A `Tensor`. Must be one of the following types:
+            `float16`, `float32`, `float64`.
+        approximate: bool, whether to enable approximation.
+    Returns:
+        A `Tensor`. Has the same type as `x`.
+    """
+  warnings.warn(
+      "gelu activation has been migrated to core TensorFlow, "
+      "and will be deprecated in Addons 0.13. "
+      "Note that the default of `approximate` changed to `False` in `tf.nn.gelu`.",
+      DeprecationWarning,
+  )
+
+  return tf.nn.gelu(x, approximate)

deepray/activations/tests/activations_test.py

@@ -18,6 +18,7 @@
 from deepray import activations
 
 ALL_ACTIVATIONS = [
+    "gelu",
     "hardshrink",
     "lisht",
     "mish",

deepray/activations/tests/gelu_test.py

@@ -0,0 +1,31 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import pytest
+
+import numpy as np
+import tensorflow as tf
+from deepray.activations import gelu
+from deepray.utils import test_utils
+
+
+@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.float64])
+def test_gelu(dtype):
+  x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0], dtype=dtype)
+  expected_result = tf.constant([-0.04540229, -0.158808, 0.0, 0.841192, 1.9545977], dtype=dtype)
+  test_utils.assert_allclose_according_to_type(gelu(x), expected_result)
+
+  expected_result = tf.constant([-0.04550028, -0.15865526, 0.0, 0.8413447, 1.9544997], dtype=dtype)
+  test_utils.assert_allclose_according_to_type(gelu(x, False), expected_result)

deepray/layers/__init__.py

@@ -22,15 +22,13 @@
     AdaptiveAveragePooling3D,
     AdaptiveMaxPooling3D,
 )
-from deepray.layers.crf import CRF
-from deepray.layers.embedding import Embedding
+
 from deepray.layers.embedding_bag import EmbeddingBag
-from deepray.layers.esn import ESN
+from deepray.layers.gelu import GELU
 from deepray.layers.max_unpooling_2d import MaxUnpooling2D
 from deepray.layers.max_unpooling_2d_v2 import MaxUnpooling2DV2
 from deepray.layers.maxout import Maxout
 from deepray.layers.multihead_attention import MultiHeadAttention
-from deepray.layers.noisy_dense import NoisyDense
 from deepray.layers.normalizations import FilterResponseNormalization
 from deepray.layers.normalizations import GroupNormalization
 from deepray.layers.normalizations import InstanceNormalization
@@ -39,8 +37,11 @@
 from deepray.layers.polynomial import PolynomialCrossing
 from deepray.layers.snake import Snake
 from deepray.layers.sparsemax import Sparsemax
-from deepray.layers.spatial_pyramid_pooling import SpatialPyramidPooling2D
 from deepray.layers.spectral_normalization import SpectralNormalization
-from deepray.layers.stochastic_depth import StochasticDepth
+from deepray.layers.spatial_pyramid_pooling import SpatialPyramidPooling2D
 from deepray.layers.tlu import TLU
 from deepray.layers.wrappers import WeightNormalization
+from deepray.layers.esn import ESN
+from deepray.layers.stochastic_depth import StochasticDepth
+from deepray.layers.noisy_dense import NoisyDense
+from deepray.layers.crf import CRF