diff --git a/Cargo.toml b/Cargo.toml
index 13c39e06..abcc6910 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,7 +27,7 @@ flate2 = { version = "1.0", features = ["cloudflare_zlib"], default-features = f
 shellwords = "1.1.0"
 blas = "0.22"
 intel-mkl-src = {version= "0.7.0", default-features = false, features=["download", "mkl-static-lp64-seq"]}
-
+libm = "0.2.6"
 [build-dependencies]
 cbindgen = "0.23.0"
 
diff --git a/src/block_leaky_relu.rs b/src/block_leaky_relu.rs
new file mode 100644
index 00000000..7e31f698
--- /dev/null
+++ b/src/block_leaky_relu.rs
@@ -0,0 +1,180 @@
+use std::any::Any;
+use std::error::Error;
+
+use crate::block_helpers;
+use crate::feature_buffer;
+use crate::graph;
+use crate::graph::BlockGraph;
+use crate::model_instance;
+use crate::port_buffer;
+use crate::regressor;
+use regressor::BlockTrait;
+
+pub struct BlockLeakyRELU {
+    pub num_inputs: usize,
+    pub input_offset: usize,
+    pub output_offset: usize,
+    pub alpha: f32,
+}
+
+pub fn new_leaky_relu_block(
+    bg: &mut BlockGraph,
+    input: graph::BlockPtrOutput,
+) -> Result<graph::BlockPtrOutput, Box<dyn Error>> {
+    let num_inputs = bg.get_num_output_values(vec![&input]);
+    assert!(num_inputs != 0);
+    let block = Box::new(BlockLeakyRELU {
+        output_offset: usize::MAX,
+        input_offset: usize::MAX,
+        num_inputs: num_inputs,
+        alpha: 0.3, // TODO consider how to extract this and make configurable
+    });
+    let mut block_outputs = bg.add_node(block, vec![input])?;
+    assert_eq!(block_outputs.len(), 1);
+    Ok(block_outputs.pop().unwrap())
+}
+
+impl BlockTrait for BlockLeakyRELU {
+    fn as_any(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    #[inline(always)]
+    fn forward_backward(
+        &mut self,
+        further_blocks: &mut [Box<dyn BlockTrait>],
+        fb: &feature_buffer::FeatureBuffer,
+        pb: &mut port_buffer::PortBuffer,
+        update: bool,
+    ) {
+        debug_assert!(self.output_offset != usize::MAX);
+        debug_assert!(self.input_offset != usize::MAX);
+        debug_assert!(self.num_inputs > 0);
+
+        unsafe {
+            for i in 0..self.num_inputs as usize {
+                let x = *pb.tape.get_unchecked_mut(self.input_offset + i);
+                if x < 0.0 {
+                    *pb.tape.get_unchecked_mut(self.output_offset + i) = self.alpha * x;
+                } else {
+                    *pb.tape.get_unchecked_mut(self.output_offset + i) = x;
+                }
+
+                if x <= 0.0 {
+                    *pb.tape.get_unchecked_mut(self.input_offset + i) = self.alpha;
+                } else {
+                    *pb.tape.get_unchecked_mut(self.input_offset + i) = 1.0;
+                }
+            }
+
+            block_helpers::forward_backward(further_blocks, fb, pb, update);
+
+            if update {
+                for i in 0..self.num_inputs as usize {
+                    let gradient = *pb.tape.get_unchecked(self.output_offset + i);
+                    *pb.tape.get_unchecked_mut(self.input_offset + i) *= gradient;
+                }
+            }
+        } // unsafe end
+    }
+
+    fn forward(
+        &self,
+        further_blocks: &[Box<dyn BlockTrait>],
+        fb: &feature_buffer::FeatureBuffer,
+        pb: &mut port_buffer::PortBuffer,
+    ) {
+        debug_assert!(self.output_offset != usize::MAX);
+        debug_assert!(self.input_offset != usize::MAX);
+        debug_assert!(self.num_inputs > 0);
+
+        unsafe {
+            for i in 0..self.num_inputs as usize {
+                let x = *pb.tape.get_unchecked_mut(self.input_offset + i);
+                if x < 0.0 {
+                    *pb.tape.get_unchecked_mut(self.output_offset + i) = self.alpha * x;
+                } else {
+                    *pb.tape.get_unchecked_mut(self.output_offset + i) = x;
+                }
+            }
+            block_helpers::forward(further_blocks, fb, pb);
+        } // unsafe end
+    }
+
+    fn allocate_and_init_weights(&mut self, _mi: &model_instance::ModelInstance) {}
+
+    fn get_num_output_values(&self, output: graph::OutputSlot) -> usize {
+        assert!(output.get_output_index() == 0);
+        return self.num_inputs;
+    }
+
+    fn get_num_output_slots(&self) -> usize {
+        1
+    }
+
+    fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) {
+        assert!(input.get_input_index() == 0);
+        self.input_offset = offset;
+    }
+
+    fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) {
+        assert!(output.get_output_index() == 0);
+        self.output_offset = offset;
+    }
+}
+
+mod tests {
+    // Note this useful idiom: importing names from outer (for mod tests) scope.
+    use super::*;
+    use crate::assert_epsilon;
+    use crate::block_misc;
+    use crate::feature_buffer;
+    use block_helpers::slearn2;
+    use block_misc::Observe;
+
+    fn fb_vec() -> feature_buffer::FeatureBuffer {
+        feature_buffer::FeatureBuffer {
+            label: 0.0,
+            example_importance: 1.0,
+            example_number: 0,
+            lr_buffer: Vec::new(),
+            ffm_buffer: Vec::new(),
+            ffm_fields_count: 0,
+        }
+    }
+
+    #[test]
+    fn test_simple_positive() {
+        let mut mi = model_instance::ModelInstance::new_empty().unwrap();
+        let mut bg = BlockGraph::new();
+        let input_block = block_misc::new_const_block(&mut bg, vec![2.0]).unwrap();
+        let leaky_relu_block = new_leaky_relu_block(&mut bg, input_block).unwrap();
+        block_misc::new_observe_block(&mut bg, leaky_relu_block, Observe::Forward, Some(1.0))
+            .unwrap();
+        bg.finalize();
+        bg.allocate_and_init_weights(&mi);
+
+        let mut pb = bg.new_port_buffer();
+
+        let fb = fb_vec();
+        assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 2.0);
+        assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 2.0); // leaky_relu doesn't learn
+    }
+
+    fn test_simple_negative() {
+        let mi = model_instance::ModelInstance::new_empty().unwrap();
+        let mut bg = BlockGraph::new();
+        let input_block = block_misc::new_const_block(&mut bg, vec![-2.0]).unwrap();
+        let leaky_relu_block = new_leaky_relu_block(&mut bg, input_block).unwrap();
+        block_misc::new_observe_block(&mut bg, leaky_relu_block, Observe::Forward, Some(1.0))
+            .unwrap();
+        bg.finalize();
+        bg.allocate_and_init_weights(&mi);
+
+        let mut pb = bg.new_port_buffer();
+
+        let fb = fb_vec();
+        assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.0);
+        assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.0); // leaky_relu doesn't learn
+    }
+}
diff --git a/src/block_relu.rs b/src/block_relu.rs
index 9ead178c..16250926 100644
--- a/src/block_relu.rs
+++ b/src/block_relu.rs
@@ -1,6 +1,3 @@
-use std::any::Any;
-use std::error::Error;
-
 use crate::block_helpers;
 use crate::feature_buffer;
 use crate::graph;
@@ -9,6 +6,8 @@ use crate::model_instance;
 use crate::port_buffer;
 use crate::regressor;
 use regressor::BlockTrait;
+use std::any::Any;
+use std::error::Error;
 
 pub struct BlockRELU {
     pub num_inputs: usize,
diff --git a/src/block_sigmoid.rs b/src/block_sigmoid.rs
new file mode 100644
index 00000000..68124ef5
--- /dev/null
+++ b/src/block_sigmoid.rs
@@ -0,0 +1,174 @@
+use libm::expf;
+use std::any::Any;
+use std::error::Error;
+use crate::block_helpers;
+use crate::feature_buffer;
+use crate::graph;
+use crate::graph::BlockGraph;
+use crate::model_instance;
+use crate::port_buffer;
+use crate::regressor;
+use regressor::BlockTrait;
+
+pub struct BlockSigmoid {
+    pub num_inputs: usize,
+    pub input_offset: usize,
+    pub output_offset: usize,
+}
+
+pub fn new_sigmoid_block(
+    bg: &mut graph::BlockGraph,
+    input: graph::BlockPtrOutput,
+) -> Result<graph::BlockPtrOutput, Box<dyn Error>> {
+    let num_inputs = bg.get_num_output_values(vec![&input]);
+    assert!(num_inputs != 0);
+    let block = Box::new(BlockSigmoid {
+        output_offset: usize::MAX,
+        input_offset: usize::MAX,
+        num_inputs: num_inputs,
+    });
+    let mut block_outputs = bg.add_node(block, vec![input])?;
+    assert_eq!(block_outputs.len(), 1);
+    Ok(block_outputs.pop().unwrap())
+}
+
+impl BlockTrait for BlockSigmoid {
+    fn as_any(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn allocate_and_init_weights(&mut self, _mi: &model_instance::ModelInstance) {}
+
+    fn get_num_output_slots(&self) -> usize {
+        1
+    }
+
+    fn get_num_output_values(&self, output: graph::OutputSlot) -> usize {
+        assert!(output.get_output_index() == 0);
+        return self.num_inputs;
+    }
+
+    fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) {
+        assert!(input.get_input_index() == 0);
+        self.input_offset = offset;
+    }
+
+    fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) {
+        assert!(output.get_output_index() == 0);
+        self.output_offset = offset;
+    }
+
+    #[inline(always)]
+    fn forward_backward(
+        &mut self,
+        further_blocks: &mut [Box<dyn BlockTrait>],
+        fb: &feature_buffer::FeatureBuffer,
+        pb: &mut port_buffer::PortBuffer,
+        update: bool,
+    ) {
+        debug_assert!(self.output_offset != usize::MAX);
+        debug_assert!(self.input_offset != usize::MAX);
+        debug_assert!(self.num_inputs > 0);
+
+        unsafe {
+            for i in 0..self.num_inputs as usize {
+                let x = *pb.tape.get_unchecked_mut(self.input_offset + i);
+
+                // for now doing the actual slow sigmoid computation. once we establish a baseline,
+                // we can replace with a fast approximation or a lookup table
+                if x < 0.0 {
+                    let epx = expf(x);
+                    let s = epx / (1.0 + epx);
+                    *pb.tape.get_unchecked_mut(self.output_offset + i) = s;
+                    *pb.tape.get_unchecked_mut(self.input_offset + i) = s * (1.0 - s);
+                } else {
+                    let s = 1.0 / (1.0 + expf(-x));
+                    *pb.tape.get_unchecked_mut(self.output_offset + i) = s;
+                    *pb.tape.get_unchecked_mut(self.input_offset + i) = s * (1.0 - s);
+                }
+            }
+
+            block_helpers::forward_backward(further_blocks, fb, pb, update);
+
+            if update {
+                for i in 0..self.num_inputs as usize {
+                    let gradient = *pb.tape.get_unchecked(self.output_offset + i);
+                    *pb.tape.get_unchecked_mut(self.input_offset + i) *= gradient;
+                }
+            }
+        } // unsafe end
+    }
+
+    fn forward(
+        &self,
+        further_blocks: &[Box<dyn BlockTrait>],
+        fb: &feature_buffer::FeatureBuffer,
+        pb: &mut port_buffer::PortBuffer,
+    ) {
+        debug_assert!(self.output_offset != usize::MAX);
+        debug_assert!(self.input_offset != usize::MAX);
+        debug_assert!(self.num_inputs > 0);
+
+        unsafe {
+            for i in 0..self.num_inputs as usize {
+                let x = *pb.tape.get_unchecked_mut(self.input_offset + i);
+                *pb.tape.get_unchecked_mut(self.output_offset + i) = 1.0 / (1.0 + expf(-x));
+            }
+            block_helpers::forward(further_blocks, fb, pb);
+        } // unsafe end
+    }
+}
+
+mod tests {
+    // Note this useful idiom: importing names from outer (for mod tests) scope.
+    use super::*;
+    use crate::assert_epsilon;
+    use crate::block_misc;
+    use crate::feature_buffer;
+    use block_helpers::slearn2;
+    use block_misc::Observe;
+
+    fn fb_vec() -> feature_buffer::FeatureBuffer {
+        feature_buffer::FeatureBuffer {
+            label: 0.0,
+            example_importance: 1.0,
+            example_number: 0,
+            lr_buffer: Vec::new(),
+            ffm_buffer: Vec::new(),
+            ffm_fields_count: 0,
+        }
+    }
+
+    #[test]
+    fn test_simple_positive() {
+        let mut mi = model_instance::ModelInstance::new_empty().unwrap();
+        let mut bg = BlockGraph::new();
+        let input_block = block_misc::new_const_block(&mut bg, vec![2.0]).unwrap();
+        let sigmoid_block = new_sigmoid_block(&mut bg, input_block).unwrap();
+        block_misc::new_observe_block(&mut bg, sigmoid_block, Observe::Forward, Some(1.0)).unwrap();
+        bg.finalize();
+        bg.allocate_and_init_weights(&mi);
+
+        let mut pb = bg.new_port_buffer();
+
+        let fb = fb_vec();
+        assert_epsilon!(slearn2  (&mut bg, &fb, &mut pb, true), 0.880797);
+        assert_epsilon!(slearn2  (&mut bg, &fb, &mut pb, true), 0.880797); // sigmoid doesn't learn
+    }
+
+    fn test_simple_negative() {
+        let mi = model_instance::ModelInstance::new_empty().unwrap();
+        let mut bg = BlockGraph::new();
+        let input_block = block_misc::new_const_block(&mut bg, vec![-2.0]).unwrap();
+        let sigmoid_block = new_sigmoid_block(&mut bg, input_block).unwrap();
+        block_misc::new_observe_block(&mut bg, sigmoid_block, Observe::Forward, Some(1.0)).unwrap();
+        bg.finalize();
+        bg.allocate_and_init_weights(&mi);
+
+        let mut pb = bg.new_port_buffer();
+
+        let fb = fb_vec();
+        assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.0);
+        assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.0); // sigmoid doesn't learn
+    }
+}
diff --git a/src/block_tanh.rs b/src/block_tanh.rs
new file mode 100644
index 00000000..3035757d
--- /dev/null
+++ b/src/block_tanh.rs
@@ -0,0 +1,166 @@
+use libm::tanhf;
+use std::any::Any;
+use std::error::Error;
+use crate::block_helpers;
+use crate::feature_buffer;
+use crate::graph;
+use crate::graph::BlockGraph;
+use crate::model_instance;
+use crate::port_buffer;
+use crate::regressor;
+use regressor::BlockTrait;
+
+pub struct BlockTanh {
+    pub num_inputs: usize,
+    pub input_offset: usize,
+    pub output_offset: usize,
+}
+
+pub fn new_tanh_block(
+    bg: &mut graph::BlockGraph,
+    input: graph::BlockPtrOutput,
+) -> Result<graph::BlockPtrOutput, Box<dyn Error>> {
+    let num_inputs = bg.get_num_output_values(vec![&input]);
+    assert!(num_inputs != 0);
+    let block = Box::new(BlockTanh {
+        output_offset: usize::MAX,
+        input_offset: usize::MAX,
+        num_inputs: num_inputs,
+    });
+    let mut block_outputs = bg.add_node(block, vec![input])?;
+    assert_eq!(block_outputs.len(), 1);
+    Ok(block_outputs.pop().unwrap())
+}
+
+impl BlockTrait for BlockTanh {
+    fn as_any(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn allocate_and_init_weights(&mut self, _mi: &model_instance::ModelInstance) {}
+
+    fn get_num_output_slots(&self) -> usize {
+        1
+    }
+
+    fn get_num_output_values(&self, output: graph::OutputSlot) -> usize {
+        assert!(output.get_output_index() == 0);
+        return self.num_inputs;
+    }
+
+    fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) {
+        assert!(input.get_input_index() == 0);
+        self.input_offset = offset;
+    }
+
+    fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) {
+        assert!(output.get_output_index() == 0);
+        self.output_offset = offset;
+    }
+
+    #[inline(always)]
+    fn forward_backward(
+        &mut self,
+        further_blocks: &mut [Box<dyn BlockTrait>],
+        fb: &feature_buffer::FeatureBuffer,
+        pb: &mut port_buffer::PortBuffer,
+        update: bool,
+    ) {
+        debug_assert!(self.output_offset != usize::MAX);
+        debug_assert!(self.input_offset != usize::MAX);
+        debug_assert!(self.num_inputs > 0);
+
+        unsafe {
+            for i in 0..self.num_inputs as usize {
+                let x = *pb.tape.get_unchecked_mut(self.input_offset + i);
+
+                // for now using libm tanh computation. once we establish a baseline,
+                // we can try to replace with a lookup table
+                let t = tanhf(x);
+                *pb.tape.get_unchecked_mut(self.output_offset + i) = t;
+                *pb.tape.get_unchecked_mut(self.input_offset + i) = 1. - t * t; // derivative
+            }
+
+            block_helpers::forward_backward(further_blocks, fb, pb, update);
+
+            if update {
+                for i in 0..self.num_inputs as usize {
+                    let gradient = *pb.tape.get_unchecked(self.output_offset + i);
+                    *pb.tape.get_unchecked_mut(self.input_offset + i) *= gradient;
+                }
+            }
+        } // unsafe end
+    }
+
+    fn forward(
+        &self,
+        further_blocks: &[Box<dyn BlockTrait>],
+        fb: &feature_buffer::FeatureBuffer,
+        pb: &mut port_buffer::PortBuffer,
+    ) {
+        debug_assert!(self.output_offset != usize::MAX);
+        debug_assert!(self.input_offset != usize::MAX);
+        debug_assert!(self.num_inputs > 0);
+
+        unsafe {
+            for i in 0..self.num_inputs as usize {
+                let x = *pb.tape.get_unchecked_mut(self.input_offset + i);
+                *pb.tape.get_unchecked_mut(self.output_offset + i) = tanhf(x);
+            }
+            block_helpers::forward(further_blocks, fb, pb);
+        } // unsafe end
+    }
+}
+
+mod tests {
+    // Note this useful idiom: importing names from outer (for mod tests) scope.
+    use super::*;
+    use crate::assert_epsilon;
+    use crate::block_misc;
+    use crate::feature_buffer;
+    use block_helpers::slearn2;
+    use block_misc::Observe;
+
+    fn fb_vec() -> feature_buffer::FeatureBuffer {
+        feature_buffer::FeatureBuffer {
+            label: 0.0,
+            example_importance: 1.0,
+            example_number: 0,
+            lr_buffer: Vec::new(),
+            ffm_buffer: Vec::new(),
+            ffm_fields_count: 0,
+        }
+    }
+
+    #[test]
+    fn test_simple_positive() {
+        let mut mi = model_instance::ModelInstance::new_empty().unwrap();
+        let mut bg = BlockGraph::new();
+        let input_block = block_misc::new_const_block(&mut bg, vec![2.0]).unwrap();
+        let tanh_block = new_tanh_block(&mut bg, input_block).unwrap();
+        block_misc::new_observe_block(&mut bg, tanh_block, Observe::Forward, Some(1.0)).unwrap();
+        bg.finalize();
+        bg.allocate_and_init_weights(&mi);
+
+        let mut pb = bg.new_port_buffer();
+
+        let fb = fb_vec();
+        assert_epsilon!(slearn2  (&mut bg, &fb, &mut pb, true), 0.9640276);
+        assert_epsilon!(slearn2  (&mut bg, &fb, &mut pb, true), 0.9640276); // tanh doesn't learn
+    }
+    fn test_simple_negative() {
+        let mi = model_instance::ModelInstance::new_empty().unwrap();
+        let mut bg = BlockGraph::new();
+        let input_block = block_misc::new_const_block(&mut bg, vec![-2.0]).unwrap();
+        let tanh_block = new_tanh_block(&mut bg, input_block).unwrap();
+        block_misc::new_observe_block(&mut bg, tanh_block, Observe::Forward, Some(1.0)).unwrap();
+        bg.finalize();
+        bg.allocate_and_init_weights(&mi);
+
+        let mut pb = bg.new_port_buffer();
+
+        let fb = fb_vec();
+        assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.0);
+        assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.0); // tanh desnt learn
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 47542585..d46a86fb 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -6,6 +6,9 @@ mod block_misc;
 mod block_neural;
 mod block_normalize;
 mod block_relu;
+mod block_leaky_relu;
+mod block_tanh;
+mod block_sigmoid;
 mod cache;
 mod cmdline;
 mod consts;
diff --git a/src/main.rs b/src/main.rs
index 2c7ceeab..5a809855 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -30,6 +30,9 @@ mod block_misc;
 mod block_neural;
 mod block_normalize;
 mod block_relu;
+mod block_leaky_relu;
+mod block_tanh;
+mod block_sigmoid;
 mod cache;
 mod cmdline;
 mod consts;
diff --git a/src/regressor.rs b/src/regressor.rs
index ed365a3a..5d7e2939 100644
--- a/src/regressor.rs
+++ b/src/regressor.rs
@@ -6,6 +6,7 @@ use std::io::Cursor;
 
 use crate::block_ffm;
 use crate::block_helpers;
+use crate::block_leaky_relu;
 use crate::block_loss_functions;
 use crate::block_lr;
 use crate::block_misc;
@@ -13,6 +14,8 @@ use crate::block_neural;
 use crate::block_neural::InitType;
 use crate::block_normalize;
 use crate::block_relu;
+use crate::block_sigmoid;
+use crate::block_tanh;
 use crate::feature_buffer;
 use crate::graph;
 use crate::model_instance;
@@ -104,13 +107,16 @@ pub fn get_regressor_with_weights(mi: &model_instance::ModelInstance) -> Regress
 enum NNActivation {
     None,
     Relu,
+    LeakyRelu,
+    Tanh,
+    Sigmoid,
 }
 
 #[derive(PartialEq)]
 enum NNLayerNorm {
     None,
-    BeforeRelu,
-    AfterRelu,
+    BeforeActivation,
+    AfterActivation,
 }
 
 impl Regressor {
@@ -204,6 +210,9 @@ impl Regressor {
                 let activation = match &*activation_str {
                     "none" => NNActivation::None,
                     "relu" => NNActivation::Relu,
+                    "leaky_relu" => NNActivation::LeakyRelu,
+                    "tanh" => NNActivation::Tanh,
+                    "sigmoid" => NNActivation::Sigmoid,
                     _ => Err(format!(
                         "unknown nn activation type: \"{}\"",
                         activation_str
@@ -213,8 +222,8 @@ impl Regressor {
 
                 let layernorm = match &*layernorm_str {
                     "none" => NNLayerNorm::None,
-                    "before" => NNLayerNorm::BeforeRelu,
-                    "after" => NNLayerNorm::AfterRelu,
+                    "before" => NNLayerNorm::BeforeActivation,
+                    "after" => NNLayerNorm::AfterActivation,
                     _ => Err(format!("unknown nn layer norm: \"{}\"", layernorm_str)).unwrap(),
                 };
 
@@ -245,14 +254,32 @@ impl Regressor {
                 )
                 .unwrap();
 
-                if layernorm == NNLayerNorm::BeforeRelu {
+                if layernorm == NNLayerNorm::BeforeActivation {
                     output =
                         block_normalize::new_normalize_layer_block(&mut bg, &mi, output).unwrap();
                 }
-                if activation == NNActivation::Relu {
-                    output = block_relu::new_relu_block(&mut bg, &mi, output).unwrap();
+
+                match activation {
+                    NNActivation::None => {}
+                    NNActivation::Relu => {
+                        output = block_relu::new_relu_block(&mut bg, &mi, output).unwrap();
+                        println!("Relu layer");
+                    }
+                    NNActivation::LeakyRelu => {
+                        output = block_leaky_relu::new_leaky_relu_block(&mut bg, output).unwrap();
+                        println!("LeakyRelu layer");
+                    }
+                    NNActivation::Tanh => {
+                        output = block_tanh::new_tanh_block(&mut bg, output).unwrap();
+                        println!("Tanh layer");
+                    }
+                    NNActivation::Sigmoid => {
+                        output = block_sigmoid::new_sigmoid_block(&mut bg, output).unwrap();
+                        println!("Sigmoid layer");
+                    }
                 }
-                if layernorm == NNLayerNorm::AfterRelu {
+
+                if layernorm == NNLayerNorm::AfterActivation {
                     output =
                         block_normalize::new_normalize_layer_block(&mut bg, &mi, output).unwrap();
                 }