Merge branch 'master' into Jingqing-patch

Author: JingqingZ

docs/user/get_start_model.rst

@@ -23,7 +23,7 @@ Static model
       nn = Dropout(keep=0.8)(nn)
       nn = Dense(n_units=800, act=tf.nn.relu)(nn)
       nn = Dropout(keep=0.8)(nn)
-      nn = Dense(n_units=10, act=tf.nn.relu)(nn)
+      nn = Dense(n_units=10, act=None)(nn)
       M = Model(inputs=ni, outputs=nn, name="mlp") # “name" is optional
       return M
 
@@ -49,7 +49,7 @@ In this case, you need to manually input the output shape of the previous layer
           self.dropout2 = Dropout(keep=0.8)
           self.dense2 = Dense(n_units=800, act=tf.nn.relu, in_channels=800)
           self.dropout3 = Dropout(keep=0.8)
-          self.dense3 = Dense(n_units=10, act=tf.nn.relu, in_channels=800)
+          self.dense3 = Dense(n_units=10, act=None, in_channels=800)
 
       def forward(self, x, foo=False):
           z = self.dropout1(x)
@@ -156,7 +156,7 @@ Print model information
   #   (dropout_1): Dropout(keep=0.8, name='dropout_1')
   #   (dense_1): Dense(n_units=800, relu, in_channels='800', name='dense_1')
   #   (dropout_2): Dropout(keep=0.8, name='dropout_2')
-  #   (dense_2): Dense(n_units=10, relu, in_channels='800', name='dense_2')
+  #   (dense_2): Dense(n_units=10, None, in_channels='800', name='dense_2')
   # )
   
   import pprint
@@ -195,7 +195,7 @@ Print model information
   #                                   'name': 'dropout_3'},
   #                          'class': 'Dropout',
   #                          'prev_layer': ['dense_2_node_0']},
-  #                         {'args': {'act': 'relu',
+  #                         {'args': {'act': None,
   #                                   'layer_type': 'normal',
   #                                   'n_units': 10,
   #                                   'name': 'dense_3'},

examples/basic_tutorials/tutorial_cifar10_cnn_static.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import multiprocessing
 import time
-
 import numpy as np
-
+import multiprocessing
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import (BatchNorm, Conv2d, Dense, Flatten, Input, LocalResponseNorm, MaxPool2d)
@@ -80,14 +78,11 @@ def get_model_batchnorm(inputs_shape):
 print_freq = 5
 n_step_epoch = int(len(y_train) / batch_size)
 n_step = n_epoch * n_step_epoch
-shuffle_buffer_size = 128  # 100
-# init_learning_rate = 0.1
-# learning_rate_decay_factor = 0.1
-# num_epoch_decay = 350
+shuffle_buffer_size = 128
 
 train_weights = net.trainable_weights
-# learning_rate = tf.Variable(init_learning_rate)
 optimizer = tf.optimizers.Adam(learning_rate)
+# looking for decay learning rate? see https://github.com/tensorlayer/srgan/blob/master/train.py
 
 
 def generator_train():
@@ -182,14 +177,10 @@ def _map_fn_test(img, target):
 
     # use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         print("   train loss: {}".format(train_loss / n_iter))
         print("   train acc:  {}".format(train_acc / n_iter))
-
         net.eval()
-
         val_loss, val_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in test_ds:
             _logits = net(X_batch)  # is_train=False, disable dropout
@@ -199,10 +190,6 @@ def _map_fn_test(img, target):
         print("   val loss: {}".format(val_loss / n_iter))
         print("   val acc:  {}".format(val_acc / n_iter))
 
-    # FIXME : how to apply lr decay in eager mode?
-    # learning_rate.assign(tf.train.exponential_decay(init_learning_rate, epoch, num_epoch_decay,
-    #                                                 learning_rate_decay_factor))
-
 # use testing data to evaluate the model
 net.eval()
 test_loss, test_acc, n_iter = 0, 0, 0

examples/basic_tutorials/tutorial_mnist_mlp_dynamic.py

@@ -1,7 +1,5 @@
 import time
-
 import numpy as np
-
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import Dense, Dropout, Input
@@ -19,7 +17,6 @@ class CustomModel(Model):
 
     def __init__(self):
         super(CustomModel, self).__init__()
-
         self.dropout1 = Dropout(keep=0.8)  #(self.innet)
         self.dense1 = Dense(n_units=800, act=tf.nn.relu, in_channels=784)  #(self.dropout1)
         self.dropout2 = Dropout(keep=0.8)  #(self.dense1)
@@ -52,27 +49,20 @@ def forward(self, x, foo=None):
 for epoch in range(n_epoch):  ## iterate the dataset n_epoch times
     start_time = time.time()
     ## iterate over the entire training set once (shuffle the data via training)
-
     for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
-
         MLP.train()  # enable dropout
-
         with tf.GradientTape() as tape:
             ## compute outputs
             _logits = MLP(X_batch, foo=1)
             ## compute loss and update model
             _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
-
         grad = tape.gradient(_loss, train_weights)
         optimizer.apply_gradients(zip(grad, train_weights))
 
     ## use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         MLP.eval()  # disable dropout
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         train_loss, train_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
             _logits = MLP(X_batch, foo=1)
@@ -81,7 +71,6 @@ def forward(self, x, foo=None):
             n_iter += 1
         print("   train foo=1 loss: {}".format(train_loss / n_iter))
         print("   train foo=1 acc:  {}".format(train_acc / n_iter))
-
         val_loss, val_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=False):
             _logits = MLP(X_batch, foo=1)  # is_train=False, disable dropout
@@ -90,7 +79,6 @@ def forward(self, x, foo=None):
             n_iter += 1
         print("   val foo=1 loss: {}".format(val_loss / n_iter))
         print("   val foo=1 acc:  {}".format(val_acc / n_iter))
-
         val_loss, val_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=False):
             _logits = MLP(X_batch)  # is_train=False, disable dropout

examples/basic_tutorials/tutorial_mnist_mlp_dynamic_2.py

@@ -1,7 +1,5 @@
 import time
-
 import numpy as np
-
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import Dense, Dropout, Input, LayerList
@@ -19,17 +17,14 @@ class CustomModelHidden(Model):
 
     def __init__(self):
         super(CustomModelHidden, self).__init__()
-
         self.dropout1 = Dropout(keep=0.8)  #(self.innet)
-
         self.seq = LayerList(
             [
                 Dense(n_units=800, act=tf.nn.relu, in_channels=784),
                 Dropout(keep=0.8),
                 Dense(n_units=800, act=tf.nn.relu, in_channels=800),
             ]
         )
-
         self.dropout3 = Dropout(keep=0.8)  #(self.seq)
 
     def forward(self, x):
@@ -43,7 +38,6 @@ class CustomModelOut(Model):
 
     def __init__(self):
         super(CustomModelOut, self).__init__()
-
         self.dense3 = Dense(n_units=10, act=tf.nn.relu, in_channels=800)
 
     def forward(self, x, foo=None):
@@ -74,30 +68,23 @@ def forward(self, x, foo=None):
 for epoch in range(n_epoch):  ## iterate the dataset n_epoch times
     start_time = time.time()
     ## iterate over the entire training set once (shuffle the data via training)
-
     for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
-
         MLP1.train()  # enable dropout
         MLP2.train()
-
         with tf.GradientTape() as tape:
             ## compute outputs
             _hidden = MLP1(X_batch)
             _logits = MLP2(_hidden, foo=1)
             ## compute loss and update model
             _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
-
         grad = tape.gradient(_loss, train_weights)
         optimizer.apply_gradients(zip(grad, train_weights))
 
     ## use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         MLP1.eval()  # disable dropout
         MLP2.eval()
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         train_loss, train_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
             _hidden = MLP1(X_batch)

examples/basic_tutorials/tutorial_mnist_mlp_static.py

@@ -1,7 +1,5 @@
 import time
-
 import numpy as np
-
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import Dense, Dropout, Input
@@ -21,7 +19,7 @@
 def get_model(inputs_shape):
     ni = Input(inputs_shape)
     nn = Dropout(keep=0.8)(ni)
-    nn = Dense(n_units=800, act=tf.nn.relu)(nn)  
+    nn = Dense(n_units=800, act=tf.nn.relu)(nn)
     nn = Dropout(keep=0.8)(nn)
     nn = Dense(n_units=800, act=tf.nn.relu)(nn)
     nn = Dropout(keep=0.8)(nn)
@@ -45,31 +43,23 @@ def get_model(inputs_shape):
 for epoch in range(n_epoch):  ## iterate the dataset n_epoch times
     start_time = time.time()
     ## iterate over the entire training set once (shuffle the data via training)
-
     for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
-
         MLP.train()  # enable dropout
-
         with tf.GradientTape() as tape:
             ## compute outputs
-            _logits = MLP(X_batch)  # alternatively, you can use MLP(x, is_train=True) and remove MLP.train()
+            _logits = MLP(X_batch)
             ## compute loss and update model
             _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
-
         grad = tape.gradient(_loss, train_weights)
         optimizer.apply_gradients(zip(grad, train_weights))
 
     ## use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         MLP.eval()  # disable dropout
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         train_loss, train_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
-
-            _logits = MLP(X_batch)  # alternatively, you can use MLP(x, is_train=False) and remove MLP.eval()
+            _logits = MLP(X_batch)
             train_loss += tl.cost.cross_entropy(_logits, y_batch, name='eval_loss')
             train_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
             n_iter += 1

examples/basic_tutorials/tutorial_mnist_mlp_static_2.py

@@ -1,7 +1,5 @@
 import time
-
 import numpy as np
-
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import Dense, Dropout, Input
@@ -55,27 +53,20 @@ def get_model(inputs_shape, hmodel):
 for epoch in range(n_epoch):  ## iterate the dataset n_epoch times
     start_time = time.time()
     ## iterate over the entire training set once (shuffle the data via training)
-
     for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
-
         MLP.train()  # enable dropout
-
         with tf.GradientTape() as tape:
             ## compute outputs
             _logits = MLP(X_batch)  # alternatively, you can use MLP(x, is_train=True) and remove MLP.train()
             ## compute loss and update model
             _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
-
         grad = tape.gradient(_loss, train_weights)
         optimizer.apply_gradients(zip(grad, train_weights))
 
     ## use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         MLP.eval()  # disable dropout
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         train_loss, train_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
 
@@ -85,7 +76,6 @@ def get_model(inputs_shape, hmodel):
             n_iter += 1
         print("   train loss: {}".format(train_loss / n_iter))
         print("   train acc:  {}".format(train_acc / n_iter))
-
         val_loss, val_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=False):
             _logits = MLP(X_batch)  # is_train=False, disable dropout