refactor: add a FC layer after RNN (#74)

* refactor: add a FC layer

1. Add a FC Layer
  - #73
2. Add a numpy doc style
3. Explicitly define an activation function in LSTMCell
  - default is tanh
4. Change to AdamOptimizer
5. Use `with` statement else the session must be closed explicitly
6. No linebreak for the 79 char limit

* refactor: nn.tanh -> tanh

Author: kkweon

lab-12-5-rnn_stock_prediction.py

@@ -1,19 +1,45 @@
+'''
+This script shows how to predict stock prices using a basic RNN
+'''
 import tensorflow as tf
 import numpy as np
 import matplotlib.pyplot as plt
 tf.set_random_seed(777)  # reproducibility
 
 
 def MinMaxScaler(data):
+    ''' Min Max Normalization
+
+    Parameters
+    ----------
+    data : numpy.ndarray
+        input data to be normalized 
+        shape: [Batch size, dimension]
+
+    Returns
+    ----------
+    data : numpy.ndarry
+        normalized data
+        shape: [Batch size, dimension]
+
+    References
+    ----------
+    .. [1] http://sebastianraschka.com/Articles/2014_about_feature_scaling.html
+
+    '''
     numerator = data - np.min(data, 0)
     denominator = np.max(data, 0) - np.min(data, 0)
     # noise term prevents the zero division
     return numerator / (denominator + 1e-7)
 
 
+# train Parameters
 timesteps = seq_length = 7
 data_dim = 5
+hidden_dim = 10
 output_dim = 1
+learing_rate = 0.01
+iterations = 500
 
 # Open, High, Low, Volume, Close
 xy = np.loadtxt('data-02-stock_daily.csv', delimiter=',')
@@ -22,6 +48,7 @@ def MinMaxScaler(data):
 x = xy
 y = xy[:, [-1]]  # Close as label
 
+# build a dataset
 dataX = []
 dataY = []
 for i in range(0, len(y) - seq_length):
@@ -31,47 +58,49 @@ def MinMaxScaler(data):
     dataX.append(_x)
     dataY.append(_y)
 
-# split to train and testing
+# train/test split
 train_size = int(len(dataY) * 0.7)
 test_size = len(dataY) - train_size
-trainX, testX = np.array(dataX[0:train_size]), np.array(
-    dataX[train_size:len(dataX)])
-trainY, testY = np.array(dataY[0:train_size]), np.array(
-    dataY[train_size:len(dataY)])
+trainX, testX = np.array(dataX[0:train_size]), np.array(dataX[train_size:len(dataX)])
+trainY, testY = np.array(dataY[0:train_size]), np.array(dataY[train_size:len(dataY)])
 
 # input place holders
 X = tf.placeholder(tf.float32, [None, seq_length, data_dim])
 Y = tf.placeholder(tf.float32, [None, 1])
 
-cell = tf.contrib.rnn.BasicLSTMCell(num_units=output_dim, state_is_tuple=True)
+# build a LSTM network
+cell = tf.contrib.rnn.BasicLSTMCell(num_units=hidden_dim, state_is_tuple=True, activation=tf.tanh)
 outputs, _states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)
-Y_pred = outputs[:, -1]  # We use the last cell's output
-
-print(outputs[:, -1])
+Y_pred = tf.contrib.layers.fully_connected(outputs[:, -1], output_dim, activation_fn=None)  # We use the last cell's output
 
 # cost/loss
 loss = tf.reduce_sum(tf.square(Y_pred - Y))  # sum of the squares
 # optimizer
-optimizer = tf.train.GradientDescentOptimizer(0.01)
+optimizer = tf.train.AdamOptimizer(learing_rate)
 train = optimizer.minimize(loss)
 
 # RMSE
 targets = tf.placeholder(tf.float32, [None, 1])
 predictions = tf.placeholder(tf.float32, [None, 1])
 rmse = tf.sqrt(tf.reduce_mean(tf.square(targets - predictions)))
 
-sess = tf.Session()
-sess.run(tf.global_variables_initializer())
-
-for i in range(500):
-    _, step_loss = sess.run([train, loss], feed_dict={X: trainX, Y: trainY})
-    print(i, step_loss)
-
-testPredict = sess.run(Y_pred, feed_dict={X: testX})
-print("RMSE", sess.run(rmse, feed_dict={
-      targets: testY, predictions: testPredict}))
-plt.plot(testY)
-plt.plot(testPredict)
-plt.xlabel("Time Period")
-plt.ylabel("Stock Price")
-plt.show()
+with tf.Session() as sess:
+    init = tf.global_variables_initializer()
+    sess.run(init)
+
+    # Training step
+    for i in range(iterations):
+        _, step_loss = sess.run([train, loss], feed_dict={X: trainX, Y: trainY})
+        print("[step: {}] loss: {}".format(i, step_loss))
+
+    # Test step
+    test_predict = sess.run(Y_pred, feed_dict={X: testX})
+    rmse = sess.run(rmse, feed_dict={targets: testY, predictions: test_predict})
+    print("RMSE: {}".format(rmse))
+
+    # Plot predictions
+    plt.plot(testY)
+    plt.plot(test_predict)
+    plt.xlabel("Time Period")
+    plt.ylabel("Stock Price")
+    plt.show()