ok so we now have working nets for 0, 1, 2 hidden layers

Author: caetra

neural_network.py

@@ -80,7 +80,7 @@ def feed_forward(self, inputs):
 
     # Back propagates a single example to update weights
     # correct_outputs is a numpy array of size num_outputs
-    def back_propagate(self, inputs, correct_outputs):
+    def back_propagate(self, inputs, correct_outputs, learning_rate):
         # get the outputs we currently get for that input
         (sums, activations) = self.feed_forward(inputs)
         network_outputs = activations[self.num_hidden_layers]
@@ -133,22 +133,22 @@ def back_propagate(self, inputs, correct_outputs):
         for layer in range(1, self.num_hidden_layers):
             for prev_neuron in range(self.neurons_per_layer):
                 for next_neuron in range(self.neurons_per_layer):
-                    self.weights[layer][prev_neuron, next_neuron] += activations[layer-1][prev_neuron] * deltas[layer][next_neuron] # and this
+                    self.weights[layer][prev_neuron, next_neuron] += learning_rate * activations[layer-1][prev_neuron] * deltas[layer][next_neuron] # and this
 
         # Update output weights
         for output in range(self.num_outputs):
             for prev_neuron in range(self.neurons_per_layer):
-                self.weights[self.num_hidden_layers][prev_neuron, output] += activations[self.num_hidden_layers-1][prev_neuron] * deltas[self.num_hidden_layers][output] # and this
+                self.weights[self.num_hidden_layers][prev_neuron, output] += learning_rate * activations[self.num_hidden_layers-1][prev_neuron] * deltas[self.num_hidden_layers][output] # and this
 
         # doesn't return anything (weights update in place)
 
 
 
     # Train a network by repeatedly backpropagating all the examples
-    def train_network(self, num_examples, inputs_list, outputs_list, num_iterations):
+    def train_network(self, num_examples, inputs_list, outputs_list, num_iterations, learning_rate):
         for i in range(num_iterations):
             for j in range(num_examples):
-                self.back_propagate(inputs_list[j], outputs_list[j])
+                self.back_propagate(inputs_list[j], outputs_list[j], learning_rate)
 
     # Convenience function for feeding forward then returning only the output
     def get_output(self, inputs):
@@ -168,4 +168,9 @@ def get_output(self, inputs):
 examples = [a, b, c, d]
 labels = [np.array([0.]), np.array([1.]), np.array([1.]), np.array([0.])]
 
-xor_net.train_network(4, examples, labels, 10000)
+xor_net.train_network(4, examples, labels, 10000, 0.3)
+
+print xor_net.get_output(a)
+print xor_net.get_output(b)
+print xor_net.get_output(c)
+print xor_net.get_output(d)

neural_network_1_hidden.py

@@ -51,6 +51,7 @@ def back_propagate(self, inputs, given_outputs, learning_rate):
 
         deltas = [np.random.rand(self.num_hidden_neurons), np.random.rand(self.num_outputs)]
 
+        # back propagate deltas
         for output in range(self.num_outputs):
             deltas[1][output] = sigmoid_derivative(sums[1][output]) * (given_outputs[output] - activations[1][output])
 
@@ -60,6 +61,7 @@ def back_propagate(self, inputs, given_outputs, learning_rate):
                 sum += (self.weights[1][neuron, output] * deltas[1][output])
             deltas[0][neuron] = (sigmoid_derivative(sums[0][neuron]) * sum)
 
+        # use deltas to update weights
         for input in range(self.num_inputs + 1):
             for neuron in range(self.num_hidden_neurons):
                 self.weights[0][input, neuron] += (learning_rate * inputs[input] * deltas[0][neuron])

neural_network_2_hidden.py

@@ -0,0 +1,114 @@
+# modification of neural network code to only be able to have 2 hidden layers
+
+import numpy as np
+
+# Sigmoid function for activation
+def sigmoid(x):
+    return 1/(1 + np.exp(-x))
+    
+# Derivative of sigmoid for back-propagating
+# Actually has a pretty cool derivative
+def sigmoid_derivative(x):
+    return sigmoid(x) * (1 - sigmoid(x))
+    
+class NeuralNetwork:
+    
+    def __init__(self, num_inputs, num_hidden_first, num_hidden_second, num_outputs):
+        self.num_inputs = num_inputs
+        self.num_hidden_first = num_hidden_first
+        self.num_hidden_second = num_hidden_second
+        self.num_outputs = num_outputs
+        
+        # +1 for bias
+        self.weights = [np.random.rand(self.num_inputs + 1, self.num_hidden_first), np.random.rand(self.num_hidden_first, self.num_hidden_second), np.random.rand(self.num_hidden_second, self.num_outputs)]
+        
+    def feed_forward(self, inputs):
+        # add 1 to end of inputs for bias
+        inputs = np.append(inputs, 1.)
+        
+        sums = [np.random.rand(self.num_hidden_first), np.random.rand(self.num_hidden_second), np.random.rand(self.num_outputs)]
+        activations = [np.random.rand(self.num_hidden_first), np.random.rand(self.num_hidden_second), np.random.rand(self.num_outputs)]
+        
+        for neuron in range(self.num_hidden_first):
+            sum = 0
+            for input in range(self.num_inputs + 1):
+                sum += (self.weights[0][input, neuron] * inputs[input])
+            sums[0][neuron] = sum
+            activations[0][neuron] = sigmoid(sum)
+            
+        for next_neuron in range(self.num_hidden_second):
+            sum = 0
+            for prev_neuron in range(self.num_hidden_second):
+                sum += (self.weights[1][prev_neuron, next_neuron] * activations[0][prev_neuron])
+            sums[1][next_neuron] = sum
+            activations[1][next_neuron] = sigmoid(sum)
+            
+        for output in range(self.num_outputs):
+            sum = 0
+            for neuron in range(self.num_hidden_second):
+                sum += (self.weights[2][neuron, output] * activations[1][neuron])
+            sums[2][output] = sum
+            activations[2][output] = sigmoid(sum)
+            
+        return (sums, activations)
+        
+    def back_propagate(self, inputs, given_outputs, learning_rate):
+        (sums, activations) = self.feed_forward(inputs)
+        inputs = np.append(inputs, 1.)
+        
+        deltas = [np.random.rand(self.num_hidden_first), np.random.rand(self.num_hidden_second), np.random.rand(self.num_outputs)]
+        
+        # back propagate deltas
+        for output in range(self.num_outputs):
+            deltas[2][output] = sigmoid_derivative(sums[2][output]) * (given_outputs[output] - activations[2][output])
+            
+        for neuron in range(self.num_hidden_second):
+            sum = 0
+            for output in range(self.num_outputs):
+                sum += (self.weights[2][neuron, output] * deltas[2][output])
+            deltas[1][neuron] = (sigmoid_derivative(sums[1][neuron]) * sum)
+            
+        for prev_neuron in range(self.num_hidden_first):
+            sum = 0
+            for next_neuron in range(self.num_hidden_second):
+                sum += (self.weights[1][prev_neuron, next_neuron] * deltas[1][next_neuron])
+            deltas[0][prev_neuron] = (sigmoid_derivative(sums[0][prev_neuron]) * sum)
+            
+        # use deltas to update weights
+        for input in range(self.num_inputs + 1):
+            for neuron in range(self.num_hidden_first):
+                self.weights[0][input, neuron] += (learning_rate * inputs[input] * deltas[0][neuron])
+                
+        for prev_neuron in range(self.num_hidden_first):
+            for next_neuron in range(self.num_hidden_second):
+                self.weights[1][prev_neuron, next_neuron] += (learning_rate * activations[0][prev_neuron] * deltas[1][next_neuron])
+
+        for neuron in range(self.num_hidden_second):
+            for output in range(self.num_outputs):
+                self.weights[2][neuron, output] += (learning_rate * activations[1][neuron] * deltas[2][output])
+                
+                
+    # Train a network by repeatedly backpropagating all the examples
+    def train_network(self, num_examples, inputs_list, outputs_list, num_iterations, learning_rate):
+        for i in range(num_iterations):
+            for j in range(num_examples):
+                self.back_propagate(inputs_list[j], outputs_list[j], learning_rate)  
+                
+    def get_output(self, inputs):
+        return self.feed_forward(inputs)[1][2]
+ 
+# main program ########################################################################## 
+xor_net = NeuralNetwork(2, 3, 3, 1)
+a = np.array([1.,1.])
+b = np.array([1.,0.])
+c = np.array([0.,1.])
+d = np.array([0.,0.])
+examples = [a, b, c, d]
+labels = [np.array([0.]), np.array([1.]), np.array([1.]), np.array([0.])]
+
+xor_net.train_network(4, examples, labels, 5000, 0.3)
+
+print xor_net.get_output(a)
+print xor_net.get_output(b)
+print xor_net.get_output(c)
+print xor_net.get_output(d)

perceptron.py

@@ -0,0 +1,90 @@
+# Perceptron (neural net with no hidden layers)
+# Can't learn things that aren't linearly separable
+# But tends to work well on most real life data
+
+import numpy as np
+
+# Sigmoid function for activation
+def sigmoid(x):
+    return 1/(1 + np.exp(-x))
+    
+# Derivative of sigmoid for back-propagating
+# Actually has a pretty cool derivative
+def sigmoid_derivative(x):
+    return sigmoid(x) * (1 - sigmoid(x))
+    
+class Perceptron:
+
+    def __init__(self, num_inputs, num_outputs):
+        self.num_inputs = num_inputs
+        self.num_outputs = num_outputs
+        
+        self.weights = np.random.rand(self.num_inputs + 1, self.num_outputs)
+        
+    def feed_forward(self, inputs):
+        inputs = np.append(inputs, 1.) # bias
+        sums = np.random.rand(self.num_outputs)
+        activations = np.random.rand(self.num_outputs)
+        
+        for output in range(self.num_outputs):
+            sum = 0
+            for input in range(self.num_inputs + 1):
+                sum += self.weights[input, output] * inputs[input]
+            sums[output] = sum
+            activations[output] = sigmoid(sum)
+            
+        return (sums, activations)
+        
+    def back_propagate(self, inputs, given_outputs, learning_rate):
+        (sums, activations) = self.feed_forward(inputs)
+        inputs = np.append(inputs, 1.)
+        deltas = np.random.rand(self.num_outputs)
+        
+        for output in range(self.num_outputs):
+            deltas[output] = sigmoid_derivative(sums[output]) * (given_outputs[output] - activations[output])
+            
+        for input in range(self.num_inputs + 1):
+            for output in range(self.num_outputs):
+                self.weights[input, output] += (learning_rate * inputs[input] * deltas[output])
+                
+    def train_network(self, num_examples, inputs_list, outputs_list, num_iterations, learning_rate):
+        for i in range(num_iterations):
+            for j in range(num_examples):
+                self.back_propagate(inputs_list[j], outputs_list[j], learning_rate)  
+                
+    def get_output(self, inputs):
+        return self.feed_forward(inputs)[1]
+        
+#### Main program ####
+
+print "doing and"
+and_net = Perceptron(2, 1)
+a = np.array([1.,1.])
+b = np.array([1.,0.])
+c = np.array([0.,1.])
+d = np.array([0.,0.])
+examples = [a, b, c, d]
+labels = [np.array([1.]), np.array([0.]), np.array([0.]), np.array([0.])]
+
+and_net.train_network(4, examples, labels, 5000, 0.3)
+
+print and_net.get_output(a)
+print and_net.get_output(b)
+print and_net.get_output(c)
+print and_net.get_output(d)
+
+print "doing or"
+or_net = Perceptron(2, 1)
+a = np.array([1.,1.])
+b = np.array([1.,0.])
+c = np.array([0.,1.])
+d = np.array([0.,0.])
+examples = [a, b, c, d]
+labels = [np.array([1.]), np.array([1.]), np.array([1.]), np.array([0.])]
+
+or_net.train_network(4, examples, labels, 5000, 0.3)
+
+print or_net.get_output(a)
+print or_net.get_output(b)
+print or_net.get_output(c)
+print or_net.get_output(d)