added nonlinearity

Author: culurciello

seq-learning-basic/train.py

@@ -102,6 +102,7 @@ def __init__(self):
         self.doc1 = 8 # number of intermediate features
         self.c1 = nn.Conv1d(1, self.doc1, self.L)
         self.l1 = nn.Linear(self.doc1, 1)
+        self.relu = nn.ReLU(inplace=True)
         print('CNN model ops per sample:', self.L*self.doc1+self.doc1)
 
     def forward(self, input, future = 0):
@@ -111,15 +112,15 @@ def forward(self, input, future = 0):
 
         for i in range(0, input.size(1)-self.L):
             input_b = input[:,i:i+self.L].unsqueeze(1) # torch.Size([97, 1, 10])
-            o1 = self.c1(input_b)
+            o1 = self.relu( self.c1(input_b) )
             o1= o1.view(input.size(0), self.doc1)
             output = self.l1(o1)
             outputs += [output]
 
         for i in range(future):# if we should predict the future
             output_b = outputs[input.size(1)-self.L-1+i:input.size(1)-1+i]
             output_b = torch.stack(output_b, 1).squeeze().unsqueeze(1)
-            o1 = self.c1(output_b)
+            o1 = self.relu( self.c1(output_b) )
             o1 = o1.view(input.size(0), self.doc1)
             output = self.l1(o1)
             outputs += [output]

seq-learning-char/model.py

@@ -45,6 +45,7 @@ def __init__(self, input_size, hidden_size, pint, output_size):
 
         self.encoder = nn.Embedding(input_size, hidden_size)
         self.c1 = nn.Conv2d(1, hidden_size, [pint, hidden_size])
+        self.relu = nn.ReLU(inplace=True)
         self.decoder = nn.Linear(hidden_size, output_size)
 
     def forward(self, input, position):
@@ -54,7 +55,7 @@ def forward(self, input, position):
         inn = enc + position
         # print('enc+pos', inn, position)
         # print('inn', inn.shape) 
-        oc1 = self.c1(inn.view(1, 1, inn.size(0), inn.size(1)))
+        oc1 = self.relu( self.c1(inn.view(1, 1, inn.size(0), inn.size(1))) )
         output = self.decoder(oc1.view(1, -1))
         # print('out', output)
         return output