Fixed comments, and fixed discrepancies to match torch perplexity

Author: adamlerer

word_language_model/main.py

@@ -23,7 +23,7 @@
 parser.add_argument('-nhid'      , type=int, default=200          , help='Number of hidden units per layer.'        )
 parser.add_argument('-nlayers'   , type=int, default=2            , help='Number of layers.'                        )
 # Optimization parameters.
-parser.add_argument('-lr'        , type=float, default=20         , help='Initial learning rate.'                   )
+parser.add_argument('-lr'        , type=float, default=1          , help='Initial learning rate.'                   )
 parser.add_argument('-clip'      , type=float, default=0.5        , help='Gradient clipping.'                       )
 parser.add_argument('-maxepoch'  , type=int,   default=6          , help='Upper epoch limit.'                       )
 parser.add_argument('-batchsize' , type=int,   default=20         , help='Batch size.'                              )
@@ -68,8 +68,6 @@ def batchify(data, bsz, bptt):
 # MAKE MODEL
 ###############################################################################
 
-initrange = 0.1
-
 class RNNModel(nn.Container):
     """A container module with an encoder, an RNN (one of several flavors),
     and a decoder. Runs one RNN step at a time.
@@ -96,11 +94,12 @@ def __init__(self, rnnType, ntoken, ninp, nhid, nlayers):
 
         # FIXME: is this better than the standard init? probably
         # FIXME: we need better reset_parameters methods in stdlib
+        initrange = 0.1
         self.encoder.weight.data.uniform_(-initrange, initrange)
         self.decoder.bias.data.fill_(0)
         self.decoder.weight.data.uniform_(-initrange, initrange)
 
-    def __call__(self, hidden, input):
+    def forward(self, hidden, input):
         emb = self.encoder(input)
         hidden, output = self.rnn(hidden, emb)
         decoded = self.decoder(output)
@@ -130,9 +129,9 @@ def evaluate(model, data, criterion):
     # Loop over validation data.
     for i in range(0, data.size(0) - 1):
         hidden, output = model(hidden, Variable(data[i], requires_grad=False))
-        loss += criterion(output, Variable(data[i+1], requires_grad=False)).data[0]
+        loss += criterion(output, Variable(data[i+1], requires_grad=False)).data
 
-    return loss / data.size(0)
+    return loss[0] / data.size(0)
 
 # simple gradient clipping, using the total norm of the gradient
 def clipGradient(model, clip):
@@ -193,7 +192,8 @@ def repackageHidden(h):
             print(
                     ('| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.6f} | ms/batch {:5.2f} | '
                     + 'train loss {:5.2f} | train ppl {:8.2f}').format(
-                epoch, i / bptt, train.size(0) / bptt, lr, elapsed * 1000 / reportinterval,
+                epoch, i / bptt, train.size(0) / bptt, lr,
+                elapsed * 1000 / reportinterval * bptt,
                 cur_loss, math.exp(cur_loss)
             ))
             total_loss = 0
@@ -204,7 +204,7 @@ def repackageHidden(h):
     # ps = pstats.Stats(pr, stream=s).sort_stats("time")
     # ps.print_stats()
     # print(s.getvalue())
-    # val_loss = evaluate(model, valid, criterion)
+    val_loss = evaluate(model, valid, criterion)
 
     print(
         '| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | valid ppl {:8.2f}'.format(

word_language_model/rnn_modules.py

@@ -12,8 +12,8 @@ class RNN(nn.Container):
 
     def __init__(self, ninp, nhid):
         super(RNN, self).__init__(
-            i2h=nn.Linear(ninp, nhid),
-            h2h=nn.Linear(nhid, nhid),
+            i2h=nn.Linear(ninp, nhid, bias=False),
+            h2h=nn.Linear(nhid, nhid, bias=False),
             sigmoid=nn.Sigmoid(),
         )
         self.ninp = ninp
@@ -31,15 +31,15 @@ class LSTM(nn.Container):
 
     def __init__(self, ninp, nhid):
         super(LSTM, self).__init__(
-            i2h=nn.Linear(ninp, 4 * nhid), 
-            h2h=nn.Linear(nhid, 4 * nhid), 
+            i2h=nn.Linear(ninp, 4 * nhid, bias=False),
+            h2h=nn.Linear(nhid, 4 * nhid, bias=False),
             sigmoid=nn.Sigmoid(),
             tanh=nn.Tanh(),
         )
         self.ninp = ninp
         self.nhid = nhid
 
-    def __call__(self, hidden, input):
+    def forward(self, hidden, input):
         c, h = hidden
         gates = self.h2h(h) + self.i2h(input)
         gates      = gates.view(input.size(0), 4, self.nhid).transpose(0, 1)
@@ -63,23 +63,23 @@ class GRU(nn.Container):
 
     def __init__(self, ninp, nhid):
         super(GRU, self).__init__(
-            i2h=nn.Linear(ninp, 3 * nhid),
-            h2h=nn.Linear(nhid, 3 * nhid),
+            i2h=nn.Linear(ninp, 3 * nhid, bias=False),
+            h2h=nn.Linear(nhid, 3 * nhid, bias=False),
             sigmoid=nn.Sigmoid(),
             tanh=nn.Tanh(),
         )
         self.ninp = ninp
         self.nhid = nhid
 
-    def __call__(self, hidden, input):
-        gi = i2h(input).view(3, input.size(0), self.nhid).transpose(0, 1)
-        gh = h2h(hidden).view(3, input.size(0), self.nhid).transpose(0, 1)
+    def forward(self, hidden, input):
+        gi = self.i2h(input).view(input.size(0), 3, self.nhid).transpose(0, 1)
+        gh = self.h2h(hidden).view(input.size(0), 3, self.nhid).transpose(0, 1)
 
         resetgate  = self.sigmoid(gi[0] + gh[0])
         updategate = self.sigmoid(gi[1] + gh[1])
 
         output = self.tanh(gi[2] + resetgate * gh[2])
-        nexth = hidden + updategate * (output - h)
+        nexth = hidden + updategate * (output - hidden)
 
         return nexth, output
 
@@ -97,7 +97,7 @@ def __init__(self, rnnClass, ninp, nhid, nlayers):
             self.layers += [layer]
             self.add_module('layer' + str(i), layer)
 
-    def __call__(self, hidden, input):
+    def forward(self, hidden, input):
         output = input
         new_hidden = [None] * self.nlayers
         for i in range(self.nlayers):