Update ResNeXt model and accuracy

Author: kuangliu

README.md

@@ -13,14 +13,15 @@ Cons:
 - No built-in log.
 
 ## Accuracy
-| Model        | Acc.    |
-| ------------ |-----------  |
-| VGG16        | 92.64%      |
-| ResNet18     | 93.02%      |
-| ResNet50     | 93.62%      |
-| ResNet101    | 93.75%      |
-| ResNeXt29    | 94.73%      |
-| DenseNet121  | 95.04%      |
+| Model            | Acc.        |
+| ------------     | ----------- |
+| VGG16            | 92.64%      |
+| ResNet18         | 93.02%      |
+| ResNet50         | 93.62%      |
+| ResNet101        | 93.75%      |
+| ResNeXt29(32x4d) | 94.73%      |
+| ResNeXt29(2x64d) | 94.82%      |
+| DenseNet121      | 95.04%      |
 
 ## Learning rate adjustment
 I manually change the `lr` during training:

main.py

@@ -59,7 +59,7 @@
     # net = ResNet18()
     # net = GoogLeNet()
     # net = DenseNet121()
-    net = ResNeXt29()
+    net = ResNeXt29_2x64d()
 
 if use_cuda:
     net.cuda()

models/resnext.py

@@ -4,7 +4,6 @@
 '''
 import torch
 import torch.nn as nn
-import torch.nn.init as init
 import torch.nn.functional as F
 
 from torch.autograd import Variable
@@ -14,20 +13,21 @@ class Block(nn.Module):
     '''Grouped convolution block.'''
     expansion = 2
 
-    def __init__(self, in_planes, planes, stride=1, cardinality=32):
+    def __init__(self, in_planes, cardinality=32, bottleneck_width=4, stride=1):
         super(Block, self).__init__()
-        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
-        self.bn1 = nn.BatchNorm2d(planes)
-        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, groups=cardinality, bias=False)
-        self.bn2 = nn.BatchNorm2d(planes)
-        self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False)
-        self.bn3 = nn.BatchNorm2d(self.expansion*planes)
+        group_width = cardinality * bottleneck_width
+        self.conv1 = nn.Conv2d(in_planes, group_width, kernel_size=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(group_width)
+        self.conv2 = nn.Conv2d(group_width, group_width, kernel_size=3, stride=stride, padding=1, groups=cardinality, bias=False)
+        self.bn2 = nn.BatchNorm2d(group_width)
+        self.conv3 = nn.Conv2d(group_width, self.expansion*group_width, kernel_size=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(self.expansion*group_width)
 
         self.shortcut = nn.Sequential()
-        if stride != 1 or in_planes != self.expansion*planes:
+        if stride != 1 or in_planes != self.expansion*group_width:
             self.shortcut = nn.Sequential(
-                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
-                nn.BatchNorm2d(self.expansion*planes)
+                nn.Conv2d(in_planes, self.expansion*group_width, kernel_size=1, stride=stride, bias=False),
+                nn.BatchNorm2d(self.expansion*group_width)
             )
 
     def forward(self, x):
@@ -40,26 +40,28 @@ def forward(self, x):
 
 
 class ResNeXt(nn.Module):
-    def __init__(self, block, num_blocks, cardinality=32, num_classes=10):
+    def __init__(self, num_blocks, cardinality, bottleneck_width, num_classes=10):
         super(ResNeXt, self).__init__()
+        self.cardinality = cardinality
+        self.bottleneck_width = bottleneck_width
         self.in_planes = 64
 
         self.conv1 = nn.Conv2d(3, 64, kernel_size=1, bias=False)
         self.bn1 = nn.BatchNorm2d(64)
-        self.layer1 = self._make_layer(block, 64, num_blocks[0], 1, cardinality)
-        self.layer2 = self._make_layer(block, 128, num_blocks[1], 2, cardinality)
-        self.layer3 = self._make_layer(block, 256, num_blocks[2], 2, cardinality)
-        # self.layer4 = self._make_layer(block, 512, num_blocks[3], 2, cardinality)
-        self.linear = nn.Linear(512, num_classes)
+        self.layer1 = self._make_layer(num_blocks[0], 1)
+        self.layer2 = self._make_layer(num_blocks[1], 2)
+        self.layer3 = self._make_layer(num_blocks[2], 2)
+        # self.layer4 = self._make_layer(num_blocks[3], 2)
+        self.linear = nn.Linear(cardinality*bottleneck_width*8, num_classes)
 
-        self.init_params()
-
-    def _make_layer(self, block, planes, num_blocks, stride, cardinality):
+    def _make_layer(self, num_blocks, stride):
         strides = [stride] + [1]*(num_blocks-1)
         layers = []
         for stride in strides:
-            layers.append(block(self.in_planes, planes, stride, cardinality))
-            self.in_planes = planes * block.expansion
+            layers.append(Block(self.in_planes, self.cardinality, self.bottleneck_width, stride))
+            self.in_planes = Block.expansion * self.cardinality * self.bottleneck_width
+        # Increase bottleneck_width by 2 after each stage.
+        self.bottleneck_width *= 2
         return nn.Sequential(*layers)
 
     def forward(self, x):
@@ -73,27 +75,23 @@ def forward(self, x):
         out = self.linear(out)
         return out
 
-    def init_params(self):
-        '''Init layer parameters.'''
-        for m in self.modules():
-            if isinstance(m, nn.Conv2d):
-                init.kaiming_normal(m.weight, mode='fan_out')
-                if m.bias:
-                    init.constant(m.bias, 0)
-            elif isinstance(m, nn.BatchNorm2d):
-                init.constant(m.weight, 1)
-                init.constant(m.bias, 0)
-            elif isinstance(m, nn.Linear):
-                init.normal(m.weight, std=1e-3)
-                if m.bias:
-                    init.constant(m.bias, 0)
-
-
-def ResNeXt29():
-    return ResNeXt(Block, [3,3,3])
-
-
-# net = resnext_cifar()
-# x = torch.randn(1,3,32,32)
-# y = net(Variable(x))
-# print(y.size())
+
+def ResNeXt29_2x64d():
+    return ResNeXt(num_blocks=[3,3,3], cardinality=2, bottleneck_width=64)
+
+def ResNeXt29_4x64d():
+    return ResNeXt(num_blocks=[3,3,3], cardinality=4, bottleneck_width=64)
+
+def ResNeXt29_8x64d():
+    return ResNeXt(num_blocks=[3,3,3], cardinality=8, bottleneck_width=64)
+
+def ResNeXt29_32x4d():
+    return ResNeXt(num_blocks=[3,3,3], cardinality=32, bottleneck_width=4)
+
+def test_resnext():
+    net = ResNeXt29_2x64d()
+    x = torch.randn(1,3,32,32)
+    y = net(Variable(x))
+    print(y.size())
+
+# test_resnext()

utils.py

@@ -9,6 +9,7 @@
 import math
 
 import torch.nn as nn
+import torch.nn.init as init
 
 
 def get_mean_and_std(dataset):
@@ -34,19 +35,20 @@ def get_mean_and_std(dataset):
 # print(mean)
 # print(std)
 
-
-def msr_init(net):
-    '''Initialize layer parameters.'''
-    for layer in net:
-        if type(layer) == nn.Conv2d:
-            n = layer.kernel_size[0]*layer.kernel_size[1]*layer.out_channels
-            layer.weight.data.normal_(0, math.sqrt(2./n))
-            layer.bias.data.zero_()
-        elif type(layer) == nn.BatchNorm2d:
-            layer.weight.data.fill_(1)
-            layer.bias.data.zero_()
-        elif type(layer) == nn.Linear:
-            layer.bias.data.zero_()
+def init_params(net):
+    '''Init layer parameters.'''
+    for m in net.modules():
+        if isinstance(m, nn.Conv2d):
+            init.kaiming_normal(m.weight, mode='fan_out')
+            if m.bias:
+                init.constant(m.bias, 0)
+        elif isinstance(m, nn.BatchNorm2d):
+            init.constant(m.weight, 1)
+            init.constant(m.bias, 0)
+        elif isinstance(m, nn.Linear):
+            init.normal(m.weight, std=1e-3)
+            if m.bias:
+                init.constant(m.bias, 0)
 
 
 _, term_width = os.popen('stty size', 'r').read().split()