Adding a ResNet56-CIFAR10 benchmark (#431)

* Added ResNet56 model and benchmark.

* inputFilters now switches its default value based on whether or not CIFAR-10-sized images are used for ResNet.

* On second thought, let's just remove that option and only use the defaults.

Author: BradLarson

Benchmarks/Models.swift

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-let benchmarkModels = [
+let benchmarkModels: [String: BenchmarkModel] = [
     "LeNetMNIST": LeNetMNIST(),
+    "ResNetCIFAR10": ResNetCIFAR10(),
 ]

Benchmarks/Models/ImageClassificationInference.swift

@@ -21,6 +21,7 @@ protocol ImageClassificationModel: Layer where Input == Tensor<Float>, Output ==
 }
 
 extension LeNet: ImageClassificationModel {}
+extension ResNet56: ImageClassificationModel {}
 
 class ImageClassificationInference<Model, ClassificationDataset>: Benchmark
 where Model: ImageClassificationModel, ClassificationDataset: ImageClassificationDataset {

Benchmarks/Models/ResNetCIFAR10.swift

@@ -0,0 +1,49 @@
+// Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Datasets
+import ImageClassificationModels
+import TensorFlow
+
+struct ResNetCIFAR10: BenchmarkModel {
+
+    var defaultInferenceSettings: BenchmarkSettings {
+        return BenchmarkSettings(batches: 1000, batchSize: 1, iterations: 10, epochs: -1)
+    }
+
+    func makeInferenceBenchmark(settings: BenchmarkSettings) -> Benchmark {
+        return ImageClassificationInference<ResNet56, CIFAR10>(settings: settings)
+    }
+
+    var defaultTrainingSettings: BenchmarkSettings {
+        return BenchmarkSettings(batches: -1, batchSize: 128, iterations: 10, epochs: 1)
+    }
+
+    func makeTrainingBenchmark(settings: BenchmarkSettings) -> Benchmark {
+        return ImageClassificationTraining<ResNet56, CIFAR10>(settings: settings)
+    }
+}
+
+struct ResNet56: Layer {
+    var model: ResNet
+    
+    init() {
+        model = ResNet(classCount: 10, depth: .resNet56, downsamplingInFirstStage: false)
+    }
+    
+    @differentiable
+    public func callAsFunction(_ input: Tensor<Float>) -> Tensor<Float> {
+        return model(input)
+    }
+}

Examples/ResNet-CIFAR10/main.swift

@@ -21,7 +21,7 @@ let batchSize = 10
 let dataset = CIFAR10(batchSize: batchSize)
 
 // Use the network sized for CIFAR-10
-var model = ResNet(classCount: 10, depth: .resNet50, downsamplingInFirstStage: true)
+var model = ResNet(classCount: 10, depth: .resNet56, downsamplingInFirstStage: false)
 
 // the classic ImageNet optimizer setting diverges on CIFAR-10
 // let optimizer = SGD(for: model, learningRate: 0.1, momentum: 0.9)

Models/ImageClassification/ResNet.swift

@@ -127,13 +127,17 @@ public struct ResNet: Layer {
     ///     3x3 convolution, corresponding to the v1.5 variant of the architecture. 
     public init(
         classCount: Int, depth: Depth, downsamplingInFirstStage: Bool = true,
-        inputFilters: Int = 64, useLaterStride: Bool = true
+        useLaterStride: Bool = true
     ) {
+        let inputFilters: Int
+        
         if downsamplingInFirstStage {
+            inputFilters = 64
             initialLayer = ConvBN(
                 filterShape: (7, 7, 3, inputFilters), strides: (2, 2), padding: .same)
             maxPool = MaxPool2D(poolSize: (3, 3), strides: (2, 2), padding: .same)
         } else {
+            inputFilters = 16
             initialLayer = ConvBN(filterShape: (3, 3, 3, inputFilters), padding: .same)
             maxPool = MaxPool2D(poolSize: (1, 1), strides: (1, 1))  // no-op
         }
@@ -151,7 +155,10 @@ public struct ResNet: Layer {
             }
         }
 
-        classifier = Dense(inputSize: depth.usesBasicBlocks ? 512 : 2048, outputSize: classCount)
+        let finalFilters = inputFilters * Int(pow(2.0, Double(depth.layerBlockSizes.count - 1)))
+        classifier = Dense(
+            inputSize: depth.usesBasicBlocks ? finalFilters : finalFilters * 4,
+            outputSize: classCount)
     }
 
     @differentiable
@@ -169,12 +176,13 @@ extension ResNet {
         case resNet18
         case resNet34
         case resNet50
+        case resNet56
         case resNet101
         case resNet152
 
         var usesBasicBlocks: Bool {
             switch self {
-            case .resNet18, .resNet34: return true
+            case .resNet18, .resNet34, .resNet56: return true
             default: return false
             }
         }
@@ -184,6 +192,7 @@ extension ResNet {
             case .resNet18: return [2, 2, 2, 2]
             case .resNet34: return [3, 4, 6, 3]
             case .resNet50: return [3, 4, 6, 3]
+            case .resNet56: return [9, 9, 9]
             case .resNet101: return [3, 4, 23, 3]
             case .resNet152: return [3, 8, 36, 3]
             }