Merge pull request #12070 from atrick/benchinfo

Author: swift-ci

benchmark/CMakeLists.txt

@@ -25,6 +25,7 @@ set(SWIFT_BENCH_MODULES
     single-source/ArrayOfGenericRef
     single-source/ArrayOfPOD
     single-source/ArrayOfRef
+    single-source/ArraySetElement
     single-source/ArraySubscript
     single-source/BitCount
     single-source/ByteSwap

benchmark/single-source/ArraySetElement.swift

@@ -0,0 +1,39 @@
+//===--- ArraySetElement.swift ---------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import TestsUtils
+
+// 33% isUniquelyReferenced
+//   15% swift_rt_swift_isUniquelyReferencedOrPinned_nonNull_native
+//   18% swift_isUniquelyReferencedOrPinned_nonNull_native
+public var ArraySetElement = BenchmarkInfo(
+  name: "ArraySetElement",
+  runFunction: run_ArraySetElement,
+  tags: [.runtime, .cpubench]
+)
+
+// This is an effort to defeat isUniquelyReferenced optimization. Ideally
+// microbenchmarks list this should be written in C.
+@inline(never)
+func storeArrayElement(_ array: inout [Int], _ i: Int) {
+  array[i] = i
+}
+
+public func run_ArraySetElement(_ N: Int) {
+  let scale = 10
+  var array = [Int](repeating: 0, count: 10000)
+  for _ in 0..<N*scale {
+    for i in 0..<array.count {
+      storeArrayElement(&array, i)
+    }
+  }
+}

benchmark/single-source/ObjectAllocation.swift

@@ -13,10 +13,13 @@
 import TestsUtils
 
 // This test checks the performance of allocations.
+// 53% _swift_release_dealloc
+// 30% _swift_alloc_object
+// 10% retain/release
 public var ObjectAllocation = BenchmarkInfo(
   name: "ObjectAllocation",
   runFunction: run_ObjectAllocation,
-  tags: [.runtime]
+  tags: [.runtime, .cpubench]
 )
 
 final class XX {

benchmark/utils/DriverUtils.swift

@@ -182,8 +182,11 @@ struct TestConfig {
     if onlyRegistered {
       allTests = registeredBenchmarks.map {
         bench -> (key: String, value: (Int)-> ()) in
-        (bench.name, bench.runFunction)
+        return (bench.name, bench.runFunction)
       }
+      // FIXME: for now unstable/extra benchmarks are not registered at all, but
+      // soon they will be handled with a default exclude list.
+      onlyPrecommit = false
     }
     else {
       allTests = [precommitTests, otherTests, stringTests]

benchmark/utils/TestsUtils.swift

@@ -17,16 +17,6 @@ import Darwin
 #endif
 
 public enum BenchmarkCategories {
-// Optimized benchmarks cover performance areas that have already been
-// investigated and optimized to some reasonable level. We expect these to be
-// "representative" of Swift perforance.
-case optimized
-
-// Most benchmarks are assumed to be "stable" and will be regularly tracked at
-// each commit. A handful may be marked unstable if continually tracking them is
-// counterproductive.
-case unstable
-
 // Validation "micro" benchmarks test a specific operation or critical path that
 // we know is important to measure.
 case validation
@@ -53,9 +43,28 @@ case miniapplication
 // report. We want to make sure the optimizer as a whole continues to handle
 // this case, but don't know how applicable it is to general Swift performance
 // relative to the other micro-benchmarks. In particular, these aren't weighted
-// as highly as "validation" benchmarks and likely won't be the subject of future
-// investigation unless they significantly regress.
+// as highly as "validation" benchmarks and likely won't be the subject of
+// future investigation unless they significantly regress.
 case regression
+
+// Most benchmarks are assumed to be "stable" and will be regularly tracked at
+// each commit. A handful may be marked unstable if continually tracking them is
+// counterproductive.
+case unstable
+
+// CPU benchmarks represent instrinsic Swift performance. They are useful for
+// measuring a fully baked Swift implementation across different platforms and
+// hardware. The benchmark should also be reasonably applicable to real Swift
+// code--it should exercise a known performance critical area. Typically these
+// will be drawn from the validation benchmarks once the language and standard
+// library implementation of the benchmark meets a reasonable efficiency
+// baseline. A benchmark should only be tagged "cpubench" after a full
+// performance investigation of the benchmark has been completed to determine
+// that it is a good representation of future Swift performance. Benchmarks
+// should not be tagged if they make use of an API that we plan on
+// reimplementing or call into code paths that have known opportunities for
+// significant optimization.
+case cpubench
 }
 
 public struct BenchmarkInfo {

benchmark/utils/main.swift

@@ -30,6 +30,7 @@ import ArrayOfGenericPOD
 import ArrayOfGenericRef
 import ArrayOfPOD
 import ArrayOfRef
+import ArraySetElement
 import ArraySubscript
 import BitCount
 import ByteSwap
@@ -137,6 +138,7 @@ private func registerBenchmark(_ bench: BenchmarkInfo) {
   registeredBenchmarks.append(bench)
 }
 
+registerBenchmark(ArraySetElement)
 registerBenchmark(ObjectAllocation)
 
 // The main test suite: precommit tests