Skip to content

Commit be9ff08

Browse files
vaindvaind
authored
Perf metrics (#2067)
See the overall issue at getsentry/team-mobile#5 and the sibling PR at sentry-java for the approach & some more details: getsentry/sentry-java#2171
1 parent f626eac commit be9ff08

File tree

22 files changed

+975
-12
lines changed

22 files changed

+975
-12
lines changed

‎.github/workflows/benchmarking.yml

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ jobs:
7979
name: Run benchmarks on Sauce Labs
8080
runs-on: ubuntu-latest
8181
needs: build-benchmark-test-target
82-
strategy:
82+
strategy:
8383
fail-fast: false
8484
matrix:
8585
suite: ["High-end device", "Mid-range device", "Low-end device"]
@@ -94,3 +94,43 @@ jobs:
9494
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
9595
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
9696
run: saucectl run --select-suite "${{matrix.suite}}" --config .sauce/benchmarking-config.yml --tags benchmark
97+
98+
app-metrics:
99+
name: Collect app metrics
100+
runs-on: macos-11
101+
steps:
102+
- name: Git checkout
103+
uses: actions/checkout@v3
104+
- run: ./scripts/ci-select-xcode.sh
105+
- uses: actions/cache@v3
106+
id: app-plain-cache
107+
with:
108+
path: Tests/Perf/test-app-plain.ipa
109+
key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ hashFiles('fastlane/Fastfile', 'Tests/Perf/test-app-plain/**') }}
110+
- name: Build test app plain
111+
if: steps.app-plain-cache.outputs['cache-hit'] != 'true'
112+
run: fastlane build_perf_test_app_plain
113+
env:
114+
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
115+
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
116+
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
117+
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
118+
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
119+
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
120+
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
121+
- name: Build test app with sentry
122+
run: fastlane build_perf_test_app_sentry
123+
env:
124+
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
125+
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
126+
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
127+
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
128+
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
129+
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
130+
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
131+
- name: Collect app metrics
132+
uses: getsentry/action-app-sdk-overhead-metrics@v1
133+
with:
134+
config: Tests/Perf/metrics-test.yml
135+
sauce-user: ${{ secrets.SAUCE_USERNAME }}
136+
sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }}

‎.swiftlint.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ excluded:
66
- Sources/SentryCrash
77
- vendor
88
- test-server/.build/
9+
- Tests/Perf
910

1011
only_rules:
1112
#- attributes

‎Tests/Perf/metrics-test.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apps:
2+
- name: io.sentry.cocoa.perf-test-app-plain
3+
path: test-app-plain.ipa
4+
- name: io.sentry.cocoa.perf-test-app-sentry
5+
path: test-app-sentry.ipa
6+
7+
startupTimeTest:
8+
runs: 50
9+
diffMin: 0
10+
diffMax: 150
11+
12+
binarySizeTest:
13+
diffMin: 200 KiB
14+
diffMax: 400 KiB

‎Tests/Perf/test-app-plain/test-app-plain.xcodeproj/project.pbxproj

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
// !$*UTF8*$!
2+
{
3+
archiveVersion = 1;
4+
classes = {
5+
};
6+
objectVersion = 55;
7+
objects = {
8+
9+
/* Begin PBXBuildFile section */
10+
2C101B5428A2953600D320BC /* test_app_plainApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2C101B5328A2953600D320BC /* test_app_plainApp.swift */; };
11+
2C101B5628A2953600D320BC /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2C101B5528A2953600D320BC /* ContentView.swift */; };
12+
2C101B5828A2953700D320BC /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 2C101B5728A2953700D320BC /* Assets.xcassets */; };
13+
2C101B5B28A2953700D320BC /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 2C101B5A28A2953700D320BC /* Preview Assets.xcassets */; };
14+
/* End PBXBuildFile section */
15+
16+
/* Begin PBXFileReference section */
17+
2C101B5028A2953600D320BC /* test-app-plain.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "test-app-plain.app"; sourceTree = BUILT_PRODUCTS_DIR; };
18+
2C101B5328A2953600D320BC /* test_app_plainApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = test_app_plainApp.swift; sourceTree = "<group>"; };
19+
2C101B5528A2953600D320BC /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
20+
2C101B5728A2953700D320BC /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
21+
2C101B5A28A2953700D320BC /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = "<group>"; };
22+
/* End PBXFileReference section */
23+
24+
/* Begin PBXFrameworksBuildPhase section */
25+
2C101B4D28A2953600D320BC /* Frameworks */ = {
26+
isa = PBXFrameworksBuildPhase;
27+
buildActionMask = 2147483647;
28+
files = (
29+
);
30+
runOnlyForDeploymentPostprocessing = 0;
31+
};
32+
/* End PBXFrameworksBuildPhase section */
33+
34+
/* Begin PBXGroup section */
35+
2C101B4728A2953600D320BC = {
36+
isa = PBXGroup;
37+
children = (
38+
2C101B5228A2953600D320BC /* test-app-plain */,
39+
2C101B5128A2953600D320BC /* Products */,
40+
);
41+
sourceTree = "<group>";
42+
};
43+
2C101B5128A2953600D320BC /* Products */ = {
44+
isa = PBXGroup;
45+
children = (
46+
2C101B5028A2953600D320BC /* test-app-plain.app */,
47+
);
48+
name = Products;
49+
sourceTree = "<group>";
50+
};
51+
2C101B5228A2953600D320BC /* test-app-plain */ = {
52+
isa = PBXGroup;
53+
children = (
54+
2C101B5328A2953600D320BC /* test_app_plainApp.swift */,
55+
2C101B5528A2953600D320BC /* ContentView.swift */,
56+
2C101B5728A2953700D320BC /* Assets.xcassets */,
57+
2C101B5928A2953700D320BC /* Preview Content */,
58+
);
59+
path = "test-app-plain";
60+
sourceTree = "<group>";
61+
};
62+
2C101B5928A2953700D320BC /* Preview Content */ = {
63+
isa = PBXGroup;
64+
children = (
65+
2C101B5A28A2953700D320BC /* Preview Assets.xcassets */,
66+
);
67+
path = "Preview Content";
68+
sourceTree = "<group>";
69+
};
70+
/* End PBXGroup section */
71+
72+
/* Begin PBXNativeTarget section */
73+
2C101B4F28A2953600D320BC /* test-app-plain */ = {
74+
isa = PBXNativeTarget;
75+
buildConfigurationList = 2C101B5E28A2953700D320BC /* Build configuration list for PBXNativeTarget "test-app-plain" */;
76+
buildPhases = (
77+
2C101B4C28A2953600D320BC /* Sources */,
78+
2C101B4D28A2953600D320BC /* Frameworks */,
79+
2C101B4E28A2953600D320BC /* Resources */,
80+
);
81+
buildRules = (
82+
);
83+
dependencies = (
84+
);
85+
name = "test-app-plain";
86+
productName = "test-app-plain";
87+
productReference = 2C101B5028A2953600D320BC /* test-app-plain.app */;
88+
productType = "com.apple.product-type.application";
89+
};
90+
/* End PBXNativeTarget section */
91+
92+
/* Begin PBXProject section */
93+
2C101B4828A2953600D320BC /* Project object */ = {
94+
isa = PBXProject;
95+
attributes = {
96+
BuildIndependentTargetsInParallel = 1;
97+
LastSwiftUpdateCheck = 1340;
98+
LastUpgradeCheck = 1340;
99+
TargetAttributes = {
100+
2C101B4F28A2953600D320BC = {
101+
CreatedOnToolsVersion = 13.4;
102+
};
103+
};
104+
};
105+
buildConfigurationList = 2C101B4B28A2953600D320BC /* Build configuration list for PBXProject "test-app-plain" */;
106+
compatibilityVersion = "Xcode 13.0";
107+
developmentRegion = en;
108+
hasScannedForEncodings = 0;
109+
knownRegions = (
110+
en,
111+
Base,
112+
);
113+
mainGroup = 2C101B4728A2953600D320BC;
114+
productRefGroup = 2C101B5128A2953600D320BC /* Products */;
115+
projectDirPath = "";
116+
projectRoot = "";
117+
targets = (
118+
2C101B4F28A2953600D320BC /* test-app-plain */,
119+
);
120+
};
121+
/* End PBXProject section */
122+
123+
/* Begin PBXResourcesBuildPhase section */
124+
2C101B4E28A2953600D320BC /* Resources */ = {
125+
isa = PBXResourcesBuildPhase;
126+
buildActionMask = 2147483647;
127+
files = (
128+
2C101B5B28A2953700D320BC /* Preview Assets.xcassets in Resources */,
129+
2C101B5828A2953700D320BC /* Assets.xcassets in Resources */,
130+
);
131+
runOnlyForDeploymentPostprocessing = 0;
132+
};
133+
/* End PBXResourcesBuildPhase section */
134+
135+
/* Begin PBXSourcesBuildPhase section */
136+
2C101B4C28A2953600D320BC /* Sources */ = {
137+
isa = PBXSourcesBuildPhase;
138+
buildActionMask = 2147483647;
139+
files = (
140+
2C101B5628A2953600D320BC /* ContentView.swift in Sources */,
141+
2C101B5428A2953600D320BC /* test_app_plainApp.swift in Sources */,
142+
);
143+
runOnlyForDeploymentPostprocessing = 0;
144+
};
145+
/* End PBXSourcesBuildPhase section */
146+
147+
/* Begin XCBuildConfiguration section */
148+
2C101B5D28A2953700D320BC /* Release */ = {
149+
isa = XCBuildConfiguration;
150+
buildSettings = {
151+
ALWAYS_SEARCH_USER_PATHS = NO;
152+
CLANG_ANALYZER_NONNULL = YES;
153+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
154+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
155+
CLANG_ENABLE_MODULES = YES;
156+
CLANG_ENABLE_OBJC_ARC = YES;
157+
CLANG_ENABLE_OBJC_WEAK = YES;
158+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
159+
CLANG_WARN_BOOL_CONVERSION = YES;
160+
CLANG_WARN_COMMA = YES;
161+
CLANG_WARN_CONSTANT_CONVERSION = YES;
162+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
163+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
164+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
165+
CLANG_WARN_EMPTY_BODY = YES;
166+
CLANG_WARN_ENUM_CONVERSION = YES;
167+
CLANG_WARN_INFINITE_RECURSION = YES;
168+
CLANG_WARN_INT_CONVERSION = YES;
169+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
170+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
171+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
172+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
173+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
174+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
175+
CLANG_WARN_STRICT_PROTOTYPES = YES;
176+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
177+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
178+
CLANG_WARN_UNREACHABLE_CODE = YES;
179+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
180+
CODE_SIGN_IDENTITY = "Apple Development";
181+
COPY_PHASE_STRIP = NO;
182+
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
183+
DEVELOPMENT_TEAM = 97JCY7859U;
184+
ENABLE_NS_ASSERTIONS = NO;
185+
ENABLE_STRICT_OBJC_MSGSEND = YES;
186+
GCC_C_LANGUAGE_STANDARD = gnu11;
187+
GCC_NO_COMMON_BLOCKS = YES;
188+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
189+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
190+
GCC_WARN_UNDECLARED_SELECTOR = YES;
191+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
192+
GCC_WARN_UNUSED_FUNCTION = YES;
193+
GCC_WARN_UNUSED_VARIABLE = YES;
194+
IPHONEOS_DEPLOYMENT_TARGET = 14.0;
195+
MTL_ENABLE_DEBUG_INFO = NO;
196+
MTL_FAST_MATH = YES;
197+
SDKROOT = iphoneos;
198+
SWIFT_COMPILATION_MODE = wholemodule;
199+
SWIFT_OPTIMIZATION_LEVEL = "-O";
200+
VALIDATE_PRODUCT = YES;
201+
};
202+
name = Release;
203+
};
204+
2C101B6028A2953700D320BC /* Release */ = {
205+
isa = XCBuildConfiguration;
206+
buildSettings = {
207+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
208+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
209+
CODE_SIGN_IDENTITY = "iPhone Developer";
210+
CODE_SIGN_STYLE = Manual;
211+
CURRENT_PROJECT_VERSION = 1;
212+
DEVELOPMENT_ASSET_PATHS = "\"test-app-plain/Preview Content\"";
213+
DEVELOPMENT_TEAM = 97JCY7859U;
214+
ENABLE_PREVIEWS = YES;
215+
GENERATE_INFOPLIST_FILE = YES;
216+
INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
217+
INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
218+
INFOPLIST_KEY_UILaunchScreen_Generation = YES;
219+
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
220+
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
221+
LD_RUNPATH_SEARCH_PATHS = (
222+
"$(inherited)",
223+
"@executable_path/Frameworks",
224+
);
225+
MARKETING_VERSION = 1.0;
226+
PRODUCT_BUNDLE_IDENTIFIER = "io.sentry.cocoa.perf-test-app-plain";
227+
PRODUCT_NAME = "$(TARGET_NAME)";
228+
PROVISIONING_PROFILE_SPECIFIER = "match Development io.sentry.cocoa.perf-test-app-plain";
229+
SWIFT_EMIT_LOC_STRINGS = YES;
230+
SWIFT_VERSION = 5.0;
231+
TARGETED_DEVICE_FAMILY = "1,2";
232+
};
233+
name = Release;
234+
};
235+
/* End XCBuildConfiguration section */
236+
237+
/* Begin XCConfigurationList section */
238+
2C101B4B28A2953600D320BC /* Build configuration list for PBXProject "test-app-plain" */ = {
239+
isa = XCConfigurationList;
240+
buildConfigurations = (
241+
2C101B5D28A2953700D320BC /* Release */,
242+
);
243+
defaultConfigurationIsVisible = 0;
244+
defaultConfigurationName = Release;
245+
};
246+
2C101B5E28A2953700D320BC /* Build configuration list for PBXNativeTarget "test-app-plain" */ = {
247+
isa = XCConfigurationList;
248+
buildConfigurations = (
249+
2C101B6028A2953700D320BC /* Release */,
250+
);
251+
defaultConfigurationIsVisible = 0;
252+
defaultConfigurationName = Release;
253+
};
254+
/* End XCConfigurationList section */
255+
};
256+
rootObject = 2C101B4828A2953600D320BC /* Project object */;
257+
}

‎Tests/Perf/test-app-plain/test-app-plain.xcodeproj/project.xcworkspace/contents.xcworkspacedata

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Tests/Perf/test-app-plain/test-app-plain.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3+
<plist version="1.0">
4+
<dict>
5+
<key>IDEDidComputeMac32BitWarning</key>
6+
<true/>
7+
</dict>
8+
</plist>

‎Tests/Perf/test-app-plain/test-app-plain/Assets.xcassets/AccentColor.colorset/Contents.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"colors" : [
3+
{
4+
"idiom" : "universal"
5+
}
6+
],
7+
"info" : {
8+
"author" : "xcode",
9+
"version" : 1
10+
}
11+
}

0 commit comments

Comments
 (0)