From 09a4e08b95725eef617eee28c8983182099558ec Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 14 Aug 2025 11:42:46 +0200 Subject: [PATCH] Update Android benchmark configurations to include newer API levels and devices increased number of benchmark runs to increase accuracy increased passing threshold from 5% to 5.5% --- .../sentry-uitest-android-benchmark-lite.yml | 5 ++- .sauce/sentry-uitest-android-benchmark.yml | 41 ++++++------------- .sauce/sentry-uitest-android-ui.yml | 12 +++--- .../android/benchmark/SentryBenchmarkTest.kt | 5 ++- 4 files changed, 25 insertions(+), 38 deletions(-) diff --git a/.sauce/sentry-uitest-android-benchmark-lite.yml b/.sauce/sentry-uitest-android-benchmark-lite.yml index b9408053ccb..fec4a141def 100644 --- a/.sauce/sentry-uitest-android-benchmark-lite.yml +++ b/.sauce/sentry-uitest-android-benchmark-lite.yml @@ -18,12 +18,13 @@ espresso: suites: - - name: "Android 11 (api 30)" + - name: "Android 15 Benchmark lite (api 35)" testOptions: clearPackageData: true useTestOrchestrator: true devices: - - id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11) + - name: ".*" + platformVersion: "15" artifacts: download: diff --git a/.sauce/sentry-uitest-android-benchmark.yml b/.sauce/sentry-uitest-android-benchmark.yml index 48737b5fa50..12995ea5e07 100644 --- a/.sauce/sentry-uitest-android-benchmark.yml +++ b/.sauce/sentry-uitest-android-benchmark.yml @@ -19,49 +19,34 @@ espresso: suites: # Devices are chosen so that there is a high-end and a low-end device for each api level - - name: "Android 12 (api 31)" + - name: "Android 15 (api 35)" testOptions: clearPackageData: true useTestOrchestrator: true devices: - - id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12) - high end - - id: Google_Pixel_5_12_real_us # Google Pixel 5 - api 31 (12) - low end + - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end + - id: Samsung_Galaxy_S23_15_real_sjc1 # Samsung Galaxy S23 - api 35 (15) - mid end + - id: Google_Pixel_6a_15_real_sjc1 # Google Pixel 6a - api 35 (15) - low end - - name: "Android 11 (api 30)" + - name: "Android 14 (api 34)" testOptions: clearPackageData: true useTestOrchestrator: true devices: - - id: Samsung_Galaxy_S10_Plus_11_real_us # Samsung Galaxy S10+ - api 30 (11) - high end - - id: Google_Pixel_4a_real_us # Google Pixel 4a - api 30 (11) - mid end - - id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11) - low end + - id: Google_Pixel_9_Pro_XL_real_sjc1 # Google Pixel 9 Pro XL - api 34 (14) - high end + - id: Samsung_Galaxy_A54_real_sjc1 # Samsung Galaxy A54 - api 34 (14) - low end - - name: "Android 10 (api 29)" + - name: "Android 13 (api 33)" testOptions: clearPackageData: true useTestOrchestrator: true devices: - - id: Google_Pixel_3a_XL_real # Google Pixel 3a XL - api 29 (10) - - id: OnePlus_6T_real # OnePlus 6T - api 29 (10) + - id: Google_Pixel_7_Pro_real_us # Google Pixel 7 Pro - api 33 (13) - high end + - id: Samsung_Galaxy_A32_5G_real_sjc1 # Samsung Galaxy A32 5G - api 33 (13) - low end -# At the time of writing (July, 4, 2022), the market share per android version is: -# 12.0 = 17.54%, 11.0 = 31.65%, 10.0 = 21.92% -# Using these 3 versions we cover 71,11% of all devices out there. Currently, this is enough for benchmarking scope -# Leaving these devices here in case we change mind on them -# devices: -# - id: Samsung_Galaxy_S8_plus_real_us # Samsung Galaxy S8+ - api 28 (9) -# - id: LG_G8_ThinQ_real_us # LG G8 ThinQ - api 28 (9) -# - id: OnePlus_5_real_us # OnePlus 5 - api 27 (8.1.0) -# - id: LG_K30_real_us1 # LG K30 - api 27 (8.1.0) -# - id: HTC_10_real_us # HTC 10 - api 26 (8.0.0) -# - id: Samsung_A3_real # Samsung Galaxy A3 2017 - api 26 (8.0.0) -# - id: ZTE_Axon_7_real2_us # ZTE Axon 7 - api 25 (7.1.1) -# - id: Motorola_Moto_X_Play_real # Motorola Moto X Play - api 25 (7.1.1) -# - id: Samsung_note_5_real_us # Samsung Galaxy Note 5 - api 24 (7.0) -# - id: LG_K10_real # LG K10 - api 24 (7.0) -# - id: Samsung_Galaxy_S6_Edge_Plus_real # Samsung Galaxy S6 Edge+ - api 23 (6.0.1) -# - id: Samsung_Tab_E_real_us # Samsung Tab E - api 23 (6.0.1) -# - id: Amazon_Kindle_Fire_HD_8_real_us # Amazon Kindle Fire HD 8 - api 22 (5.1.1) +# At the time of writing (August, 13, 2025), the market share per android version is: +# 15.0 = 26.75%, 14.0 = 19.5%, 13 = 15.95% +# Using these 3 versions we cover 62.2% of all devices out there. Currently, this is enough for benchmarking scope artifacts: download: diff --git a/.sauce/sentry-uitest-android-ui.yml b/.sauce/sentry-uitest-android-ui.yml index 5cec16cf7c5..be371d0c314 100644 --- a/.sauce/sentry-uitest-android-ui.yml +++ b/.sauce/sentry-uitest-android-ui.yml @@ -18,29 +18,29 @@ espresso: testApp: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk suites: - - name: "Android 14 Ui test (api 34)" + - name: "Android 15 Ui test (api 35)" testOptions: clearPackageData: true useTestOrchestrator: true devices: - name: ".*" - platformVersion: "14" + platformVersion: "15" - - name: "Android 13 Ui test (api 33)" + - name: "Android 14 Ui test (api 34)" testOptions: clearPackageData: true useTestOrchestrator: true devices: - name: ".*" - platformVersion: "13" + platformVersion: "14" - - name: "Android 11 Ui test (api 31)" + - name: "Android 13 Ui test (api 33)" testOptions: clearPackageData: true useTestOrchestrator: true devices: - name: ".*" - platformVersion: "11" + platformVersion: "13" # Controls what artifacts to fetch when the suite on Sauce Cloud has finished. artifacts: diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt index dbed89029a3..f2066577c6e 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt @@ -101,6 +101,7 @@ class SentryBenchmarkTest : BaseBenchmarkTest() { benchmarkOperationProfiled, "ProfiledTransaction", refreshRate, + measuredIterations = 40, ) comparisonResults.printAllRuns("Profiling Benchmark") val comparisonResult = comparisonResults.getSummaryResult() @@ -108,8 +109,8 @@ class SentryBenchmarkTest : BaseBenchmarkTest() { // Currently we just want to assert the cpu overhead assertTrue( - comparisonResult.cpuTimeIncreasePercentage in 0F..5F, - "Expected ${comparisonResult.cpuTimeIncreasePercentage} to be in range 0 < x < 5", + comparisonResult.cpuTimeIncreasePercentage in 0F..5.5F, + "Expected ${comparisonResult.cpuTimeIncreasePercentage} to be in range 0 < x < 5.5", ) }