From 5aab325a7c8daaf23910b675b09379d35c60f016 Mon Sep 17 00:00:00 2001 From: Aaron Clarke Date: Fri, 10 Jul 2020 13:37:03 -0700 Subject: [PATCH 1/7] Moved to RMSE for image comparison to account for slight variations in golden image production. --- .../ios/Scenarios/ScenariosUITests/GoldenImage.m | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m index e1b27c9bb845a..ec574a1f1eff9 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m @@ -6,6 +6,8 @@ #import #include +static const double kRmseThreshold = 0.5; + @interface GoldenImage () @end @@ -67,8 +69,17 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { CGContextDrawImage(contextB, CGRectMake(0, 0, widthA, heightA), imageRefB); CGContextRelease(contextB); - BOOL isSame = memcmp(rawA.mutableBytes, rawB.mutableBytes, size) == 0; - return isSame; + const char* apos = rawA.mutableBytes; + const char* bpos = rawB.mutableBytes; + double sum = 0.0; + for(size_t i = 0; i < size; ++i, ++apos, ++bpos) { + double aval = *apos; + double bval = *bpos; + double diff = aval - bval; + sum += diff * diff; + } + double rmse = sqrt(sum / size); + return rmse <= kRmseThreshold; } NS_INLINE NSString* _platformName() { From 5eb302f58853bfa6d444e19c9dad86461a9cb744 Mon Sep 17 00:00:00 2001 From: Aaron Clarke Date: Fri, 10 Jul 2020 13:45:37 -0700 Subject: [PATCH 2/7] ran formatter --- fml/platform/darwin/scoped_nsobject.h | 18 ------------------ .../Scenarios/ScenariosUITests/GoldenImage.m | 2 +- testing/scenario_app/run_ios_tests.sh | 2 +- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/fml/platform/darwin/scoped_nsobject.h b/fml/platform/darwin/scoped_nsobject.h index 50b08ebb3603d..0cf493812ca9c 100644 --- a/fml/platform/darwin/scoped_nsobject.h +++ b/fml/platform/darwin/scoped_nsobject.h @@ -70,24 +70,6 @@ class scoped_nsprotocol { NST get() const { return object_; } - void swap(scoped_nsprotocol& that) { - NST temp = that.object_; - that.object_ = object_; - object_ = temp; - } - - // scoped_nsprotocol<>::release() is like scoped_ptr<>::release. It is NOT a - // wrapper for [object_ release]. To force a scoped_nsprotocol<> to call - // [object_ release], use scoped_nsprotocol<>::reset(). - [[nodiscard]] NST release() { - NST temp = object_; - object_ = nil; - return temp; - } - - // Shift reference to the autorelease pool to be released later. - NST autorelease() { return [release() autorelease]; } - private: NST object_; }; diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m index ec574a1f1eff9..51973b43d718a 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m @@ -72,7 +72,7 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { const char* apos = rawA.mutableBytes; const char* bpos = rawB.mutableBytes; double sum = 0.0; - for(size_t i = 0; i < size; ++i, ++apos, ++bpos) { + for (size_t i = 0; i < size; ++i, ++apos, ++bpos) { double aval = *apos; double bval = *bpos; double diff = aval - bval; diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index 14e3a31b9f22f..1b6d269960af3 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -22,7 +22,7 @@ pushd ios/Scenarios set -o pipefail && xcodebuild -sdk iphonesimulator \ -scheme Scenarios \ - -destination 'platform=iOS Simulator,name=iPhone 8' \ + -destination 'platform=iOS Simulator,name=iPhone 8,OS=13.0' \ test \ FLUTTER_ENGINE=$FLUTTER_ENGINE | $PRETTY From d15ad8db7e6bf70c34576e3c60deaf7c99b6e432 Mon Sep 17 00:00:00 2001 From: Aaron Clarke Date: Fri, 10 Jul 2020 13:46:47 -0700 Subject: [PATCH 3/7] reverted accidental commit --- fml/platform/darwin/scoped_nsobject.h | 18 ++++++++++++++++++ testing/scenario_app/run_ios_tests.sh | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/fml/platform/darwin/scoped_nsobject.h b/fml/platform/darwin/scoped_nsobject.h index 0cf493812ca9c..50b08ebb3603d 100644 --- a/fml/platform/darwin/scoped_nsobject.h +++ b/fml/platform/darwin/scoped_nsobject.h @@ -70,6 +70,24 @@ class scoped_nsprotocol { NST get() const { return object_; } + void swap(scoped_nsprotocol& that) { + NST temp = that.object_; + that.object_ = object_; + object_ = temp; + } + + // scoped_nsprotocol<>::release() is like scoped_ptr<>::release. It is NOT a + // wrapper for [object_ release]. To force a scoped_nsprotocol<> to call + // [object_ release], use scoped_nsprotocol<>::reset(). + [[nodiscard]] NST release() { + NST temp = object_; + object_ = nil; + return temp; + } + + // Shift reference to the autorelease pool to be released later. + NST autorelease() { return [release() autorelease]; } + private: NST object_; }; diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index 1b6d269960af3..14e3a31b9f22f 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -22,7 +22,7 @@ pushd ios/Scenarios set -o pipefail && xcodebuild -sdk iphonesimulator \ -scheme Scenarios \ - -destination 'platform=iOS Simulator,name=iPhone 8,OS=13.0' \ + -destination 'platform=iOS Simulator,name=iPhone 8' \ test \ FLUTTER_ENGINE=$FLUTTER_ENGINE | $PRETTY From 98797e44989954c9219b3ac510f49297f02db5f4 Mon Sep 17 00:00:00 2001 From: Aaron Clarke Date: Mon, 13 Jul 2020 14:39:44 -0700 Subject: [PATCH 4/7] started ignoring transparent pixels and fixed a flakey test --- .../Scenarios/ScenariosUITests/GoldenImage.m | 17 +++++++++++++---- .../UnobstructedPlatformViewTests.m | 6 +----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m index 51973b43d718a..cbd8e3c4ea7b7 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m @@ -6,7 +6,7 @@ #import #include -static const double kRmseThreshold = 0.5; +static const double kRmseThreshold = 1.0; @interface GoldenImage () @@ -72,13 +72,22 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { const char* apos = rawA.mutableBytes; const char* bpos = rawB.mutableBytes; double sum = 0.0; + int64_t opaqueCount = 0; for (size_t i = 0; i < size; ++i, ++apos, ++bpos) { double aval = *apos; double bval = *bpos; - double diff = aval - bval; - sum += diff * diff; + // Skip transparent pixels. + if (aval == 0 && bval == 0 && i % 4 == 0) { + i += 3; + apos += 3; + bpos += 3; + } else { + double diff = aval - bval; + sum += diff * diff; + opaqueCount += 1; + } } - double rmse = sqrt(sum / size); + double rmse = sqrt(sum / opaqueCount); return rmse <= kRmseThreshold; } diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/UnobstructedPlatformViewTests.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/UnobstructedPlatformViewTests.m index 375220361a8c6..2d3db20b4e7a9 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/UnobstructedPlatformViewTests.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/UnobstructedPlatformViewTests.m @@ -243,12 +243,8 @@ - (void)testPlatformViewsMaxOverlays { XCUIElement* overlay = app.otherElements[@"platform_view[0].overlay[0]"]; XCTAssertTrue(overlay.exists); - XCTAssertEqual(overlay.frame.origin.x, 75); - XCTAssertEqual(overlay.frame.origin.y, 85); - XCTAssertEqual(overlay.frame.size.width, 150); - XCTAssertEqual(overlay.frame.size.height, 190); - XCTAssertFalse(app.otherElements[@"platform_view[0].overlay[1]"].exists); + XCTAssertTrue(CGRectContainsRect(platform_view.frame, overlay.frame)); } @end From 71a0d0ab8d6845ed29b0d0fca5f65b1989f66589 Mon Sep 17 00:00:00 2001 From: Aaron Clarke Date: Mon, 13 Jul 2020 14:43:42 -0700 Subject: [PATCH 5/7] stopped ignoring transparent pixels when calculating the rmse --- .../ios/Scenarios/ScenariosUITests/GoldenImage.m | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m index cbd8e3c4ea7b7..a7ea0649e67d1 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m @@ -6,7 +6,7 @@ #import #include -static const double kRmseThreshold = 1.0; +static const double kRmseThreshold = 0.5; @interface GoldenImage () @@ -72,7 +72,6 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { const char* apos = rawA.mutableBytes; const char* bpos = rawB.mutableBytes; double sum = 0.0; - int64_t opaqueCount = 0; for (size_t i = 0; i < size; ++i, ++apos, ++bpos) { double aval = *apos; double bval = *bpos; @@ -84,10 +83,10 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { } else { double diff = aval - bval; sum += diff * diff; - opaqueCount += 1; } } - double rmse = sqrt(sum / opaqueCount); + double rmse = sqrt(sum / size); + NSLog(@"rmse: %f", rmse); return rmse <= kRmseThreshold; } From 1e3c8acd0545bd0b5be75e254b3dd3a6288859db Mon Sep 17 00:00:00 2001 From: Aaron Clarke Date: Mon, 13 Jul 2020 14:45:26 -0700 Subject: [PATCH 6/7] removed print statement --- .../scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m index a7ea0649e67d1..4b18212d77591 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m @@ -86,7 +86,6 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { } } double rmse = sqrt(sum / size); - NSLog(@"rmse: %f", rmse); return rmse <= kRmseThreshold; } From 7811b9bd05ec68abdd2fc9d83c884d933bb24c03 Mon Sep 17 00:00:00 2001 From: Aaron Clarke Date: Mon, 13 Jul 2020 14:51:41 -0700 Subject: [PATCH 7/7] tweaked the test for transparency --- .../ios/Scenarios/ScenariosUITests/GoldenImage.m | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m index 4b18212d77591..9961d1a13faf8 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m @@ -73,14 +73,14 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { const char* bpos = rawB.mutableBytes; double sum = 0.0; for (size_t i = 0; i < size; ++i, ++apos, ++bpos) { - double aval = *apos; - double bval = *bpos; // Skip transparent pixels. - if (aval == 0 && bval == 0 && i % 4 == 0) { + if (*apos == 0 && *bpos == 0 && i % 4 == 0) { i += 3; apos += 3; bpos += 3; } else { + double aval = *apos; + double bval = *bpos; double diff = aval - bval; sum += diff * diff; }