Add a golden scenario test for fallback font rendering on iOS take 3 (flutter#22736)

Author: xster

testing/scenario_app/.gitignore

@@ -3,6 +3,7 @@ pubspec.lock   # This only has local dependencies in it.
 build/
 
 ios/Scenarios/*.framework/
+ios/Scenarios/*.xcframework/
 android/app/libs/flutter.jar
 android/app/src/main/jniLibs/arm64-v8a/libapp.so
 android/gradle-home/.cache

testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.pbxproj

@@ -28,15 +28,6 @@
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
       shouldUseLaunchSchemeArgsEnv = "YES"
       systemAttachmentLifetime = "keepNever">
-      <MacroExpansion>
-         <BuildableReference
-            BuildableIdentifier = "primary"
-            BlueprintIdentifier = "248D76C622E388370012F0C1"
-            BuildableName = "Scenarios.app"
-            BlueprintName = "Scenarios"
-            ReferencedContainer = "container:Scenarios.xcodeproj">
-         </BuildableReference>
-      </MacroExpansion>
       <Testables>
          <TestableReference
             skipped = "NO"
@@ -87,6 +78,10 @@
             argument = "--screen-before-flutter"
             isEnabled = "NO">
          </CommandLineArgument>
+         <CommandLineArgument
+            argument = "--bogus-font-text"
+            isEnabled = "NO">
+         </CommandLineArgument>
          <CommandLineArgument
             argument = "--text-semantics-focus"
             isEnabled = "NO">

testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/xcshareddata/xcschemes/Scenarios.xcscheme

@@ -28,7 +28,7 @@ - (BOOL)application:(UIApplication*)application
     self.window.tintColor = UIColor.systemPinkColor;
   }
   NSDictionary<NSString*, NSString*>* launchArgsMap = @{
-    // The Platform view golden test args should match `PlatformViewGoldenTestManager`.
+    // The golden test args should match `GoldenTestManager`.
     @"--locale-initialization" : @"locale_initialization",
     @"--platform-view" : @"platform_view",
     @"--platform-view-no-overlay-intersection" : @"platform_view_no_overlay_intersection",
@@ -53,7 +53,8 @@ - (BOOL)application:(UIApplication*)application
     @"--tap-status-bar" : @"tap_status_bar",
     @"--text-semantics-focus" : @"text_semantics_focus",
     @"--animated-color-square" : @"animated_color_square",
-    @"--platform-view-with-continuous-texture" : @"platform_view_with_continuous_texture"
+    @"--platform-view-with-continuous-texture" : @"platform_view_with_continuous_texture",
+    @"--bogus-font-text" : @"bogus_font_text"
   };
   __block NSString* flutterViewControllerTestName = nil;
   [launchArgsMap
@@ -104,16 +105,18 @@ - (void)setupFlutterViewControllerTest:(NSString*)scenarioIdentifier {
                                   codec:[FlutterJSONMethodCodec sharedInstance]];
               [channel invokeMethod:@"set_scenario" arguments:@{@"name" : scenarioIdentifier}];
             }];
+  // Can be used to synchronize timing in the test for a signal from Dart.
   [engine.binaryMessenger
-      setMessageHandlerOnChannel:@"touches_scenario"
+      setMessageHandlerOnChannel:@"display_data"
             binaryMessageHandler:^(NSData* _Nullable message, FlutterBinaryReply _Nonnull reply) {
               NSDictionary* dict = [NSJSONSerialization JSONObjectWithData:message
                                                                    options:0
                                                                      error:nil];
-              UITextField* text = [[UITextField alloc] initWithFrame:CGRectMake(0, 0, 300, 100)];
-              text.text = dict[@"change"];
+              UITextField* text = [[UITextField alloc] initWithFrame:CGRectMake(0, 400, 300, 100)];
+              text.text = dict[@"data"];
               [flutterViewController.view addSubview:text];
             }];
+
   TextPlatformViewFactory* textPlatformViewFactory =
       [[TextPlatformViewFactory alloc] initWithMessenger:engine.binaryMessenger];
   NSObject<FlutterPluginRegistrar>* registrar =

testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m

@@ -0,0 +1,31 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import <Flutter/Flutter.h>
+#import <XCTest/XCTest.h>
+#import "GoldenTestManager.h"
+
+FLUTTER_ASSERT_ARC
+
+@interface BogusFontTextTest : XCTestCase
+
+@end
+
+@implementation BogusFontTextTest
+
+- (void)testFontRenderingWhenSuppliedWithBogusFont {
+  self.continueAfterFailure = NO;
+
+  XCUIApplication* application = [[XCUIApplication alloc] init];
+  application.launchArguments = @[ @"--bogus-font-text" ];
+  [application launch];
+
+  XCUIElement* addTextField = application.textFields[@"ready"];
+  XCTAssertTrue([addTextField waitForExistenceWithTimeout:30]);
+
+  GoldenTestManager* manager = [[GoldenTestManager alloc] initWithLaunchArg:@"--bogus-font-text"];
+  [manager checkGoldenForTest:self];
+}
+
+@end

testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m

@@ -3,7 +3,7 @@
 // found in the LICENSE file.
 
 #import <XCTest/XCTest.h>
-#import "PlatformViewGoldenTestManager.h"
+#import "GoldenTestManager.h"
 
 NS_ASSUME_NONNULL_BEGIN
 
@@ -12,20 +12,19 @@ NS_ASSUME_NONNULL_BEGIN
 // A new PlatformView golden tests can subclass this and override the `-initiWithInvocation:`
 // method, which then retun the `-initWithManager:invocation:`
 //
-// Then in any test method, call `checkGolden` to perform the golden test.
+// Then in any test method, call `checkPlatformViewGolden` to perform the golden test.
 //
 // This base class doesn't run any test case on its own.
 @interface GoldenPlatformViewTests : XCTestCase
 
 @property(nonatomic, strong) XCUIApplication* application;
 
-// Initialize with a `PlatformViewGoldenTestManager`.
-- (instancetype)initWithManager:(PlatformViewGoldenTestManager*)manager
-                     invocation:(NSInvocation*)invocation;
+// Initialize with a `GoldenTestManager`.
+- (instancetype)initWithManager:(GoldenTestManager*)manager invocation:(NSInvocation*)invocation;
 
 // Take a sceenshot of the test app and check it has the same pixels with goldenImage inside the
-// `PlatformViewGoldenTestManager`.
-- (void)checkGolden;
+// `GoldenTestManager`.
+- (void)checkPlatformViewGolden;
 
 @end
 

testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h

@@ -5,23 +5,21 @@
 #import "GoldenPlatformViewTests.h"
 
 #include <sys/sysctl.h>
-
-#import "PlatformViewGoldenTestManager.h"
+#import "GoldenTestManager.h"
 
 static const NSInteger kSecondsToWaitForPlatformView = 30;
 
 @interface GoldenPlatformViewTests ()
 
 @property(nonatomic, copy) NSString* goldenName;
 
-@property(nonatomic, strong) PlatformViewGoldenTestManager* manager;
+@property(nonatomic, strong) GoldenTestManager* manager;
 
 @end
 
 @implementation GoldenPlatformViewTests
 
-- (instancetype)initWithManager:(PlatformViewGoldenTestManager*)manager
-                     invocation:(NSInvocation*)invocation {
+- (instancetype)initWithManager:(GoldenTestManager*)manager invocation:(NSInvocation*)invocation {
   self = [super initWithInvocation:invocation];
   _manager = manager;
   return self;
@@ -37,7 +35,7 @@ - (void)setUp {
 }
 
 // Note: don't prefix with "test" or GoldenPlatformViewTests will run instead of the subclasses.
-- (void)checkGolden {
+- (void)checkPlatformViewGolden {
   XCUIElement* element = self.application.textViews.firstMatch;
   BOOL exists = [element waitForExistenceWithTimeout:kSecondsToWaitForPlatformView];
   if (!exists) {
@@ -47,29 +45,6 @@ - (void)checkGolden {
             @(kSecondsToWaitForPlatformView));
   }
 
-  GoldenImage* golden = self.manager.goldenImage;
-
-  XCUIScreenshot* screenshot = [[XCUIScreen mainScreen] screenshot];
-  if (!golden.image) {
-    XCTAttachment* attachment = [XCTAttachment attachmentWithScreenshot:screenshot];
-    attachment.name = @"new_golden";
-    attachment.lifetime = XCTAttachmentLifetimeKeepAlways;
-    [self addAttachment:attachment];
-    XCTFail(@"This test will fail - no golden named %@ found. Follow the steps in the "
-            @"README to add a new golden.",
-            golden.goldenName);
-  }
-
-  if (![golden compareGoldenToImage:screenshot.image]) {
-    XCTAttachment* screenshotAttachment;
-    screenshotAttachment = [XCTAttachment attachmentWithImage:screenshot.image];
-    screenshotAttachment.name = golden.goldenName;
-    screenshotAttachment.lifetime = XCTAttachmentLifetimeKeepAlways;
-    [self addAttachment:screenshotAttachment];
-
-    XCTFail(@"Goldens to not match. Follow the steps in the "
-            @"README to update golden named %@ if needed.",
-            golden.goldenName);
-  }
+  [self.manager checkGoldenForTest:self];
 }
 @end

testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m

@@ -3,7 +3,7 @@
 // found in the LICENSE file.
 
 #import <Foundation/Foundation.h>
-
+#import <XCTest/XCTest.h>
 #import "GoldenImage.h"
 
 NS_ASSUME_NONNULL_BEGIN
@@ -14,7 +14,7 @@ extern NSDictionary* launchArgsMap;
 //
 // It creates the correct `identifer` based on the `launchArg`.
 // It also generates the correct GoldenImage based on the `identifier`.
-@interface PlatformViewGoldenTestManager : NSObject
+@interface GoldenTestManager : NSObject
 
 @property(readonly, strong, nonatomic) GoldenImage* goldenImage;
 @property(readonly, copy, nonatomic) NSString* identifier;
@@ -25,6 +25,10 @@ extern NSDictionary* launchArgsMap;
 // Crahes if the launchArg is not mapped in `Appdelegate.launchArgsMap`.
 - (instancetype)initWithLaunchArg:(NSString*)launchArg;
 
+// Take a sceenshot of the test app and check it has the same pixels with
+// goldenImage inside the `GoldenTestManager`.
+- (void)checkGoldenForTest:(XCTestCase*)test;
+
 @end
 
 NS_ASSUME_NONNULL_END

testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewGoldenTestManager.h renamed to testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h

@@ -2,17 +2,16 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#import "PlatformViewGoldenTestManager.h"
-
+#import "GoldenTestManager.h"
 #import <XCTest/XCTest.h>
 
-@interface PlatformViewGoldenTestManager ()
+@interface GoldenTestManager ()
 
 @property(readwrite, strong, nonatomic) GoldenImage* goldenImage;
 
 @end
 
-@implementation PlatformViewGoldenTestManager
+@implementation GoldenTestManager
 
 NSDictionary* launchArgsMap;
 
@@ -34,6 +33,7 @@ - (instancetype)initWithLaunchArg:(NSString*)launchArg {
         @"--platform-view-transform" : @"platform_view_transform",
         @"--platform-view-opacity" : @"platform_view_opacity",
         @"--platform-view-rotate" : @"platform_view_rotate",
+        @"--bogus-font-text" : @"bogus_font_text",
       };
     });
     _identifier = launchArgsMap[launchArg];
@@ -44,4 +44,32 @@ - (instancetype)initWithLaunchArg:(NSString*)launchArg {
   return self;
 }
 
+- (void)checkGoldenForTest:(XCTestCase*)test {
+  XCUIScreenshot* screenshot = [[XCUIScreen mainScreen] screenshot];
+  if (!_goldenImage.image) {
+    XCTAttachment* attachment = [XCTAttachment attachmentWithScreenshot:screenshot];
+    attachment.name = [_goldenImage.goldenName stringByAppendingString:@"_new"];
+    attachment.lifetime = XCTAttachmentLifetimeKeepAlways;
+    [test addAttachment:attachment];
+    // Instead of XCTFail because that definition changed between Xcode 11 and 12 whereas this impl
+    // is stable.
+    _XCTPrimitiveFail(test,
+                      @"This test will fail - no golden named %@ found. "
+                      @"Follow the steps in the README to add a new golden.",
+                      _goldenImage.goldenName);
+  }
+
+  if (![_goldenImage compareGoldenToImage:screenshot.image]) {
+    XCTAttachment* screenshotAttachment = [XCTAttachment attachmentWithImage:screenshot.image];
+    screenshotAttachment.name = [_goldenImage.goldenName stringByAppendingString:@"_actual"];
+    screenshotAttachment.lifetime = XCTAttachmentLifetimeKeepAlways;
+    [test addAttachment:screenshotAttachment];
+
+    _XCTPrimitiveFail(test,
+                      @"Goldens do not match. Follow the steps in the "
+                      @"README to update golden named %@ if needed.",
+                      _goldenImage.goldenName);
+  }
+}
+
 @end