Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

[iOS][Keyboard] Wait vsync on UI thread and update viewport inset to avoid jitter. #42312

Merged
merged 29 commits into from
Jun 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion shell/platform/darwin/ios/framework/Source/FlutterEngine.mm
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,12 @@ - (void)dispatchPointerDataPacket:(std::unique_ptr<flutter::PointerDataPacket>)p
return _shell->GetTaskRunners().GetPlatformTaskRunner();
}

- (fml::RefPtr<fml::TaskRunner>)RasterTaskRunner {
- (fml::RefPtr<fml::TaskRunner>)uiTaskRunner {
FML_DCHECK(_shell);
return _shell->GetTaskRunners().GetUITaskRunner();
}

- (fml::RefPtr<fml::TaskRunner>)rasterTaskRunner {
FML_DCHECK(_shell);
return _shell->GetTaskRunners().GetRasterTaskRunner();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ extern NSString* const kFlutterEngineWillDealloc;
- (void)dispatchPointerDataPacket:(std::unique_ptr<flutter::PointerDataPacket>)packet;

- (fml::RefPtr<fml::TaskRunner>)platformTaskRunner;
- (fml::RefPtr<fml::TaskRunner>)RasterTaskRunner;
- (fml::RefPtr<fml::TaskRunner>)uiTaskRunner;
- (fml::RefPtr<fml::TaskRunner>)rasterTaskRunner;

- (fml::WeakPtr<flutter::PlatformView>)platformView;

Expand Down
115 changes: 73 additions & 42 deletions shell/platform/darwin/ios/framework/Source/FlutterViewController.mm
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,11 @@ @interface FlutterViewController () <FlutterBinaryMessenger, UIScrollViewDelegat
/**
* Keyboard animation properties
*/
@property(nonatomic, assign) double targetViewInsetBottom;
@property(nonatomic, assign) CGFloat targetViewInsetBottom;
@property(nonatomic, assign) CGFloat originalViewInsetBottom;
@property(nonatomic, retain) VSyncClient* keyboardAnimationVSyncClient;
@property(nonatomic, assign) BOOL keyboardAnimationIsShowing;
@property(nonatomic, assign) fml::TimePoint keyboardAnimationStartTime;
@property(nonatomic, assign) CGFloat originalViewInsetBottom;
@property(nonatomic, assign) BOOL isKeyboardInOrTransitioningFromBackground;

/// VSyncClient for touch events delivery frame rate correction.
Expand Down Expand Up @@ -574,8 +574,8 @@ - (void)installFirstFrameCallback {
// Start on the platform thread.
weakPlatformView->SetNextFrameCallback([weakSelf = [self getWeakPtr],
platformTaskRunner = [_engine.get() platformTaskRunner],
RasterTaskRunner = [_engine.get() RasterTaskRunner]]() {
FML_DCHECK(RasterTaskRunner->RunsTasksOnCurrentThread());
rasterTaskRunner = [_engine.get() rasterTaskRunner]]() {
FML_DCHECK(rasterTaskRunner->RunsTasksOnCurrentThread());
// Get callback on raster thread and jump back to platform thread.
platformTaskRunner->PostTask([weakSelf]() {
if (weakSelf) {
Expand Down Expand Up @@ -1596,7 +1596,55 @@ - (void)startKeyBoardAnimation:(NSTimeInterval)duration {

// Invalidate old vsync client if old animation is not completed.
[self invalidateKeyboardAnimationVSyncClient];
[self setupKeyboardAnimationVsyncClient];

fml::WeakPtr<FlutterViewController> weakSelf = [self getWeakPtr];
FlutterKeyboardAnimationCallback keyboardAnimationCallback = ^(
fml::TimePoint keyboardAnimationTargetTime) {
if (!weakSelf) {
return;
}
fml::scoped_nsobject<FlutterViewController> flutterViewController(
[(FlutterViewController*)weakSelf.get() retain]);
if (!flutterViewController) {
return;
}

// If the view controller's view is not loaded, bail out.
if (!flutterViewController.get().isViewLoaded) {
return;
}
// If the view for tracking keyboard animation is nil, means it is not
// created, bail out.
if ([flutterViewController keyboardAnimationView] == nil) {
return;
}
// If keyboardAnimationVSyncClient is nil, means the animation ends.
// And should bail out.
if (flutterViewController.get().keyboardAnimationVSyncClient == nil) {
return;
}

if ([flutterViewController keyboardAnimationView].superview == nil) {
// Ensure the keyboardAnimationView is in view hierarchy when animation running.
[flutterViewController.get().view addSubview:[flutterViewController keyboardAnimationView]];
}

if ([flutterViewController keyboardSpringAnimation] == nil) {
if (flutterViewController.get().keyboardAnimationView.layer.presentationLayer) {
flutterViewController.get()->_viewportMetrics.physical_view_inset_bottom =
flutterViewController.get()
.keyboardAnimationView.layer.presentationLayer.frame.origin.y;
[flutterViewController updateViewportMetricsIfNeeded];
}
} else {
fml::TimeDelta timeElapsed =
keyboardAnimationTargetTime - flutterViewController.get().keyboardAnimationStartTime;
flutterViewController.get()->_viewportMetrics.physical_view_inset_bottom =
[[flutterViewController keyboardSpringAnimation] curveFunction:timeElapsed.ToSecondsF()];
[flutterViewController updateViewportMetricsIfNeeded];
}
};
[self setupKeyboardAnimationVsyncClient:keyboardAnimationCallback];
VSyncClient* currentVsyncClient = _keyboardAnimationVSyncClient;

[UIView animateWithDuration:duration
Expand Down Expand Up @@ -1639,45 +1687,28 @@ - (void)setupKeyboardSpringAnimationIfNeeded:(CAAnimation*)keyboardAnimation {
toValue:self.targetViewInsetBottom]);
}

- (void)setupKeyboardAnimationVsyncClient {
auto callback = [weakSelf =
[self getWeakPtr]](std::unique_ptr<flutter::FrameTimingsRecorder> recorder) {
if (!weakSelf) {
return;
}
fml::scoped_nsobject<FlutterViewController> flutterViewController(
[(FlutterViewController*)weakSelf.get() retain]);
if (!flutterViewController) {
return;
}

if ([flutterViewController keyboardAnimationView].superview == nil) {
// Ensure the keyboardAnimationView is in view hierarchy when animation running.
[flutterViewController.get().view addSubview:[flutterViewController keyboardAnimationView]];
}

if ([flutterViewController keyboardSpringAnimation] == nil) {
if (flutterViewController.get().keyboardAnimationView.layer.presentationLayer) {
flutterViewController.get()->_viewportMetrics.physical_view_inset_bottom =
flutterViewController.get()
.keyboardAnimationView.layer.presentationLayer.frame.origin.y;
[flutterViewController updateViewportMetricsIfNeeded];
}
} else {
fml::TimeDelta timeElapsed = recorder.get()->GetVsyncTargetTime() -
flutterViewController.get().keyboardAnimationStartTime;

flutterViewController.get()->_viewportMetrics.physical_view_inset_bottom =
[[flutterViewController keyboardSpringAnimation] curveFunction:timeElapsed.ToSecondsF()];
[flutterViewController updateViewportMetricsIfNeeded];
}
};
flutter::Shell& shell = [_engine.get() shell];
- (void)setupKeyboardAnimationVsyncClient:
(FlutterKeyboardAnimationCallback)keyboardAnimationCallback {
if (!keyboardAnimationCallback) {
return;
}
NSAssert(_keyboardAnimationVSyncClient == nil,
@"_keyboardAnimationVSyncClient must be nil when setup");
_keyboardAnimationVSyncClient =
[[VSyncClient alloc] initWithTaskRunner:shell.GetTaskRunners().GetPlatformTaskRunner()
callback:callback];

// Make sure the new viewport metrics get sent after the begin frame event has processed.
fml::scoped_nsprotocol<FlutterKeyboardAnimationCallback> animationCallback(
[keyboardAnimationCallback copy]);
auto uiCallback = [animationCallback,
engine = _engine](std::unique_ptr<flutter::FrameTimingsRecorder> recorder) {
fml::TimeDelta frameInterval = recorder->GetVsyncTargetTime() - recorder->GetVsyncStartTime();
fml::TimePoint keyboardAnimationTargetTime = recorder->GetVsyncTargetTime() + frameInterval;
[engine platformTaskRunner]->PostTask([animationCallback, keyboardAnimationTargetTime] {
animationCallback.get()(keyboardAnimationTargetTime);
});
};

_keyboardAnimationVSyncClient = [[VSyncClient alloc] initWithTaskRunner:[_engine uiTaskRunner]
callback:uiCallback];
_keyboardAnimationVSyncClient.allowPauseAfterVsync = NO;
[_keyboardAnimationVSyncClient await];
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ - (FlutterTextInputPlugin*)textInputPlugin;
- (void)sendKeyEvent:(const FlutterKeyEvent&)event
callback:(nullable FlutterKeyEventCallback)callback
userData:(nullable void*)userData;
- (fml::RefPtr<fml::TaskRunner>)uiTaskRunner;
@end

/// Sometimes we have to use a custom mock to avoid retain cycles in OCMock.
Expand Down Expand Up @@ -135,10 +136,11 @@ - (BOOL)shouldIgnoreKeyboardNotification:(NSNotification*)notification;
- (FlutterKeyboardMode)calculateKeyboardAttachMode:(NSNotification*)notification;
- (CGFloat)calculateMultitaskingAdjustment:(CGRect)screenRect keyboardFrame:(CGRect)keyboardFrame;
- (void)startKeyBoardAnimation:(NSTimeInterval)duration;
- (void)setupKeyboardAnimationVsyncClient;
- (UIView*)keyboardAnimationView;
- (SpringAnimation*)keyboardSpringAnimation;
- (void)setupKeyboardSpringAnimationIfNeeded:(CAAnimation*)keyboardAnimation;
- (void)setupKeyboardAnimationVsyncClient:
(FlutterKeyboardAnimationCallback)keyboardAnimationCallback;
- (void)ensureViewportMetricsIsCorrect;
- (void)invalidateKeyboardAnimationVSyncClient;
- (void)addInternalPlugins;
Expand Down Expand Up @@ -197,18 +199,6 @@ - (void)testViewDidLoadWillInvokeCreateTouchRateCorrectionVSyncClient {
OCMVerify([viewControllerMock createTouchRateCorrectionVSyncClientIfNeeded]);
}

- (void)testStartKeyboardAnimationWillInvokeSetupKeyboardAnimationVsyncClient {
FlutterEngine* engine = [[FlutterEngine alloc] init];
[engine runWithEntrypoint:nil];
FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engine
nibName:nil
bundle:nil];
FlutterViewController* viewControllerMock = OCMPartialMock(viewController);
viewControllerMock.targetViewInsetBottom = 100;
[viewControllerMock startKeyBoardAnimation:0.25];
OCMVerify([viewControllerMock setupKeyboardAnimationVsyncClient]);
}

- (void)testStartKeyboardAnimationWillInvokeSetupKeyboardSpringAnimationIfNeeded {
FlutterEngine* engine = [[FlutterEngine alloc] init];
[engine runWithEntrypoint:nil];
Expand Down Expand Up @@ -451,6 +441,34 @@ - (void)testShouldIgnoreKeyboardNotification {
}
}

- (void)testKeyboardAnimationWillWaitUIThreadVsync {
// We need to make sure the new viewport metrics get sent after the
// begin frame event has processed. And this test is to expect that the callback
// will sync with UI thread. So just simulate a lot of works on UI thread and
// test the keyboard animation callback will execute until UI task completed.
// Related issue: https://github.com/flutter/flutter/issues/120555.

FlutterEngine* engine = [[FlutterEngine alloc] init];
[engine runWithEntrypoint:nil];
FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engine
nibName:nil
bundle:nil];
// Post a task to UI thread to block the thread.
const int delayTime = 1;
[engine uiTaskRunner]->PostTask([] { sleep(delayTime); });
XCTestExpectation* expectation = [self expectationWithDescription:@"keyboard animation callback"];

__block CFTimeInterval fulfillTime;
FlutterKeyboardAnimationCallback callback = ^(fml::TimePoint targetTime) {
fulfillTime = CACurrentMediaTime();
[expectation fulfill];
};
CFTimeInterval startTime = CACurrentMediaTime();
[viewController setupKeyboardAnimationVsyncClient:callback];
[self waitForExpectationsWithTimeout:5.0 handler:nil];
XCTAssertTrue(fulfillTime - startTime > delayTime);
}

- (void)testCalculateKeyboardAttachMode {
FlutterEngine* mockEngine = OCMPartialMock([[FlutterEngine alloc] init]);
[mockEngine createShell:@"" libraryURI:@"" initialRoute:nil];
Expand Down Expand Up @@ -629,9 +647,9 @@ - (void)testCalculateKeyboardInset {
}

- (void)testHandleKeyboardNotification {
FlutterEngine* mockEngine = OCMPartialMock([[FlutterEngine alloc] init]);
[mockEngine createShell:@"" libraryURI:@"" initialRoute:nil];
FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:mockEngine
FlutterEngine* engine = [[FlutterEngine alloc] init];
[engine runWithEntrypoint:nil];
FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engine
nibName:nil
bundle:nil];
// keyboard is empty
Expand All @@ -652,11 +670,9 @@ - (void)testHandleKeyboardNotification {
[self setupMockMainScreenAndView:viewControllerMock viewFrame:viewFrame convertedFrame:viewFrame];
viewControllerMock.targetViewInsetBottom = 0;
XCTestExpectation* expectation = [self expectationWithDescription:@"update viewport"];
OCMStub([mockEngine updateViewportMetrics:flutter::ViewportMetrics()])
.ignoringNonObjectArgs()
.andDo(^(NSInvocation* invocation) {
[expectation fulfill];
});
OCMStub([viewControllerMock updateViewportMetricsIfNeeded]).andDo(^(NSInvocation* invocation) {
[expectation fulfill];
});

[viewControllerMock handleKeyboardNotification:notification];
XCTAssertTrue(viewControllerMock.targetViewInsetBottom == 320 * UIScreen.mainScreen.scale);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
#import "flutter/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h"
#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h"
#import "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h"

FLUTTER_ASSERT_NOT_ARC
Expand All @@ -31,7 +32,9 @@ @interface FlutterViewController (Testing)
@property(nonatomic, retain) VSyncClient* touchRateCorrectionVSyncClient;

- (void)createTouchRateCorrectionVSyncClientIfNeeded;
- (void)setupKeyboardAnimationVsyncClient;
- (void)setupKeyboardAnimationVsyncClient:
(FlutterKeyboardAnimationCallback)keyboardAnimationCallback;
- (void)startKeyBoardAnimation:(NSTimeInterval)duration;
- (void)triggerTouchRateCorrectionIfNeeded:(NSSet*)touches;

@end
Expand All @@ -53,7 +56,9 @@ - (void)testSetupKeyboardAnimationVsyncClientWillCreateNewVsyncClientForFlutterV
FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engine
nibName:nil
bundle:nil];
[viewController setupKeyboardAnimationVsyncClient];
FlutterKeyboardAnimationCallback callback = ^(fml::TimePoint targetTime) {
};
[viewController setupKeyboardAnimationVsyncClient:callback];
XCTAssertNotNil(viewController.keyboardAnimationVSyncClient);
CADisplayLink* link = [viewController.keyboardAnimationVSyncClient getDisplayLink];
XCTAssertNotNil(link);
Expand Down Expand Up @@ -173,4 +178,26 @@ - (void)testTriggerTouchRateCorrectionVSyncClientCorrectly {
XCTAssertFalse(link.isPaused);
}

- (void)testFlutterViewControllerStartKeyboardAnimationWillCreateVsyncClientCorrectly {
FlutterEngine* engine = [[FlutterEngine alloc] init];
[engine runWithEntrypoint:nil];
FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engine
nibName:nil
bundle:nil];
viewController.targetViewInsetBottom = 100;
[viewController startKeyBoardAnimation:0.25];
XCTAssertNotNil(viewController.keyboardAnimationVSyncClient);
}

- (void)
testSetupKeyboardAnimationVsyncClientWillNotCreateNewVsyncClientWhenKeyboardAnimationCallbackIsNil {
FlutterEngine* engine = [[FlutterEngine alloc] init];
[engine runWithEntrypoint:nil];
FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engine
nibName:nil
bundle:nil];
[viewController setupKeyboardAnimationVsyncClient:nil];
XCTAssertNil(viewController.keyboardAnimationVSyncClient);
}

@end
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ typedef NS_ENUM(NSInteger, FlutterKeyboardMode) {
FlutterKeyboardModeFloating = 2,
};

typedef void (^FlutterKeyboardAnimationCallback)(fml::TimePoint);

@interface FlutterViewController () <FlutterViewResponder>

@property(class, nonatomic, readonly) BOOL accessibilityIsOnOffSwitchLabelsEnabled;
Expand Down