Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Win32: Handle OnAccessibilityEvent #30176

Merged
cbracken merged 1 commit into from
Dec 9, 2021
Merged

Win32: Handle OnAccessibilityEvent #30176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 120 additions & 1 deletion shell/platform/windows/accessibility_bridge_delegate_win32.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,120 @@ AccessibilityBridgeDelegateWin32::AccessibilityBridgeDelegateWin32(

void AccessibilityBridgeDelegateWin32::OnAccessibilityEvent(
ui::AXEventGenerator::TargetedEvent targeted_event) {
// TODO(cbracken): https://github.com/flutter/flutter/issues/77838
ui::AXNode* ax_node = targeted_event.node;
ui::AXEventGenerator::Event event_type = targeted_event.event_params.event;

// Look up the flutter platform node delegate.
auto bridge = engine_->accessibility_bridge().lock();
assert(bridge);
auto node_delegate =
bridge->GetFlutterPlatformNodeDelegateFromID(ax_node->id()).lock();
assert(node_delegate);
std::shared_ptr<FlutterPlatformNodeDelegateWin32> win_delegate =
std::static_pointer_cast<FlutterPlatformNodeDelegateWin32>(node_delegate);

switch (event_type) {
case ui::AXEventGenerator::Event::ALERT:
DispatchWinAccessibilityEvent(win_delegate, EVENT_SYSTEM_ALERT);
break;
case ui::AXEventGenerator::Event::CHILDREN_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_REORDER);
break;
case ui::AXEventGenerator::Event::FOCUS_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_FOCUS);
break;
case ui::AXEventGenerator::Event::IGNORED_CHANGED:
if (ax_node->IsIgnored()) {
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_HIDE);
}
break;
case ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_NAMECHANGE);
break;
case ui::AXEventGenerator::Event::LIVE_REGION_CHANGED:
DispatchWinAccessibilityEvent(win_delegate,
EVENT_OBJECT_LIVEREGIONCHANGED);
break;
case ui::AXEventGenerator::Event::NAME_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_NAMECHANGE);
break;
case ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_SYSTEM_SCROLLINGEND);
break;
case ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_SYSTEM_SCROLLINGEND);
break;
case ui::AXEventGenerator::Event::SELECTED_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_VALUECHANGE);
break;
case ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_SELECTIONWITHIN);
break;
case ui::AXEventGenerator::Event::SUBTREE_CREATED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_SHOW);
break;
case ui::AXEventGenerator::Event::VALUE_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_VALUECHANGE);
break;
case ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED:
DispatchWinAccessibilityEvent(win_delegate, EVENT_OBJECT_STATECHANGE);
break;
case ui::AXEventGenerator::Event::ACCESS_KEY_CHANGED:
case ui::AXEventGenerator::Event::ACTIVE_DESCENDANT_CHANGED:
case ui::AXEventGenerator::Event::ATK_TEXT_OBJECT_ATTRIBUTE_CHANGED:
case ui::AXEventGenerator::Event::ATOMIC_CHANGED:
case ui::AXEventGenerator::Event::AUTO_COMPLETE_CHANGED:
case ui::AXEventGenerator::Event::BUSY_CHANGED:
case ui::AXEventGenerator::Event::CHECKED_STATE_CHANGED:
case ui::AXEventGenerator::Event::CLASS_NAME_CHANGED:
case ui::AXEventGenerator::Event::COLLAPSED:
case ui::AXEventGenerator::Event::CONTROLS_CHANGED:
case ui::AXEventGenerator::Event::DESCRIBED_BY_CHANGED:
case ui::AXEventGenerator::Event::DESCRIPTION_CHANGED:
case ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED:
case ui::AXEventGenerator::Event::DOCUMENT_TITLE_CHANGED:
case ui::AXEventGenerator::Event::DROPEFFECT_CHANGED:
case ui::AXEventGenerator::Event::ENABLED_CHANGED:
case ui::AXEventGenerator::Event::EXPANDED:
case ui::AXEventGenerator::Event::FLOW_FROM_CHANGED:
case ui::AXEventGenerator::Event::FLOW_TO_CHANGED:
case ui::AXEventGenerator::Event::GRABBED_CHANGED:
case ui::AXEventGenerator::Event::HASPOPUP_CHANGED:
case ui::AXEventGenerator::Event::HIERARCHICAL_LEVEL_CHANGED:
case ui::AXEventGenerator::Event::INVALID_STATUS_CHANGED:
case ui::AXEventGenerator::Event::KEY_SHORTCUTS_CHANGED:
case ui::AXEventGenerator::Event::LABELED_BY_CHANGED:
case ui::AXEventGenerator::Event::LANGUAGE_CHANGED:
case ui::AXEventGenerator::Event::LAYOUT_INVALIDATED:
case ui::AXEventGenerator::Event::LIVE_REGION_CREATED:
case ui::AXEventGenerator::Event::LIVE_REGION_NODE_CHANGED:
case ui::AXEventGenerator::Event::LIVE_RELEVANT_CHANGED:
case ui::AXEventGenerator::Event::LIVE_STATUS_CHANGED:
case ui::AXEventGenerator::Event::LOAD_COMPLETE:
case ui::AXEventGenerator::Event::LOAD_START:
case ui::AXEventGenerator::Event::MENU_ITEM_SELECTED:
case ui::AXEventGenerator::Event::MULTILINE_STATE_CHANGED:
case ui::AXEventGenerator::Event::MULTISELECTABLE_STATE_CHANGED:
case ui::AXEventGenerator::Event::OBJECT_ATTRIBUTE_CHANGED:
case ui::AXEventGenerator::Event::OTHER_ATTRIBUTE_CHANGED:
case ui::AXEventGenerator::Event::PLACEHOLDER_CHANGED:
case ui::AXEventGenerator::Event::PORTAL_ACTIVATED:
case ui::AXEventGenerator::Event::POSITION_IN_SET_CHANGED:
case ui::AXEventGenerator::Event::READONLY_CHANGED:
case ui::AXEventGenerator::Event::RELATED_NODE_CHANGED:
case ui::AXEventGenerator::Event::REQUIRED_STATE_CHANGED:
case ui::AXEventGenerator::Event::ROLE_CHANGED:
case ui::AXEventGenerator::Event::ROW_COUNT_CHANGED:
case ui::AXEventGenerator::Event::SET_SIZE_CHANGED:
case ui::AXEventGenerator::Event::SORT_CHANGED:
case ui::AXEventGenerator::Event::STATE_CHANGED:
case ui::AXEventGenerator::Event::TEXT_ATTRIBUTE_CHANGED:
case ui::AXEventGenerator::Event::VALUE_MAX_CHANGED:
case ui::AXEventGenerator::Event::VALUE_MIN_CHANGED:
case ui::AXEventGenerator::Event::VALUE_STEP_CHANGED:
// Unhandled event type.
break;
}
}

void AccessibilityBridgeDelegateWin32::DispatchAccessibilityAction(
Expand All @@ -33,4 +146,10 @@ AccessibilityBridgeDelegateWin32::CreateFlutterPlatformNodeDelegate() {
return std::make_shared<FlutterPlatformNodeDelegateWin32>(engine_);
}

void AccessibilityBridgeDelegateWin32::DispatchWinAccessibilityEvent(
std::shared_ptr<FlutterPlatformNodeDelegateWin32> node_delegate,
DWORD event_type) {
node_delegate->DispatchWinAccessibilityEvent(event_type);
}

} // namespace flutter
8 changes: 8 additions & 0 deletions shell/platform/windows/accessibility_bridge_delegate_win32.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
#define FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_BRIDGE_DELEGATE_H_

#include "flutter/shell/platform/common/accessibility_bridge.h"

#include "flutter/shell/platform/windows/flutter_platform_node_delegate_win32.h"
#include "flutter/shell/platform/windows/flutter_windows_engine.h"

namespace flutter {
Expand Down Expand Up @@ -38,6 +40,12 @@ class AccessibilityBridgeDelegateWin32
std::shared_ptr<FlutterPlatformNodeDelegate>
CreateFlutterPlatformNodeDelegate() override;

// Dispatches a Windows accessibility event of the specified type, generated
// by the accessibility node associated with the specified semantics node.
virtual void DispatchWinAccessibilityEvent(
std::shared_ptr<FlutterPlatformNodeDelegateWin32> node_delegate,
DWORD event_type);

private:
FlutterWindowsEngine* engine_;
};
Expand Down
129 changes: 129 additions & 0 deletions shell/platform/windows/accessibility_bridge_delegate_win32_unittests.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,34 @@ namespace testing {

namespace {

// A structure representing a Win32 MSAA event targeting a specified node.
struct MsaaEvent {
std::shared_ptr<FlutterPlatformNodeDelegateWin32> node_delegate;
DWORD event_type;
};

// Accessibility bridge delegate that captures events dispatched to the OS.
class AccessibilityBridgeDelegateWin32Spy
: public AccessibilityBridgeDelegateWin32 {
public:
explicit AccessibilityBridgeDelegateWin32Spy(FlutterWindowsEngine* engine)
: AccessibilityBridgeDelegateWin32(engine) {}

void DispatchWinAccessibilityEvent(
std::shared_ptr<FlutterPlatformNodeDelegateWin32> node_delegate,
DWORD event_type) override {
dispatched_events_.push_back({node_delegate, event_type});
}

void Reset() { dispatched_events_.clear(); }
const std::vector<MsaaEvent>& dispatched_events() const {
return dispatched_events_;
};

private:
std::vector<MsaaEvent> dispatched_events_;
};

// Returns an engine instance configured with dummy project path values, and
// overridden methods for sending platform messages, so that the engine can
// respond as if the framework were connected.
Expand Down Expand Up @@ -96,6 +124,31 @@ void PopulateAXTree(std::shared_ptr<AccessibilityBridge> bridge) {
bridge->CommitUpdates();
}

ui::AXNode* AXNodeFromID(std::shared_ptr<AccessibilityBridge> bridge,
int32_t id) {
auto node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(id).lock();
return node_delegate ? node_delegate->GetAXNode() : nullptr;
}

void ExpectWinEventFromAXEvent(int32_t node_id,
ui::AXEventGenerator::Event ax_event,
DWORD expected_event) {
auto window_binding_handler =
std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
FlutterWindowsView view(std::move(window_binding_handler));
view.SetEngine(GetTestEngine());
view.OnUpdateSemanticsEnabled(true);

auto bridge = view.GetEngine()->accessibility_bridge().lock();
PopulateAXTree(bridge);

AccessibilityBridgeDelegateWin32Spy spy(view.GetEngine());
spy.OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
{ax_event, ax::mojom::EventFrom::kNone, {}}});
ASSERT_EQ(spy.dispatched_events().size(), 1);
EXPECT_EQ(spy.dispatched_events()[0].event_type, expected_event);
}

} // namespace

TEST(AccessibilityBridgeDelegateWin32, NodeDelegateHasUniqueId) {
Expand Down Expand Up @@ -139,5 +192,81 @@ TEST(AccessibilityBridgeDelegateWin32, DispatchAccessibilityAction) {
EXPECT_EQ(actual_action, kFlutterSemanticsActionCopy);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityEventAlert) {
ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::ALERT,
EVENT_SYSTEM_ALERT);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityEventChildrenChanged) {
ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::CHILDREN_CHANGED,
EVENT_OBJECT_REORDER);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityEventFocusChanged) {
ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::FOCUS_CHANGED,
EVENT_OBJECT_FOCUS);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityEventIgnoredChanged) {
// Static test nodes with no text, hint, or scrollability are ignored.
ExpectWinEventFromAXEvent(4, ui::AXEventGenerator::Event::IGNORED_CHANGED,
EVENT_OBJECT_HIDE);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityImageAnnotationChanged) {
ExpectWinEventFromAXEvent(
1, ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED,
EVENT_OBJECT_NAMECHANGE);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityLiveRegionChanged) {
ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::LIVE_REGION_CHANGED,
EVENT_OBJECT_LIVEREGIONCHANGED);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityNameChanged) {
ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::NAME_CHANGED,
EVENT_OBJECT_NAMECHANGE);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityHScrollPosChanged) {
ExpectWinEventFromAXEvent(
1, ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED,
EVENT_SYSTEM_SCROLLINGEND);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityVScrollPosChanged) {
ExpectWinEventFromAXEvent(
1, ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED,
EVENT_SYSTEM_SCROLLINGEND);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilitySelectedChanged) {
ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::SELECTED_CHANGED,
EVENT_OBJECT_VALUECHANGE);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilitySelectedChildrenChanged) {
ExpectWinEventFromAXEvent(
2, ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED,
EVENT_OBJECT_SELECTIONWITHIN);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilitySubtreeCreated) {
ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::SUBTREE_CREATED,
EVENT_OBJECT_SHOW);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityValueChanged) {
ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::VALUE_CHANGED,
EVENT_OBJECT_VALUECHANGE);
}

TEST(AccessibilityBridgeDelegateWin32, OnAccessibilityStateChanged) {
ExpectWinEventFromAXEvent(
1, ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED,
EVENT_OBJECT_STATECHANGE);
}

} // namespace testing
} // namespace flutter
15 changes: 15 additions & 0 deletions shell/platform/windows/flutter_platform_node_delegate_win32.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,4 +77,19 @@ gfx::Rect FlutterPlatformNodeDelegateWin32::GetBoundsRect(
extent.y - origin.y);
}

void FlutterPlatformNodeDelegateWin32::DispatchWinAccessibilityEvent(
DWORD event_type) {
FlutterWindowsView* view = engine_->view();
Copy link
Contributor

@gspencergoog gspencergoog Dec 7, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could be auto.

if (!view) {
return;
}
HWND hwnd = view->GetPlatformWindow();
if (!hwnd) {
return;
}
assert(ax_platform_node_);
::NotifyWinEvent(event_type, hwnd, OBJID_CLIENT,
-ax_platform_node_->GetUniqueId());
}

} // namespace flutter
5 changes: 5 additions & 0 deletions shell/platform/windows/flutter_platform_node_delegate_win32.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ class FlutterPlatformNodeDelegateWin32 : public FlutterPlatformNodeDelegate {

const ui::AXUniqueId& GetUniqueId() const override { return unique_id_; }

// Dispatches a Windows accessibility event of the specified type, generated
// by the accessibility node associated with this object. This is a
// convenience wrapper around |NotifyWinEvent|.
virtual void DispatchWinAccessibilityEvent(DWORD event_type);

private:
ui::AXPlatformNode* ax_platform_node_;
FlutterWindowsEngine* engine_;
Expand Down
7 changes: 2 additions & 5 deletions shell/platform/windows/window_win32.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,11 +179,8 @@ LRESULT WindowWin32::OnGetObject(UINT const message,

gfx::NativeViewAccessible root_view = GetNativeViewAccessible();
if (is_uia_request && root_view) {
Microsoft::WRL::ComPtr<IRawElementProviderSimple> root;
root_view->QueryInterface(IID_PPV_ARGS(&root));
LRESULT lresult =
UiaReturnRawElementProvider(window_handle_, wparam, lparam, root.Get());
return lresult;
// TODO(cbracken): https://github.com/flutter/flutter/issues/94782
// Implement when we adopt UIA support.
} else if (is_msaa_request && root_view) {
// Return the IAccessible for the root view.
Microsoft::WRL::ComPtr<IAccessible> root(root_view);
Expand Down