firka/flutter
3
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch backend to consume new semantics API (flutter/engine#3103)

Browse Source
This commit is contained in:
Adam Barth
2016-10-11 10:52:48 -07:00
committed by GitHub
parent be2271e05e
commit 42b02cda80
25 changed files with 583 additions and 573 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
engine/src/flutter/common/threads.cc
View File

@@ -26,6 +26,10 @@ Threads::Threads(ftl::RefPtr<ftl::TaskRunner> platform,
Threads::~Threads() {}
const ftl::RefPtr<ftl::TaskRunner>& Threads::Platform() {
return Get().platform_;
}
const ftl::RefPtr<ftl::TaskRunner>& Threads::Gpu() {
return Get().gpu_;
}

8
engine/src/flutter/lib/ui/semantics/semantics_node.cc
View File

@@ -12,4 +12,12 @@ SemanticsNode::SemanticsNode() = default;
SemanticsNode::~SemanticsNode() = default;
bool SemanticsNode::HasAction(SemanticsAction action) {
return (actions & static_cast<int32_t>(action)) != 0;
}
bool SemanticsNode::HasFlag(SemanticsFlags flag) {
return (flags & static_cast<int32_t>(flag)) != 0;
}
} // namespace blink

3
engine/src/flutter/lib/ui/semantics/semantics_node.h
View File

@@ -37,6 +37,9 @@ struct SemanticsNode {
SemanticsNode();
~SemanticsNode();
bool HasAction(SemanticsAction action);
bool HasFlag(SemanticsFlags flag);
int32_t id = 0;
int32_t flags = 0;
int32_t actions = 0;

31
engine/src/flutter/shell/common/animator.cc
View File

@@ -12,8 +12,8 @@
namespace shell {
Animator::Animator(Rasterizer* rasterizer, Engine* engine)
: rasterizer_(rasterizer->GetWeakRasterizerPtr()),
Animator::Animator(ftl::WeakPtr<Rasterizer> rasterizer, Engine* engine)
: rasterizer_(rasterizer),
engine_(engine),
layer_tree_pipeline_(ftl::MakeRefCounted<LayerTreePipeline>(3)),
pending_frame_semaphore_(1),
@@ -77,16 +77,12 @@ void Animator::Render(std::unique_ptr<flow::LayerTree> layer_tree) {
// Commit the pending continuation.
producer_continuation_.Complete(std::move(layer_tree));
// Notify the rasterizer that the pipeline has items it may consume.
auto weak_rasterizer(rasterizer_);
auto pipeline = layer_tree_pipeline_;
blink::Threads::Gpu()->PostTask([weak_rasterizer, pipeline]() {
if (!weak_rasterizer) {
return;
}
weak_rasterizer->Draw(pipeline);
});
blink::Threads::Gpu()->PostTask(
[ rasterizer = rasterizer_, pipeline = layer_tree_pipeline_ ]() {
if (!rasterizer.get())
return;
rasterizer->Draw(pipeline);
});
}
void Animator::RequestFrame() {
@@ -107,16 +103,11 @@ void Animator::RequestFrame() {
// started an expensive operation right after posting this message however.
// To support that, we need edge triggered wakes on VSync.
auto weak = weak_factory_.GetWeakPtr();
blink::Threads::UI()->PostTask([weak]() {
if (!weak) {
blink::Threads::UI()->PostTask([self = weak_factory_.GetWeakPtr()]() {
if (!self.get())
return;
}
TRACE_EVENT_INSTANT0("flutter", "RequestFrame", TRACE_EVENT_SCOPE_PROCESS);
weak->AwaitVSync(base::Bind(&Animator::BeginFrame, weak));
self->AwaitVSync(base::Bind(&Animator::BeginFrame, self));
});
}

2
engine/src/flutter/shell/common/animator.h
View File

@@ -19,7 +19,7 @@ namespace shell {
class Animator {
public:
explicit Animator(Rasterizer* rasterizer, Engine* engine);
explicit Animator(ftl::WeakPtr<Rasterizer> rasterizer, Engine* engine);
~Animator();

28
engine/src/flutter/shell/common/engine.cc
View File

@@ -20,8 +20,10 @@
#include "flutter/runtime/dart_init.h"
#include "flutter/runtime/runtime_init.h"
#include "flutter/shell/common/animator.h"
#include "flutter/shell/common/platform_view.h"
#include "flutter/sky/engine/public/web/Sky.h"
#include "lib/ftl/files/path.h"
#include "lib/ftl/functional/make_copyable.h"
#include "mojo/public/cpp/application/connect.h"
#include "third_party/skia/include/core/SkCanvas.h"
#include "third_party/skia/include/core/SkPictureRecorder.h"
@@ -47,8 +49,10 @@ std::string FindPackagesPath(const std::string& main_dart) {
} // namespace
Engine::Engine(Rasterizer* rasterizer)
: animator_(new Animator(rasterizer, this)),
Engine::Engine(PlatformView* platform_view)
: platform_view_(platform_view->GetWeakPtr()),
animator_(new Animator(platform_view->rasterizer().GetWeakRasterizerPtr(),
this)),
binding_(this),
activity_running_(false),
have_surface_(false),
@@ -160,6 +164,18 @@ void Engine::DispatchPointerDataPacket(const PointerDataPacket& packet) {
runtime_->DispatchPointerDataPacket(packet);
}
void Engine::DispatchSemanticsAction(int id, blink::SemanticsAction action) {
TRACE_EVENT0("flutter", "Engine::DispatchPointerDataPacket");
if (runtime_)
runtime_->DispatchSemanticsAction(id, action);
}
void Engine::SetSemanticsEnabled(bool enabled) {
TRACE_EVENT0("flutter", "Engine::DispatchPointerDataPacket");
if (runtime_)
runtime_->SetSemanticsEnabled(enabled);
}
void Engine::RunFromSnapshotStream(
const std::string& script_uri,
mojo::ScopedDataPipeConsumerHandle snapshot) {
@@ -334,6 +350,12 @@ void Engine::Render(std::unique_ptr<flow::LayerTree> layer_tree) {
animator_->Render(std::move(layer_tree));
}
void Engine::UpdateSemantics(std::vector<blink::SemanticsNode> update) {}
void Engine::UpdateSemantics(std::vector<blink::SemanticsNode> update) {
blink::Threads::Platform()->PostTask(ftl::MakeCopyable(
[ platform_view = platform_view_, update = std::move(update) ]() mutable {
if (platform_view)
platform_view->UpdateSemantics(std::move(update));
}));
}
} // namespace shell

6
engine/src/flutter/shell/common/engine.h
View File

@@ -24,12 +24,13 @@
#include "third_party/skia/include/core/SkPicture.h"
namespace shell {
class PlatformView;
class Animator;
using PointerDataPacket = blink::PointerDataPacket;
class Engine : public sky::SkyEngine, public blink::RuntimeDelegate {
public:
explicit Engine(Rasterizer* rasterizer);
explicit Engine(PlatformView* platform_view);
~Engine() override;
@@ -51,6 +52,8 @@ class Engine : public sky::SkyEngine, public blink::RuntimeDelegate {
void OnOutputSurfaceCreated(const ftl::Closure& gpu_continuation);
void OnOutputSurfaceDestroyed(const ftl::Closure& gpu_continuation);
void DispatchPointerDataPacket(const PointerDataPacket& packet);
void DispatchSemanticsAction(int id, blink::SemanticsAction action);
void SetSemanticsEnabled(bool enabled);
private:
// SkyEngine implementation:
@@ -91,6 +94,7 @@ class Engine : public sky::SkyEngine, public blink::RuntimeDelegate {
void ConfigureAssetBundle(const std::string& path);
void ConfigureRuntime(const std::string& script_uri);
ftl::WeakPtr<PlatformView> platform_view_;
std::unique_ptr<Animator> animator_;
sky::ServicesDataPtr services_;

32
engine/src/flutter/shell/common/platform_view.cc
View File

@@ -15,8 +15,10 @@
namespace shell {
PlatformView::PlatformView(std::unique_ptr<Rasterizer> rasterizer)
: rasterizer_(std::move(rasterizer)), size_(SkISize::Make(0, 0)) {
engine_.reset(new Engine(rasterizer_.get()));
: rasterizer_(std::move(rasterizer)),
size_(SkISize::Make(0, 0)),
weak_factory_(this) {
engine_.reset(new Engine(this));
}
PlatformView::~PlatformView() {
@@ -30,10 +32,28 @@ PlatformView::~PlatformView() {
blink::Threads::UI()->PostTask([engine]() { delete engine; });
}
void PlatformView::DispatchSemanticsAction(int32_t id,
blink::SemanticsAction action) {
blink::Threads::UI()->PostTask(
[ engine = engine_->GetWeakPtr(), id, action ] {
if (engine.get()) {
engine->DispatchSemanticsAction(
id, static_cast<blink::SemanticsAction>(action));
}
});
}
void PlatformView::SetSemanticsEnabled(bool enabled) {
blink::Threads::UI()->PostTask([ engine = engine_->GetWeakPtr(), enabled ] {
if (engine.get())
engine->SetSemanticsEnabled(enabled);
});
}
void PlatformView::ConnectToEngine(
mojo::InterfaceRequest<sky::SkyEngine> request) {
blink::Threads::UI()->PostTask(ftl::MakeCopyable([
view = GetWeakViewPtr(), engine = engine().GetWeakPtr(),
view = GetWeakPtr(), engine = engine().GetWeakPtr(),
request = std::move(request)
]() mutable {
if (engine.get())
@@ -90,6 +110,10 @@ void PlatformView::NotifyDestroyed() {
latch.Wait();
}
ftl::WeakPtr<PlatformView> PlatformView::GetWeakPtr() {
return weak_factory_.GetWeakPtr();
}
SkISize PlatformView::GetSize() {
return size_;
}
@@ -98,6 +122,8 @@ void PlatformView::Resize(const SkISize& size) {
size_ = size;
}
void PlatformView::UpdateSemantics(std::vector<blink::SemanticsNode> update) {}
void PlatformView::SetupResourceContextOnIOThread() {
ftl::AutoResetWaitableEvent latch;

20
engine/src/flutter/shell/common/platform_view.h
View File

@@ -7,6 +7,7 @@
#include <memory>
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/shell/common/engine.h"
#include "flutter/shell/common/shell.h"
#include "flutter/shell/common/surface.h"
@@ -35,6 +36,9 @@ class PlatformView {
virtual ~PlatformView();
void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action);
void SetSemanticsEnabled(bool enabled);
void ConnectToEngine(mojo::InterfaceRequest<sky::SkyEngine> request);
void NotifyCreated(std::unique_ptr<Surface> surface);
@@ -44,7 +48,7 @@ class PlatformView {
void NotifyDestroyed();
virtual ftl::WeakPtr<PlatformView> GetWeakViewPtr() = 0;
ftl::WeakPtr<PlatformView> GetWeakPtr();
virtual bool ResourceContextMakeCurrent() = 0;
@@ -52,6 +56,8 @@ class PlatformView {
virtual void Resize(const SkISize& size);
virtual void UpdateSemantics(std::vector<blink::SemanticsNode> update);
Rasterizer& rasterizer() { return *rasterizer_; }
Engine& engine() { return *engine_; }
@@ -60,17 +66,19 @@ class PlatformView {
const std::string& assets_directory) = 0;
protected:
SurfaceConfig surface_config_;
std::unique_ptr<Rasterizer> rasterizer_;
std::unique_ptr<Engine> engine_;
SkISize size_;
explicit PlatformView(std::unique_ptr<Rasterizer> rasterizer);
void SetupResourceContextOnIOThreadPerform(
ftl::AutoResetWaitableEvent* event);
SurfaceConfig surface_config_;
std::unique_ptr<Rasterizer> rasterizer_;
std::unique_ptr<Engine> engine_;
SkISize size_;
private:
ftl::WeakPtrFactory<PlatformView> weak_factory_;
FTL_DISALLOW_COPY_AND_ASSIGN(PlatformView);
};

492
engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java
View File

@@ -7,41 +7,54 @@ package io.flutter.view;
import android.graphics.Rect;
import android.opengl.Matrix;
import android.os.Bundle;
import android.util.Log;
import android.view.View;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityManager;
import android.view.accessibility.AccessibilityNodeInfo;
import android.view.accessibility.AccessibilityNodeProvider;
import org.chromium.mojo.system.MojoException;
import org.chromium.mojom.semantics.SemanticAction;
import org.chromium.mojom.semantics.SemanticsListener;
import org.chromium.mojom.semantics.SemanticsNode;
import org.chromium.mojom.semantics.SemanticsServer;
import org.chromium.mojom.sky.ViewportMetrics;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
class AccessibilityBridge extends AccessibilityNodeProvider implements SemanticsListener {
private Map<Integer, SemanticObject> mObjects;
private FlutterView mOwner;
private SemanticsServer.Proxy mSemanticsServer;
private boolean mAccessibilityEnabled = false;
private SemanticObject mFocusedObject;
private SemanticObject mHoveredObject;
class AccessibilityBridge extends AccessibilityNodeProvider {
private static final String TAG = "FlutterView";
AccessibilityBridge(FlutterView owner, SemanticsServer.Proxy semanticsServer) {
private Map<Integer, SemanticsObject> mObjects;
private FlutterView mOwner;
private boolean mAccessibilityEnabled = false;
private SemanticsObject mFocusedObject;
private SemanticsObject mHoveredObject;
private static final int SEMANTICS_ACTION_TAP = 1 << 0;
private static final int SEMANTICS_ACTION_LONG_PRESS = 1 << 1;
private static final int SEMANTICS_ACTION_SCROLL_LEFT = 1 << 2;
private static final int SEMANTICS_ACTION_SCROLL_RIGHT = 1 << 3;
private static final int SEMANTICS_ACTION_SCROLL_UP = 1 << 4;
private static final int SEMANTICS_ACTION_SCROLL_DOWN = 1 << 5;
private static final int SEMANTICS_ACTION_INCREASE = 1 << 6;
private static final int SEMANTICS_ACTION_DECREASE = 1 << 7;
private static final int SEMANTICS_ACTION_SCROLLABLE = SEMANTICS_ACTION_SCROLL_LEFT |
SEMANTICS_ACTION_SCROLL_RIGHT |
SEMANTICS_ACTION_SCROLL_UP |
SEMANTICS_ACTION_SCROLL_DOWN;
private static final int SEMANTICS_FLAG_HAS_CHECKED_STATE = 1 << 0;
private static final int SEMANTICS_FLAG_IS_CHECKED = 1 << 1;
AccessibilityBridge(FlutterView owner) {
assert owner != null;
assert semanticsServer != null;
mOwner = owner;
mObjects = new HashMap<Integer, SemanticObject>();
mSemanticsServer = semanticsServer;
mSemanticsServer.addSemanticsListener(this);
mObjects = new HashMap<Integer, SemanticsObject>();
}
void setAccessibilityEnabled(boolean accessibilityEnabled) {
@@ -58,7 +71,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
return result;
}
SemanticObject object = mObjects.get(virtualViewId);
SemanticsObject object = mObjects.get(virtualViewId);
if (object == null)
return null;
@@ -88,15 +101,15 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
result.setVisibleToUser(true);
result.setEnabled(true); // TODO(ianh): Expose disabled subtrees
if (object.canBeTapped) {
if ((object.actions & SEMANTICS_ACTION_TAP) != 0) {
result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK);
result.setClickable(true);
}
if (object.canBeLongPressed) {
if ((object.actions & SEMANTICS_ACTION_LONG_PRESS) != 0) {
result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_LONG_CLICK);
result.setLongClickable(true);
}
if (object.canBeScrolledHorizontally || object.canBeScrolledVertically) {
if ((object.actions & SEMANTICS_ACTION_SCROLLABLE) != 0) {
// TODO(ianh): Once we're on SDK v23+, call addAction to
// expose AccessibilityAction.ACTION_SCROLL_LEFT, _RIGHT,
// _UP, and _DOWN when appropriate.
@@ -106,8 +119,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
result.setScrollable(true);
}
result.setCheckable(object.hasCheckedState);
result.setChecked(object.isChecked);
result.setCheckable((object.flags & SEMANTICS_FLAG_HAS_CHECKED_STATE) != 0);
result.setChecked((object.flags & SEMANTICS_FLAG_IS_CHECKED) != 0);
result.setText(object.label);
// TODO(ianh): use setTraversalBefore/setTraversalAfter to set
@@ -124,7 +137,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
}
if (object.children != null) {
for (SemanticObject child : object.children) {
for (SemanticsObject child : object.children) {
result.addChild(mOwner, child.id);
}
}
@@ -134,36 +147,36 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
@Override
public boolean performAction(int virtualViewId, int action, Bundle arguments) {
SemanticObject object = mObjects.get(virtualViewId);
SemanticsObject object = mObjects.get(virtualViewId);
if (object == null) {
return false;
}
switch (action) {
case AccessibilityNodeInfo.ACTION_CLICK: {
mSemanticsServer.performAction(virtualViewId, SemanticAction.TAP);
mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_TAP);
return true;
}
case AccessibilityNodeInfo.ACTION_LONG_CLICK: {
mSemanticsServer.performAction(virtualViewId, SemanticAction.LONG_PRESS);
mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_LONG_PRESS);
return true;
}
case AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD: {
if (object.canBeScrolledVertically) {
mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_UP);
} else if (object.canBeScrolledHorizontally) {
if ((object.actions & SEMANTICS_ACTION_SCROLL_UP) != 0) {
mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_SCROLL_UP);
} else if ((object.actions & SEMANTICS_ACTION_SCROLL_LEFT) != 0) {
// TODO(ianh): bidi support
mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_LEFT);
mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_SCROLL_LEFT);
} else {
return false;
}
return true;
}
case AccessibilityNodeInfo.ACTION_SCROLL_FORWARD: {
if (object.canBeScrolledVertically) {
mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_DOWN);
} else if (object.canBeScrolledHorizontally) {
if ((object.actions & SEMANTICS_ACTION_SCROLL_DOWN) != 0) {
mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_SCROLL_DOWN);
} else if ((object.actions & SEMANTICS_ACTION_SCROLL_RIGHT) != 0) {
// TODO(ianh): bidi support
mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_RIGHT);
mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_SCROLL_RIGHT);
} else {
return false;
}
@@ -192,10 +205,21 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
// TODO(ianh): implement findAccessibilityNodeInfosByText()
// TODO(ianh): implement findFocus()
private SemanticObject getRootObject() {
private SemanticsObject getRootObject() {
assert mObjects.containsKey(0);
return mObjects.get(0);
}
private SemanticsObject getOrCreateObject(int id) {
SemanticsObject object = mObjects.get(id);
if (object == null) {
object = new SemanticsObject();
object.id = id;
mObjects.put(id, object);
}
return object;
}
void handleTouchExplorationExit() {
if (mHoveredObject != null) {
sendAccessibilityEvent(mHoveredObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
@@ -207,8 +231,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
if (mObjects.isEmpty()) {
return;
}
assert mObjects.containsKey(0);
SemanticObject newObject = getRootObject().hitTest(Math.round(x), Math.round(y));
SemanticsObject newObject = getRootObject().hitTest(new float[]{ x, y, 0, 1 });
if (newObject != mHoveredObject) {
// sending ENTER before EXIT is how Android wants it
if (newObject != null) {
@@ -221,57 +244,35 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
}
}
@Override
public void updateSemanticsTree(SemanticsNode[] nodes) {
Set<SemanticObject> updatedObjects = new HashSet<SemanticObject>();
Set<SemanticObject> removedObjects = new HashSet<SemanticObject>();
for (SemanticsNode node : nodes) {
updateSemanticObject(node, updatedObjects, removedObjects);
sendAccessibilityEvent(node.id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
void updateSemantics(ByteBuffer buffer, String[] strings) {
buffer.order(ByteOrder.LITTLE_ENDIAN);
ArrayList<Integer> updated = new ArrayList<Integer>();
while (buffer.hasRemaining()) {
int id = buffer.getInt();
getOrCreateObject(id).updateWith(buffer, strings);
updated.add(id);
}
for (SemanticObject object : removedObjects) {
if (!updatedObjects.contains(object)) {
removeSemanticObject(object, updatedObjects);
}
}
}
private SemanticObject updateSemanticObject(SemanticsNode node,
Set<SemanticObject> updatedObjects,
Set<SemanticObject> removedObjects) {
SemanticObject object = mObjects.get(node.id);
if (object == null) {
object = new SemanticObject();
mObjects.put(node.id, object);
Set<SemanticsObject> visitedObjects = new HashSet<SemanticsObject>();
SemanticsObject rootObject = getRootObject();
if (rootObject != null) {
final float[] identity = new float[16];
Matrix.setIdentityM(identity, 0);
rootObject.updateRecursively(identity, visitedObjects, false);
}
object.updateWith(node);
updatedObjects.add(object);
if (node.children != null) {
if (node.children.length == 0) {
if (object.children != null) {
removedObjects.addAll(object.children);
}
object.children = null;
} else {
if (object.children == null) {
object.children = new ArrayList<SemanticObject>(node.children.length);
} else {
removedObjects.addAll(object.children);
object.children.clear();
}
for (SemanticsNode childNode : node.children) {
SemanticObject childObject = updateSemanticObject(childNode, updatedObjects, removedObjects);
childObject.parent = object;
object.children.add(childObject);
}
Iterator<Map.Entry<Integer, SemanticsObject>> it = mObjects.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Integer, SemanticsObject> entry = it.next();
if (!visitedObjects.contains(entry.getKey())) {
willRemoveSemanticsObject(entry.getValue());
it.remove();
}
}
if (node.geometry != null) {
// has to be done after children are updated
// since they also get marked dirty
object.invalidateGlobalGeometry();
for (Integer id : updated) {
sendAccessibilityEvent(id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
}
return object;
}
private void sendAccessibilityEvent(int virtualViewId, int eventType) {
@@ -288,11 +289,10 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
}
}
private void removeSemanticObject(SemanticObject object, Set<SemanticObject> updatedObjects) {
private void willRemoveSemanticsObject(SemanticsObject object) {
assert mObjects.containsKey(object.id);
assert mObjects.get(object.id) == object;
object.parent = null;
mObjects.remove(object.id);
if (mFocusedObject == object) {
sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
mFocusedObject = null;
@@ -300,153 +300,183 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
if (mHoveredObject == object) {
mHoveredObject = null;
}
if (object.children != null) {
for (SemanticObject child : object.children) {
if (!updatedObjects.contains(child)) {
assert child.parent == object;
removeSemanticObject(child, updatedObjects);
}
}
}
}
void reset(SemanticsServer.Proxy newSemanticsServer) {
void reset() {
mObjects.clear();
sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
if (mFocusedObject != null)
sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
mFocusedObject = null;
mHoveredObject = null;
mSemanticsServer.close();
sendAccessibilityEvent(0, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
mSemanticsServer = newSemanticsServer;
mSemanticsServer.addSemanticsListener(this);
}
private class SemanticObject {
SemanticObject() { }
private class SemanticsObject {
SemanticsObject() { }
void updateWith(SemanticsNode node) {
if (id == -1) {
id = node.id;
assert node.flags != null;
assert node.strings != null;
assert node.geometry != null;
assert node.children != null;
}
assert id == node.id;
if (node.flags != null) {
hasCheckedState = node.flags.hasCheckedState;
isChecked = node.flags.isChecked;
}
if (node.strings != null) {
label = node.strings.label;
}
if (node.geometry != null) {
transform = node.geometry.transform;
left = node.geometry.left;
top = node.geometry.top;
width = node.geometry.width;
height = node.geometry.height;
}
if (node.actions != null) {
canBeTapped = false;
canBeLongPressed = false;
canBeScrolledHorizontally = false;
canBeScrolledVertically = false;
for (int action : node.actions) {
switch (action) {
case SemanticAction.TAP:
canBeTapped = true;
break;
case SemanticAction.LONG_PRESS:
canBeLongPressed = true;
break;
case SemanticAction.SCROLL_LEFT:
canBeScrolledHorizontally = true;
break;
case SemanticAction.SCROLL_RIGHT:
canBeScrolledHorizontally = true;
break;
case SemanticAction.SCROLL_UP:
canBeScrolledVertically = true;
break;
case SemanticAction.SCROLL_DOWN:
canBeScrolledVertically = true;
break;
case SemanticAction.INCREASE:
// Not implemented.
break;
case SemanticAction.DECREASE:
// Not implemented.
break;
}
}
}
}
// fields that we pass straight to the Android accessibility API
int id = -1;
SemanticObject parent;
boolean canBeTapped;
boolean canBeLongPressed;
boolean canBeScrolledHorizontally;
boolean canBeScrolledVertically;
boolean hasCheckedState;
boolean isChecked;
String label;
List<SemanticObject> children;
// geometry, which we have to convert to global coordinates to send to Android
private float[] transform; // can be null, meaning identity transform
int actions;
int flags;
String label;
private float left;
private float top;
private float width;
private float height;
private float right;
private float bottom;
private float[] transform;
private boolean geometryDirty = true;
private void invalidateGlobalGeometry() {
if (geometryDirty) {
return;
}
geometryDirty = true;
// TODO(ianh): if we are the AccessibilityBridge.this.mFocusedObject
// then we may have to unfocus and refocus ourselves to get Android to update the focus rect
if (children != null) {
for (SemanticObject child : children) {
child.invalidateGlobalGeometry();
SemanticsObject parent;
List<SemanticsObject> children;
private boolean inverseTransformDirty = true;
private float[] inverseTransform;
private boolean globalGeometryDirty = true;
private float[] globalTransform;
private Rect globalRect;
void updateWith(ByteBuffer buffer, String[] strings) {
actions = buffer.getInt();
flags = buffer.getInt();
final int stringIndex = buffer.getInt();
if (stringIndex == -1)
label = null;
else
label = strings[stringIndex];
left = buffer.getFloat();
top = buffer.getFloat();
right = buffer.getFloat();
bottom = buffer.getFloat();
if (transform == null)
transform = new float[16];
for (int i = 0; i < 16; ++i)
transform[i] = buffer.getFloat();
inverseTransformDirty = true;
globalGeometryDirty = true;
final int childCount = buffer.getInt();
if (childCount == 0) {
children = null;
} else {
if (children == null)
children = new ArrayList<SemanticsObject>(childCount);
else
children.clear();
for (int i = 0; i < childCount; ++i) {
SemanticsObject child = getOrCreateObject(buffer.getInt());
children.add(child);
child.parent = this;
}
}
}
private float[] globalTransform; // cached transform from the root node to this node
private Rect globalRect; // cached Rect of bounds of this node in coordinate space of the root node
private void ensureInverseTransform() {
if (!inverseTransformDirty)
return;
inverseTransformDirty = false;
if (inverseTransform == null)
inverseTransform = new float[16];
if (!Matrix.invertM(inverseTransform, 0, transform, 0))
Arrays.fill(inverseTransform, 0);
}
private float[] getGlobalTransform() {
if (geometryDirty) {
if (parent == null) {
globalTransform = transform;
} else {
float[] parentTransform = parent.getGlobalTransform();
if (transform == null) {
globalTransform = parentTransform;
} else if (parentTransform == null) {
globalTransform = transform;
} else {
globalTransform = new float[16];
Matrix.multiplyMM(globalTransform, 0, transform, 0, parentTransform, 0);
Rect getGlobalRect() {
assert !globalGeometryDirty;
return globalRect;
}
SemanticsObject hitTest(float[] point) {
final float w = point[3];
final float x = point[0] / w;
final float y = point[1] / w;
if (x < left || x >= right || y < top || y >= bottom)
return null;
if (children != null) {
final float[] transformedPoint = new float[4];
for (int i = children.size() - 1; i >= 0; i -= 1) {
final SemanticsObject child = children.get(i);
child.ensureInverseTransform();
Matrix.multiplyMV(transformedPoint, 0, child.inverseTransform, 0, point, 0);
final SemanticsObject result = child.hitTest(transformedPoint);
if (result != null) {
return result;
}
}
}
return globalTransform;
return this;
}
private float[] transformPoint(float[] transform, float[] point) {
if (transform == null)
return point; // this is a 4-item array but the caller will ignore all but the first two items
float[] transformedPoint = new float[4];
Matrix.multiplyMV(transformedPoint, 0, transform, 0, point, 0);
assert transformedPoint[2] == 0;
return new float[]{transformedPoint[0] / transformedPoint[3],
transformedPoint[1] / transformedPoint[3]};
void updateRecursively(float[] ancestorTransform, Set<SemanticsObject> visitedObjects, boolean forceUpdate) {
visitedObjects.add(this);
if (globalGeometryDirty)
forceUpdate = true;
if (forceUpdate) {
if (globalTransform == null)
globalTransform = new float[16];
Matrix.multiplyMM(globalTransform, 0, transform, 0, ancestorTransform, 0);
final float[] sample = new float[4];
sample[2] = 0;
sample[3] = 1;
final float[] point1 = new float[4];
final float[] point2 = new float[4];
final float[] point3 = new float[4];
final float[] point4 = new float[4];
sample[0] = left;
sample[1] = top;
transformPoint(point1, globalTransform, sample);
sample[0] = right;
sample[1] = top;
transformPoint(point2, globalTransform, sample);
sample[0] = right;
sample[1] = bottom;
transformPoint(point3, globalTransform, sample);
sample[0] = left;
sample[1] = bottom;
transformPoint(point4, globalTransform, sample);
if (globalRect == null)
globalRect = new Rect();
globalRect.set(
Math.round(min(point1[0], point2[0], point3[0], point4[0])),
Math.round(min(point1[1], point2[1], point3[1], point4[1])),
Math.round(max(point1[0], point2[0], point3[0], point4[0])),
Math.round(max(point1[1], point2[1], point3[1], point4[1]))
);
globalGeometryDirty = false;
}
assert globalTransform != null;
assert globalRect != null;
if (children != null) {
for (int i = 0; i < children.size(); ++i) {
children.get(i).updateRecursively(globalTransform, visitedObjects, forceUpdate);
}
}
}
private void transformPoint(float[] result, float[] transform, float[] point) {
Matrix.multiplyMV(result, 0, transform, 0, point, 0);
final float w = result[3];
result[0] /= w;
result[1] /= w;
result[2] /= w;
result[3] = 0;
}
private float min(float a, float b, float c, float d) {
@@ -456,47 +486,5 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics
private float max(float a, float b, float c, float d) {
return Math.max(a, Math.max(b, Math.max(c, d)));
}
Rect getGlobalRect() {
if (geometryDirty) {
float[] transform = getGlobalTransform();
float[] point1 = transformPoint(transform, new float[]{left, top, 0, 1});
float[] point2 = transformPoint(transform, new float[]{left + width, top, 0, 1});
float[] point3 = transformPoint(transform, new float[]{left + width, top + height, 0, 1});
float[] point4 = transformPoint(transform, new float[]{left, top + height, 0, 1});
// TODO(ianh): Scaling here is a hack to work around #1360.
float scale = mOwner.getDevicePixelRatio();
globalRect = new Rect(
Math.round(min(point1[0], point2[0], point3[0], point4[0]) * scale),
Math.round(min(point1[1], point2[1], point3[1], point4[1]) * scale),
Math.round(max(point1[0], point2[0], point3[0], point4[0]) * scale),
Math.round(max(point1[1], point2[1], point3[1], point4[1]) * scale)
);
}
return globalRect;
}
SemanticObject hitTest(int x, int y) {
Rect rect = getGlobalRect();
if (!rect.contains(x, y))
return null;
if (children != null) {
for (int index = children.size()-1; index >= 0; index -= 1) {
SemanticObject child = children.get(index);
SemanticObject result = child.hitTest(x, y);
if (result != null) {
return result;
}
}
}
return this;
}
}
@Override
public void close() {}
@Override
public void onConnectionError(MojoException e) {}
}

27
engine/src/flutter/shell/platform/android/io/flutter/view/FlutterView.java
View File

@@ -46,7 +46,6 @@ import org.chromium.mojom.editing.Keyboard;
import org.chromium.mojom.flutter.platform.ApplicationMessages;
import org.chromium.mojom.mojo.ServiceProvider;
import org.chromium.mojom.raw_keyboard.RawKeyboardService;
import org.chromium.mojom.semantics.SemanticsServer;
import org.chromium.mojom.sky.AppLifecycleState;
import org.chromium.mojom.sky.ServicesData;
import org.chromium.mojom.sky.SkyEngine;
@@ -578,7 +577,8 @@ public class FlutterView extends SurfaceView
private static native Bitmap nativeGetBitmap(long nativePlatformViewAndroid);
private static native void nativeDispatchPointerDataPacket(long nativePlatformViewAndroid, ByteBuffer buffer, int position);
private static native void nativeDispatchSemanticsAction(long nativePlatformViewAndroid, int id, int action);
private static native void nativeSetSemanticsEnabled(long nativePlatformViewAndroid, boolean enabled);
private static native void nativeInvokePlatformMessageResponseCallback(long nativePlatformViewAndroid, int responseId, String buffer);
@CalledByNative
@@ -603,11 +603,21 @@ public class FlutterView extends SurfaceView
nativeInvokePlatformMessageResponseCallback(mNativePlatformView, responseId, null);
}
@CalledByNative
private void updateSemantics(ByteBuffer buffer, String[] strings) {
if (mAccessibilityNodeProvider != null)
mAccessibilityNodeProvider.updateSemantics(buffer, strings);
}
// ACCESSIBILITY
private boolean mAccessibilityEnabled = false;
private boolean mTouchExplorationEnabled = false;
protected void dispatchSemanticsAction(int id, int action) {
nativeDispatchSemanticsAction(mNativePlatformView, id, action);
}
@Override
protected void onAttachedToWindow() {
super.onAttachedToWindow();
@@ -669,21 +679,14 @@ public class FlutterView extends SurfaceView
void ensureAccessibilityEnabled() {
if (mAccessibilityNodeProvider == null) {
mAccessibilityNodeProvider = new AccessibilityBridge(this, createSemanticsServer());
mAccessibilityNodeProvider = new AccessibilityBridge(this);
nativeSetSemanticsEnabled(mNativePlatformView, true);
}
}
private SemanticsServer.Proxy createSemanticsServer() {
Core core = CoreImpl.getInstance();
Pair<SemanticsServer.Proxy, InterfaceRequest<SemanticsServer>> server =
SemanticsServer.MANAGER.getInterfaceRequest(core);
mDartServiceProvider.connectToService(SemanticsServer.MANAGER.getName(), server.second.passHandle());
return server.first;
}
void resetAccessibilityTree() {
if (mAccessibilityNodeProvider != null) {
mAccessibilityNodeProvider.reset(createSemanticsServer());
mAccessibilityNodeProvider.reset();
}
}

73
engine/src/flutter/shell/platform/android/platform_view_android.cc
View File

@@ -9,9 +9,12 @@
#include <sys/resource.h>
#include <sys/time.h>
#include <sys/types.h>
#include <memory>
#include <utility>
#include "base/android/jni_android.h"
#include "base/android/jni_array.h"
#include "base/android/jni_string.h"
#include "base/bind.h"
#include "base/location.h"
@@ -29,7 +32,7 @@
namespace shell {
PlatformViewAndroid::PlatformViewAndroid()
: PlatformView(std::make_unique<GPURasterizer>()), weak_factory_(this) {}
: PlatformView(std::make_unique<GPURasterizer>()) {}
PlatformViewAndroid::~PlatformViewAndroid() = default;
@@ -159,6 +162,20 @@ void PlatformViewAndroid::HandlePlatformMessage(
}
}
void PlatformViewAndroid::DispatchSemanticsAction(JNIEnv* env,
jobject obj,
jint id,
jint action) {
PlatformView::DispatchSemanticsAction(
id, static_cast<blink::SemanticsAction>(action));
}
void PlatformViewAndroid::SetSemanticsEnabled(JNIEnv* env,
jobject obj,
jboolean enabled) {
PlatformView::SetSemanticsEnabled(enabled);
}
void PlatformViewAndroid::ReleaseSurface() {
if (surface_gl_) {
NotifyDestroyed();
@@ -166,10 +183,6 @@ void PlatformViewAndroid::ReleaseSurface() {
}
}
ftl::WeakPtr<shell::PlatformView> PlatformViewAndroid::GetWeakViewPtr() {
return weak_factory_.GetWeakPtr();
}
bool PlatformViewAndroid::ResourceContextMakeCurrent() {
return surface_gl_ ? surface_gl_->GLOffscreenContextMakeCurrent() : false;
}
@@ -184,6 +197,56 @@ void PlatformViewAndroid::Resize(const SkISize& size) {
}
}
void PlatformViewAndroid::UpdateSemantics(
std::vector<blink::SemanticsNode> update) {
constexpr size_t kBytesPerNode = 25 * sizeof(int32_t);
constexpr size_t kBytesPerChild = sizeof(int32_t);
JNIEnv* env = base::android::AttachCurrentThread();
{
base::android::ScopedJavaLocalRef<jobject> view = flutter_view_.get(env);
if (view.is_null())
return;
size_t num_bytes = 0;
for (const blink::SemanticsNode& node : update) {
num_bytes += kBytesPerNode;
num_bytes += node.children.size() * kBytesPerChild;
}
std::vector<char> buffer(num_bytes);
int32_t* buffer_int32 = reinterpret_cast<int32_t*>(&buffer[0]);
float* buffer_float32 = reinterpret_cast<float*>(&buffer[0]);
std::vector<std::string> strings;
size_t position = 0;
for (const blink::SemanticsNode& node : update) {
buffer_int32[position++] = node.id;
buffer_int32[position++] = node.flags;
buffer_int32[position++] = node.actions;
if (node.label.empty()) {
buffer_int32[position++] = -1;
} else {
buffer_int32[position++] = strings.size();
strings.push_back(node.label);
}
buffer_float32[position++] = node.rect.left();
buffer_float32[position++] = node.rect.top();
buffer_float32[position++] = node.rect.right();
buffer_float32[position++] = node.rect.bottom();
node.transform.asColMajorf(&buffer_float32[position]);
position += 16;
buffer_int32[position++] = node.children.size();
for (int32_t child : node.children)
buffer_int32[position++] = child;
}
Java_FlutterView_updateSemantics(
env, view.obj(), env->NewDirectByteBuffer(buffer.data(), buffer.size()),
base::android::ToJavaArrayOfStrings(env, strings).obj());
}
}
void PlatformViewAndroid::RunFromSource(const std::string& main,
const std::string& packages,
const std::string& assets_directory) {

19
engine/src/flutter/shell/platform/android/platform_view_android.h
View File

@@ -44,20 +44,24 @@ class PlatformViewAndroid : public PlatformView {
jint response_id,
jstring response);
void DispatchSemanticsAction(JNIEnv* env, jobject obj, jint id, jint action);
void SetSemanticsEnabled(JNIEnv* env, jobject obj, jboolean enabled);
base::android::ScopedJavaLocalRef<jobject> GetBitmap(JNIEnv* env,
jobject obj);
ftl::WeakPtr<shell::PlatformView> GetWeakViewPtr() override;
bool ResourceContextMakeCurrent() override;
virtual SkISize GetSize();
SkISize GetSize() override;
virtual void Resize(const SkISize& size);
void Resize(const SkISize& size) override;
virtual void RunFromSource(const std::string& main,
const std::string& packages,
const std::string& assets_directory);
void UpdateSemantics(std::vector<blink::SemanticsNode> update) override;
void RunFromSource(const std::string& main,
const std::string& packages,
const std::string& assets_directory) override;
void set_flutter_view(const JavaObjectWeakGlobalRef& flutter_view) {
flutter_view_ = flutter_view;
@@ -66,7 +70,6 @@ class PlatformViewAndroid : public PlatformView {
private:
std::unique_ptr<AndroidSurfaceGL> surface_gl_;
JavaObjectWeakGlobalRef flutter_view_;
ftl::WeakPtrFactory<PlatformViewAndroid> weak_factory_;
// We use id 0 to mean that no response is expected.
int next_response_id_ = 1;

3
engine/src/flutter/shell/platform/darwin/desktop/platform_view_mac.h
View File

@@ -25,8 +25,6 @@ class PlatformViewMac : public PlatformView, public GPUSurfaceGLDelegate {
sky::SkyEnginePtr& engineProxy();
ftl::WeakPtr<PlatformView> GetWeakViewPtr() override;
bool GLContextMakeCurrent() override;
bool GLContextClearCurrent() override;
@@ -45,7 +43,6 @@ class PlatformViewMac : public PlatformView, public GPUSurfaceGLDelegate {
base::scoped_nsobject<NSOpenGLView> opengl_view_;
base::scoped_nsobject<NSOpenGLContext> resource_loading_context_;
sky::SkyEnginePtr sky_engine_;
ftl::WeakPtrFactory<PlatformViewMac> weak_factory_;
bool IsValid() const;

7
engine/src/flutter/shell/platform/darwin/desktop/platform_view_mac.mm
View File

@@ -23,8 +23,7 @@ PlatformViewMac::PlatformViewMac(NSOpenGLView* gl_view)
opengl_view_([gl_view retain]),
resource_loading_context_([[NSOpenGLContext alloc]
initWithFormat:gl_view.pixelFormat
shareContext:gl_view.openGLContext]),
weak_factory_(this) {
shareContext:gl_view.openGLContext]) {
NSArray* paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory,
NSUserDomainMask, YES);
if (paths.count > 0) {
@@ -93,10 +92,6 @@ sky::SkyEnginePtr& PlatformViewMac::engineProxy() {
return sky_engine_;
}
ftl::WeakPtr<PlatformView> PlatformViewMac::GetWeakViewPtr() {
return weak_factory_.GetWeakPtr();
}
intptr_t PlatformViewMac::GLContextFBO() const {
// Default window bound framebuffer FBO 0.
return 0;

1
engine/src/flutter/shell/platform/darwin/ios/BUILD.gn
View File

@@ -44,6 +44,7 @@ shared_library("flutter_framework_dylib") {
deps = [
"//base:base",
"//dart/runtime:libdart",
"//flutter/lib/ui",
"//flutter/services/activity",
"//flutter/services/editing",
"//flutter/services/engine:interfaces",

6
engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm
View File

@@ -364,11 +364,11 @@ static inline PointerChangeMapperPhase PointerChangePhaseFromUITouchPhase(
// There doesn't appear to be any way to determine whether the accessibility
// inspector is enabled on the simulator. We conservatively always turn on the
// accessibility bridge in the simulator.
bool enable = true;
bool enabled = true;
#else
bool enable = UIAccessibilityIsVoiceOverRunning();
bool enabled = UIAccessibilityIsVoiceOverRunning();
#endif
_platformView->ToggleAccessibility(self.view, enable);
_platformView->ToggleAccessibility(self.view, enabled);
}
#pragma mark - Locale updates

46
engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h
View File

@@ -6,16 +6,13 @@
#define SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_ACCESSIBILITY_BRIDGE_H_
#include <memory>
#include <set>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "flutter/services/semantics/semantics.mojom.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h"
#include "flutter/sky/engine/platform/geometry/FloatRect.h"
#include "lib/ftl/macros.h"
#include "mojo/public/cpp/bindings/array.h"
#include "mojo/public/cpp/bindings/strong_binding.h"
#include "mojo/public/interfaces/application/service_provider.mojom.h"
#include "third_party/skia/include/core/SkMatrix44.h"
#include "third_party/skia/include/core/SkRect.h"
@@ -23,50 +20,47 @@ namespace shell {
class AccessibilityBridge;
} // namespace shell
@interface SemanticObject : NSObject
@interface SemanticsObject : NSObject
/**
* The globally unique identifier for this node.
*/
@property(nonatomic, readonly) uint32_t uid;
@property(nonatomic, readonly) int32_t uid;
/**
* The parent of this node in the node tree. Will be nil for the root node and
* during transient state changes.
*/
@property(nonatomic, assign) SemanticObject* parent;
@property(nonatomic, assign) SemanticsObject* parent;
- (instancetype)init __attribute__((unavailable("Use initWithBridge instead")));
- (instancetype)initWithBridge:(shell::AccessibilityBridge*)bridge
uid:(uint32_t)uid NS_DESIGNATED_INITIALIZER;
uid:(int32_t)uid NS_DESIGNATED_INITIALIZER;
@end
namespace shell {
class PlatformViewIOS;
class AccessibilityBridge final : public semantics::SemanticsListener {
class AccessibilityBridge final {
public:
AccessibilityBridge(UIView*, mojo::ServiceProvider*);
~AccessibilityBridge() override;
AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view);
~AccessibilityBridge();
void UpdateSemanticsTree(mojo::Array<semantics::SemanticsNodePtr>) override;
void UpdateSemantics(std::vector<blink::SemanticsNode> nodes);
void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action);
UIView* view() { return view_; }
semantics::SemanticsServer* server() { return semantics_server_.get(); }
UIView* view() const { return view_; }
private:
SemanticObject* UpdateSemanticObject(
const semantics::SemanticsNodePtr& node,
std::set<SemanticObject*>* updated_objects,
std::set<SemanticObject*>* removed_objects);
void RemoveSemanticObject(SemanticObject* node,
std::set<SemanticObject*>* updated_objects);
SemanticsObject* GetOrCreateObject(int32_t id);
void VisitObjectsRecursively(SemanticsObject* object,
std::unordered_set<int>* visited_objects);
void ReleaseObjects(const std::unordered_map<int, SemanticsObject*>& objects);
UIView* view_;
semantics::SemanticsServerPtr semantics_server_;
std::unordered_map<int, SemanticObject*> objects_;
mojo::Binding<semantics::SemanticsListener> binding_;
PlatformViewIOS* platform_view_;
std::unordered_map<int, SemanticsObject*> objects_;
FTL_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge);
};

296
engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm
View File

@@ -4,45 +4,42 @@
#include "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
#import <UIKit/UIKit.h>
#include <vector>
#include <utility>
#include "base/logging.h"
#include "mojo/public/cpp/application/connect.h"
#import <UIKit/UIKit.h>
#include "lib/ftl/logging.h"
#include "flutter/shell/platform/darwin/ios/platform_view_ios.h"
namespace {
constexpr uint32_t kRootNodeId = 0;
constexpr int32_t kRootNodeId = 0;
// Contains better abstractions than the raw Mojo data structure
struct Geometry {
Geometry& operator=(const semantics::SemanticGeometryPtr& other) {
if (!other->transform.is_null()) {
transform.setColMajorf(other->transform.data());
}
rect.setXYWH(other->left, other->top, other->width, other->height);
return *this;
blink::SemanticsAction GetSemanticsActionForScrollDirection(
UIAccessibilityScrollDirection direction) {
switch (direction) {
case UIAccessibilityScrollDirectionRight:
case UIAccessibilityScrollDirectionPrevious: // TODO(abarth): Support RTL.
return blink::SemanticsAction::kScrollRight;
case UIAccessibilityScrollDirectionLeft:
case UIAccessibilityScrollDirectionNext: // TODO(abarth): Support RTL.
return blink::SemanticsAction::kScrollLeft;
case UIAccessibilityScrollDirectionUp:
return blink::SemanticsAction::kScrollUp;
case UIAccessibilityScrollDirectionDown:
return blink::SemanticsAction::kScrollDown;
}
SkMatrix44 transform = SkMatrix44(SkMatrix44::kIdentity_Constructor);
SkRect rect;
};
FTL_DCHECK(false); // Unreachable
return blink::SemanticsAction::kScrollDown;
}
} // namespace
@implementation SemanticObject {
@implementation SemanticsObject {
shell::AccessibilityBridge* _bridge;
semantics::SemanticFlagsPtr _flags;
semantics::SemanticStringsPtr _strings;
Geometry _geometry;
bool _canBeTapped;
bool _canBeLongPressed;
bool _canBeScrolledHorizontally;
bool _canBeScrolledVertically;
bool _canBeAdjusted;
std::vector<SemanticObject*> _children;
blink::SemanticsNode _node;
std::vector<SemanticsObject*> _children;
}
#pragma mark - Override base class designated initializers
@@ -57,9 +54,9 @@ struct Geometry {
#pragma mark - Designated initializers
- (instancetype)initWithBridge:(shell::AccessibilityBridge*)bridge
uid:(uint32_t)uid {
DCHECK(bridge != nil) << "bridge must be set";
DCHECK(uid >= kRootNodeId);
uid:(int32_t)uid {
FTL_DCHECK(bridge != nil) << "bridge must be set";
FTL_DCHECK(uid >= kRootNodeId);
self = [super init];
if (self) {
@@ -72,52 +69,11 @@ struct Geometry {
#pragma mark - Semantic object methods
- (void)updateWith:(const semantics::SemanticsNodePtr&)node {
DCHECK(_uid == node->id);
if (!node->flags.is_null()) {
_flags = node->flags.Pass();
}
if (!node->strings.is_null()) {
_strings = node->strings.Pass();
}
if (!node->geometry.is_null()) {
_geometry = node->geometry;
}
if (!node->actions.is_null()) {
_canBeTapped = false;
_canBeLongPressed = false;
_canBeScrolledHorizontally = false;
_canBeScrolledVertically = false;
for (int action : node->actions) {
switch (static_cast<semantics::SemanticAction>(action)) {
case semantics::SemanticAction::TAP:
_canBeTapped = true;
break;
case semantics::SemanticAction::LONG_PRESS:
_canBeLongPressed = true;
break;
case semantics::SemanticAction::SCROLL_LEFT:
case semantics::SemanticAction::SCROLL_RIGHT:
_canBeScrolledHorizontally = true;
break;
case semantics::SemanticAction::SCROLL_UP:
case semantics::SemanticAction::SCROLL_DOWN:
_canBeScrolledVertically = true;
break;
case semantics::SemanticAction::INCREASE:
case semantics::SemanticAction::DECREASE:
_canBeAdjusted = true;
break;
}
}
}
- (void)setSemanticsNode:(const blink::SemanticsNode*)node {
_node = *node;
}
- (std::vector<SemanticObject*>*)children {
- (std::vector<SemanticsObject*>*)children {
return &_children;
}
@@ -133,39 +89,40 @@ struct Geometry {
// Note: hit detection will only apply to elements that report
// -isAccessibilityElement of YES. The framework will continue scanning the
// entire element tree looking for such a hit.
return _canBeTapped || _children.empty();
return _node.HasAction(blink::SemanticsAction::kTap) || _children.empty();
}
- (NSString*)accessibilityLabel {
if (_strings.is_null() || _strings->label.get().empty()) {
if (_node.label.empty()) {
return nil;
}
return @(_strings->label.data());
return @(_node.label.data());
}
- (UIAccessibilityTraits)accessibilityTraits {
UIAccessibilityTraits traits = UIAccessibilityTraitNone;
if (_canBeTapped) {
if (_node.HasAction(blink::SemanticsAction::kTap)) {
traits |= UIAccessibilityTraitButton;
}
if (_canBeAdjusted) {
if (_node.HasAction(blink::SemanticsAction::kIncrease)
|| _node.HasAction(blink::SemanticsAction::kDecrease)) {
traits |= UIAccessibilityTraitAdjustable;
}
return traits;
}
- (CGRect)accessibilityFrame {
SkMatrix44 globalTransform = _geometry.transform;
for (SemanticObject* parent = _parent; parent; parent = parent.parent) {
globalTransform = globalTransform * parent->_geometry.transform;
SkMatrix44 globalTransform = _node.transform;
for (SemanticsObject* parent = _parent; parent; parent = parent.parent) {
globalTransform = globalTransform * parent->_node.transform;
}
SkPoint quad[4];
_geometry.rect.toQuad(quad);
_node.rect.toQuad(quad);
for (auto& point : quad) {
SkScalar vector[4] = {point.x(), point.y(), 0, 1};
globalTransform.mapScalars(vector);
point.set(vector[0], vector[1]);
point.set(vector[0] / vector[3], vector[1] / vector[3]);
}
SkRect rect;
rect.set(quad, 4);
@@ -210,61 +167,22 @@ struct Geometry {
}
- (void)accessibilityIncrement {
if (_canBeAdjusted) {
_bridge->server()->PerformAction(_uid, semantics::SemanticAction::INCREASE);
if (_node.HasAction(blink::SemanticsAction::kIncrease)) {
_bridge->DispatchSemanticsAction(_uid, blink::SemanticsAction::kIncrease);
}
}
- (void)accessibilityDecrement {
if (_canBeAdjusted) {
_bridge->server()->PerformAction(_uid, semantics::SemanticAction::DECREASE);
if (_node.HasAction(blink::SemanticsAction::kDecrease)) {
_bridge->DispatchSemanticsAction(_uid, blink::SemanticsAction::kDecrease);
}
}
- (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
BOOL canBeScrolled = NO;
switch (direction) {
case UIAccessibilityScrollDirectionRight:
case UIAccessibilityScrollDirectionLeft:
canBeScrolled = _canBeScrolledHorizontally;
break;
case UIAccessibilityScrollDirectionUp:
case UIAccessibilityScrollDirectionDown:
canBeScrolled = _canBeScrolledVertically;
break;
default:
// Note: page turning of reading content is not currently supported
// (UIAccessibilityScrollDirectionNext,
// UIAccessibilityScrollDirectionPrevious)
canBeScrolled = NO;
break;
}
if (!canBeScrolled) {
blink::SemanticsAction action = GetSemanticsActionForScrollDirection(direction);
if (_node.HasAction(action))
return NO;
}
switch (direction) {
case UIAccessibilityScrollDirectionRight:
_bridge->server()->PerformAction(_uid,
semantics::SemanticAction::SCROLL_RIGHT);
break;
case UIAccessibilityScrollDirectionLeft:
_bridge->server()->PerformAction(_uid,
semantics::SemanticAction::SCROLL_LEFT);
break;
case UIAccessibilityScrollDirectionUp:
_bridge->server()->PerformAction(_uid,
semantics::SemanticAction::SCROLL_UP);
break;
case UIAccessibilityScrollDirectionDown:
_bridge->server()->PerformAction(_uid,
semantics::SemanticAction::SCROLL_DOWN);
break;
default:
DCHECK(false) << "Unsupported scroll direction: " << direction;
}
_bridge->DispatchSemanticsAction(_uid, action);
// TODO(tvolkert): provide meaningful string (e.g. "page 2 of 5")
UIAccessibilityPostNotification(UIAccessibilityPageScrolledNotification, nil);
return YES;
@@ -287,38 +205,46 @@ struct Geometry {
namespace shell {
AccessibilityBridge::AccessibilityBridge(UIView* view,
mojo::ServiceProvider* serviceProvider)
: view_(view), binding_(this) {
mojo::ConnectToService(serviceProvider, mojo::GetProxy(&semantics_server_));
mojo::InterfaceHandle<semantics::SemanticsListener> listener;
binding_.Bind(&listener);
semantics_server_->AddSemanticsListener(listener.Pass());
}
PlatformViewIOS* platform_view)
: view_(view), platform_view_(platform_view) {}
AccessibilityBridge::~AccessibilityBridge() {
for (const auto& entry : objects_) {
SemanticObject* object = entry.second;
[object neuter];
[object release];
}
ReleaseObjects(objects_);
objects_.clear();
}
void AccessibilityBridge::UpdateSemanticsTree(
mojo::Array<semantics::SemanticsNodePtr> nodes) {
std::set<SemanticObject*> updated_objects;
std::set<SemanticObject*> removed_objects;
for (const semantics::SemanticsNodePtr& node : nodes) {
UpdateSemanticObject(node, &updated_objects, &removed_objects);
}
for (SemanticObject* object : removed_objects) {
if (!updated_objects.count(object)) {
RemoveSemanticObject(object, &updated_objects);
void AccessibilityBridge::UpdateSemantics(std::vector<blink::SemanticsNode> nodes) {
for (const blink::SemanticsNode& node : nodes) {
SemanticsObject* object = GetOrCreateObject(node.id);
[object setSemanticsNode:&node];
const size_t childrenCount = node.children.size();
auto& children = *[object children];
children.resize(childrenCount);
for (size_t i = 0; i < childrenCount; ++i) {
SemanticsObject* child = GetOrCreateObject(node.children[i]);
child.parent = object;
children[i] = child;
}
}
SemanticObject* root = objects_[kRootNodeId];
SemanticsObject* root = objects_[kRootNodeId];
std::unordered_set<int> visited_objects;
if (root)
VisitObjectsRecursively(root, &visited_objects);
std::unordered_map<int, SemanticsObject*> doomed_objects;
doomed_objects.swap(objects_);
for (int uid : visited_objects) {
auto it = doomed_objects.find(uid);
objects_.insert(*it);
doomed_objects.erase(it);
// TODO(abarth): Use extract once we're at C++17.
}
ReleaseObjects(doomed_objects);
doomed_objects.clear();
if (root) {
if (!view_.accessibilityElements) {
view_.accessibilityElements = @[ root ];
@@ -330,46 +256,36 @@ void AccessibilityBridge::UpdateSemanticsTree(
nil);
}
SemanticObject* AccessibilityBridge::UpdateSemanticObject(
const semantics::SemanticsNodePtr& node,
std::set<SemanticObject*>* updated_objects,
std::set<SemanticObject*>* removed_objects) {
SemanticObject* object = objects_[node->id];
void AccessibilityBridge::DispatchSemanticsAction(
int32_t uid,
blink::SemanticsAction action) {
platform_view_->DispatchSemanticsAction(uid, action);
}
SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid) {
SemanticsObject* object = objects_[uid];
if (!object) {
object = [[SemanticObject alloc] initWithBridge:this uid:node->id];
objects_[node->id] = object;
}
[object updateWith:node];
updated_objects->insert(object);
if (!node->children.is_null()) {
std::vector<SemanticObject*>* children = [object children];
removed_objects->insert(children->begin(), children->end());
children->clear();
children->reserve(node->children.size());
for (const auto& child_node : node->children) {
SemanticObject* child_object =
UpdateSemanticObject(child_node, updated_objects, removed_objects);
child_object.parent = object;
children->push_back(child_object);
}
object = [[SemanticsObject alloc] initWithBridge:this uid:uid];
objects_[uid] = object;
}
return object;
}
void AccessibilityBridge::RemoveSemanticObject(
SemanticObject* object,
std::set<SemanticObject*>* updated_objects) {
DCHECK(objects_[object.uid] == object);
objects_.erase(object.uid);
for (SemanticObject* child : *[object children]) {
if (!updated_objects->count(child)) {
DCHECK(child.parent == object);
child.parent = nil;
RemoveSemanticObject(child, updated_objects);
}
void AccessibilityBridge::VisitObjectsRecursively(
SemanticsObject* object,
std::unordered_set<int>* visited_objects) {
visited_objects->insert(object.uid);
for (SemanticsObject* child : *[object children])
VisitObjectsRecursively(child, visited_objects);
}
void AccessibilityBridge::ReleaseObjects(
const std::unordered_map<int, SemanticsObject*>& objects) {
for (const auto& entry : objects) {
SemanticsObject* object = entry.second;
[object neuter];
[object release];
}
[object neuter];
[object release];
}
} // namespace shell

7
engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.h
View File

@@ -29,7 +29,7 @@ class PlatformViewIOS : public PlatformView, public GPUSurfaceGLDelegate {
~PlatformViewIOS() override;
void ToggleAccessibility(UIView* view, bool enable);
void ToggleAccessibility(UIView* view, bool enabled);
void ConnectToEngineAndSetupServices();
@@ -39,8 +39,6 @@ class PlatformViewIOS : public PlatformView, public GPUSurfaceGLDelegate {
ApplicationMessagesImpl& AppMessageReceiver();
ftl::WeakPtr<PlatformView> GetWeakViewPtr() override;
bool ResourceContextMakeCurrent() override;
bool GLContextMakeCurrent() override;
@@ -55,6 +53,8 @@ class PlatformViewIOS : public PlatformView, public GPUSurfaceGLDelegate {
const std::string& packages,
const std::string& assets_directory) override;
void UpdateSemantics(std::vector<blink::SemanticsNode> update) override;
private:
std::unique_ptr<IOSGLContext> context_;
sky::SkyEnginePtr engine_;
@@ -62,7 +62,6 @@ class PlatformViewIOS : public PlatformView, public GPUSurfaceGLDelegate {
flutter::platform::ApplicationMessagesPtr app_message_sender_;
ApplicationMessagesImpl app_message_receiver_;
std::unique_ptr<AccessibilityBridge> accessibility_bridge_;
ftl::WeakPtrFactory<PlatformViewIOS> weak_factory_;
void SetupAndLoadFromSource(const std::string& main,
const std::string& packages,

22
engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm
View File

@@ -273,8 +273,7 @@ class IOSGLContext {
PlatformViewIOS::PlatformViewIOS(CAEAGLLayer* layer)
: PlatformView(std::make_unique<GPURasterizer>()),
context_(std::make_unique<IOSGLContext>(surface_config_, layer)),
weak_factory_(this) {
context_(std::make_unique<IOSGLContext>(surface_config_, layer)) {
NSArray* paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory,
NSUserDomainMask, YES);
shell::Shell::Shared().tracing_controller().set_traces_base_path(
@@ -295,15 +294,16 @@ shell::ApplicationMessagesImpl& PlatformViewIOS::AppMessageReceiver() {
return app_message_receiver_;
}
void PlatformViewIOS::ToggleAccessibility(UIView* view, bool enable) {
if (enable) {
if (!accessibility_bridge_ && dart_services_.get() != nullptr) {
void PlatformViewIOS::ToggleAccessibility(UIView* view, bool enabled) {
if (enabled) {
if (!accessibility_bridge_) {
accessibility_bridge_.reset(
new shell::AccessibilityBridge(view, dart_services_.get()));
new shell::AccessibilityBridge(view, this));
}
} else {
accessibility_bridge_ = nullptr;
}
SetSemanticsEnabled(enabled);
}
void PlatformViewIOS::ConnectToEngineAndSetupServices() {
@@ -347,10 +347,6 @@ void PlatformViewIOS::SetupAndLoadFromSource(
engine_->RunFromFile(main, packages, assets_directory);
}
ftl::WeakPtr<PlatformView> PlatformViewIOS::GetWeakViewPtr() {
return weak_factory_.GetWeakPtr();
}
bool PlatformViewIOS::ResourceContextMakeCurrent() {
return context_ != nullptr ? context_->ResourceMakeCurrent() : false;
}
@@ -387,4 +383,10 @@ void PlatformViewIOS::RunFromSource(const std::string& main,
delete latch;
}
void PlatformViewIOS::UpdateSemantics(
std::vector<blink::SemanticsNode> update) {
if (accessibility_bridge_)
accessibility_bridge_->UpdateSemantics(std::move(update));
}
} // namespace shell

7
engine/src/flutter/shell/platform/linux/platform_view_glfw.cc
View File

@@ -18,8 +18,7 @@ PlatformViewGLFW::PlatformViewGLFW()
: PlatformView(std::make_unique<GPURasterizer>()),
valid_(false),
glfw_window_(nullptr),
buttons_(0),
weak_factory_(this) {
buttons_(0) {
if (!glfwInit()) {
return;
}
@@ -78,10 +77,6 @@ bool PlatformViewGLFW::IsValid() const {
return valid_;
}
ftl::WeakPtr<PlatformView> PlatformViewGLFW::GetWeakViewPtr() {
return weak_factory_.GetWeakPtr();
}
intptr_t PlatformViewGLFW::GLContextFBO() const {
// The default window bound FBO.
return 0;

3
engine/src/flutter/shell/platform/linux/platform_view_glfw.h
View File

@@ -26,8 +26,6 @@ class PlatformViewGLFW : public PlatformView, public GPUSurfaceGLDelegate {
bool IsValid() const;
ftl::WeakPtr<PlatformView> GetWeakViewPtr() override;
bool ResourceContextMakeCurrent() override;
bool GLContextMakeCurrent() override;
@@ -47,7 +45,6 @@ class PlatformViewGLFW : public PlatformView, public GPUSurfaceGLDelegate {
GLFWwindow* glfw_window_;
sky::SkyEnginePtr engine_;
int buttons_;
ftl::WeakPtrFactory<PlatformViewGLFW> weak_factory_;
void OnWindowSizeChanged(int width, int height);

9
engine/src/flutter/shell/testing/platform_view_test.cc
View File

@@ -4,21 +4,16 @@
#include "flutter/shell/testing/platform_view_test.h"
#include "flutter/shell/common/shell.h"
#include "flutter/shell/common/null_rasterizer.h"
#include "flutter/shell/common/shell.h"
namespace shell {
PlatformViewTest::PlatformViewTest()
: PlatformView(std::unique_ptr<Rasterizer>(new NullRasterizer())),
weak_factory_(this) {}
: PlatformView(std::unique_ptr<Rasterizer>(new NullRasterizer())) {}
PlatformViewTest::~PlatformViewTest() = default;
ftl::WeakPtr<PlatformView> PlatformViewTest::GetWeakViewPtr() {
return weak_factory_.GetWeakPtr();
}
bool PlatformViewTest::ResourceContextMakeCurrent() {
return false;
}

4
engine/src/flutter/shell/testing/platform_view_test.h
View File

@@ -19,8 +19,6 @@ class PlatformViewTest : public PlatformView {
~PlatformViewTest();
ftl::WeakPtr<PlatformView> GetWeakViewPtr() override;
bool ResourceContextMakeCurrent() override;
void RunFromSource(const std::string& main,
@@ -28,8 +26,6 @@ class PlatformViewTest : public PlatformView {
const std::string& assets_directory) override;
private:
ftl::WeakPtrFactory<PlatformViewTest> weak_factory_;
FTL_DISALLOW_COPY_AND_ASSIGN(PlatformViewTest);
};