[Impeller] Add external synchronization to Vulkan queues access. (flutter/engine#40848)

Fixes https://github.com/flutter/flutter/issues/123883

engine/src/flutter/ci/licenses_golden/licenses_flutter

@@ -1492,6 +1492,8 @@ ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_library_vk.cc
 ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_library_vk.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_vk.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_vk.h + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/queue_vk.cc + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/queue_vk.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/render_pass_vk.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/render_pass_vk.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/sampler_library_vk.cc + ../../../flutter/LICENSE
@@ -4069,6 +4071,8 @@ FILE: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_library_vk.cc
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_library_vk.h
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_vk.cc
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_vk.h
+FILE: ../../../flutter/impeller/renderer/backend/vulkan/queue_vk.cc
+FILE: ../../../flutter/impeller/renderer/backend/vulkan/queue_vk.h
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/render_pass_vk.cc
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/render_pass_vk.h
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/sampler_library_vk.cc

engine/src/flutter/impeller/renderer/backend/vulkan/BUILD.gn

@@ -38,6 +38,8 @@ impeller_component("vulkan") {
     "pipeline_library_vk.h",
     "pipeline_vk.cc",
     "pipeline_vk.h",
+    "queue_vk.cc",
+    "queue_vk.h",
     "render_pass_vk.cc",
     "render_pass_vk.h",
     "sampler_library_vk.cc",

engine/src/flutter/impeller/renderer/backend/vulkan/command_encoder_vk.cc

@@ -82,12 +82,12 @@ class TrackedObjectsVK {
 };
 
 CommandEncoderVK::CommandEncoderVK(vk::Device device,
-                                   vk::Queue queue,
+                                   const std::shared_ptr<QueueVK>& queue,
                                    const std::shared_ptr<CommandPoolVK>& pool,
                                    std::shared_ptr<FenceWaiterVK> fence_waiter)
     : fence_waiter_(std::move(fence_waiter)),
       tracked_objects_(std::make_shared<TrackedObjectsVK>(device, pool)) {
-  if (!fence_waiter_ || !tracked_objects_->IsValid()) {
+  if (!fence_waiter_ || !tracked_objects_->IsValid() || !queue) {
     return;
   }
   vk::CommandBufferBeginInfo begin_info;
@@ -131,7 +131,7 @@ bool CommandEncoderVK::Submit() {
   vk::SubmitInfo submit_info;
   std::vector<vk::CommandBuffer> buffers = {command_buffer};
   submit_info.setCommandBuffers(buffers);
-  if (queue_.submit(submit_info, *fence) != vk::Result::eSuccess) {
+  if (queue_->Submit(submit_info, *fence) != vk::Result::eSuccess) {
     return false;
   }
 
@@ -207,9 +207,6 @@ void CommandEncoderVK::PushDebugGroup(const char* label) const {
   if (auto command_buffer = GetCommandBuffer()) {
     command_buffer.beginDebugUtilsLabelEXT(label_info);
   }
-  if (queue_) {
-    queue_.beginDebugUtilsLabelEXT(label_info);
-  }
 }
 
 void CommandEncoderVK::PopDebugGroup() const {
@@ -219,9 +216,6 @@ void CommandEncoderVK::PopDebugGroup() const {
   if (auto command_buffer = GetCommandBuffer()) {
     command_buffer.endDebugUtilsLabelEXT();
   }
-  if (queue_) {
-    queue_.endDebugUtilsLabelEXT();
-  }
 }
 
 void CommandEncoderVK::InsertDebugMarker(const char* label) const {
@@ -234,7 +228,7 @@ void CommandEncoderVK::InsertDebugMarker(const char* label) const {
     command_buffer.insertDebugUtilsLabelEXT(label_info);
   }
   if (queue_) {
-    queue_.insertDebugUtilsLabelEXT(label_info);
+    queue_->InsertDebugMarker(label);
   }
 }
 

engine/src/flutter/impeller/renderer/backend/vulkan/command_encoder_vk.h

@@ -10,6 +10,7 @@
 #include "flutter/fml/macros.h"
 #include "impeller/renderer/backend/vulkan/command_pool_vk.h"
 #include "impeller/renderer/backend/vulkan/descriptor_pool_vk.h"
+#include "impeller/renderer/backend/vulkan/queue_vk.h"
 #include "impeller/renderer/backend/vulkan/shared_object_vk.h"
 #include "impeller/renderer/backend/vulkan/vk.h"
 
@@ -53,14 +54,13 @@ class CommandEncoderVK {
   friend class ContextVK;
 
   vk::Device device_ = {};
-  vk::Queue queue_ = {};
-
+  std::shared_ptr<QueueVK> queue_;
   std::shared_ptr<FenceWaiterVK> fence_waiter_;
   std::shared_ptr<TrackedObjectsVK> tracked_objects_;
   bool is_valid_ = false;
 
   CommandEncoderVK(vk::Device device,
-                   vk::Queue queue,
+                   const std::shared_ptr<QueueVK>& queue,
                    const std::shared_ptr<CommandPoolVK>& pool,
                    std::shared_ptr<FenceWaiterVK> fence_waiter);
 

engine/src/flutter/impeller/renderer/backend/vulkan/command_pool_vk.cc

@@ -70,7 +70,7 @@ CommandPoolVK::CommandPoolVK(const ContextVK* context)
     : owner_id_(std::this_thread::get_id()) {
   vk::CommandPoolCreateInfo pool_info;
 
-  pool_info.queueFamilyIndex = context->GetGraphicsQueueInfo().index;
+  pool_info.queueFamilyIndex = context->GetGraphicsQueue()->GetIndex().family;
   pool_info.flags = vk::CommandPoolCreateFlagBits::eTransient;
   auto pool = context->GetDevice().createCommandPoolUnique(pool_info);
   if (pool.result != vk::Result::eSuccess) {

engine/src/flutter/impeller/renderer/backend/vulkan/context_vk.cc

@@ -50,7 +50,7 @@ static std::optional<vk::PhysicalDevice> PickPhysicalDevice(
 }
 
 static std::vector<vk::DeviceQueueCreateInfo> GetQueueCreateInfos(
-    std::initializer_list<QueueVK> queues) {
+    std::initializer_list<QueueIndexVK> queues) {
   std::map<size_t /* family */, size_t /* index */> family_index_map;
   for (const auto& queue : queues) {
     family_index_map[queue.family] = 0;
@@ -72,8 +72,8 @@ static std::vector<vk::DeviceQueueCreateInfo> GetQueueCreateInfos(
   return infos;
 }
 
-static std::optional<QueueVK> PickQueue(const vk::PhysicalDevice& device,
-                                        vk::QueueFlagBits flags) {
+static std::optional<QueueIndexVK> PickQueue(const vk::PhysicalDevice& device,
+                                             vk::QueueFlagBits flags) {
   // This can be modified to ensure that dedicated queues are returned for each
   // queue type depending on support.
   const auto families = device.getQueueFamilyProperties();
@@ -81,7 +81,7 @@ static std::optional<QueueVK> PickQueue(const vk::PhysicalDevice& device,
     if (!(families[i].queueFlags & flags)) {
       continue;
     }
-    return QueueVK{.family = i, .index = 0};
+    return QueueIndexVK{.family = i, .index = 0};
   }
   return std::nullopt;
 }
@@ -331,6 +331,19 @@ void ContextVK::Setup(Settings settings) {
     return;
   }
 
+  //----------------------------------------------------------------------------
+  /// Fetch the queues.
+  ///
+  QueuesVK queues(device.value.get(),      //
+                  graphics_queue.value(),  //
+                  compute_queue.value(),   //
+                  transfer_queue.value()   //
+  );
+  if (!queues.IsValid()) {
+    VALIDATION_LOG << "Could not fetch device queues.";
+    return;
+  }
+
   //----------------------------------------------------------------------------
   /// All done!
   ///
@@ -342,15 +355,7 @@ void ContextVK::Setup(Settings settings) {
   shader_library_ = std::move(shader_library);
   sampler_library_ = std::move(sampler_library);
   pipeline_library_ = std::move(pipeline_library);
-  graphics_queue_ =
-      device_->getQueue(graphics_queue->family, graphics_queue->index);
-  compute_queue_ =
-      device_->getQueue(compute_queue->family, compute_queue->index);
-  transfer_queue_ =
-      device_->getQueue(transfer_queue->family, transfer_queue->index);
-  graphics_queue_info_ = graphics_queue.value();
-  compute_queue_info_ = compute_queue.value();
-  transfer_queue_info_ = transfer_queue.value();
+  queues_ = std::move(queues);
   device_capabilities_ = std::move(caps);
   fence_waiter_ = std::move(fence_waiter);
   is_valid_ = true;
@@ -360,11 +365,6 @@ void ContextVK::Setup(Settings settings) {
   /// messengers have had a chance to be setup.
   ///
   SetDebugName(device_.get(), device_.get(), "ImpellerDevice");
-  SetDebugName(device_.get(), graphics_queue_, "ImpellerGraphicsQ");
-  SetDebugName(device_.get(), compute_queue_, "ImpellerComputeQ");
-  if (transfer_queue_ != graphics_queue_) {
-    SetDebugName(device_.get(), transfer_queue_, "ImpellerTransferQ");
-  }
 }
 
 bool ContextVK::IsValid() const {
@@ -451,12 +451,8 @@ const std::shared_ptr<const Capabilities>& ContextVK::GetCapabilities() const {
   return device_capabilities_;
 }
 
-vk::Queue ContextVK::GetGraphicsQueue() const {
-  return graphics_queue_;
-}
-
-QueueVK ContextVK::GetGraphicsQueueInfo() const {
-  return graphics_queue_info_;
+const std::shared_ptr<QueueVK>& ContextVK::GetGraphicsQueue() const {
+  return queues_.graphics_queue;
 }
 
 vk::PhysicalDevice ContextVK::GetPhysicalDevice() const {
@@ -474,10 +470,10 @@ std::unique_ptr<CommandEncoderVK> ContextVK::CreateGraphicsCommandEncoder()
     return nullptr;
   }
   auto encoder = std::unique_ptr<CommandEncoderVK>(new CommandEncoderVK(
-      *device_,         //
-      graphics_queue_,  //
-      tls_pool,         //
-      fence_waiter_     //
+      *device_,                //
+      queues_.graphics_queue,  //
+      tls_pool,                //
+      fence_waiter_            //
       ));
   if (!encoder->IsValid()) {
     return nullptr;

engine/src/flutter/impeller/renderer/backend/vulkan/context_vk.h

@@ -13,6 +13,7 @@
 #include "impeller/base/backend_cast.h"
 #include "impeller/core/formats.h"
 #include "impeller/renderer/backend/vulkan/pipeline_library_vk.h"
+#include "impeller/renderer/backend/vulkan/queue_vk.h"
 #include "impeller/renderer/backend/vulkan/sampler_library_vk.h"
 #include "impeller/renderer/backend/vulkan/shader_library_vk.h"
 #include "impeller/renderer/backend/vulkan/swapchain_vk.h"
@@ -110,9 +111,7 @@ class ContextVK final : public Context, public BackendCast<ContextVK, Context> {
   vk::UniqueSurfaceKHR CreateAndroidSurface(ANativeWindow* window) const;
 #endif  // FML_OS_ANDROID
 
-  vk::Queue GetGraphicsQueue() const;
-
-  QueueVK GetGraphicsQueueInfo() const;
+  const std::shared_ptr<QueueVK>& GetGraphicsQueue() const;
 
   vk::PhysicalDevice GetPhysicalDevice() const;
 
@@ -127,12 +126,7 @@ class ContextVK final : public Context, public BackendCast<ContextVK, Context> {
   std::shared_ptr<ShaderLibraryVK> shader_library_;
   std::shared_ptr<SamplerLibraryVK> sampler_library_;
   std::shared_ptr<PipelineLibraryVK> pipeline_library_;
-  vk::Queue graphics_queue_ = {};
-  vk::Queue compute_queue_ = {};
-  vk::Queue transfer_queue_ = {};
-  QueueVK graphics_queue_info_ = {};
-  QueueVK compute_queue_info_ = {};
-  QueueVK transfer_queue_info_ = {};
+  QueuesVK queues_;
   std::shared_ptr<SwapchainVK> swapchain_;
   std::shared_ptr<const Capabilities> device_capabilities_;
   std::shared_ptr<FenceWaiterVK> fence_waiter_;

engine/src/flutter/impeller/renderer/backend/vulkan/fence_waiter_vk.cc

@@ -7,6 +7,7 @@
 #include <chrono>
 
 #include "flutter/fml/thread.h"
+#include "flutter/fml/trace_event.h"
 
 namespace impeller {
 
@@ -78,6 +79,7 @@ FenceWaiterVK::TrimAndCreateWaitSetLocked() {
   if (terminate_) {
     return std::nullopt;
   }
+  TRACE_EVENT0("impeller", "TrimFences");
   std::vector<vk::Fence> fences;
   fences.reserve(wait_set_.size());
   for (auto it = wait_set_.begin(); it != wait_set_.end();) {

engine/src/flutter/impeller/renderer/backend/vulkan/queue_vk.cc

@@ -0,0 +1,74 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "impeller/renderer/backend/vulkan/queue_vk.h"
+
+#include "impeller/renderer/backend/vulkan/context_vk.h"
+
+namespace impeller {
+
+QueueVK::QueueVK(QueueIndexVK index, vk::Queue queue)
+    : index_(index), queue_(queue) {}
+
+QueueVK::~QueueVK() = default;
+
+const QueueIndexVK& QueueVK::GetIndex() const {
+  return index_;
+}
+
+vk::Result QueueVK::Submit(const vk::SubmitInfo& submit_info,
+                           const vk::Fence& fence) const {
+  Lock lock(queue_mutex_);
+  return queue_.submit(submit_info, fence);
+}
+
+void QueueVK::InsertDebugMarker(const char* label) const {
+  if (!HasValidationLayers()) {
+    return;
+  }
+  vk::DebugUtilsLabelEXT label_info;
+  label_info.pLabelName = label;
+  Lock lock(queue_mutex_);
+  queue_.insertDebugUtilsLabelEXT(label_info);
+}
+
+QueuesVK::QueuesVK() = default;
+
+QueuesVK::QueuesVK(const vk::Device& device,
+                   QueueIndexVK graphics,
+                   QueueIndexVK compute,
+                   QueueIndexVK transfer) {
+  auto vk_graphics = device.getQueue(graphics.family, graphics.index);
+  auto vk_compute = device.getQueue(compute.family, compute.index);
+  auto vk_transfer = device.getQueue(transfer.family, transfer.index);
+
+  // Always setup the graphics queue.
+  graphics_queue = std::make_shared<QueueVK>(graphics, vk_graphics);
+  ContextVK::SetDebugName(device, vk_graphics, "ImpellerGraphicsQ");
+
+  // Setup the compute queue if its different from the graphics queue.
+  if (compute == graphics) {
+    compute_queue = graphics_queue;
+  } else {
+    compute_queue = std::make_shared<QueueVK>(compute, vk_compute);
+    ContextVK::SetDebugName(device, vk_compute, "ImpellerComputeQ");
+  }
+
+  // Setup the transfer queue if its different from the graphics or compute
+  // queues.
+  if (transfer == graphics) {
+    transfer_queue = graphics_queue;
+  } else if (transfer == compute) {
+    transfer_queue = compute_queue;
+  } else {
+    transfer_queue = std::make_shared<QueueVK>(transfer, vk_transfer);
+    ContextVK::SetDebugName(device, vk_transfer, "ImpellerTransferQ");
+  }
+}
+
+bool QueuesVK::IsValid() const {
+  return graphics_queue && compute_queue && transfer_queue;
+}
+
+}  // namespace impeller

engine/src/flutter/impeller/renderer/backend/vulkan/queue_vk.h

@@ -0,0 +1,72 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#pragma once
+
+#include <memory>
+
+#include "flutter/fml/macros.h"
+#include "impeller/base/thread.h"
+#include "impeller/renderer/backend/vulkan/vk.h"
+
+namespace impeller {
+
+struct QueueIndexVK {
+  size_t family = 0;
+  size_t index = 0;
+
+  constexpr bool operator==(const QueueIndexVK& other) const {
+    return family == other.family && index == other.index;
+  }
+};
+
+//------------------------------------------------------------------------------
+/// @brief      A thread safe object that can be used to access device queues.
+///             If multiple objects are created with the same underlying queue,
+///             then the external synchronization guarantees of Vulkan queues
+///             cannot be met. So care must be taken the same device queue
+///             doesn't form the basis of multiple `QueueVK`s.
+///
+class QueueVK {
+ public:
+  QueueVK(QueueIndexVK index, vk::Queue queue);
+
+  ~QueueVK();
+
+  const QueueIndexVK& GetIndex() const;
+
+  vk::Result Submit(const vk::SubmitInfo& submit_info,
+                    const vk::Fence& fence) const;
+
+  void InsertDebugMarker(const char* label) const;
+
+ private:
+  mutable Mutex queue_mutex_;
+
+  const QueueIndexVK index_;
+  const vk::Queue queue_ IPLR_GUARDED_BY(queue_mutex_);
+
+  FML_DISALLOW_COPY_AND_ASSIGN(QueueVK);
+};
+
+//------------------------------------------------------------------------------
+/// @brief      The collection of queues used by the context. The queues may all
+///             be the same.
+///
+struct QueuesVK {
+  std::shared_ptr<QueueVK> graphics_queue;
+  std::shared_ptr<QueueVK> compute_queue;
+  std::shared_ptr<QueueVK> transfer_queue;
+
+  QueuesVK();
+
+  QueuesVK(const vk::Device& device,
+           QueueIndexVK graphics,
+           QueueIndexVK compute,
+           QueueIndexVK transfer);
+
+  bool IsValid() const;
+};
+
+}  // namespace impeller

engine/src/flutter/impeller/renderer/backend/vulkan/swapchain_impl_vk.cc

@@ -418,7 +418,7 @@ bool SwapchainImplVK::Present(const std::shared_ptr<SwapchainImageVK>& image,
     submit_info.setWaitSemaphores(*sync->render_ready);
     submit_info.setSignalSemaphores(*sync->present_ready);
     auto result =
-        context.GetGraphicsQueue().submit(submit_info, *sync->acquire);
+        context.GetGraphicsQueue()->Submit(submit_info, *sync->acquire);
     if (result != vk::Result::eSuccess) {
       VALIDATION_LOG << "Could not wait on render semaphore: "
                      << vk::to_string(result);

engine/src/flutter/impeller/renderer/backend/vulkan/vk.h

@@ -69,12 +69,3 @@
 static_assert(VK_HEADER_VERSION >= 215, "Vulkan headers must not be too old.");
 
 #include "flutter/flutter_vma/flutter_vma.h"
-
-namespace impeller {
-
-struct QueueVK {
-  size_t family = 0;
-  size_t index = 0;
-};
-
-}  // namespace impeller