[Impeller] Adjustments to SubmitKHR and queries. (flutter/engine#47249)

Removes the rest of the validation errors by increasing the size of the query ring buffer. Because we rely on callbacks fired from the fence waiter, when the phone is under load these can be substantially delayed from the frame finishing.

Adjusts presentation to use a dedicated default priority/slow core affinity/single threaded task runner. From the ARM video guide, submitKHR will block until all previously submitted work has been scheduled. This needs to happen in order, and doesn't need to run at high priority or even on medium speed cores.

engine/src/flutter/impeller/renderer/backend/vulkan/context_vk.cc

@@ -4,6 +4,8 @@
 
 #include "impeller/renderer/backend/vulkan/context_vk.h"
 
+#include "fml/concurrent_message_loop.h"
+
 #ifdef FML_OS_ANDROID
 #include <pthread.h>
 #include <sys/resource.h>
@@ -128,6 +130,12 @@ void ContextVK::Setup(Settings settings) {
     return;
   }
 
+  queue_submit_thread_ = std::make_unique<fml::Thread>("QueueSubmitThread");
+  queue_submit_thread_->GetTaskRunner()->PostTask([]() {
+    // submitKHR is extremely cheap and mostly blocks on an internal fence.
+    fml::RequestAffinity(fml::CpuAffinity::kEfficiency);
+  });
+
   raster_message_loop_ = fml::ConcurrentMessageLoop::Create(
       std::min(4u, std::thread::hardware_concurrency()));
   raster_message_loop_->PostTaskToAllWorkers([]() {
@@ -486,6 +494,10 @@ const vk::Device& ContextVK::GetDevice() const {
   return device_holder_->device.get();
 }
 
+const fml::RefPtr<fml::TaskRunner> ContextVK::GetQueueSubmitRunner() const {
+  return queue_submit_thread_->GetTaskRunner();
+}
+
 const std::shared_ptr<fml::ConcurrentTaskRunner>
 ContextVK::GetConcurrentWorkerTaskRunner() const {
   return raster_message_loop_->GetTaskRunner();
@@ -500,6 +512,7 @@ void ContextVK::Shutdown() {
   fence_waiter_.reset();
   resource_manager_.reset();
 
+  queue_submit_thread_->Join();
   raster_message_loop_->Terminate();
 }
 

engine/src/flutter/impeller/renderer/backend/vulkan/context_vk.h

@@ -10,6 +10,7 @@
 #include "flutter/fml/macros.h"
 #include "flutter/fml/mapping.h"
 #include "flutter/fml/unique_fd.h"
+#include "fml/thread.h"
 #include "impeller/base/backend_cast.h"
 #include "impeller/core/formats.h"
 #include "impeller/renderer/backend/vulkan/command_pool_vk.h"
@@ -133,6 +134,18 @@ class ContextVK final : public Context,
   const std::shared_ptr<fml::ConcurrentTaskRunner>
   GetConcurrentWorkerTaskRunner() const;
 
+  /// @brief A single-threaded task runner that should only be used for
+  ///        submitKHR.
+  ///
+  /// SubmitKHR will block until all previously submitted command buffers have
+  /// been scheduled. If there are no platform views in the scene (excluding
+  /// texture backed platform views). Then it is safe for SwapchainImpl::Present
+  /// to return before submit has completed. To do so, we offload the submit
+  /// command to a specialized single threaded task runner. The single thread
+  /// ensures that we do not queue up too much work and that the submissions
+  /// proceed in order.
+  const fml::RefPtr<fml::TaskRunner> GetQueueSubmitRunner() const;
+
   std::shared_ptr<SurfaceContextVK> CreateSurfaceContext();
 
   const std::shared_ptr<QueueVK>& GetGraphicsQueue() const;
@@ -176,6 +189,7 @@ class ContextVK final : public Context,
   std::shared_ptr<CommandPoolRecyclerVK> command_pool_recycler_;
   std::string device_name_;
   std::shared_ptr<fml::ConcurrentMessageLoop> raster_message_loop_;
+  std::unique_ptr<fml::Thread> queue_submit_thread_;
   std::shared_ptr<GPUTracerVK> gpu_tracer_;
 
   bool sync_presentation_ = false;

engine/src/flutter/impeller/renderer/backend/vulkan/gpu_tracer_vk.cc

@@ -16,7 +16,7 @@
 
 namespace impeller {
 
-static constexpr uint32_t kPoolSize = 64u;
+static constexpr uint32_t kPoolSize = 1024u;
 
 GPUTracerVK::GPUTracerVK(const std::shared_ptr<DeviceHolder>& device_holder)
     : device_holder_(device_holder) {

engine/src/flutter/impeller/renderer/backend/vulkan/swapchain_impl_vk.cc

@@ -522,7 +522,7 @@ bool SwapchainImplVK::Present(const std::shared_ptr<SwapchainImageVK>& image,
   if (context.GetSyncPresentation()) {
     task();
   } else {
-    context.GetConcurrentWorkerTaskRunner()->PostTask(task);
+    context.GetQueueSubmitRunner()->PostTask(task);
   }
   return true;
 }

engine/src/flutter/impeller/renderer/context.h

@@ -175,7 +175,9 @@ class Context {
   ///
   ///             This is required for correct rendering on Android when using
   ///             the hybrid composition mode. This has no effect on other
-  ///             backends.
+  ///             backends. This is analogous to the check for isMainThread in
+  ///             surface_mtl.mm to block presentation on scheduling of all
+  ///             pending work.
   virtual void SetSyncPresentation(bool value) {}
 
   //----------------------------------------------------------------------------