From 2c651a9de5e4cd2dec5615501c968b28d62d1c7a Mon Sep 17 00:00:00 2001 From: Jason Simmons Date: Mon, 18 Jun 2018 11:37:47 -0700 Subject: [PATCH] Create images from uncompressed pixel data (flutter/engine#5550) Fixes https://github.com/flutter/flutter/issues/9184 --- engine/src/flutter/lib/ui/painting.dart | 45 ++++++- engine/src/flutter/lib/ui/painting/codec.cc | 131 ++++++++++++++++++-- 2 files changed, 162 insertions(+), 14 deletions(-) diff --git a/engine/src/flutter/lib/ui/painting.dart b/engine/src/flutter/lib/ui/painting.dart index 7699f847b7..b8aa56c718 100644 --- a/engine/src/flutter/lib/ui/painting.dart +++ b/engine/src/flutter/lib/ui/painting.dart @@ -1367,6 +1367,26 @@ enum ImageByteFormat { png, } +/// The format of pixel data given to [decodeImageFromPixels]. +enum PixelFormat { + /// Each pixel is 32 bits, with the highest 8 bits encoding red, the next 8 + /// bits encoding green, the next 8 bits encoding blue, and the lowest 8 bits + /// encoding alpha. + rgba8888, + + /// Each pixel is 32 bits, with the highest 8 bits encoding blue, the next 8 + /// bits encoding green, the next 8 bits encoding red, and the lowest 8 bits + /// encoding alpha. + bgra8888, +} + +class _ImageInfo { + _ImageInfo(this.width, this.height, this.format); + int width; + int height; + int format; +} + /// Opaque handle to raw decoded image data (pixels). /// /// To obtain an [Image] object, use [instantiateImageCodec]. @@ -1481,14 +1501,14 @@ class Codec extends NativeFieldWrapperClass2 { /// failed. Future instantiateImageCodec(Uint8List list) { return _futurize( - (_Callback callback) => _instantiateImageCodec(list, callback) + (_Callback callback) => _instantiateImageCodec(list, callback, null) ); } /// Instantiates a [Codec] object for an image binary data. /// /// Returns an error message if the instantiation has failed, null otherwise. -String _instantiateImageCodec(Uint8List list, _Callback callback) +String _instantiateImageCodec(Uint8List list, _Callback callback, _ImageInfo imageInfo) native 'instantiateImageCodec'; /// Loads a single image frame from a byte array into an [Image] object. @@ -1499,12 +1519,31 @@ void decodeImageFromList(Uint8List list, ImageDecoderCallback callback) { _decodeImageFromListAsync(list, callback); } -Future _decodeImageFromListAsync(Uint8List list, ImageDecoderCallback callback) async { +Future _decodeImageFromListAsync(Uint8List list, + ImageDecoderCallback callback) async { final Codec codec = await instantiateImageCodec(list); final FrameInfo frameInfo = await codec.getNextFrame(); callback(frameInfo.image); } +/// Convert an array of pixel values into an [Image] object. +/// +/// [pixels] is the pixel data in the encoding described by [format]. +void decodeImageFromPixels( + Uint8List pixels, + int width, + int height, + PixelFormat format, + ImageDecoderCallback callback +) { + final _ImageInfo imageInfo = new _ImageInfo(width, height, format.index); + final Future codecFuture = _futurize( + (_Callback callback) => _instantiateImageCodec(pixels, callback, imageInfo) + ); + codecFuture.then((Codec codec) => codec.getNextFrame()) + .then((FrameInfo frameInfo) => callback(frameInfo.image)); +} + /// Determines the winding rule that decides how the interior of a [Path] is /// calculated. /// diff --git a/engine/src/flutter/lib/ui/painting/codec.cc b/engine/src/flutter/lib/ui/painting/codec.cc index c75ec5fef3..ddee0dedc0 100644 --- a/engine/src/flutter/lib/ui/painting/codec.cc +++ b/engine/src/flutter/lib/ui/painting/codec.cc @@ -32,6 +32,12 @@ namespace { static constexpr const char* kInitCodecTraceTag = "InitCodec"; static constexpr const char* kCodecNextFrameTraceTag = "CodecNextFrame"; +// This must be kept in sync with the enum in painting.dart +enum PixelFormat { + kRGBA8888, + kBGRA8888, +}; + static void InvokeCodecCallback(fxl::RefPtr codec, std::unique_ptr callback, size_t trace_id) { @@ -104,15 +110,52 @@ fxl::RefPtr InitCodec(fml::WeakPtr context, return fxl::MakeRefCounted(std::move(frameInfo)); } +fxl::RefPtr InitCodecUncompressed( + fml::WeakPtr context, + sk_sp buffer, + SkImageInfo image_info, + fxl::RefPtr unref_queue, + size_t trace_id) { + TRACE_FLOW_STEP("flutter", kInitCodecTraceTag, trace_id); + TRACE_EVENT0("blink", "InitCodecUncompressed"); + + if (buffer == nullptr || buffer->isEmpty()) { + FXL_LOG(ERROR) << "InitCodecUncompressed failed - buffer was empty"; + return nullptr; + } + + sk_sp skImage; + if (context) { + SkPixmap pixmap(image_info, buffer->data(), image_info.minRowBytes()); + skImage = SkImage::MakeCrossContextFromPixmap(context.get(), pixmap, false, + nullptr, true); + } else { + skImage = SkImage::MakeRasterData(image_info, std::move(buffer), + image_info.minRowBytes()); + } + + auto image = CanvasImage::Create(); + image->set_image({skImage, unref_queue}); + auto frameInfo = fxl::MakeRefCounted(std::move(image), 0); + return fxl::MakeRefCounted(std::move(frameInfo)); +} + void InitCodecAndInvokeCodecCallback( fxl::RefPtr ui_task_runner, fml::WeakPtr context, fxl::RefPtr unref_queue, std::unique_ptr callback, sk_sp buffer, + std::unique_ptr image_info, size_t trace_id) { - auto codec = - InitCodec(context, std::move(buffer), std::move(unref_queue), trace_id); + fxl::RefPtr codec; + if (image_info) { + codec = InitCodecUncompressed(context, std::move(buffer), *image_info, + std::move(unref_queue), trace_id); + } else { + codec = + InitCodec(context, std::move(buffer), std::move(unref_queue), trace_id); + } ui_task_runner->PostTask( fxl::MakeCopyable([callback = std::move(callback), codec = std::move(codec), trace_id]() mutable { @@ -120,13 +163,74 @@ void InitCodecAndInvokeCodecCallback( })); } +bool ConvertImageInfo(Dart_Handle image_info_handle, + Dart_NativeArguments args, + SkImageInfo* image_info) { + Dart_Handle width_handle = Dart_GetField(image_info_handle, ToDart("width")); + if (!Dart_IsInteger(width_handle)) { + Dart_SetReturnValue(args, ToDart("ImageInfo.width must be an integer")); + return false; + } + Dart_Handle height_handle = + Dart_GetField(image_info_handle, ToDart("height")); + if (!Dart_IsInteger(height_handle)) { + Dart_SetReturnValue(args, ToDart("ImageInfo.height must be an integer")); + return false; + } + Dart_Handle format_handle = + Dart_GetField(image_info_handle, ToDart("format")); + if (!Dart_IsInteger(format_handle)) { + Dart_SetReturnValue(args, ToDart("ImageInfo.format must be an integer")); + return false; + } + + PixelFormat pixel_format = static_cast( + tonic::DartConverter::FromDart(format_handle)); + SkColorType color_type = kUnknown_SkColorType; + switch (pixel_format) { + case kRGBA8888: + color_type = kRGBA_8888_SkColorType; + break; + case kBGRA8888: + color_type = kBGRA_8888_SkColorType; + break; + } + if (color_type == kUnknown_SkColorType) { + Dart_SetReturnValue(args, ToDart("Invalid pixel format")); + return false; + } + + *image_info = + SkImageInfo::Make(tonic::DartConverter::FromDart(width_handle), + tonic::DartConverter::FromDart(height_handle), + color_type, kPremul_SkAlphaType); + + return true; +} + void InstantiateImageCodec(Dart_NativeArguments args) { static size_t trace_counter = 1; const size_t trace_id = trace_counter++; TRACE_FLOW_BEGIN("flutter", kInitCodecTraceTag, trace_id); - Dart_Handle exception = nullptr; + Dart_Handle callback_handle = Dart_GetNativeArgument(args, 1); + if (!Dart_IsClosure(callback_handle)) { + TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); + Dart_SetReturnValue(args, ToDart("Callback must be a function")); + return; + } + Dart_Handle image_info_handle = Dart_GetNativeArgument(args, 2); + std::unique_ptr image_info; + if (!Dart_IsNull(image_info_handle)) { + image_info = std::make_unique(); + if (!ConvertImageInfo(image_info_handle, args, image_info.get())) { + TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); + return; + } + } + + Dart_Handle exception = nullptr; tonic::Uint8List list = tonic::DartConverter::FromArguments(args, 0, exception); if (exception) { @@ -135,11 +239,15 @@ void InstantiateImageCodec(Dart_NativeArguments args) { return; } - Dart_Handle callback_handle = Dart_GetNativeArgument(args, 1); - if (!Dart_IsClosure(callback_handle)) { - TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); - Dart_SetReturnValue(args, ToDart("Callback must be a function")); - return; + if (image_info) { + int expected_size = image_info->minRowBytes() * image_info->height(); + if (list.num_elements() < expected_size) { + TRACE_FLOW_END("flutter", kInitCodecTraceTag, trace_id); + list.Release(); + Dart_SetReturnValue( + args, ToDart("Pixel buffer size does not match image size")); + return; + } } auto buffer = SkData::MakeWithCopy(list.data(), list.num_elements()); @@ -150,13 +258,14 @@ void InstantiateImageCodec(Dart_NativeArguments args) { task_runners.GetIOTaskRunner()->PostTask(fxl::MakeCopyable( [callback = std::make_unique( tonic::DartState::Current(), callback_handle), - buffer = std::move(buffer), trace_id, + buffer = std::move(buffer), trace_id, image_info = std::move(image_info), ui_task_runner = task_runners.GetUITaskRunner(), context = dart_state->GetResourceContext(), queue = UIDartState::Current()->GetSkiaUnrefQueue()]() mutable { InitCodecAndInvokeCodecCallback(std::move(ui_task_runner), context, std::move(queue), std::move(callback), - std::move(buffer), trace_id); + std::move(buffer), + std::move(image_info), trace_id); })); } @@ -349,7 +458,7 @@ Dart_Handle SingleFrameCodec::getNextFrame(Dart_Handle callback_handle) { void Codec::RegisterNatives(tonic::DartLibraryNatives* natives) { natives->Register({ - {"instantiateImageCodec", InstantiateImageCodec, 2, true}, + {"instantiateImageCodec", InstantiateImageCodec, 3, true}, }); natives->Register({FOR_EACH_BINDING(DART_REGISTER_NATIVE)}); }