From f10bdfbf77e8d07531d36af1880adb7401907dcb Mon Sep 17 00:00:00 2001 From: Todd Volkert Date: Mon, 16 Apr 2018 12:39:46 -0700 Subject: [PATCH] Return raw (unencoded) bytes in Image.toByteData() (flutter/engine#5008) Building image encoding into the engine bloated the binary size. This change will return raw bytes, and callers who use this functionality can take on the dependency on image encoding in their apps (via a Dart package or a platform plugin). Fixes https://github.com/flutter/flutter/issues/16537 --- engine/src/flutter/lib/ui/painting.dart | 88 ++-------------- engine/src/flutter/lib/ui/painting/image.cc | 6 +- engine/src/flutter/lib/ui/painting/image.h | 2 +- .../flutter/lib/ui/painting/image_encoding.cc | 91 +++++++++-------- .../flutter/lib/ui/painting/image_encoding.h | 6 +- .../flutter/testing/dart/encoding_test.dart | 96 +++++++++++------- engine/src/flutter/testing/dart/pubspec.yaml | 7 +- engine/src/flutter/testing/resources/4x4.png | Bin 0 -> 71 bytes .../flutter/testing/resources/square-80.jpg | Bin 928 -> 0 bytes .../flutter/testing/resources/square-80.webp | Bin 666 -> 0 bytes .../src/flutter/testing/resources/square.png | Bin 125 -> 0 bytes 11 files changed, 127 insertions(+), 169 deletions(-) create mode 100644 engine/src/flutter/testing/resources/4x4.png delete mode 100644 engine/src/flutter/testing/resources/square-80.jpg delete mode 100644 engine/src/flutter/testing/resources/square-80.webp delete mode 100644 engine/src/flutter/testing/resources/square.png diff --git a/engine/src/flutter/lib/ui/painting.dart b/engine/src/flutter/lib/ui/painting.dart index 02c4b4986e..96af472629 100644 --- a/engine/src/flutter/lib/ui/painting.dart +++ b/engine/src/flutter/lib/ui/painting.dart @@ -1196,80 +1196,6 @@ class Paint { } } -/// An encoding format to use with the [Image.toByteData]. -class EncodingFormat { - /// PNG format. - /// - /// A loss-less compression format for images. This format is well suited for - /// images with hard edges, such as screenshots or sprites, and images with - /// text. Transparency is supported. The PNG format supports images up to - /// 2,147,483,647 pixels in either dimension, though in practice available - /// memory provides a more immediate limitation on maximum image size. - /// - /// PNG images normally use the `.png` file extension and the `image/png` MIME - /// type. - /// - /// See also: - /// - /// * , the Wikipedia page on PNG. - /// * , the PNG standard. - const EncodingFormat.png() - : _format = _pngFormat, - _quality = 0; - - /// JPEG format. - /// - /// This format, strictly speaking called JFIF, is a lossy compression - /// graphics format that can handle images up to 65,535 pixels in either - /// dimension. The [quality] metric is a value in the range 0 to 100 that - /// controls the compression ratio. Values in the range of about 50 to 90 are - /// somewhat reasonable; values above 95 increase the file size with little - /// noticeable improvement to the quality, values below 50 drop the quality - /// substantially. - /// - /// This format is well suited for photographs. It is very poorly suited for - /// images with hard edges or text. It does not support transparency. - /// - /// JPEG images normally use the `.jpeg` file extension and the `image/jpeg` - /// MIME type. - /// - /// See also: - /// - /// * , the Wikipedia page on JPEG. - const EncodingFormat.jpeg({int quality = 80}) - : _format = _jpegFormat, - _quality = quality; - - /// WebP format. - /// - /// The WebP format supports both lossy and lossless compression; however, the - /// [Image.toByteData] method always uses lossy compression when [webp] is - /// specified. The [quality] metric is a value in the range 0 to 100 that - /// controls the compression ratio; higher values result in better quality but - /// larger file sizes, and vice versa. WebP images are limited to 16,383 - /// pixels in each direction (width and height). - /// - /// WebP images normally use the `.webp` file extension and the `image/webp` - /// MIME type. - /// - /// See also: - /// - /// * , the Wikipedia page on WebP. - const EncodingFormat.webp({int quality = 80}) - : _format = _webpFormat, - _quality = quality; - - final int _format; - final int _quality; - - // Be conservative with the formats we expose. It is easy to add new formats - // in future but difficult to remove. - // These values must be kept in sync with the logic in ToSkEncodedImageFormat. - static const int _jpegFormat = 0; - static const int _pngFormat = 1; - static const int _webpFormat = 2; -} - /// Opaque handle to raw decoded image data (pixels). /// /// To obtain an [Image] object, use [instantiateImageCodec]. @@ -1291,20 +1217,20 @@ class Image extends NativeFieldWrapperClass2 { /// Converts the [Image] object into a byte array. /// - /// The [format] is encoding format to be used. + /// The image bytes will be RGBA form, 8 bits per channel, row-primary. /// - /// Returns a future which complete with the binary image data (e.g a PNG or JPEG binary data) or - /// an error if encoding fails. - Future toByteData({EncodingFormat format: const EncodingFormat.jpeg()}) { + /// Returns a future that completes with the binary image data or an error + /// if encoding fails. + Future toByteData() { return _futurize((_Callback callback) { - return _toByteData(format._format, format._quality, (Uint8List encoded) { - callback(encoded.buffer.asByteData()); + return _toByteData((Uint8List encoded) { + callback(encoded?.buffer?.asByteData()); }); }); } /// Returns an error message on failure, null on success. - String _toByteData(int format, int quality, _Callback callback) native 'Image_toByteData'; + String _toByteData(_Callback callback) native 'Image_toByteData'; /// Release the resources used by this object. The object is no longer usable /// after this method is called. diff --git a/engine/src/flutter/lib/ui/painting/image.cc b/engine/src/flutter/lib/ui/painting/image.cc index a1b9b45cdc..4704a9ae66 100644 --- a/engine/src/flutter/lib/ui/painting/image.cc +++ b/engine/src/flutter/lib/ui/painting/image.cc @@ -32,10 +32,8 @@ CanvasImage::CanvasImage() = default; CanvasImage::~CanvasImage() = default; -Dart_Handle CanvasImage::toByteData(int format, - int quality, - Dart_Handle callback) { - return EncodeImage(this, format, quality, callback); +Dart_Handle CanvasImage::toByteData(Dart_Handle callback) { + return GetImageBytes(this, callback); } void CanvasImage::dispose() { diff --git a/engine/src/flutter/lib/ui/painting/image.h b/engine/src/flutter/lib/ui/painting/image.h index aeec2a0149..3c6620fda2 100644 --- a/engine/src/flutter/lib/ui/painting/image.h +++ b/engine/src/flutter/lib/ui/painting/image.h @@ -31,7 +31,7 @@ class CanvasImage final : public fxl::RefCountedThreadSafe, int height() { return image_.get()->height(); } - Dart_Handle toByteData(int format, int quality, Dart_Handle callback); + Dart_Handle toByteData(Dart_Handle callback); void dispose(); diff --git a/engine/src/flutter/lib/ui/painting/image_encoding.cc b/engine/src/flutter/lib/ui/painting/image_encoding.cc index f010fce893..952e77edda 100644 --- a/engine/src/flutter/lib/ui/painting/image_encoding.cc +++ b/engine/src/flutter/lib/ui/painting/image_encoding.cc @@ -8,6 +8,7 @@ #include #include "flutter/common/task_runners.h" +#include "flutter/glue/trace_event.h" #include "flutter/lib/ui/painting/image.h" #include "flutter/lib/ui/ui_dart_state.h" #include "lib/fxl/build_config.h" @@ -17,6 +18,7 @@ #include "lib/tonic/typed_data/uint8_list.h" #include "third_party/skia/include/core/SkEncodedImageFormat.h" #include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/core/SkSurface.h" using tonic::DartInvoke; using tonic::DartPersistentValue; @@ -41,65 +43,70 @@ void InvokeDataCallback(std::unique_ptr callback, } } -sk_sp EncodeImage(sk_sp image, - SkEncodedImageFormat format, - int quality) { +sk_sp GetImageBytesAsRGBA(sk_sp image) { + TRACE_EVENT0("flutter", __FUNCTION__); + if (image == nullptr) { return nullptr; } - return image->encodeToData(format, quality); + + // Copy the GPU image snapshot into CPU memory. + auto cpu_snapshot = image->makeRasterImage(); + if (!cpu_snapshot) { + FXL_LOG(ERROR) << "Pixel copy failed."; + return nullptr; + } + + SkPixmap pixmap; + if (!cpu_snapshot->peekPixels(&pixmap)) { + FXL_LOG(ERROR) << "Pixel address is not available."; + return nullptr; + } + + if (pixmap.colorType() != kRGBA_8888_SkColorType) { + TRACE_EVENT0("flutter", "ConvertToRGBA"); + + // Convert the pixel data to N32 to adhere to our API contract. + const auto image_info = SkImageInfo::MakeN32Premul(image->width(), + image->height()); + auto surface = SkSurface::MakeRaster(image_info); + surface->writePixels(pixmap, 0, 0); + if (!surface->peekPixels(&pixmap)) { + FXL_LOG(ERROR) << "Pixel address is not available."; + return nullptr; + } + ASSERT(pixmap.colorType() == kRGBA_8888_SkColorType); + + const size_t pixmap_size = pixmap.computeByteSize(); + return SkData::MakeWithCopy(pixmap.addr32(), pixmap_size); + } else { + const size_t pixmap_size = pixmap.computeByteSize(); + return SkData::MakeWithCopy(pixmap.addr32(), pixmap_size); + } } -void EncodeImageAndInvokeDataCallback( +void GetImageBytesAndInvokeDataCallback( std::unique_ptr callback, sk_sp image, - SkEncodedImageFormat format, - int quality, fxl::RefPtr ui_task_runner) { - sk_sp encoded = EncodeImage(std::move(image), format, quality); + sk_sp buffer = GetImageBytesAsRGBA(std::move(image)); ui_task_runner->PostTask( - fxl::MakeCopyable([callback = std::move(callback), encoded]() mutable { - InvokeDataCallback(std::move(callback), std::move(encoded)); + fxl::MakeCopyable([callback = std::move(callback), buffer]() mutable { + InvokeDataCallback(std::move(callback), std::move(buffer)); })); } -SkEncodedImageFormat ToSkEncodedImageFormat(int format) { - // Map the formats exposed in flutter to formats supported in Skia. - // See: - // https://github.com/google/skia/blob/master/include/core/SkEncodedImageFormat.h - switch (format) { - case 0: - return SkEncodedImageFormat::kJPEG; - case 1: - return SkEncodedImageFormat::kPNG; - case 2: - return SkEncodedImageFormat::kWEBP; - default: - /* NOTREACHED */ - return SkEncodedImageFormat::kWEBP; - } -} - } // namespace -Dart_Handle EncodeImage(CanvasImage* canvas_image, - int format, - int quality, - Dart_Handle callback_handle) { +Dart_Handle GetImageBytes(CanvasImage* canvas_image, + Dart_Handle callback_handle) { if (!canvas_image) return ToDart("encode called with non-genuine Image."); if (!Dart_IsClosure(callback_handle)) return ToDart("Callback must be a function."); - SkEncodedImageFormat image_format = ToSkEncodedImageFormat(format); - - if (quality > 100) - quality = 100; - if (quality < 0) - quality = 0; - auto callback = std::make_unique( tonic::DartState::Current(), callback_handle); sk_sp image = canvas_image->image(); @@ -107,11 +114,11 @@ Dart_Handle EncodeImage(CanvasImage* canvas_image, const auto& task_runners = UIDartState::Current()->GetTaskRunners(); task_runners.GetIOTaskRunner()->PostTask(fxl::MakeCopyable( - [callback = std::move(callback), image, image_format, quality, + [callback = std::move(callback), image, ui_task_runner = task_runners.GetUITaskRunner()]() mutable { - EncodeImageAndInvokeDataCallback(std::move(callback), std::move(image), - image_format, quality, - std::move(ui_task_runner)); + GetImageBytesAndInvokeDataCallback(std::move(callback), + std::move(image), + std::move(ui_task_runner)); })); return Dart_Null(); diff --git a/engine/src/flutter/lib/ui/painting/image_encoding.h b/engine/src/flutter/lib/ui/painting/image_encoding.h index a121d597cc..5081bc4ee5 100644 --- a/engine/src/flutter/lib/ui/painting/image_encoding.h +++ b/engine/src/flutter/lib/ui/painting/image_encoding.h @@ -11,10 +11,8 @@ namespace blink { class CanvasImage; -Dart_Handle EncodeImage(CanvasImage* canvas_image, - int format, - int quality, - Dart_Handle callback_handle); +Dart_Handle GetImageBytes(CanvasImage* canvas_image, + Dart_Handle callback_handle); } // namespace blink diff --git a/engine/src/flutter/testing/dart/encoding_test.dart b/engine/src/flutter/testing/dart/encoding_test.dart index 5fbbb4eb63..22c0a28465 100644 --- a/engine/src/flutter/testing/dart/encoding_test.dart +++ b/engine/src/flutter/testing/dart/encoding_test.dart @@ -7,57 +7,81 @@ import 'dart:ui'; import 'dart:typed_data'; import 'dart:io'; -import 'package:test/test.dart'; +import 'package:flutter_test/flutter_test.dart'; import 'package:path/path.dart' as path; +const int _kWidth = 10; +const int _kRadius = 2; + +const Color _kBlack = const Color.fromRGBO(0, 0, 0, 1.0); +const Color _kGreen = const Color.fromRGBO(0, 255, 0, 1.0); + void main() { - final Image testImage = createSquareTestImage(); + group('Image.toByteData', () { + test('Encode with default arguments', () async { + Image testImage = createSquareTestImage(); + ByteData data = await testImage.toByteData(); + expect(new Uint8List.view(data.buffer), getExpectedBytes()); + }); - test('Encode with default arguments', () async { - ByteData data = await testImage.toByteData(); - List expected = readFile('square-80.jpg'); - expect(new Uint8List.view(data.buffer), expected); - }); - - test('Encode JPEG', () async { - ByteData data = await testImage.toByteData( - format: new EncodingFormat.jpeg(quality: 80)); - List expected = readFile('square-80.jpg'); - expect(new Uint8List.view(data.buffer), expected); - }); - - test('Encode PNG', () async { - ByteData data = - await testImage.toByteData(format: new EncodingFormat.png()); - List expected = readFile('square.png'); - expect(new Uint8List.view(data.buffer), expected); - }); - - test('Encode WEBP', () async { - ByteData data = await testImage.toByteData( - format: new EncodingFormat.webp(quality: 80)); - List expected = readFile('square-80.webp'); - expect(new Uint8List.view(data.buffer), expected); + test('Handles greyscale images', () async { + Uint8List png = await new File('../resources/4x4.png').readAsBytes(); + Completer completer = new Completer(); + decodeImageFromList(png, (Image image) => completer.complete(image)); + Image image = await completer.future; + ByteData data = await image.toByteData(); + Uint8List bytes = data.buffer.asUint8List(); + expect(bytes, hasLength(16)); + expect(bytes, [ + 255, 255, 255, 255, + 127, 127, 127, 255, + 127, 127, 127, 255, + 0, 0, 0, 255, + ]); + }); }); } Image createSquareTestImage() { + double width = _kWidth.toDouble(); + double radius = _kRadius.toDouble(); + double innerWidth = (_kWidth - 2 * _kRadius).toDouble(); + PictureRecorder recorder = new PictureRecorder(); - Canvas canvas = new Canvas(recorder, new Rect.fromLTWH(0.0, 0.0, 10.0, 10.0)); + Canvas canvas = + new Canvas(recorder, new Rect.fromLTWH(0.0, 0.0, width, width)); Paint black = new Paint() ..strokeWidth = 1.0 - ..color = const Color.fromRGBO(0, 0, 0, 1.0); + ..color = _kBlack; Paint green = new Paint() ..strokeWidth = 1.0 - ..color = const Color.fromRGBO(0, 255, 0, 1.0); + ..color = _kGreen; - canvas.drawRect(new Rect.fromLTWH(0.0, 0.0, 10.0, 10.0), black); - canvas.drawRect(new Rect.fromLTWH(2.0, 2.0, 6.0, 6.0), green); - return recorder.endRecording().toImage(10, 10); + canvas.drawRect(new Rect.fromLTWH(0.0, 0.0, width, width), black); + canvas.drawRect( + new Rect.fromLTWH(radius, radius, innerWidth, innerWidth), green); + return recorder.endRecording().toImage(_kWidth, _kWidth); } -List readFile(fileName) { - final file = new File(path.join('flutter', 'testing', 'resources', fileName)); - return file.readAsBytesSync(); +List getExpectedBytes() { + int bytesPerChannel = 4; + List result = new List(_kWidth * _kWidth * bytesPerChannel); + + fillWithColor(Color color, int min, int max) { + for (int i = min; i < max; i++) { + for (int j = min; j < max; j++) { + int offset = i * bytesPerChannel + j * _kWidth * bytesPerChannel; + result[offset] = color.red; + result[offset + 1] = color.green; + result[offset + 2] = color.blue; + result[offset + 3] = color.alpha; + } + } + } + + fillWithColor(_kBlack, 0, _kWidth); + fillWithColor(_kGreen, _kRadius, _kWidth - _kRadius); + + return result; } diff --git a/engine/src/flutter/testing/dart/pubspec.yaml b/engine/src/flutter/testing/dart/pubspec.yaml index 43f35ec12a..2c79b49c89 100644 --- a/engine/src/flutter/testing/dart/pubspec.yaml +++ b/engine/src/flutter/testing/dart/pubspec.yaml @@ -1,3 +1,8 @@ name: engine_tests dependencies: - test: 0.12.15+4 + flutter: + sdk: flutter +dev_dependencies: + flutter_test: + sdk: flutter + path: any diff --git a/engine/src/flutter/testing/resources/4x4.png b/engine/src/flutter/testing/resources/4x4.png new file mode 100644 index 0000000000000000000000000000000000000000..21b295abc16c89cdc87d86c10625d94352bab3f2 GIT binary patch literal 71 zcmeAS@N?(olHy`uVBq!ia0y~yU|<4a4h9AWhVZ*VKNuJo_&i-4LpZJ{|2uym;5-8Z Xn;>J;gq>e}K`K05{an^LB{Ts50uK?K literal 0 HcmV?d00001 diff --git a/engine/src/flutter/testing/resources/square-80.jpg b/engine/src/flutter/testing/resources/square-80.jpg deleted file mode 100644 index 1140c33bd39039758e39b057c1da6ea0746bee4e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 928 zcmex= zB}GB*P6`o`Q3@b27*0$sE`YJI(2NWW->!j7XJELji$$K05NAqiaWVq~!v_WihLWO) z$S4K|#uNqy29b0Ky8^^cg0Oo)?4pn$X9fnw1q=)f8tF*vBqVluNkIt%1LFyh`s9Kl z5c>`T1A|R&PASMPkee787#Mg`^Fo6e7#JcL7#L(2iW!0!+!>relGMbA@XTM#z+mx> zfq{7Ra*6Qq^9YEGNJxl_h>1zbs>(}AE6a$9Dd;FD ztEp>hX-dlL8t7^msA_0xfDB<|XJ_Z);1cHI64sCslhPm={6D}T$icwHz{Sj{#K0uT z$SlbC{|JK^10yI^!@Lio+1URdVNeiYU|?lo2E{E%41^h(m|0oa1clj!6b=6$VGza8 z$in*n2!p&J0}~@NE0RV=W+oOvMFu6qz(lA2w-|Vs85x)anFSf_8CoA#i`|*I>GA3( zi?@gF-n{GkjyVsHs_an}tWlCEVBpo&7C7dYVmJ$(bU=hK^z`&ruz`(!>MxM^j0TN6M3=9lR5@1q*fgv}qq$tSU zNg*OK3ZxJu491Dc#RXsnc7lOf3=tPN{0fR&83uQTe1?36bcP&;R0ahGPlh~(WClG31qLGq0|r9|v#)N9O(1eajNR|GCv& q-+k)*zw*DYbpOWI?*9{gd-at1znTBetNxQ~KJxz3eKvFR1_l6%4|8Au diff --git a/engine/src/flutter/testing/resources/square.png b/engine/src/flutter/testing/resources/square.png deleted file mode 100644 index a042cece5516363a3d4c81eb781e5ced2456df80..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 125 zcmeAS@N?(olHy`uVBq!ia0y~yU;weXIM^5%7