Reland: "Added wide-gamut color support for ui.Image.toByteData and ui.Image.colorSpace" (flutter/engine#40312)

Reland: "Added wide-gamut color support for `ui.Image.toByteData` and `ui.Image.colorSpace`"

engine/src/flutter/lib/ui/dart_ui.cc

@@ -190,6 +190,7 @@ typedef CanvasPath Path;
   V(Image, width, 1)                                   \
   V(Image, height, 1)                                  \
   V(Image, toByteData, 3)                              \
+  V(Image, colorSpace, 1)                              \
   V(ImageDescriptor, bytesPerPixel, 1)                 \
   V(ImageDescriptor, dispose, 1)                       \
   V(ImageDescriptor, height, 1)                        \

engine/src/flutter/lib/ui/painting.dart

@@ -1568,6 +1568,31 @@ class Paint {
   }
 }
 
+/// The color space describes the colors that are available to an [Image].
+///
+/// This value can help decide which [ImageByteFormat] to use with
+/// [Image.toByteData]. Images that are in the [extendedSRGB] color space
+/// should use something like [ImageByteFormat.rawExtendedRgba128] so that
+/// colors outside of the sRGB gamut aren't lost.
+///
+/// This is also the result of [Image.colorSpace].
+///
+/// See also: https://en.wikipedia.org/wiki/Color_space
+enum ColorSpace {
+  /// The sRGB color space.
+  ///
+  /// You may know this as the standard color space for the web or the color
+  /// space of non-wide-gamut Flutter apps.
+  ///
+  /// See also: https://en.wikipedia.org/wiki/SRGB
+  sRGB,
+  /// A color space that is backwards compatible with sRGB but can represent
+  /// colors outside of that gamut with values outside of [0..1]. In order to
+  /// see the extended values an [ImageByteFormat] like
+  /// [ImageByteFormat.rawExtendedRgba128] must be used.
+  extendedSRGB,
+}
+
 /// The format in which image bytes should be returned when using
 /// [Image.toByteData].
 // We do not expect to add more encoding formats to the ImageByteFormat enum,
@@ -1591,6 +1616,31 @@ enum ImageByteFormat {
   /// image may use a single 8-bit channel for each pixel.
   rawUnmodified,
 
+  /// Raw extended range RGBA format.
+  ///
+  /// Unencoded bytes, in RGBA row-primary form with straight alpha, 32 bit
+  /// float (IEEE 754 binary32) per channel.
+  ///
+  /// Example usage:
+  ///
+  /// ```dart
+  /// import 'dart:ui' as ui;
+  /// import 'dart:typed_data';
+  ///
+  /// Future<Map<String, double>> getFirstPixel(ui.Image image) async {
+  ///   final ByteData data =
+  ///       (await image.toByteData(format: ui.ImageByteFormat.rawExtendedRgba128))!;
+  ///   final Float32List floats = Float32List.view(data.buffer);
+  ///   return <String, double>{
+  ///     'r': floats[0],
+  ///     'g': floats[1],
+  ///     'b': floats[2],
+  ///     'a': floats[3],
+  ///   };
+  /// }
+  /// ```
+  rawExtendedRgba128,
+
   /// PNG format.
   ///
   /// A loss-less compression format for images. This format is well suited for
@@ -1726,6 +1776,10 @@ class Image {
   /// The [format] argument specifies the format in which the bytes will be
   /// returned.
   ///
+  /// Using [ImageByteFormat.rawRgba] on an image in the color space
+  /// [ColorSpace.extendedSRGB] will result in the gamut being squished to fit
+  /// into the sRGB gamut, resulting in the loss of wide-gamut colors.
+  ///
   /// Returns a future that completes with the binary image data or an error
   /// if encoding fails.
   // We do not expect to add more encoding formats to the ImageByteFormat enum,
@@ -1737,6 +1791,29 @@ class Image {
     return _image.toByteData(format: format);
   }
 
+  /// The color space that is used by the [Image]'s colors.
+  ///
+  /// This value is a consequence of how the [Image] has been created.  For
+  /// example, loading a PNG that is in the Display P3 color space will result
+  /// in a [ColorSpace.extendedSRGB] image.
+  ///
+  /// On rendering backends that don't support wide gamut colors (anything but
+  /// iOS impeller), wide gamut images will still report [ColorSpace.sRGB] if
+  /// rendering wide gamut colors isn't supported.
+  // Note: The docstring will become outdated as new platforms support wide
+  // gamut color, please keep it up to date.
+  ColorSpace get colorSpace {
+    final int colorSpaceValue = _image.colorSpace;
+    switch (colorSpaceValue) {
+      case 0:
+        return ColorSpace.sRGB;
+      case 1:
+        return ColorSpace.extendedSRGB;
+      default:
+        throw UnsupportedError('Unrecognized color space: $colorSpaceValue');
+    }
+  }
+
   /// If asserts are enabled, returns the [StackTrace]s of each open handle from
   /// [clone], in creation order.
   ///
@@ -1903,6 +1980,9 @@ class _Image extends NativeFieldWrapperClass1 {
 
   final Set<Image> _handles = <Image>{};
 
+  @Native<Int32 Function(Pointer<Void>)>(symbol: 'Image::colorSpace')
+  external int get colorSpace;
+
   @override
   String toString() => '[$width\u00D7$height]';
 }

engine/src/flutter/lib/ui/painting/image.cc

@@ -7,6 +7,9 @@
 #include <algorithm>
 #include <limits>
 
+#if IMPELLER_SUPPORTS_RENDERING
+#include "flutter/lib/ui/painting/image_encoding_impeller.h"
+#endif
 #include "flutter/lib/ui/painting/image_encoding.h"
 #include "third_party/tonic/converter/dart_converter.h"
 #include "third_party/tonic/dart_args.h"
@@ -35,4 +38,16 @@ void CanvasImage::dispose() {
   ClearDartWrapper();
 }
 
+int CanvasImage::colorSpace() {
+  if (image_->skia_image()) {
+    return ColorSpace::kSRGB;
+  } else if (image_->impeller_texture()) {
+#if IMPELLER_SUPPORTS_RENDERING
+    return ImageEncodingImpeller::GetColorSpace(image_->impeller_texture());
+#endif  // IMPELLER_SUPPORTS_RENDERING
+  }
+
+  return -1;
+}
+
 }  // namespace flutter

engine/src/flutter/lib/ui/painting/image.h

@@ -14,6 +14,12 @@
 
 namespace flutter {
 
+// Must be kept in sync with painting.dart.
+enum ColorSpace {
+  kSRGB,
+  kExtendedSRGB,
+};
+
 class CanvasImage final : public RefCountedDartWrappable<CanvasImage> {
   DEFINE_WRAPPERTYPEINFO();
   FML_FRIEND_MAKE_REF_COUNTED(CanvasImage);
@@ -37,6 +43,8 @@ class CanvasImage final : public RefCountedDartWrappable<CanvasImage> {
 
   void set_image(sk_sp<DlImage> image) { image_ = image; }
 
+  int colorSpace();
+
  private:
   CanvasImage();
 

engine/src/flutter/lib/ui/painting/image_encoding.cc

@@ -38,6 +38,7 @@ enum ImageByteFormat {
   kRawRGBA,
   kRawStraightRGBA,
   kRawUnmodified,
+  kRawExtendedRgba128,
   kPNG,
 };
 
@@ -123,19 +124,21 @@ sk_sp<SkData> EncodeImage(const sk_sp<SkImage>& raster_image,
         return nullptr;
       };
       return png_image;
-    } break;
-    case kRawRGBA: {
+    }
+    case kRawRGBA:
       return CopyImageByteData(raster_image, kRGBA_8888_SkColorType,
                                kPremul_SkAlphaType);
-    } break;
-    case kRawStraightRGBA: {
+
+    case kRawStraightRGBA:
       return CopyImageByteData(raster_image, kRGBA_8888_SkColorType,
                                kUnpremul_SkAlphaType);
-    } break;
-    case kRawUnmodified: {
+
+    case kRawUnmodified:
       return CopyImageByteData(raster_image, raster_image->colorType(),
                                raster_image->alphaType());
-    } break;
+    case kRawExtendedRgba128:
+      return CopyImageByteData(raster_image, kRGBA_F32_SkColorType,
+                               kUnpremul_SkAlphaType);
   }
 
   FML_LOG(ERROR) << "Unknown error encoding image.";

engine/src/flutter/lib/ui/painting/image_encoding_impeller.cc

@@ -163,4 +163,16 @@ void ImageEncodingImpeller::ConvertImageToRaster(
   });
 }
 
+int ImageEncodingImpeller::GetColorSpace(
+    const std::shared_ptr<impeller::Texture>& texture) {
+  const impeller::TextureDescriptor& desc = texture->GetTextureDescriptor();
+  switch (desc.format) {
+    case impeller::PixelFormat::kB10G10R10XR:  // intentional_fallthrough
+    case impeller::PixelFormat::kR16G16B16A16Float:
+      return ColorSpace::kExtendedSRGB;
+    default:
+      return ColorSpace::kSRGB;
+  }
+}
+
 }  // namespace flutter

engine/src/flutter/lib/ui/painting/image_encoding_impeller.h

@@ -17,6 +17,8 @@ namespace flutter {
 
 class ImageEncodingImpeller {
  public:
+  static int GetColorSpace(const std::shared_ptr<impeller::Texture>& texture);
+
   /// Converts a DlImage to a SkImage.
   /// This should be called from the thread that corresponds to
   /// `dl_image->owning_context()` when gpu access is guaranteed.

engine/src/flutter/lib/web_ui/lib/painting.dart

@@ -351,6 +351,8 @@ abstract class Image {
 
   List<StackTrace>? debugGetOpenHandleStackTraces() => null;
 
+  ColorSpace get colorSpace => ColorSpace.sRGB;
+
   @override
   String toString() => '[$width\u00D7$height]';
 }
@@ -431,6 +433,11 @@ class ImageFilter {
     engine.renderer.composeImageFilters(outer: outer, inner: inner);
 }
 
+enum ColorSpace {
+  sRGB,
+  extendedSRGB,
+}
+
 enum ImageByteFormat {
   rawRgba,
   rawStraightRgba,

engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart

@@ -378,6 +378,9 @@ class CkImage implements ui.Image, StackTraceDebugger {
     }
   }
 
+  @override
+  ui.ColorSpace get colorSpace => ui.ColorSpace.sRGB;
+
   Future<ByteData> _readPixelsFromSkImage(ui.ImageByteFormat format) {
     final SkAlphaType alphaType = format == ui.ImageByteFormat.rawStraightRgba ? canvasKit.AlphaType.Unpremul : canvasKit.AlphaType.Premul;
     final ByteData? data = _encodeImage(

engine/src/flutter/lib/web_ui/lib/src/engine/html_image_codec.dart

@@ -204,6 +204,9 @@ class HtmlImage implements ui.Image {
     }
   }
 
+  @override
+  ui.ColorSpace get colorSpace => ui.ColorSpace.sRGB;
+
   DomHTMLImageElement cloneImageElement() {
     if (!_didClone) {
       _didClone = true;

engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart

@@ -23,6 +23,9 @@ class SkwasmImage implements ui.Image {
     throw UnimplementedError();
   }
 
+  @override
+  ui.ColorSpace get colorSpace => ui.ColorSpace.sRGB;
+
   @override
   void dispose() {
     throw UnimplementedError();

engine/src/flutter/lib/web_ui/test/html/recording_canvas_golden_test.dart

@@ -7,7 +7,7 @@ import 'dart:typed_data';
 
 import 'package:test/bootstrap/browser.dart';
 import 'package:test/test.dart';
-import 'package:ui/src/engine.dart';
+import 'package:ui/src/engine.dart' hide ColorSpace;
 import 'package:ui/ui.dart' hide TextStyle;
 import 'package:web_engine_tester/golden_tester.dart';
 
@@ -780,6 +780,9 @@ class TestImage implements Image {
 
   @override
   List<StackTrace>/*?*/ debugGetOpenHandleStackTraces() => <StackTrace>[];
+
+  @override
+  ColorSpace get colorSpace => ColorSpace.sRGB;
 }
 
 Paragraph createTestParagraph() {

engine/src/flutter/testing/dart/encoding_test.dart

@@ -67,6 +67,21 @@ void main() {
     final List<int> expected = await readFile('square.png');
     expect(Uint8List.view(data.buffer), expected);
   });
+
+  test('Image.toByteData ExtendedRGBA128', () async {
+    final Image image = await Square4x4Image.image;
+    final ByteData data = (await image.toByteData(format: ImageByteFormat.rawExtendedRgba128))!;
+    expect(image.width, _kWidth);
+    expect(image.height, _kWidth);
+    expect(data.lengthInBytes, _kWidth * _kWidth * 4 * 4);
+    // Top-left pixel should be black.
+    final Float32List floats = Float32List.view(data.buffer);
+    expect(floats[0], 0.0);
+    expect(floats[1], 0.0);
+    expect(floats[2], 0.0);
+    expect(floats[3], 1.0);
+    expect(image.colorSpace, ColorSpace.sRGB);
+  });
 }
 
 class Square4x4Image {