[Impeller Scene] Add animation/PBR descriptions to ipscene (flutter/engine#38397)

engine/src/flutter/impeller/scene/geometry.cc

@@ -53,7 +53,17 @@ std::shared_ptr<VertexBufferGeometry> Geometry::MakeFromFlatbuffer(
       break;
   }
 
-  const size_t vertices_bytes = mesh.vertices()->size() * sizeof(fb::Vertex);
+  if (mesh.vertices_type() == fb::VertexBuffer::SkinnedVertexBuffer) {
+    VALIDATION_LOG << "Skinned meshes not yet supported.";
+    return nullptr;
+  }
+  if (mesh.vertices_type() != fb::VertexBuffer::UnskinnedVertexBuffer) {
+    VALIDATION_LOG << "Invalid vertex buffer type.";
+    return nullptr;
+  }
+
+  const auto* vertices = mesh.vertices_as_UnskinnedVertexBuffer()->vertices();
+  const size_t vertices_bytes = vertices->size() * sizeof(fb::Vertex);
   const size_t indices_bytes = mesh.indices()->data()->size();
   if (vertices_bytes == 0 || indices_bytes == 0) {
     return nullptr;
@@ -67,7 +77,7 @@ std::shared_ptr<VertexBufferGeometry> Geometry::MakeFromFlatbuffer(
   buffer->SetLabel("Mesh vertices+indices");
 
   const uint8_t* vertices_start =
-      reinterpret_cast<const uint8_t*>(mesh.vertices()->Get(0));
+      reinterpret_cast<const uint8_t*>(vertices->Get(0));
   const uint8_t* indices_start =
       reinterpret_cast<const uint8_t*>(mesh.indices()->data()->Data());
 

engine/src/flutter/impeller/scene/importer/importer_gltf.cc

@@ -93,7 +93,7 @@ static bool ProcessMeshPrimitive(const tinygltf::Model& gltf,
                                      accessor.count);            // count
     }
 
-    builder.WriteFBVertices(mesh_primitive.vertices);
+    builder.WriteFBVertices(mesh_primitive);
   }
 
   //---------------------------------------------------------------------------
@@ -136,6 +136,9 @@ static bool ProcessMeshPrimitive(const tinygltf::Model& gltf,
 static void ProcessNode(const tinygltf::Model& gltf,
                         const tinygltf::Node& in_node,
                         fb::NodeT& out_node) {
+  out_node.name = in_node.name;
+  out_node.children = in_node.children;
+
   //---------------------------------------------------------------------------
   /// Transform.
   ///
@@ -184,16 +187,6 @@ static void ProcessNode(const tinygltf::Model& gltf,
       out_node.mesh_primitives.push_back(std::move(mesh_primitive));
     }
   }
-
-  //---------------------------------------------------------------------------
-  /// Children.
-  ///
-
-  for (size_t node_i = 0; node_i < in_node.children.size(); node_i++) {
-    auto child = std::make_unique<fb::NodeT>();
-    ProcessNode(gltf, gltf.nodes[in_node.children[node_i]], *child);
-    out_node.children.push_back(std::move(child));
-  }
 }
 
 bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) {
@@ -218,10 +211,12 @@ bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) {
   }
 
   const tinygltf::Scene& scene = gltf.scenes[gltf.defaultScene];
-  for (size_t node_i = 0; node_i < scene.nodes.size(); node_i++) {
+  out_scene.children = scene.nodes;
+
+  for (size_t node_i = 0; node_i < gltf.nodes.size(); node_i++) {
     auto node = std::make_unique<fb::NodeT>();
-    ProcessNode(gltf, gltf.nodes[scene.nodes[node_i]], *node);
-    out_scene.children.push_back(std::move(node));
+    ProcessNode(gltf, gltf.nodes[node_i], *node);
+    out_scene.nodes.push_back(std::move(node));
   }
 
   return true;

engine/src/flutter/impeller/scene/importer/importer_unittests.cc

@@ -21,21 +21,23 @@ TEST(ImporterTest, CanParseGLTF) {
   ASSERT_TRUE(ParseGLTF(*mapping, scene));
 
   ASSERT_EQ(scene.children.size(), 1u);
-  auto& node = *scene.children[0];
+  auto& node = scene.nodes[scene.children[0]];
 
-  Matrix node_transform = ToMatrix(*node.transform);
+  Matrix node_transform = ToMatrix(*node->transform);
   ASSERT_MATRIX_NEAR(node_transform, Matrix());
 
-  ASSERT_EQ(node.mesh_primitives.size(), 1u);
-  auto& mesh = *node.mesh_primitives[0];
+  ASSERT_EQ(node->mesh_primitives.size(), 1u);
+  auto& mesh = *node->mesh_primitives[0];
   ASSERT_EQ(mesh.indices->count, 918u);
 
   uint16_t first_index =
       *reinterpret_cast<uint16_t*>(mesh.indices->data.data());
   ASSERT_EQ(first_index, 45u);
 
-  ASSERT_EQ(mesh.vertices.size(), 260u);
-  auto& vertex = mesh.vertices[0];
+  ASSERT_EQ(mesh.vertices.type, fb::VertexBuffer::UnskinnedVertexBuffer);
+  auto& vertices = mesh.vertices.AsUnskinnedVertexBuffer()->vertices;
+  ASSERT_EQ(vertices.size(), 260u);
+  auto& vertex = vertices[0];
 
   Vector3 position = ToVector3(vertex.position());
   ASSERT_VECTOR3_NEAR(position, Vector3(-0.0100185, -0.522907, -0.133178));

engine/src/flutter/impeller/scene/importer/scene.fbs

@@ -4,6 +4,56 @@
 
 namespace impeller.fb;
 
+//-----------------------------------------------------------------------------
+/// Materials.
+///
+
+struct Color {
+  r: float;
+  g: float;
+  b: float;
+  a: float;
+}
+
+/// A compressed texture for Flutter to decode. The `bytes` field takes
+/// precedent over the `uri` field.
+table Texture {
+  /// A Flutter asset URI for an image file to import and decode.
+  uri: string;
+  /// Compressed image bytes for Flutter to decode. If this field is not empty,
+  /// it takes precedent over the `uri` field for sourcing the texture.
+  bytes: [ubyte];
+}
+
+enum MaterialType:byte {
+  kUnlit,
+  kPhysicallyBased,
+}
+
+/// The final color of each material component is the texture color multiplied
+/// by the factor of the component.
+/// Texture fields are indices into the `Scene`->`textures` array. All textures
+/// are optional -- a texture index value of -1 indicates no texture.
+table  Material {
+  // When the `MaterialType` is `kUnlit`, only the `base_color` fields are used.
+  type: MaterialType;
+
+  base_color_factor: Color;
+  base_color_texture: int = -1;
+
+  metallic_factor: float = 0;
+  roughness_factor: float = 0.5;
+  metallic_roughness_texture: int = -1;  // Red=Metallic, Green=Roughness.
+
+  normal_texture: int = -1;  // Tangent space normal map.
+
+  occlusion_texture: int = -1;
+}
+
+//-----------------------------------------------------------------------------
+/// Geometry.
+///
+
 struct Vec2 {
   x: float;
   y: float;
@@ -22,17 +72,6 @@ struct Vec4 {
   w: float;
 }
 
-struct Color {
-  r: float;
-  g: float;
-  b: float;
-  a: float;
-}
-
-struct Matrix {
-  m: [float:16];
-}
-
 // This attribute layout is expected to be identical to that within
 // `impeller/scene/shaders/geometry.vert`.
 struct Vertex {
@@ -43,6 +82,27 @@ struct Vertex {
   color: Color;
 }
 
+table UnskinnedVertexBuffer {
+  vertices: [Vertex];
+}
+
+struct SkinnedVertex {
+  vertex: Vertex;
+  /// Four joint indices corresponding to this mesh's skin transforms. These
+  /// are floats instead of ints because this vertex data is uploaded directly
+  /// to the GPU, and float attributes work for all Impeller backends.
+  joints: Vec4;
+  /// Four weight values that specify the influence of the corresponding
+  /// joints.
+  weights: Vec4;
+}
+
+table SkinnedVertexBuffer {
+  vertices: [SkinnedVertex];
+}
+
+union VertexBuffer { UnskinnedVertexBuffer, SkinnedVertexBuffer }
+
 enum IndexType:byte {
   k16Bit,
   k32Bit,
@@ -54,31 +114,69 @@ table Indices {
   type: IndexType;
 }
 
-table Texture {
-  // TODO(bdero): Allow optional image data embedding.
-  uri: string;
-}
-
-table Material {
-  base_color_factor: Color;
-  base_color_texture: Texture;
-  // TODO(bdero): PBR textures.
-}
-
 table MeshPrimitive {
-  vertices: [Vertex];
+  vertices: VertexBuffer;
   indices: Indices;
   material: Material;
 }
 
+//-----------------------------------------------------------------------------
+/// Animations.
+///
+
+table TranslationKeyframes {
+  values: [Vec3];
+}
+
+table RotationKeyframes {
+  values: [Vec4];
+}
+
+table ScaleKeyframes {
+  values: [Vec3];
+}
+
+union Keyframes { TranslationKeyframes, RotationKeyframes, ScaleKeyframes }
+
+table Channel {
+  node: int;  // Index into `Scene`->`nodes`.
+  timeline: [float];
+  keyframes: Keyframes;
+}
+
+table Animation {
+  name: string;
+  channels: [Channel];
+}
+
+table Skin {
+  joints: [int];  // Index into `Scene`->`nodes`.
+  inverse_bind_matrices: [Matrix];
+  /// The root joint of the skeleton.
+  skeleton: int;  // Index into `Scene`->`nodes`.
+}
+
+//-----------------------------------------------------------------------------
+/// Scene graph.
+///
+
+struct Matrix {
+  m: [float:16];
+}
+
 table Node {
-  children: [Node];
+  name: string;
+  children: [int];  // Index into `Scene`->`nodes`.
   transform: Matrix;
   mesh_primitives: [MeshPrimitive];
+  skin: Skin;
 }
 
 table Scene {
-  children: [Node];
+  children: [int];  // Index into `Scene`->`nodes`.
+  nodes: [Node];
+  textures: [Texture];  // Textures may be reused across different materials.
+  animations: [Animation];
 }
 
 root_type Scene;

engine/src/flutter/impeller/scene/importer/vertices_builder.cc

@@ -6,6 +6,7 @@
 
 #include <cstring>
 #include <limits>
+#include <memory>
 #include <type_traits>
 
 #include "flutter/fml/logging.h"
@@ -18,13 +19,15 @@ namespace importer {
 
 VerticesBuilder::VerticesBuilder() = default;
 
-void VerticesBuilder::WriteFBVertices(std::vector<fb::Vertex>& vertices) const {
-  vertices.resize(0);
+void VerticesBuilder::WriteFBVertices(fb::MeshPrimitiveT& primitive) const {
+  auto vertex_buffer = fb::UnskinnedVertexBufferT();
+  vertex_buffer.vertices.resize(0);
   for (auto& v : vertices_) {
-    vertices.push_back(fb::Vertex(
+    vertex_buffer.vertices.push_back(fb::Vertex(
         ToFBVec3(v.position), ToFBVec3(v.normal), ToFBVec4(v.tangent),
         ToFBVec2(v.texture_coords), ToFBColor(v.color)));
   }
+  primitive.vertices.Set(std::move(vertex_buffer));
 }
 
 /// @brief  Reads a numeric component from `source` and returns a 32bit float.

engine/src/flutter/impeller/scene/importer/vertices_builder.h

@@ -57,7 +57,7 @@ class VerticesBuilder {
 
   VerticesBuilder();
 
-  void WriteFBVertices(std::vector<fb::Vertex>& vertices) const;
+  void WriteFBVertices(fb::MeshPrimitiveT& primitive) const;
 
   void SetAttributeFromBuffer(AttributeType attribute,
                               ComponentType component_type,

engine/src/flutter/impeller/scene/node.cc

@@ -16,50 +16,76 @@
 namespace impeller {
 namespace scene {
 
-std::optional<Node> Node::MakeFromFlatbuffer(fml::Mapping& mapping,
-                                             Allocator& allocator) {
+std::shared_ptr<Node> Node::MakeFromFlatbuffer(fml::Mapping& mapping,
+                                               Allocator& allocator) {
   flatbuffers::Verifier verifier(mapping.GetMapping(), mapping.GetSize());
   if (!fb::VerifySceneBuffer(verifier)) {
-    return std::nullopt;
+    VALIDATION_LOG << "Failed to unpack scene: Scene flatbuffer is invalid.";
+    return nullptr;
   }
 
   return Node::MakeFromFlatbuffer(*fb::GetScene(mapping.GetMapping()),
                                   allocator);
 }
 
-Node Node::MakeFromFlatbuffer(const fb::Scene& scene, Allocator& allocator) {
-  Node result;
-
-  if (!scene.children()) {
-    return result;
+std::shared_ptr<Node> Node::MakeFromFlatbuffer(const fb::Scene& scene,
+                                               Allocator& allocator) {
+  auto result = std::make_shared<Node>();
+  if (!scene.nodes() || !scene.children()) {
+    return result;  // The scene is empty.
   }
-  for (const auto* child : *scene.children()) {
-    result.AddChild(Node::MakeFromFlatbuffer(*child, allocator));
+
+  // Initialize nodes for unpacking the entire scene.
+  std::vector<std::shared_ptr<Node>> scene_nodes;
+  scene_nodes.reserve(scene.nodes()->size());
+  for (size_t node_i = 0; node_i < scene.nodes()->size(); node_i++) {
+    scene_nodes.push_back(std::make_shared<Node>());
+  }
+
+  // Connect children to the root node.
+  for (int child : *scene.children()) {
+    if (child < 0 || static_cast<size_t>(child) >= scene_nodes.size()) {
+      VALIDATION_LOG << "Scene child index out of range.";
+      continue;
+    }
+    result->AddChild(scene_nodes[child]);
+  }
+  // TODO(bdero): Unpack animations.
+
+  // Unpack each node.
+  for (size_t node_i = 0; node_i < scene.nodes()->size(); node_i++) {
+    scene_nodes[node_i]->UnpackFromFlatbuffer(*scene.nodes()->Get(node_i),
+                                              scene_nodes, allocator);
   }
 
   return result;
 }
 
-Node Node::MakeFromFlatbuffer(const fb::Node& node, Allocator& allocator) {
-  Node result;
-
-  if (node.mesh_primitives()) {
+void Node::UnpackFromFlatbuffer(
+    const fb::Node& source_node,
+    const std::vector<std::shared_ptr<Node>>& scene_nodes,
+    Allocator& allocator) {
+  if (source_node.mesh_primitives()) {
     Mesh mesh;
-    for (const auto* primitives : *node.mesh_primitives()) {
+    for (const auto* primitives : *source_node.mesh_primitives()) {
       auto geometry = Geometry::MakeFromFlatbuffer(*primitives, allocator);
       mesh.AddPrimitive({geometry, Material::MakeUnlit()});
     }
-    result.SetMesh(std::move(mesh));
+    SetMesh(std::move(mesh));
   }
 
-  if (!node.children()) {
-    return result;
-  }
-  for (const auto* child : *node.children()) {
-    result.AddChild(Node::MakeFromFlatbuffer(*child, allocator));
+  if (!source_node.children()) {
+    return;
   }
 
-  return result;
+  // Wire up graph connections.
+  for (int child : *source_node.children()) {
+    if (child < 0 || static_cast<size_t>(child) >= scene_nodes.size()) {
+      VALIDATION_LOG << "Node child index out of range.";
+      continue;
+    }
+    AddChild(scene_nodes[child]);
+  }
 }
 
 Node::Node() = default;
@@ -96,24 +122,20 @@ Matrix Node::GetGlobalTransform() const {
   return local_transform_;
 }
 
-bool Node::AddChild(Node node) {
-  if (node.parent_ != nullptr) {
+bool Node::AddChild(std::shared_ptr<Node> node) {
+  // This ensures that cycles are impossible.
+  if (node->parent_ != nullptr) {
     VALIDATION_LOG
         << "Cannot add a node as a child which already has a parent.";
     return false;
   }
-  node.parent_ = this;
+  node->parent_ = this;
   children_.push_back(std::move(node));
 
-  Node& ref = children_.back();
-  for (Node& child : ref.children_) {
-    child.parent_ = &ref;
-  }
-
   return true;
 }
 
-std::vector<Node>& Node::GetChildren() {
+std::vector<std::shared_ptr<Node>>& Node::GetChildren() {
   return children_;
 }
 
@@ -131,7 +153,7 @@ bool Node::Render(SceneEncoder& encoder, const Matrix& parent_transform) const {
   mesh_.Render(encoder, transform);
 
   for (auto& child : children_) {
-    if (!child.Render(encoder, transform)) {
+    if (!child->Render(encoder, transform)) {
       return false;
     }
   }

engine/src/flutter/impeller/scene/node.h

@@ -20,10 +20,10 @@ namespace scene {
 
 class Node final {
  public:
-  static std::optional<Node> MakeFromFlatbuffer(fml::Mapping& mapping,
-                                                Allocator& allocator);
-  static Node MakeFromFlatbuffer(const fb::Scene& scene, Allocator& allocator);
-  static Node MakeFromFlatbuffer(const fb::Node& node, Allocator& allocator);
+  static std::shared_ptr<Node> MakeFromFlatbuffer(fml::Mapping& mapping,
+                                                  Allocator& allocator);
+  static std::shared_ptr<Node> MakeFromFlatbuffer(const fb::Scene& scene,
+                                                  Allocator& allocator);
 
   Node();
   ~Node();
@@ -37,8 +37,8 @@ class Node final {
   void SetGlobalTransform(Matrix transform);
   Matrix GetGlobalTransform() const;
 
-  bool AddChild(Node child);
-  std::vector<Node>& GetChildren();
+  bool AddChild(std::shared_ptr<Node> child);
+  std::vector<std::shared_ptr<Node>>& GetChildren();
 
   void SetMesh(Mesh mesh);
   Mesh& GetMesh();
@@ -49,11 +49,19 @@ class Node final {
   Matrix local_transform_;
 
  private:
+  void UnpackFromFlatbuffer(
+      const fb::Node& node,
+      const std::vector<std::shared_ptr<Node>>& scene_nodes,
+      Allocator& allocator);
+
+  bool is_root_ = false;
   Node* parent_ = nullptr;
-  std::vector<Node> children_;
+  std::vector<std::shared_ptr<Node>> children_;
   Mesh mesh_;
 
   FML_DISALLOW_COPY_AND_ASSIGN(Node);
+
+  friend Scene;
 };
 
 }  // namespace scene

engine/src/flutter/impeller/scene/scene.cc

@@ -16,7 +16,9 @@ namespace impeller {
 namespace scene {
 
 Scene::Scene(std::shared_ptr<Context> context)
-    : scene_context_(std::make_unique<SceneContext>(std::move(context))){};
+    : scene_context_(std::make_unique<SceneContext>(std::move(context))) {
+  root_.is_root_ = true;
+};
 
 Node& Scene::GetRoot() {
   return root_;

engine/src/flutter/impeller/scene/scene_unittests.cc

@@ -73,9 +73,9 @@ TEST_P(SceneTest, FlutterLogo) {
       flutter::testing::OpenFixtureAsMapping("flutter_logo.glb.ipscene");
   ASSERT_NE(mapping, nullptr);
 
-  std::optional<Node> gltf_scene =
+  std::shared_ptr<Node> gltf_scene =
       Node::MakeFromFlatbuffer(*mapping, *allocator);
-  ASSERT_TRUE(gltf_scene.has_value());
+  ASSERT_NE(gltf_scene, nullptr);
 
   std::shared_ptr<UnlitMaterial> material = Material::MakeUnlit();
   auto color_baked = CreateTextureForFixture("flutter_logo_baked.png");
@@ -83,11 +83,12 @@ TEST_P(SceneTest, FlutterLogo) {
   material->SetVertexColorWeight(0);
 
   ASSERT_EQ(gltf_scene->GetChildren().size(), 1u);
-  ASSERT_EQ(gltf_scene->GetChildren()[0].GetMesh().GetPrimitives().size(), 1u);
-  gltf_scene->GetChildren()[0].GetMesh().GetPrimitives()[0].material = material;
+  ASSERT_EQ(gltf_scene->GetChildren()[0]->GetMesh().GetPrimitives().size(), 1u);
+  gltf_scene->GetChildren()[0]->GetMesh().GetPrimitives()[0].material =
+      material;
 
   auto scene = Scene(GetContext());
-  scene.GetRoot().AddChild(std::move(gltf_scene.value()));
+  scene.GetRoot().AddChild(std::move(gltf_scene));
   scene.GetRoot().SetLocalTransform(Matrix::MakeScale({3, 3, 3}));
 
   Renderer::RenderCallback callback = [&](RenderTarget& render_target) {
@@ -116,15 +117,15 @@ TEST_P(SceneTest, TwoTriangles) {
       flutter::testing::OpenFixtureAsMapping("two_triangles.glb.ipscene");
   ASSERT_NE(mapping, nullptr);
 
-  std::optional<Node> gltf_scene =
+  std::shared_ptr<Node> gltf_scene =
       Node::MakeFromFlatbuffer(*mapping, *allocator);
-  ASSERT_TRUE(gltf_scene.has_value());
+  ASSERT_NE(gltf_scene, nullptr);
 
   auto scene = Scene(GetContext());
-  scene.GetRoot().AddChild(std::move(gltf_scene.value()));
+  scene.GetRoot().AddChild(std::move(gltf_scene));
 
   Renderer::RenderCallback callback = [&](RenderTarget& render_target) {
-    Node& node = scene.GetRoot().GetChildren()[0];
+    Node& node = *scene.GetRoot().GetChildren()[0];
     node.SetLocalTransform(node.GetLocalTransform() *
                            Matrix::MakeRotation(0.02, {0, 1, 0, 0}));