Mesh Skinning. Attempt #3 (#4238)

# Objective
Load skeletal weights and indices from GLTF files. Animate meshes.

## Solution
 - Load skeletal weights and indices from GLTF files.
 - Added `SkinnedMesh` component and ` SkinnedMeshInverseBindPose` asset
 - Added `extract_skinned_meshes` to extract joint matrices.
 - Added queue phase systems for enqueuing the buffer writes.

Some notes:

 -  This ports part of # #2359 to the current main.
 -  This generates new `BufferVec`s and bind groups every frame. The expectation here is that the number of `Query::get` calls during extract is probably going to be the stronger bottleneck, with up to 256 calls per skinned mesh. Until that is optimized, caching buffers and bind groups is probably a non-concern.
 - Unfortunately, due to the uniform size requirements, this means a 16KB buffer is allocated for every skinned mesh every frame. There's probably a few ways to get around this, but most of them require either compute shaders or storage buffers, which are both incompatible with WebGL2.

Co-authored-by: james7132 <[email protected]>
Co-authored-by: François <[email protected]>
Co-authored-by: James Liu <[email protected]>

Author: 3 people

Cargo.toml

@@ -235,6 +235,15 @@ path = "examples/3d/update_gltf_scene.rs"
 name = "wireframe"
 path = "examples/3d/wireframe.rs"
 
+# Animation
+[[example]]
+name = "custom_skinned_mesh"
+path = "examples/animation/custom_skinned_mesh.rs"
+
+[[example]]
+name = "gltf_skinned_mesh"
+path = "examples/animation/gltf_skinned_mesh.rs"
+
 # Application
 [[example]]
 name = "custom_loop"

assets/models/SimpleSkin/SimpleSkin.gltf

@@ -0,0 +1 @@
+{"scenes":[{"nodes":[0]}],"nodes":[{"skin":0,"mesh":0,"children":[1]},{"children":[2],"translation":[0,1,0]},{"rotation":[0,0,0,1]}],"meshes":[{"primitives":[{"attributes":{"POSITION":1,"JOINTS_0":2,"WEIGHTS_0":3},"indices":0}]}],"skins":[{"inverseBindMatrices":4,"joints":[1,2]}],"animations":[{"channels":[{"sampler":0,"target":{"node":2,"path":"rotation"}}],"samplers":[{"input":5,"interpolation":"LINEAR","output":6}]}],"buffers":[{"uri":"data:application/gltf-buffer;base64,AAABAAMAAAADAAIAAgADAAUAAgAFAAQABAAFAAcABAAHAAYABgAHAAkABgAJAAgAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAAAAAAAAAAD8AAAAAAACAPwAAAD8AAAAAAAAAAAAAgD8AAAAAAACAPwAAgD8AAAAAAAAAAAAAwD8AAAAAAACAPwAAwD8AAAAAAAAAAAAAAEAAAAAAAACAPwAAAEAAAAAA","byteLength":168},{"uri":"data:application/gltf-buffer;base64,AAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAABAPwAAgD4AAAAAAAAAAAAAQD8AAIA+AAAAAAAAAAAAAAA/AAAAPwAAAAAAAAAAAAAAPwAAAD8AAAAAAAAAAAAAgD4AAEA/AAAAAAAAAAAAAIA+AABAPwAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAA=","byteLength":320},{"uri":"data:application/gltf-buffer;base64,AACAPwAAAAAAAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAvwAAgL8AAAAAAACAPwAAgD8AAAAAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAAAAAAAAAAAAAAIA/AAAAAAAAAL8AAIC/AAAAAAAAgD8=","byteLength":128},{"uri":"data:application/gltf-buffer;base64,AAAAAAAAAD8AAIA/AADAPwAAAEAAACBAAABAQAAAYEAAAIBAAACQQAAAoEAAALBAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAkxjEPkSLbD8AAAAAAAAAAPT9ND/0/TQ/AAAAAAAAAAD0/TQ/9P00PwAAAAAAAAAAkxjEPkSLbD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAkxjEvkSLbD8AAAAAAAAAAPT9NL/0/TQ/AAAAAAAAAAD0/TS/9P00PwAAAAAAAAAAkxjEvkSLbD8AAAAAAAAAAAAAAAAAAIA/","byteLength":240}],"bufferViews":[{"buffer":0,"byteOffset":0,"byteLength":48,"target":34963},{"buffer":0,"byteOffset":48,"byteLength":120,"target":34962},{"buffer":1,"byteOffset":0,"byteLength":320,"byteStride":16},{"buffer":2,"byteOffset":0,"byteLength":128},{"buffer":3,"byteOffset":0,"byteLength":240}],"accessors":[{"bufferView":0,"byteOffset":0,"componentType":5123,"count":24,"type":"SCALAR","max":[9],"min":[0]},{"bufferView":1,"byteOffset":0,"componentType":5126,"count":10,"type":"VEC3","max":[1,2,0],"min":[0,0,0]},{"bufferView":2,"byteOffset":0,"componentType":5123,"count":10,"type":"VEC4","max":[0,1,0,0],"min":[0,1,0,0]},{"bufferView":2,"byteOffset":160,"componentType":5126,"count":10,"type":"VEC4","max":[1,1,0,0],"min":[0,0,0,0]},{"bufferView":3,"byteOffset":0,"componentType":5126,"count":2,"type":"MAT4","max":[1,0,0,0,0,1,0,0,0,0,1,0,-0.5,-1,0,1],"min":[1,0,0,0,0,1,0,0,0,0,1,0,-0.5,-1,0,1]},{"bufferView":4,"byteOffset":0,"componentType":5126,"count":12,"type":"SCALAR","max":[5.5],"min":[0]},{"bufferView":4,"byteOffset":48,"componentType":5126,"count":12,"type":"VEC4","max":[0,0,0.707,1],"min":[0,0,-0.707,0.707]}],"asset":{"version":"2.0"}}

crates/bevy_gltf/src/loader.rs

@@ -3,7 +3,7 @@ use bevy_asset::{
     AssetIoError, AssetLoader, AssetPath, BoxedFuture, Handle, LoadContext, LoadedAsset,
 };
 use bevy_core::Name;
-use bevy_ecs::{prelude::FromWorld, world::World};
+use bevy_ecs::{entity::Entity, prelude::FromWorld, world::World};
 use bevy_hierarchy::{BuildWorldChildren, WorldChildBuilder};
 use bevy_log::warn;
 use bevy_math::{Mat4, Quat, Vec3};
@@ -16,7 +16,10 @@ use bevy_render::{
         Camera, Camera2d, Camera3d, CameraProjection, OrthographicProjection, PerspectiveProjection,
     },
     color::Color,
-    mesh::{Indices, Mesh, VertexAttributeValues},
+    mesh::{
+        skinning::{SkinnedMesh, SkinnedMeshInverseBindposes},
+        Indices, Mesh, VertexAttributeValues,
+    },
     primitives::{Aabb, Frustum},
     render_resource::{AddressMode, Face, FilterMode, PrimitiveTopology, SamplerDescriptor},
     renderer::RenderDevice,
@@ -249,6 +252,18 @@ async fn load_gltf<'a, 'b>(
             //     mesh.insert_attribute(Mesh::ATTRIBUTE_COLOR, vertex_attribute);
             // }
 
+            if let Some(iter) = reader.read_joints(0) {
+                let vertex_attribute = VertexAttributeValues::Uint16x4(iter.into_u16().collect());
+                mesh.insert_attribute(Mesh::ATTRIBUTE_JOINT_INDEX, vertex_attribute);
+            }
+
+            if let Some(vertex_attribute) = reader
+                .read_weights(0)
+                .map(|v| VertexAttributeValues::Float32x4(v.into_f32().collect()))
+            {
+                mesh.insert_attribute(Mesh::ATTRIBUTE_JOINT_WEIGHT, vertex_attribute);
+            }
+
             if let Some(indices) = reader.read_indices() {
                 mesh.set_indices(Some(Indices::U32(indices.into_u32().collect())));
             };
@@ -384,18 +399,45 @@ async fn load_gltf<'a, 'b>(
             });
     }
 
+    let skinned_mesh_inverse_bindposes: Vec<_> = gltf
+        .skins()
+        .map(|gltf_skin| {
+            let reader = gltf_skin.reader(|buffer| Some(&buffer_data[buffer.index()]));
+            let inverse_bindposes: Vec<Mat4> = reader
+                .read_inverse_bind_matrices()
+                .unwrap()
+                .map(|mat| Mat4::from_cols_array_2d(&mat))
+                .collect();
+
+            load_context.set_labeled_asset(
+                &skin_label(&gltf_skin),
+                LoadedAsset::new(SkinnedMeshInverseBindposes::from(inverse_bindposes)),
+            )
+        })
+        .collect();
+
     let mut scenes = vec![];
     let mut named_scenes = HashMap::default();
     for scene in gltf.scenes() {
         let mut err = None;
         let mut world = World::default();
+        let mut node_index_to_entity_map = HashMap::new();
+        let mut entity_to_skin_index_map = HashMap::new();
+
         world
             .spawn()
             .insert_bundle(TransformBundle::identity())
             .with_children(|parent| {
                 for node in scene.nodes() {
-                    let result =
-                        load_node(&node, parent, load_context, &buffer_data, &animated_nodes);
+                    let result = load_node(
+                        &node,
+                        parent,
+                        load_context,
+                        &buffer_data,
+                        &animated_nodes,
+                        &mut node_index_to_entity_map,
+                        &mut entity_to_skin_index_map,
+                    );
                     if result.is_err() {
                         err = Some(result);
                         return;
@@ -405,6 +447,21 @@ async fn load_gltf<'a, 'b>(
         if let Some(Err(err)) = err {
             return Err(err);
         }
+
+        for (&entity, &skin_index) in &entity_to_skin_index_map {
+            let mut entity = world.entity_mut(entity);
+            let skin = gltf.skins().nth(skin_index).unwrap();
+            let joint_entities: Vec<_> = skin
+                .joints()
+                .map(|node| node_index_to_entity_map[&node.index()])
+                .collect();
+
+            entity.insert(SkinnedMesh {
+                inverse_bindposes: skinned_mesh_inverse_bindposes[skin_index].clone(),
+                joints: joint_entities,
+            });
+        }
+
         let scene_handle = load_context
             .set_labeled_asset(&scene_label(&scene), LoadedAsset::new(Scene::new(world)));
 
@@ -575,6 +632,8 @@ fn load_node(
     load_context: &mut LoadContext,
     buffer_data: &[Vec<u8>],
     animated_nodes: &HashSet<usize>,
+    node_index_to_entity_map: &mut HashMap<usize, Entity>,
+    entity_to_skin_index_map: &mut HashMap<Entity, usize>,
 ) -> Result<(), GltfError> {
     let transform = gltf_node.transform();
     let mut gltf_error = None;
@@ -645,6 +704,9 @@ fn load_node(
         }
     }
 
+    // Map node index to entity
+    node_index_to_entity_map.insert(gltf_node.index(), node.id());
+
     node.with_children(|parent| {
         if let Some(mesh) = gltf_node.mesh() {
             // append primitives
@@ -660,13 +722,13 @@ fn load_node(
                 }
 
                 let primitive_label = primitive_label(&mesh, &primitive);
+                let bounds = primitive.bounding_box();
                 let mesh_asset_path =
                     AssetPath::new_ref(load_context.path(), Some(&primitive_label));
                 let material_asset_path =
                     AssetPath::new_ref(load_context.path(), Some(&material_label));
 
-                let bounds = primitive.bounding_box();
-                parent
+                let node = parent
                     .spawn_bundle(PbrBundle {
                         mesh: load_context.get_handle(mesh_asset_path),
                         material: load_context.get_handle(material_asset_path),
@@ -675,7 +737,13 @@ fn load_node(
                     .insert(Aabb::from_min_max(
                         Vec3::from_slice(&bounds.min),
                         Vec3::from_slice(&bounds.max),
-                    ));
+                    ))
+                    .id();
+
+                // Mark for adding skinned mesh
+                if let Some(skin) = gltf_node.skin() {
+                    entity_to_skin_index_map.insert(node, skin.index());
+                }
             }
         }
 
@@ -723,7 +791,15 @@ fn load_node(
 
         // append other nodes
         for child in gltf_node.children() {
-            if let Err(err) = load_node(&child, parent, load_context, buffer_data, animated_nodes) {
+            if let Err(err) = load_node(
+                &child,
+                parent,
+                load_context,
+                buffer_data,
+                animated_nodes,
+                node_index_to_entity_map,
+                entity_to_skin_index_map,
+            ) {
                 gltf_error = Some(err);
                 return;
             }
@@ -770,6 +846,10 @@ fn scene_label(scene: &gltf::Scene) -> String {
     format!("Scene{}", scene.index())
 }
 
+fn skin_label(skin: &gltf::Skin) -> String {
+    format!("Skin{}", skin.index())
+}
+
 /// Extracts the texture sampler data from the glTF texture.
 fn texture_sampler<'a>(texture: &gltf::Texture) -> SamplerDescriptor<'a> {
     let gltf_sampler = texture.sampler();

crates/bevy_pbr/Cargo.toml

@@ -29,3 +29,4 @@ bevy_window = { path = "../bevy_window", version = "0.7.0-dev" }
 bitflags = "1.2"
 # direct dependency required for derive macro
 bytemuck = { version = "1", features = ["derive"] }
+smallvec = "1.0"

crates/bevy_pbr/src/material.rs

@@ -245,11 +245,11 @@ impl<M: SpecializedMaterial> SpecializedMeshPipeline for MaterialPipeline<M> {
         if let Some(fragment_shader) = &self.fragment_shader {
             descriptor.fragment.as_mut().unwrap().shader = fragment_shader.clone();
         }
-        descriptor.layout = Some(vec![
-            self.mesh_pipeline.view_layout.clone(),
-            self.material_layout.clone(),
-            self.mesh_pipeline.mesh_layout.clone(),
-        ]);
+
+        // MeshPipeline::specialize's current implementation guarantees that the returned
+        // specialized descriptor has a populated layout
+        let descriptor_layout = descriptor.layout.as_mut().unwrap();
+        descriptor_layout.insert(1, self.material_layout.clone());
 
         M::specialize(&mut descriptor, key.material_key, layout)?;
         Ok(descriptor)

crates/bevy_pbr/src/render/depth.wgsl

@@ -12,8 +12,18 @@ var<uniform> view: View;
 [[group(1), binding(0)]]
 var<uniform> mesh: Mesh;
 
+#ifdef SKINNED
+[[group(1), binding(1)]]
+var<uniform> joint_matrices: SkinnedMesh;
+#import bevy_pbr::skinning
+#endif
+
 struct Vertex {
     [[location(0)]] position: vec3<f32>;
+#ifdef SKINNED
+    [[location(4)]] joint_indices: vec4<u32>;
+    [[location(5)]] joint_weights: vec4<f32>;
+#endif
 };
 
 struct VertexOutput {
@@ -22,7 +32,13 @@ struct VertexOutput {
 
 [[stage(vertex)]]
 fn vertex(vertex: Vertex) -> VertexOutput {
+#ifdef SKINNED
+    let model = skin_model(vertex.joint_indices, vertex.joint_weights);
+#else
+    let model = mesh.model;
+#endif
+
     var out: VertexOutput;
-    out.clip_position = view.view_proj * mesh.model * vec4<f32>(vertex.position, 1.0);
+    out.clip_position = view.view_proj * model * vec4<f32>(vertex.position, 1.0);
     return out;
 }

crates/bevy_pbr/src/render/light.rs

@@ -159,6 +159,7 @@ pub const SHADOW_FORMAT: TextureFormat = TextureFormat::Depth32Float;
 pub struct ShadowPipeline {
     pub view_layout: BindGroupLayout,
     pub mesh_layout: BindGroupLayout,
+    pub skinned_mesh_layout: BindGroupLayout,
     pub point_light_sampler: Sampler,
     pub directional_light_sampler: Sampler,
 }
@@ -187,10 +188,12 @@ impl FromWorld for ShadowPipeline {
         });
 
         let mesh_pipeline = world.get_resource::<MeshPipeline>().unwrap();
+        let skinned_mesh_layout = mesh_pipeline.skinned_mesh_layout.clone();
 
         ShadowPipeline {
             view_layout,
             mesh_layout: mesh_pipeline.mesh_layout.clone(),
+            skinned_mesh_layout,
             point_light_sampler: render_device.create_sampler(&SamplerDescriptor {
                 address_mode_u: AddressMode::ClampToEdge,
                 address_mode_v: AddressMode::ClampToEdge,
@@ -256,18 +259,31 @@ impl SpecializedMeshPipeline for ShadowPipeline {
         key: Self::Key,
         layout: &MeshVertexBufferLayout,
     ) -> Result<RenderPipelineDescriptor, SpecializedMeshPipelineError> {
-        let vertex_buffer_layout =
-            layout.get_layout(&[Mesh::ATTRIBUTE_POSITION.at_shader_location(0)])?;
+        let mut vertex_attributes = vec![Mesh::ATTRIBUTE_POSITION.at_shader_location(0)];
+
+        let mut bind_group_layout = vec![self.view_layout.clone(), self.mesh_layout.clone()];
+        let mut shader_defs = Vec::new();
+
+        if layout.contains(Mesh::ATTRIBUTE_JOINT_INDEX)
+            && layout.contains(Mesh::ATTRIBUTE_JOINT_WEIGHT)
+        {
+            shader_defs.push(String::from("SKINNED"));
+            vertex_attributes.push(Mesh::ATTRIBUTE_JOINT_INDEX.at_shader_location(4));
+            vertex_attributes.push(Mesh::ATTRIBUTE_JOINT_WEIGHT.at_shader_location(5));
+            bind_group_layout.push(self.skinned_mesh_layout.clone());
+        }
+
+        let vertex_buffer_layout = layout.get_layout(&vertex_attributes)?;
 
         Ok(RenderPipelineDescriptor {
             vertex: VertexState {
                 shader: SHADOW_SHADER_HANDLE.typed::<Shader>(),
                 entry_point: "vertex".into(),
-                shader_defs: vec![],
+                shader_defs,
                 buffers: vec![vertex_buffer_layout],
             },
             fragment: None,
-            layout: Some(vec![self.view_layout.clone(), self.mesh_layout.clone()]),
+            layout: Some(bind_group_layout),
             primitive: PrimitiveState {
                 topology: key.primitive_topology(),
                 strip_index_format: None,