mosure · mosure · Feb 18, 2025 · Feb 18, 2025 · Feb 18, 2025 · Feb 18, 2025
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "bevy_gaussian_splatting"
 description = "bevy gaussian splatting render pipeline plugin"
-version = "4.1.1"
+version = "4.2.0"
 edition = "2021"
 authors = ["mosure <mitchell@mosure.me>"]
 license = "MIT OR Apache-2.0"
@@ -186,6 +186,7 @@ default-features = false
 features = [
   "bevy_asset",
   "bevy_core_pipeline",
+  "bevy_pbr",
   "bevy_render",
   "bevy_winit",
   "x11",

diff --git a/src/gaussian/settings.rs b/src/gaussian/settings.rs
@@ -86,7 +86,7 @@ pub enum RasterizeMode {
     #[default]
     Color,
     Depth,
-    Flow,
+    OpticalFlow,
     Normal,
     Velocity,
 }

diff --git a/src/io/ply.rs b/src/io/ply.rs
@@ -98,7 +98,7 @@ pub fn parse_ply_3d(
     let required_properties = vec![
         "x", "y", "z",
         "f_dc_0", "f_dc_1", "f_dc_2",
-        "scale_0", "scale_1", "scale_2",
+        "scale_0", "scale_1",
         "opacity",
         "rot_0", "rot_1", "rot_2", "rot_3",
     ];

diff --git a/src/material/mod.rs b/src/material/mod.rs
@@ -2,6 +2,7 @@ use bevy::prelude::*;
 
 pub mod classification;
 pub mod depth;
+pub mod optical_flow;
 pub mod spherical_harmonics;
 pub mod spherindrical_harmonics;
 
@@ -21,6 +22,7 @@ impl Plugin for MaterialPlugin {
         app.add_plugins((
             classification::ClassificationMaterialPlugin,
             depth::DepthMaterialPlugin,
+            optical_flow::OpticalFlowMaterialPlugin,
             spherical_harmonics::SphericalHarmonicCoefficientsPlugin,
             spherindrical_harmonics::SpherindricalHarmonicCoefficientsPlugin,
         ));

diff --git a/src/material/optical_flow.rs b/src/material/optical_flow.rs
@@ -0,0 +1,21 @@
+use bevy::{
+    prelude::*,
+    asset::load_internal_asset,
+};
+
+
+const OPTICAL_FLOW_SHADER_HANDLE: Handle<Shader> = Handle::weak_from_u128(1451151234);
+
+
+pub struct OpticalFlowMaterialPlugin;
+
+impl Plugin for OpticalFlowMaterialPlugin {
+    fn build(&self, app: &mut App) {
+        load_internal_asset!(
+            app,
+            OPTICAL_FLOW_SHADER_HANDLE,
+            "optical_flow.wgsl",
+            Shader::from_wgsl
+        );
+    }
+}
diff --git a/src/material/optical_flow.wgsl b/src/material/optical_flow.wgsl
@@ -0,0 +1,59 @@
+#define_import_path bevy_gaussian_splatting::optical_flow
+
+#import bevy_pbr::{
+    forward_io::VertexOutput,
+    prepass_utils,
+}
+#import bevy_render::color_operations::hsv_to_rgb
+#import bevy_render::maths::PI_2
+
+#import bevy_gaussian_splatting::bindings::{
+    globals,
+    previous_view_uniforms,
+    view,
+}
+
+
+
+fn calculate_motion_vector(
+    world_position: vec3<f32>,
+    previous_world_position: vec3<f32>,
+) -> vec2<f32> {
+    let world_position_t = vec4<f32>(world_position, 1.0);
+    let previous_world_position_t = vec4<f32>(previous_world_position, 1.0);
+    let clip_position_t = view.unjittered_clip_from_world * world_position_t;
+    let clip_position = clip_position_t.xy / clip_position_t.w;
+    let previous_clip_position_t = previous_view_uniforms.clip_from_world * previous_world_position_t;
+    let previous_clip_position = previous_clip_position_t.xy / previous_clip_position_t.w;
+    // These motion vectors are used as offsets to UV positions and are stored
+    // in the range -1,1 to allow offsetting from the one corner to the
+    // diagonally-opposite corner in UV coordinates, in either direction.
+    // A difference between diagonally-opposite corners of clip space is in the
+    // range -2,2, so this needs to be scaled by 0.5. And the V direction goes
+    // down where clip space y goes up, so y needs to be flipped.
+    return (clip_position - previous_clip_position) * vec2(0.5, -0.5);
+}
+
+
+fn optical_flow_to_rgb(
+    motion_vector: vec2<f32>,
+) -> vec3<f32> {
+    let flow = motion_vector / globals.delta_time;
+
+    let radius = length(flow);
+    var angle = atan2(flow.y, flow.x);
+    if (angle < 0.0) {
+        angle += PI_2;
+    }
+
+    // let sigma: f32 = 0.15;
+    // let norm_factor = sigma * 2.0;
+    // let m = clamp(radius / norm_factor, 0.0, 1.0);
+    let m = clamp(radius, 0.0, 1.0);
+
+    let rgb = hsv_to_rgb(vec3<f32>(angle, m, 1.0));
+    return rgb;
+}
+
+// TODO: support immediate vs. persistent previous_view, aiding with no-smoothness on the pan-orbit camera (required by cpu sort)
+// TODO: set clear color to white in optical flow render mode
diff --git a/src/morph/particle.rs b/src/morph/particle.rs
@@ -261,7 +261,7 @@ impl<R: PlanarStorage> FromWorld for ParticleBehaviorPipeline<R> {
         let particle_behavior_pipeline = pipeline_cache.queue_compute_pipeline(ComputePipelineDescriptor {
             label: Some("particle_behavior_pipeline".into()),
             layout: vec![
-                gaussian_cloud_pipeline.view_layout.clone(),
+                gaussian_cloud_pipeline.compute_view_layout.clone(),
                 gaussian_cloud_pipeline.gaussian_uniform_layout.clone(),
                 gaussian_cloud_pipeline.gaussian_cloud_layout.clone(),
                 particle_behavior_layout.clone(),

diff --git a/src/render/bindings.wgsl b/src/render/bindings.wgsl
@@ -1,11 +1,13 @@
 #define_import_path bevy_gaussian_splatting::bindings
 
+#import bevy_pbr::prepass_bindings::PreviousViewUniforms
 #import bevy_render::globals::Globals
 #import bevy_render::view::View
 
 
 @group(0) @binding(0) var<uniform> view: View;
 @group(0) @binding(1) var<uniform> globals: Globals;
+@group(0) @binding(2) var<uniform> previous_view_uniforms: PreviousViewUniforms;
 
 struct GaussianUniforms {
     transform: mat4x4<f32>,

diff --git a/src/render/gaussian.wgsl b/src/render/gaussian.wgsl
@@ -5,6 +5,10 @@
 }
 #import bevy_gaussian_splatting::classification::class_to_rgb
 #import bevy_gaussian_splatting::depth::depth_to_rgb
+#import bevy_gaussian_splatting::optical_flow::{
+    calculate_motion_vector,
+    optical_flow_to_rgb,
+}
 #import bevy_gaussian_splatting::helpers::{
     get_rotation_matrix,
     get_scale_matrix,
@@ -178,6 +182,7 @@ fn vs_points(
     let position = vec4<f32>(get_position(splat_index), 1.0);
 
     var transformed_position = (gaussian_uniforms.transform * position).xyz;
+    var previous_transformed_position = transformed_position;
 
 #ifdef DRAW_SELECTED
     discard_quad |= get_visibility(splat_index) < 0.5;
@@ -254,6 +259,7 @@ fn vs_points(
 
         let position_t = vec4<f32>(position.xyz + gaussian_4d.delta_mean, 1.0);
         transformed_position = (gaussian_uniforms.transform * position_t).xyz;
+        // TODO: set previous_transformed_position based on temporal position delta
         let projected_position = world_to_clip(transformed_position);
 
         if !in_frustum(projected_position.xyz) {
@@ -346,9 +352,13 @@ fn vs_points(
         0.5 * (t.y + 1.0),
         0.5 * (t.z + 1.0)
     );
-#else ifdef RASTERIZE_FLOW
-    // TODO: optical flow rendering
-    rgb = vec3<f32>(1.0);
+#else ifdef RASTERIZE_OPTICAL_FLOW
+    let motion_vector = calculate_motion_vector(
+        transformed_position,
+        previous_transformed_position,
+    );
+
+    rgb = optical_flow_to_rgb(motion_vector);
 #else ifdef RASTERIZE_VELOCITY
     let time_delta = 1e-3;
     let future_gaussian_4d = conditional_cov3d(
-Original file line number
+Diff line change
@@ Expand Up / @@ -86,7 +86,7 @@ pub enum RasterizeMode { @@
         #[default]
         Color,
         Depth,
-        Flow,
+        OpticalFlow,
         Normal,
         Velocity,
     }
@@ Expand Down @@