From 370244ad791bf82277d4f325fd5197e639360332 Mon Sep 17 00:00:00 2001 From: mosure Date: Sun, 5 Nov 2023 22:28:19 -0600 Subject: [PATCH 1/5] chore: build for bevy 0.12 --- Cargo.toml | 10 +++---- README.md | 2 +- src/gaussian.rs | 30 ++++++++++++------- src/lib.rs | 2 +- src/ply.rs | 3 +- src/render/gaussian.wgsl | 6 ++-- src/render/mod.rs | 62 ++++++++++++++++++++-------------------- viewer/viewer.rs | 21 ++------------ 8 files changed, 64 insertions(+), 72 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b0da72a9..7274c001 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "bevy_gaussian_splatting" description = "bevy gaussian splatting render pipeline plugin" -version = "0.2.0" +version = "0.3.0" edition = "2021" authors = ["mosure "] license = "MIT" @@ -14,15 +14,15 @@ exclude = [".devcontainer", ".github", "docs", "dist", "build", "assets", "credi default-run = "viewer" [dependencies] -bevy-inspector-egui = "0.20.0" -bevy_panorbit_camera = "0.8.0" +bevy-inspector-egui = "0.21" +bevy_panorbit_camera = { git = "https://github.com/mosure/bevy_panorbit_camera.git", rev = "5b0dcb0" } bincode2 = "2.0.1" bytemuck = "1.14.0" flate2 = "1.0.28" ply-rs = "0.1.3" rand = "0.8.5" serde = "1.0.189" -wgpu = "0.16.0" +wgpu = "0.17.1" [target.'cfg(target_arch = "wasm32")'.dependencies] @@ -32,7 +32,7 @@ wasm-bindgen = "0.2.87" # TODO: use minimal bevy features [dependencies.bevy] -version = "0.11.3" +version = "0.12" default-features = true diff --git a/README.md b/README.md index dbb6d291..111a160f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ bevy gaussian splatting render pipeline plugin -`cargo run -- {path to ply or gcloud file}` +`cargo run -- scenes/icecream.gcloud` ## capabilities diff --git a/src/gaussian.rs b/src/gaussian.rs index 986d4be7..ac12cbc2 100644 --- a/src/gaussian.rs +++ b/src/gaussian.rs @@ -3,6 +3,7 @@ use std::{ io::{ BufReader, Cursor, + ErrorKind, }, marker::Copy, }; @@ -11,8 +12,9 @@ use bevy::{ prelude::*, asset::{ AssetLoader, + AsyncReadExt, LoadContext, - LoadedAsset, + io::Reader, }, reflect::TypeUuid, render::render_resource::ShaderType, @@ -136,6 +138,7 @@ pub struct Gaussian { } #[derive( + Asset, Clone, Debug, Reflect, @@ -233,12 +236,21 @@ impl Default for GaussianCloudSettings { pub struct GaussianCloudLoader; impl AssetLoader for GaussianCloudLoader { + type Asset = GaussianCloud; + type Settings = (); + type Error = std::io::Error; + fn load<'a>( &'a self, - bytes: &'a [u8], + reader: &'a mut Reader, + _settings: &'a Self::Settings, load_context: &'a mut LoadContext, - ) -> BoxedFuture<'a, Result<(), bevy::asset::Error>> { + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).await?; + match load_context.path().extension() { Some(ext) if ext == "ply" => { let cursor = Cursor::new(bytes); @@ -247,19 +259,15 @@ impl AssetLoader for GaussianCloudLoader { let ply_cloud = parse_ply(&mut f)?; let cloud = GaussianCloud(ply_cloud); - load_context.set_default_asset(LoadedAsset::new(cloud)); - - Ok(()) + Ok(cloud) }, Some(ext) if ext == "gcloud" => { - let decompressed = GzDecoder::new(bytes); + let decompressed = GzDecoder::new(bytes.as_slice()); let cloud: GaussianCloud = deserialize_from(decompressed).expect("failed to decode cloud"); - load_context.set_default_asset(LoadedAsset::new(cloud)); - - Ok(()) + Ok(cloud) }, - _ => Ok(()), + _ => Err(std::io::Error::new(ErrorKind::Other, "only .ply and .gcloud supported")), } }) } diff --git a/src/lib.rs b/src/lib.rs index 6d9bfbc4..92ad9478 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,7 +34,7 @@ impl Plugin for GaussianSplattingPlugin { // TODO: allow hot reloading of GaussianCloud handle through inspector UI app.register_type::(); app.register_type::(); - app.add_asset::(); + app.init_asset::(); app.register_asset_reflect::(); app.init_asset_loader::(); diff --git a/src/ply.rs b/src/ply.rs index aba6a883..ab5d1da3 100644 --- a/src/ply.rs +++ b/src/ply.rs @@ -1,6 +1,5 @@ use std::io::BufRead; -use bevy::asset::Error; use ply_rs::{ ply::{ Property, @@ -53,7 +52,7 @@ impl PropertyAccess for Gaussian { } } -pub fn parse_ply(mut reader: &mut dyn BufRead) -> Result, Error> { +pub fn parse_ply(mut reader: &mut dyn BufRead) -> Result, std::io::Error> { let gaussian_parser = Parser::::new(); let header = gaussian_parser.read_header(&mut reader)?; diff --git a/src/render/gaussian.wgsl b/src/render/gaussian.wgsl index ce434a7d..ab2fce92 100644 --- a/src/render/gaussian.wgsl +++ b/src/render/gaussian.wgsl @@ -1,7 +1,7 @@ -#import bevy_render::globals Globals -#import bevy_render::view View +#import bevy_render::globals::Globals +#import bevy_render::view::View -#import bevy_gaussian_splatting::spherical_harmonics spherical_harmonics_lookup +#import bevy_gaussian_splatting::spherical_harmonics::spherical_harmonics_lookup struct GaussianInput { diff --git a/src/render/mod.rs b/src/render/mod.rs index 4febd734..ca513965 100644 --- a/src/render/mod.rs +++ b/src/render/mod.rs @@ -4,7 +4,6 @@ use bevy::{ prelude::*, asset::{ load_internal_asset, - HandleUntyped, LoadState, }, core_pipeline::core_3d::{ @@ -18,7 +17,6 @@ use bevy::{ }, query::ROQueryItem, }, - reflect::TypeUuid, render::{ Extract, extract_component::{ @@ -75,8 +73,8 @@ use crate::gaussian::{ }; -const GAUSSIAN_SHADER_HANDLE: HandleUntyped = HandleUntyped::weak_from_u64(Shader::TYPE_UUID, 68294581); -const SPHERICAL_HARMONICS_SHADER_HANDLE: HandleUntyped = HandleUntyped::weak_from_u64(Shader::TYPE_UUID, 834667312); +const GAUSSIAN_SHADER_HANDLE: Handle = Handle::weak_from_u128(68294581); +const SPHERICAL_HARMONICS_SHADER_HANDLE: Handle = Handle::weak_from_u128(834667312); pub mod node { pub const RADIX_SORT: &str = "radix_sort"; @@ -269,6 +267,8 @@ fn queue_gaussians( draw_function: draw_custom, distance: 0.0, pipeline, + batch_range: 0..1, + dynamic_offset: None, }); } } @@ -436,7 +436,7 @@ impl FromWorld for GaussianCloudPipeline { gaussian_cloud_layout.clone(), radix_sort_layout.clone(), ]; - let shader = GAUSSIAN_SHADER_HANDLE.typed(); + let shader = GAUSSIAN_SHADER_HANDLE; let shader_defs = shader_defs(false, false); let pipeline_cache = render_world.resource::(); @@ -758,8 +758,10 @@ pub fn queue_gaussian_bind_group( assert!(model.size() == std::mem::size_of::() as u64); - groups.base_bind_group = Some(render_device.create_bind_group(&BindGroupDescriptor { - entries: &[ + groups.base_bind_group = Some(render_device.create_bind_group( + "gaussian_uniform_bind_group", + &gaussian_cloud_pipeline.gaussian_uniform_layout, + &[ BindGroupEntry { binding: 0, resource: BindingResource::Buffer(BufferBinding { @@ -769,16 +771,14 @@ pub fn queue_gaussian_bind_group( }), }, ], - layout: &gaussian_cloud_pipeline.gaussian_uniform_layout, - label: Some("gaussian_uniform_bind_group"), - })); + )); for (entity, cloud_handle) in gaussian_clouds.iter() { - if asset_server.get_load_state(cloud_handle) == LoadState::Loading { + if Some(LoadState::Loading) == asset_server.get_load_state(cloud_handle) { continue; } - if !gaussian_cloud_res.contains_key(cloud_handle) { + if gaussian_cloud_res.get(cloud_handle).is_none() { continue; } @@ -804,10 +804,10 @@ pub fn queue_gaussian_bind_group( let radix_sort_bind_groups: [BindGroup; 4] = (0..4) .map(|idx| { - render_device.create_bind_group(&BindGroupDescriptor { - label: format!("radix_sort_bind_group {}", idx).as_str().into(), - layout: &gaussian_cloud_pipeline.radix_sort_layout, - entries: &[ + render_device.create_bind_group( + format!("radix_sort_bind_group {}", idx).as_str(), + &gaussian_cloud_pipeline.radix_sort_layout, + &[ BindGroupEntry { binding: 0, resource: BindingResource::Buffer(BufferBinding { @@ -843,15 +843,17 @@ pub fn queue_gaussian_bind_group( }), }, ], - }) + ) }) .collect::>() .try_into() .unwrap(); commands.entity(entity).insert(GaussianCloudBindGroup { - cloud_bind_group: render_device.create_bind_group(&BindGroupDescriptor { - entries: &[ + cloud_bind_group: render_device.create_bind_group( + "gaussian_cloud_bind_group", + &gaussian_cloud_pipeline.gaussian_cloud_layout, + &[ BindGroupEntry { binding: 0, resource: BindingResource::Buffer(BufferBinding { @@ -861,12 +863,12 @@ pub fn queue_gaussian_bind_group( }), }, ], - layout: &gaussian_cloud_pipeline.gaussian_cloud_layout, - label: Some("gaussian_cloud_bind_group"), - }), + ), radix_sort_bind_groups, - sorted_bind_group: render_device.create_bind_group(&BindGroupDescriptor { - entries: &[ + sorted_bind_group: render_device.create_bind_group( + "render_sorted_bind_group", + &gaussian_cloud_pipeline.sorted_layout, + &[ BindGroupEntry { binding: 5, resource: BindingResource::Buffer(BufferBinding { @@ -876,9 +878,7 @@ pub fn queue_gaussian_bind_group( }), }, ], - layout: &gaussian_cloud_pipeline.sorted_layout, - label: Some("render_sorted_bind_group"), - }), + ), }); } } @@ -927,11 +927,11 @@ pub fn queue_gaussian_view_bind_groups( }, ]; - let view_bind_group = render_device.create_bind_group(&BindGroupDescriptor { - entries: &entries, - label: Some("gaussian_view_bind_group"), + let view_bind_group = render_device.create_bind_group( + "gaussian_view_bind_group", layout, - }); + &entries, + ); commands.entity(entity).insert(GaussianViewBindGroup { diff --git a/viewer/viewer.rs b/viewer/viewer.rs index 4e7b4fb9..fb265e38 100644 --- a/viewer/viewer.rs +++ b/viewer/viewer.rs @@ -1,13 +1,11 @@ use bevy::{ prelude::*, app::AppExit, - asset::ChangeWatcher, core::Name, diagnostic::{ DiagnosticsStore, FrameTimeDiagnosticsPlugin, }, - utils::Duration, }; use bevy_inspector_egui::quick::WorldInspectorPlugin; use bevy_panorbit_camera::{ @@ -51,8 +49,6 @@ fn setup_gaussian_cloud( mut commands: Commands, asset_server: Res, mut gaussian_assets: ResMut>, - // mut meshes: ResMut>, - // mut materials: ResMut>, ) { let cloud: Handle; let settings = GaussianCloudSettings { @@ -61,10 +57,10 @@ fn setup_gaussian_cloud( ..default() }; - let filename = std::env::args().nth(1); - if let Some(filename) = filename { + let file_arg = std::env::args().nth(1); + if let Some(filename) = file_arg { println!("loading {}", filename); - cloud = asset_server.load(filename.as_str()); + cloud = asset_server.load(filename.to_string()); } else { cloud = gaussian_assets.add(GaussianCloud::test_model()); } @@ -77,13 +73,6 @@ fn setup_gaussian_cloud( Name::new("gaussian_cloud"), )); - // commands.spawn(PbrBundle { - // mesh: meshes.add(Mesh::from(shape::Cube { size: 1.0 })), - // material: materials.add(Color::rgb(0.8, 0.3, 0.6).into()), - // transform: Transform::from_xyz(0.0, 0.0, 0.0), - // ..default() - // }); - commands.spawn(( Camera3dBundle { transform: Transform::from_translation(Vec3::new(0.0, 1.5, 5.0)), @@ -105,10 +94,6 @@ fn example_app() { app.insert_resource(ClearColor(Color::rgb_u8(0, 0, 0))); app.add_plugins( DefaultPlugins - .set(AssetPlugin { - watch_for_changes: ChangeWatcher::with_delay(Duration::from_millis(200)), - ..Default::default() - }) .set(ImagePlugin::default_nearest()) .set(WindowPlugin { primary_window: Some(Window { From 1fd6b30392407fab90d0b556e57f3efdf646f56b Mon Sep 17 00:00:00 2001 From: mosure Date: Mon, 6 Nov 2023 10:00:20 -0600 Subject: [PATCH 2/5] chore: upgrade bevy_panorbit_camera --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7274c001..e6602bf2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ default-run = "viewer" [dependencies] bevy-inspector-egui = "0.21" -bevy_panorbit_camera = { git = "https://github.com/mosure/bevy_panorbit_camera.git", rev = "5b0dcb0" } +bevy_panorbit_camera = "0.9.0" bincode2 = "2.0.1" bytemuck = "1.14.0" flate2 = "1.0.28" From b4796323ceece663d49aac2930c229f8d9d55341 Mon Sep 17 00:00:00 2001 From: mosure Date: Mon, 6 Nov 2023 15:53:49 -0600 Subject: [PATCH 3/5] fix: gate phase item addition on uniform availability --- src/render/mod.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/render/mod.rs b/src/render/mod.rs index a7540d6d..7abcf9b4 100644 --- a/src/render/mod.rs +++ b/src/render/mod.rs @@ -122,9 +122,9 @@ impl Plugin for RenderPipelinePlugin { .add_systems( Render, ( - queue_gaussian_bind_group.in_set(RenderSet::Queue), - queue_gaussian_view_bind_groups.in_set(RenderSet::Queue), - queue_gaussians.in_set(RenderSet::Queue), + queue_gaussian_bind_group.in_set(RenderSet::QueueMeshes), + queue_gaussian_view_bind_groups.in_set(RenderSet::QueueMeshes), + queue_gaussians.in_set(RenderSet::QueueMeshes), ), ); } @@ -235,6 +235,7 @@ impl RenderAsset for GaussianCloud { #[allow(clippy::too_many_arguments)] fn queue_gaussians( + gaussian_cloud_uniform: Res>, transparent_3d_draw_functions: Res>, custom_pipeline: Res, mut pipelines: ResMut>, @@ -250,6 +251,11 @@ fn queue_gaussians( &mut RenderPhase, )>, ) { + // TODO: condition this system based on GaussianCloudBindGroup attachment + if gaussian_cloud_uniform.buffer().is_none() { + return; + }; + let draw_custom = transparent_3d_draw_functions.read().id::(); for (_view, mut transparent_phase) in &mut views { @@ -262,10 +268,15 @@ fn queue_gaussians( let pipeline = pipelines.specialize(&pipeline_cache, &custom_pipeline, key); + // // TODO: distance to gaussian cloud centroid + // let rangefinder = view.rangefinder3d(); + transparent_phase.add(Transparent3d { entity, draw_function: draw_custom, distance: 0.0, + // distance: rangefinder + // .distance_translation(&mesh_instance.transforms.transform.translation), pipeline, batch_range: 0..1, dynamic_offset: None, @@ -754,8 +765,6 @@ pub fn queue_gaussian_bind_group( return; }; - assert!(model.size() == std::mem::size_of::() as u64); - groups.base_bind_group = Some(render_device.create_bind_group( "gaussian_uniform_bind_group", &gaussian_cloud_pipeline.gaussian_uniform_layout, @@ -772,6 +781,7 @@ pub fn queue_gaussian_bind_group( )); for (entity, cloud_handle) in gaussian_clouds.iter() { + // TODO: add asset loading indicator (and maybe streamed loading) if Some(LoadState::Loading) == asset_server.get_load_state(cloud_handle) { continue; } From 418307a9146c8a10fc429fb9d32e063fb8501c91 Mon Sep 17 00:00:00 2001 From: mosure Date: Mon, 6 Nov 2023 16:12:12 -0600 Subject: [PATCH 4/5] docs: bevy 0.12 support --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 11f4d417..a8ca4fea 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ to build wasm run: | `bevy_gaussian_splatting` | `bevy` | | :-- | :-- | +| `0.4` | `0.12` | | `0.1 - 0.3` | `0.11` | From baf959092bdac2b97eedacaf2488ced1573e351b Mon Sep 17 00:00:00 2001 From: mosure Date: Mon, 6 Nov 2023 16:12:38 -0600 Subject: [PATCH 5/5] chore: bump version to 0.4.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e6602bf2..416d70fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "bevy_gaussian_splatting" description = "bevy gaussian splatting render pipeline plugin" -version = "0.3.0" +version = "0.4.0" edition = "2021" authors = ["mosure "] license = "MIT"