diff --git a/.gitignore b/.gitignore index 4b365cc6..c2592bc5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ out/ *.gcloud *.ply *.ply4d +*.json .DS_Store diff --git a/Cargo.lock b/Cargo.lock index f2c51889..025bd507 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -991,6 +991,7 @@ dependencies = [ "bevy_reflect 0.15.2", "bevy_tasks 0.15.2", "bevy_utils 0.15.2", + "serde", "uuid", ] @@ -1246,7 +1247,7 @@ dependencies = [ [[package]] name = "bevy_gaussian_splatting" -version = "4.2.0" +version = "4.3.0" dependencies = [ "base64 0.22.1", "bevy 0.15.2", @@ -1274,6 +1275,7 @@ dependencies = [ "rand 0.8.5", "rayon", "serde", + "serde_json", "static_assertions", "typenum", "wasm-bindgen", @@ -1428,6 +1430,7 @@ dependencies = [ "bevy_reflect 0.15.2", "bevy_utils 0.15.2", "derive_more", + "serde", "smol_str", ] @@ -2129,6 +2132,7 @@ dependencies = [ "nonmax", "radsort", "rectangle-pack", + "serde", ] [[package]] @@ -2236,6 +2240,7 @@ dependencies = [ "bevy_reflect 0.15.2", "bevy_utils 0.15.2", "crossbeam-channel", + "serde", ] [[package]] @@ -2264,6 +2269,7 @@ dependencies = [ "bevy_math 0.15.2", "bevy_reflect 0.15.2", "derive_more", + "serde", ] [[package]] @@ -2306,6 +2312,7 @@ dependencies = [ "bytemuck", "derive_more", "nonmax", + "serde", "smallvec", "taffy", ] @@ -2408,6 +2415,7 @@ dependencies = [ "bevy_reflect 0.15.2", "bevy_utils 0.15.2", "raw-window-handle", + "serde", "smol_str", ] @@ -2466,6 +2474,7 @@ dependencies = [ "cfg-if", "crossbeam-channel", "raw-window-handle", + "serde", "wasm-bindgen", "web-sys", "wgpu-types 23.0.0", @@ -6416,6 +6425,9 @@ name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +dependencies = [ + "serde", +] [[package]] name = "smithay-client-toolkit" diff --git a/Cargo.toml b/Cargo.toml index 9aba06b2..587698a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "bevy_gaussian_splatting" description = "bevy gaussian splatting render pipeline plugin" -version = "4.2.0" +version = "4.3.0" edition = "2021" authors = ["mosure "] license = "MIT OR Apache-2.0" @@ -170,6 +170,7 @@ ply-rs = { version = "0.1", optional = true } rand = "0.8" rayon = { version = "1.8", optional = true } serde = "1.0" +serde_json = "1.0" static_assertions = "1.1" typenum = "1.17" wgpu = "23.0.1" @@ -189,6 +190,7 @@ features = [ "bevy_pbr", "bevy_render", "bevy_winit", + "serialize", "x11", ] diff --git a/README.md b/README.md index de9cbb0d..1140bd24 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ bevy_gaussian_splatting --input-file {gaussian_splat_ply_file} - [X] 2dgs - [X] 3dgs - [x] 4dgs +- [x] multi-cloud scene format +- [ ] gltf gaussian extensions - [ ] 4dgs motion blur - [ ] implicit mlp node (isotropic rotation, color) - [ ] temporal gaussian hierarchy @@ -82,7 +84,7 @@ fn setup_gaussian_cloud( CloudSettings::default(), )); - commands.spawn(Camera3dBundle::default()); + commands.spawn(Camera3d::default()); } ``` diff --git a/assets/scene.json b/assets/scene.json new file mode 100644 index 00000000..f04e3c13 --- /dev/null +++ b/assets/scene.json @@ -0,0 +1,24 @@ +{ + "bundles": [ + { + "asset_path": "./trellis.ply", + "name": "trellis_0", + "settings": {}, + "transform": { + "translation": [0.0, 0.0, 0.0], + "rotation": [0.0, 0.0, 0.0, 1.0], + "scale": [1.0, -1.0, 1.0] + } + }, + { + "asset_path": "./trellis.ply", + "name": "trellis_1", + "settings": {}, + "transform": { + "translation": [5.0, 0.0, 0.0], + "rotation": [0.0, 0.0, 0.0, 1.0], + "scale": [1.0, -1.0, 1.0] + } + } + ] +} diff --git a/examples/headless.rs b/examples/headless.rs index 20343351..39bc73d2 100644 --- a/examples/headless.rs +++ b/examples/headless.rs @@ -378,9 +378,9 @@ fn setup_gaussian_cloud( if args.gaussian_count > 0 { println!("generating {} gaussians", args.gaussian_count); cloud = gaussian_assets.add(random_gaussians_3d(args.gaussian_count)); - } else if !args.input_file.is_empty() { - println!("loading {}", args.input_file); - cloud = asset_server.load(&args.input_file); + } else if args.input_cloud.is_some() && !args.input_cloud.as_ref().unwrap().is_empty() { + println!("loading {:?}", args.input_cloud); + cloud = asset_server.load(&args.input_cloud.as_ref().unwrap().clone()); } else { cloud = gaussian_assets.add(PlanarGaussian3d::test_model()); } diff --git a/src/gaussian/settings.rs b/src/gaussian/settings.rs index 41db6326..6413b483 100644 --- a/src/gaussian/settings.rs +++ b/src/gaussian/settings.rs @@ -17,6 +17,8 @@ use crate::sort::SortMode; Hash, PartialEq, Reflect, + Serialize, + Deserialize, )] pub enum DrawMode { #[default] @@ -93,8 +95,16 @@ pub enum RasterizeMode { // TODO: breakdown into components -#[derive(Component, Reflect, Clone)] +#[derive( + Component, + Clone, + Debug, + Reflect, + Serialize, + Deserialize, +)] #[reflect(Component)] +#[serde(default)] pub struct CloudSettings { pub aabb: bool, pub global_opacity: f32, diff --git a/src/io/mod.rs b/src/io/mod.rs index cfe1eb24..27f444cb 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -1,6 +1,21 @@ +use bevy::prelude::*; + pub mod codec; pub mod gcloud; pub mod loader; +pub mod scene; #[cfg(feature = "io_ply")] pub mod ply; + + +#[derive(Default)] +pub struct IoPlugin; +impl Plugin for IoPlugin { + fn build(&self, app: &mut App) { + app.init_asset_loader::(); + app.init_asset_loader::(); + + app.add_plugins(scene::GaussianScenePlugin); + } +} diff --git a/src/io/scene.rs b/src/io/scene.rs new file mode 100644 index 00000000..fff5e58b --- /dev/null +++ b/src/io/scene.rs @@ -0,0 +1,169 @@ + +#[allow(unused_imports)] +use std::io::{ + BufReader, + Cursor, + ErrorKind, +}; + +use bevy::{ + prelude::*, + asset::{ + AssetLoader, + LoadContext, + io::Reader, + }, +}; +use serde::{ + Deserialize, + Serialize, +}; + +use crate::gaussian::{ + formats::planar_3d::{ + PlanarGaussian3d, + PlanarGaussian3dHandle, + }, + settings::CloudSettings, +}; + + +#[derive(Default)] +pub struct GaussianScenePlugin; +impl Plugin for GaussianScenePlugin { + fn build(&self, app: &mut App) { + app.register_type::(); + app.init_asset::(); + + app.init_asset_loader::(); + + app.add_systems( + Update, + ( + spawn_scene, + ) + ); + } +} + + + +#[derive( + Clone, + Debug, + Default, + Reflect, + Serialize, + Deserialize, +)] +pub struct CloudBundle { + pub asset_path: String, + pub name: String, + pub settings: CloudSettings, + pub transform: Transform, +} + +// TODO: support scene hierarchy with gaussian gltf extension +#[derive( + Asset, + Clone, + Debug, + Default, + Reflect, + Serialize, + Deserialize, +)] +pub struct GaussianScene { + pub bundles: Vec, +} + +#[derive(Component, Clone, Debug, Default, Reflect)] +#[require(Transform, Visibility)] +pub struct GaussianSceneHandle(pub Handle); + +#[derive(Component, Clone, Debug, Default, Reflect)] +pub struct GaussianSceneLoaded; + + +fn spawn_scene( + mut commands: Commands, + scene_handles: Query< + ( + Entity, + &GaussianSceneHandle, + ), + Without, + >, + asset_server: Res, + scenes: Res>, +) { + for (entity, scene_handle) in scene_handles.iter() { + if let Some(load_state) = &asset_server.get_load_state(&scene_handle.0) { + if !load_state.is_loaded() { + continue; + } + } + + if scenes.get(&scene_handle.0).is_none() { + continue; + } + + let scene = scenes.get(&scene_handle.0).unwrap(); + + let bundles = scene.bundles + .iter() + .map(|bundle|( + // TODO: switch between 3d and 4d clouds based on settings + PlanarGaussian3dHandle( + asset_server.load::(bundle.asset_path.clone()) + ), + Name::new(bundle.name.clone()), + bundle.settings.clone(), + bundle.transform, + ) + ) + .collect::>(); + + commands + .entity(entity) + .with_children(move |builder| { + for bundle in bundles { + builder.spawn(bundle); + } + }) + .insert(GaussianSceneLoaded); + } +} + + +#[derive(Default)] +pub struct GaussianSceneLoader; + +impl AssetLoader for GaussianSceneLoader { + type Asset = GaussianScene; + type Settings = (); + type Error = std::io::Error; + + async fn load( + &self, + reader: &mut dyn Reader, + _: &Self::Settings, + load_context: &mut LoadContext<'_>, + ) -> Result { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).await?; + + match load_context.path().extension() { + Some(ext) if ext == "json" => { + let scene: GaussianScene = serde_json::from_slice(&bytes) + .map_err(|err| std::io::Error::new(ErrorKind::InvalidData, err))?; + Ok(scene) + }, + _ => Err(std::io::Error::new(ErrorKind::Other, "only .json supported")), + } + } + + fn extensions(&self) -> &[&str] { + &["json"] + } +} diff --git a/src/lib.rs b/src/lib.rs index 47f4c4a8..9fb072d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,12 +28,14 @@ pub use gaussian::{ }, }; +pub use io::scene::{ + GaussianScene, + GaussianSceneHandle, +}; + pub use material::spherical_harmonics::SphericalHarmonicCoefficients; -use io::loader::{ - Gaussian3dLoader, - Gaussian4dLoader, -}; +use io::IoPlugin; pub mod camera; pub mod gaussian; @@ -58,8 +60,7 @@ impl Plugin for GaussianSplattingPlugin { // TODO: allow hot reloading of Cloud handle through inspector UI app.register_type::(); - app.init_asset_loader::(); - app.init_asset_loader::(); + app.add_plugins(IoPlugin); app.add_plugins(( camera::GaussianCameraPlugin, diff --git a/src/render/mod.rs b/src/render/mod.rs index 250fe23d..33f47f1a 100644 --- a/src/render/mod.rs +++ b/src/render/mod.rs @@ -324,6 +324,10 @@ fn queue_gaussians( render_entity, visible_entity, ) in visible_entities.iter::>() { + if gaussian_splatting_bundles.get(*render_entity).is_err() { + continue; + } + let ( _entity, cloud_handle, @@ -399,7 +403,6 @@ impl FromWorld for CloudPipeline { fn from_world(render_world: &mut World) -> Self { let render_device = render_world.resource::(); - // TODO: store both ShaderStages::all() and ShaderStages::VERTEX_FRAGMENT pipelines (for previous_view/sort nodes) let view_layout_entries = vec![ BindGroupLayoutEntry { binding: 0, diff --git a/src/sort/mod.rs b/src/sort/mod.rs index d32bb041..da7d7da0 100644 --- a/src/sort/mod.rs +++ b/src/sort/mod.rs @@ -31,6 +31,10 @@ use bytemuck::{ Pod, Zeroable, }; +use serde::{ + Deserialize, + Serialize, +}; use static_assertions::assert_cfg; use crate::{ @@ -69,6 +73,8 @@ assert_cfg!( Clone, PartialEq, Reflect, + Serialize, + Deserialize, )] pub enum SortMode { None, diff --git a/src/utils.rs b/src/utils.rs index ba947f9d..a5902fb0 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -45,8 +45,11 @@ pub struct GaussianSplattingViewer { #[arg(long, default_value = "1")] pub msaa_samples: u8, - #[arg(long, default_value = "", help = "input file path (or url/base64_url if web_asset feature is enabled)")] - pub input_file: String, + #[arg(long, default_value = None, help = "input file path (or url/base64_url if web_asset feature is enabled)")] + pub input_cloud: Option, + + #[arg(long, default_value = None, help = "input file path (or url/base64_url if web_asset feature is enabled)")] + pub input_scene: Option, #[arg(long, default_value = "0")] pub gaussian_count: usize, @@ -75,7 +78,8 @@ impl Default for GaussianSplattingViewer { height: 1080.0, name: "bevy_gaussian_splatting".to_string(), msaa_samples: 1, - input_file: "".to_string(), + input_cloud: None, + input_scene: None, gaussian_count: 0, gaussian_mode: GaussianMode::Gaussian3d, playback_mode: PlaybackMode::Sin, diff --git a/viewer/viewer.rs b/viewer/viewer.rs index 2e4dfcd2..1d350fe9 100644 --- a/viewer/viewer.rs +++ b/viewer/viewer.rs @@ -35,6 +35,8 @@ use bevy_gaussian_splatting::{ CloudSettings, GaussianCamera, GaussianMode, + GaussianScene, + GaussianSceneHandle, GaussianSplattingPlugin, PlanarGaussian3d, PlanarGaussian4d, @@ -98,14 +100,42 @@ fn setup_gaussian_cloud( mut gaussian_3d_assets: ResMut>, mut gaussian_4d_assets: ResMut>, ) { + debug!("spawning camera..."); + commands.spawn(( + Camera3d::default(), + Transform::from_translation(Vec3::new(0.0, 1.5, 5.0)), + Tonemapping::None, + MotionVectorPrepass, + PanOrbitCamera { + allow_upside_down: true, + orbit_smoothness: 0.1, + pan_smoothness: 0.1, + zoom_smoothness: 0.1, + ..default() + }, + GaussianCamera::default(), + )); + + if let Some(input_scene) = &args.input_scene { + let input_uri = parse_input_file(input_scene.as_str()); + log(&format!("loading {}", input_uri)); + + let scene: Handle = asset_server.load(&input_uri); + commands.spawn(( + GaussianSceneHandle(scene), + Name::new("gaussian_scene"), + )); + return; + } + match args.gaussian_mode { GaussianMode::Gaussian2d | GaussianMode::Gaussian3d => { let cloud: Handle; if args.gaussian_count > 0 { log(&format!("generating {} gaussians", args.gaussian_count)); cloud = gaussian_3d_assets.add(random_gaussians_3d(args.gaussian_count)); - } else if !args.input_file.is_empty() { - let input_uri = parse_input_file(&args.input_file); + } else if let Some(input_cloud) = &args.input_cloud { + let input_uri = parse_input_file(input_cloud.as_str()); log(&format!("loading {}", input_uri)); cloud = asset_server.load(&input_uri); } else { @@ -128,8 +158,8 @@ fn setup_gaussian_cloud( if args.gaussian_count > 0 { log(&format!("generating {} gaussians", args.gaussian_count)); cloud = gaussian_4d_assets.add(random_gaussians_4d(args.gaussian_count)); - } else if !args.input_file.is_empty() { - let input_uri = parse_input_file(&args.input_file); + } else if let Some(input_cloud) = &args.input_cloud { + let input_uri = parse_input_file(input_cloud.as_str()); log(&format!("loading {}", input_uri)); cloud = asset_server.load(&input_uri); } else { @@ -148,22 +178,6 @@ fn setup_gaussian_cloud( )); } } - - debug!("spawning camera..."); - commands.spawn(( - Camera3d::default(), - Transform::from_translation(Vec3::new(0.0, 1.5, 5.0)), - Tonemapping::None, - MotionVectorPrepass, - PanOrbitCamera { - allow_upside_down: true, - orbit_smoothness: 0.1, - pan_smoothness: 0.1, - zoom_smoothness: 0.1, - ..default() - }, - GaussianCamera::default(), - )); }