From 3bc874e0965dd70e05ab2e5851ad8a4bf1f931e5 Mon Sep 17 00:00:00 2001 From: OJ Kwon <1210596+kwonoj@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:01:45 -0800 Subject: [PATCH] feat(tasks-fs): allow to skip subpath invalidation --- crates/node-file-trace/src/lib.rs | 2 +- crates/turbo-tasks-fetch/tests/fetch.rs | 2 +- .../turbo-tasks-fs/examples/hash_directory.rs | 2 +- crates/turbo-tasks-fs/examples/hash_glob.rs | 2 +- crates/turbo-tasks-fs/src/embed/dir.rs | 2 +- crates/turbo-tasks-fs/src/embed/file.rs | 1 + crates/turbo-tasks-fs/src/lib.rs | 33 ++++++++++++++++++- crates/turbopack-cli/src/util.rs | 4 +-- crates/turbopack-tests/tests/execution.rs | 4 +-- crates/turbopack-tests/tests/snapshot.rs | 4 +-- crates/turbopack/benches/node_file_trace.rs | 2 +- crates/turbopack/examples/turbopack.rs | 2 +- crates/turbopack/tests/node-file-trace.rs | 4 ++- 13 files changed, 49 insertions(+), 15 deletions(-) diff --git a/crates/node-file-trace/src/lib.rs b/crates/node-file-trace/src/lib.rs index 23f0d4cfbf282..775c1f3cf765e 100644 --- a/crates/node-file-trace/src/lib.rs +++ b/crates/node-file-trace/src/lib.rs @@ -195,7 +195,7 @@ impl Args { } async fn create_fs(name: &str, root: &str, watch: bool) -> Result>> { - let fs = DiskFileSystem::new(name.to_string(), root.to_string()); + let fs = DiskFileSystem::new(name.to_string(), root.to_string(), vec![]); if watch { fs.await?.start_watching()?; } else { diff --git a/crates/turbo-tasks-fetch/tests/fetch.rs b/crates/turbo-tasks-fetch/tests/fetch.rs index 2e6603e6b6bbe..c7b7e6ef7da53 100644 --- a/crates/turbo-tasks-fetch/tests/fetch.rs +++ b/crates/turbo-tasks-fetch/tests/fetch.rs @@ -142,5 +142,5 @@ async fn errors_on_404() { } fn get_issue_context() -> Vc { - DiskFileSystem::new("root".to_owned(), "/".to_owned()).root() + DiskFileSystem::new("root".to_owned(), "/".to_owned(), vec![]).root() } diff --git a/crates/turbo-tasks-fs/examples/hash_directory.rs b/crates/turbo-tasks-fs/examples/hash_directory.rs index 219113591887f..032e4f4ae667c 100644 --- a/crates/turbo-tasks-fs/examples/hash_directory.rs +++ b/crates/turbo-tasks-fs/examples/hash_directory.rs @@ -30,7 +30,7 @@ async fn main() -> Result<()> { let task = tt.spawn_root_task(|| { Box::pin(async { let root = current_dir().unwrap().to_str().unwrap().to_string(); - let disk_fs = DiskFileSystem::new("project".to_string(), root); + let disk_fs = DiskFileSystem::new("project".to_string(), root, vec![]); disk_fs.await?.start_watching()?; // Smart Pointer cast diff --git a/crates/turbo-tasks-fs/examples/hash_glob.rs b/crates/turbo-tasks-fs/examples/hash_glob.rs index 9471718aa01b5..28e3231682d3f 100644 --- a/crates/turbo-tasks-fs/examples/hash_glob.rs +++ b/crates/turbo-tasks-fs/examples/hash_glob.rs @@ -27,7 +27,7 @@ async fn main() -> Result<()> { let task = tt.spawn_root_task(|| { Box::pin(async { let root = current_dir().unwrap().to_str().unwrap().to_string(); - let disk_fs = DiskFileSystem::new("project".to_string(), root); + let disk_fs = DiskFileSystem::new("project".to_string(), root, vec![]); disk_fs.await?.start_watching()?; // Smart Pointer cast diff --git a/crates/turbo-tasks-fs/src/embed/dir.rs b/crates/turbo-tasks-fs/src/embed/dir.rs index 840719c05f333..90e34fc5b86e7 100644 --- a/crates/turbo-tasks-fs/src/embed/dir.rs +++ b/crates/turbo-tasks-fs/src/embed/dir.rs @@ -11,7 +11,7 @@ pub async fn directory_from_relative_path( name: String, path: String, ) -> Result>> { - let disk_fs = DiskFileSystem::new(name, path); + let disk_fs = DiskFileSystem::new(name, path, vec![]); disk_fs.await?.start_watching()?; Ok(Vc::upcast(disk_fs)) diff --git a/crates/turbo-tasks-fs/src/embed/file.rs b/crates/turbo-tasks-fs/src/embed/file.rs index f4dade0519339..6dcc925f13ae1 100644 --- a/crates/turbo-tasks-fs/src/embed/file.rs +++ b/crates/turbo-tasks-fs/src/embed/file.rs @@ -21,6 +21,7 @@ pub async fn content_from_relative_path( let disk_fs = DiskFileSystem::new( root_path.to_string_lossy().to_string(), root_path.to_string_lossy().to_string(), + vec![], ); disk_fs.await?.start_watching()?; diff --git a/crates/turbo-tasks-fs/src/lib.rs b/crates/turbo-tasks-fs/src/lib.rs index bbb49f19faed7..a929733ac2360 100644 --- a/crates/turbo-tasks-fs/src/lib.rs +++ b/crates/turbo-tasks-fs/src/lib.rs @@ -189,6 +189,11 @@ pub struct DiskFileSystem { #[turbo_tasks(debug_ignore, trace_ignore)] #[serde(skip)] watcher: Arc, + /// Array of paths that should not notify invalidations. + /// `notify` currently doesn't support unwatching subpaths from the root, + /// so underlying we still watches filesystem event but only skips to + /// invalidate. + ignored_subpaths: Vec, } impl DiskFileSystem { @@ -357,6 +362,8 @@ impl DiskFileSystem { #[cfg(not(any(target_os = "macos", target_os = "windows")))] let disk_watcher = self.watcher.clone(); + let ignored_paths = self.ignored_subpaths.clone(); + spawn_thread(move || { let mut batched_invalidate_path = HashSet::new(); let mut batched_invalidate_path_dir = HashSet::new(); @@ -373,6 +380,16 @@ impl DiskFileSystem { match event { Ok(Ok(events)) => { events.iter().for_each(|DebouncedEvent { event: notify_debouncer_full::notify::Event {kind, paths, ..}, .. }| { + let paths: Vec = paths.iter().filter(|p| { + !ignored_paths.iter().any(|ignored| { + p.starts_with(ignored) + }) + }).cloned().collect(); + + if paths.is_empty() { + return; + } + // [NOTE] there is attrs in the `Event` struct, which contains few more metadata like process_id who triggered the event, // or the source we may able to utilize later. match kind { @@ -628,8 +645,21 @@ pub fn path_to_key(path: impl AsRef) -> String { #[turbo_tasks::value_impl] impl DiskFileSystem { + /// Create a new instance of `DiskFileSystem`. + /// # Arguments + /// + /// * `name` - Name of the filesystem. + /// * `root` - Path to the given filesystem's root. + /// * `ignored_subpaths` - A list of subpaths that should not trigger + /// invalidation. This should be a full path, since it is possible that + /// root & project dir is different and requires to ignore specific + /// subpaths from each. #[turbo_tasks::function] - pub async fn new(name: String, root: String) -> Result> { + pub async fn new( + name: String, + root: String, + ignored_subpaths: Vec, + ) -> Result> { mark_stateful(); // create the directory for the filesystem on disk, if it doesn't exist fs::create_dir_all(&root).await?; @@ -642,6 +672,7 @@ impl DiskFileSystem { invalidator_map: Arc::new(InvalidatorMap::new()), dir_invalidator_map: Arc::new(InvalidatorMap::new()), watcher: Default::default(), + ignored_subpaths: ignored_subpaths.iter().map(PathBuf::from).collect(), }; Ok(Self::cell(instance)) diff --git a/crates/turbopack-cli/src/util.rs b/crates/turbopack-cli/src/util.rs index bca5920c41a64..1c230e22d5be0 100644 --- a/crates/turbopack-cli/src/util.rs +++ b/crates/turbopack-cli/src/util.rs @@ -61,14 +61,14 @@ pub fn normalize_entries(entries: &Option>) -> Vec { #[turbo_tasks::function] pub async fn project_fs(project_dir: String) -> Result>> { - let disk_fs = DiskFileSystem::new("project".to_string(), project_dir.to_string()); + let disk_fs = DiskFileSystem::new("project".to_string(), project_dir.to_string(), vec![]); disk_fs.await?.start_watching()?; Ok(Vc::upcast(disk_fs)) } #[turbo_tasks::function] pub async fn output_fs(project_dir: String) -> Result>> { - let disk_fs = DiskFileSystem::new("output".to_string(), project_dir.to_string()); + let disk_fs = DiskFileSystem::new("output".to_string(), project_dir.to_string(), vec![]); disk_fs.await?.start_watching()?; Ok(Vc::upcast(disk_fs)) } diff --git a/crates/turbopack-tests/tests/execution.rs b/crates/turbopack-tests/tests/execution.rs index ad4ef70c6f827..f0a780aa7775a 100644 --- a/crates/turbopack-tests/tests/execution.rs +++ b/crates/turbopack-tests/tests/execution.rs @@ -187,8 +187,8 @@ async fn prepare_test(resource: String) -> Result> { resource_path.to_str().unwrap() ); - let root_fs = DiskFileSystem::new("workspace".to_string(), REPO_ROOT.clone()); - let project_fs = DiskFileSystem::new("project".to_string(), REPO_ROOT.clone()); + let root_fs = DiskFileSystem::new("workspace".to_string(), REPO_ROOT.clone(), vec![]); + let project_fs = DiskFileSystem::new("project".to_string(), REPO_ROOT.clone(), vec![]); let project_root = project_fs.root(); let relative_path = resource_path.strip_prefix(&*REPO_ROOT).context(format!( diff --git a/crates/turbopack-tests/tests/snapshot.rs b/crates/turbopack-tests/tests/snapshot.rs index 6bac2add586e7..04c2b39b03bf8 100644 --- a/crates/turbopack-tests/tests/snapshot.rs +++ b/crates/turbopack-tests/tests/snapshot.rs @@ -183,8 +183,8 @@ async fn run_test(resource: String) -> Result> { Err(_) => SnapshotOptions::default(), Ok(options_str) => parse_json_with_source_context(&options_str).unwrap(), }; - let root_fs = DiskFileSystem::new("workspace".to_string(), REPO_ROOT.clone()); - let project_fs = DiskFileSystem::new("project".to_string(), REPO_ROOT.clone()); + let root_fs = DiskFileSystem::new("workspace".to_string(), REPO_ROOT.clone(), vec![]); + let project_fs = DiskFileSystem::new("project".to_string(), REPO_ROOT.clone(), vec![]); let project_root = project_fs.root(); let relative_path = test_path.strip_prefix(&*REPO_ROOT)?; diff --git a/crates/turbopack/benches/node_file_trace.rs b/crates/turbopack/benches/node_file_trace.rs index cb1177ee6328e..2df32ff591e04 100644 --- a/crates/turbopack/benches/node_file_trace.rs +++ b/crates/turbopack/benches/node_file_trace.rs @@ -71,7 +71,7 @@ fn bench_emit(b: &mut Bencher, bench_input: &BenchInput) { let input = bench_input.input.clone(); async move { let task = tt.spawn_once_task(async move { - let input_fs = DiskFileSystem::new("tests".to_string(), tests_root.clone()); + let input_fs = DiskFileSystem::new("tests".to_string(), tests_root.clone(), vec![]); let input = input_fs.root().join(input.clone()); let input_dir = input.parent().parent(); diff --git a/crates/turbopack/examples/turbopack.rs b/crates/turbopack/examples/turbopack.rs index ec850187ad1c3..a3e8752b960eb 100644 --- a/crates/turbopack/examples/turbopack.rs +++ b/crates/turbopack/examples/turbopack.rs @@ -38,7 +38,7 @@ async fn main() -> Result<()> { let task = tt.spawn_root_task(|| { Box::pin(async { let root = current_dir().unwrap().to_str().unwrap().to_string(); - let disk_fs = DiskFileSystem::new(PROJECT_FILESYSTEM_NAME.to_string(), root); + let disk_fs = DiskFileSystem::new(PROJECT_FILESYSTEM_NAME.to_string(), root, vec![]); disk_fs.await?.start_watching()?; // Smart Pointer cast diff --git a/crates/turbopack/tests/node-file-trace.rs b/crates/turbopack/tests/node-file-trace.rs index 716d54bac1efd..9c050adff889e 100644 --- a/crates/turbopack/tests/node-file-trace.rs +++ b/crates/turbopack/tests/node-file-trace.rs @@ -403,6 +403,7 @@ fn node_file_trace( let workspace_fs: Vc> = Vc::upcast(DiskFileSystem::new( "workspace".to_string(), package_root.clone(), + vec![], )); let input_dir = workspace_fs.root(); let input = input_dir.join(format!("tests/{input_string}")); @@ -410,7 +411,8 @@ fn node_file_trace( #[cfg(not(feature = "bench_against_node_nft"))] let original_output = exec_node(package_root, input); - let output_fs = DiskFileSystem::new("output".to_string(), directory.clone()); + let output_fs = + DiskFileSystem::new("output".to_string(), directory.clone(), vec![]); let output_dir = output_fs.root(); let source = FileSource::new(input);