diff --git a/crates/turborepo-filewatch/src/hash_watcher.rs b/crates/turborepo-filewatch/src/hash_watcher.rs index 14897ce8a6f6e..81df500b83072 100644 --- a/crates/turborepo-filewatch/src/hash_watcher.rs +++ b/crates/turborepo-filewatch/src/hash_watcher.rs @@ -17,7 +17,7 @@ use tokio::{ use tracing::{debug, trace}; use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPath, AnchoredSystemPathBuf}; use turborepo_repository::discovery::DiscoveryResponse; -use turborepo_scm::{package_deps::INPUT_INCLUDE_DEFAULT_FILES, Error as SCMError, GitHashes, SCM}; +use turborepo_scm::{Error as SCMError, GitHashes, SCM}; use crate::{ debouncer::Debouncer, @@ -41,15 +41,15 @@ pub enum InputGlobs { } impl InputGlobs { - pub fn from_raw(mut raw: Vec) -> Result { + pub fn from_raw(raw: Vec, include_default: bool) -> Result { if raw.is_empty() { - Ok(Self::Default) - } else if let Some(default_pos) = raw.iter().position(|g| g == INPUT_INCLUDE_DEFAULT_FILES) - { - raw.remove(default_pos); - Ok(Self::DefaultWithExtras(GlobSet::from_raw_unfiltered(raw)?)) + return Ok(Self::Default); + } + let glob_set = GlobSet::from_raw_unfiltered(raw)?; + if include_default { + Ok(Self::DefaultWithExtras(glob_set)) } else { - Ok(Self::Specific(GlobSet::from_raw_unfiltered(raw)?)) + Ok(Self::Specific(glob_set)) } } @@ -64,12 +64,16 @@ impl InputGlobs { fn as_inputs(&self) -> Vec { match self { InputGlobs::Default => Vec::new(), - InputGlobs::DefaultWithExtras(glob_set) => { - let mut inputs = glob_set.as_inputs(); - inputs.push(INPUT_INCLUDE_DEFAULT_FILES.to_string()); - inputs + InputGlobs::DefaultWithExtras(glob_set) | InputGlobs::Specific(glob_set) => { + glob_set.as_inputs() } - InputGlobs::Specific(glob_set) => glob_set.as_inputs(), + } + } + + pub fn include_default_files(&self) -> bool { + match self { + InputGlobs::Default | InputGlobs::DefaultWithExtras(..) => true, + InputGlobs::Specific(..) => false, } } } @@ -542,6 +546,7 @@ impl Subscriber { &repo_root, &spec.package_path, &inputs, + spec.inputs.include_default_files(), telemetry, ); trace!("hashing complete for {:?}", spec); diff --git a/crates/turborepo-lib/src/daemon/client.rs b/crates/turborepo-lib/src/daemon/client.rs index 05f5760ca0dcb..d1d63f24e7072 100644 --- a/crates/turborepo-lib/src/daemon/client.rs +++ b/crates/turborepo-lib/src/daemon/client.rs @@ -13,7 +13,10 @@ use super::{ proto::{DiscoverPackagesResponse, GetFileHashesResponse}, Paths, }; -use crate::daemon::{proto, proto::PackageChangeEvent}; +use crate::{ + daemon::proto::{self, PackageChangeEvent}, + task_graph::TaskInputs, +}; #[derive(Debug, Clone)] pub struct DaemonClient { @@ -160,13 +163,14 @@ impl DaemonClient { pub async fn get_file_hashes( &mut self, package_path: &AnchoredSystemPath, - inputs: &[String], + inputs: &TaskInputs, ) -> Result { let response = self .client .get_file_hashes(proto::GetFileHashesRequest { package_path: package_path.to_string(), - input_globs: inputs.to_vec(), + input_globs: inputs.globs.to_vec(), + include_default: Some(inputs.default), }) .await? .into_inner(); diff --git a/crates/turborepo-lib/src/daemon/proto/turbod.proto b/crates/turborepo-lib/src/daemon/proto/turbod.proto index f59c9eccd8a8e..dc4e1ff7c08d4 100644 --- a/crates/turborepo-lib/src/daemon/proto/turbod.proto +++ b/crates/turborepo-lib/src/daemon/proto/turbod.proto @@ -141,6 +141,7 @@ message GetFileHashesRequest { // AnchoredSystemPathBuf string package_path = 1; repeated string input_globs = 2; + optional bool include_default = 3; } message GetFileHashesResponse { diff --git a/crates/turborepo-lib/src/daemon/server.rs b/crates/turborepo-lib/src/daemon/server.rs index 12f0412e90fd8..fcea42d7000c8 100644 --- a/crates/turborepo-lib/src/daemon/server.rs +++ b/crates/turborepo-lib/src/daemon/server.rs @@ -386,8 +386,9 @@ impl TurboGrpcServiceInner { &self, package_path: String, inputs: Vec, + include_default: bool, ) -> Result, RpcError> { - let inputs = InputGlobs::from_raw(inputs)?; + let inputs = InputGlobs::from_raw(inputs, include_default)?; let package_path = AnchoredSystemPathBuf::try_from(package_path.as_str()) .map_err(|e| RpcError::InvalidAnchoredPath(package_path, e))?; let hash_spec = HashSpec { @@ -549,7 +550,13 @@ impl proto::turbod_server::Turbod for TurboGrpcServiceInner { ) -> Result, tonic::Status> { let inner = request.into_inner(); let file_hashes = self - .get_file_hashes(inner.package_path, inner.input_globs) + .get_file_hashes( + inner.package_path, + inner.input_globs, + // If an old client attempts to talk to a new server, assume that we should watch + // default files + inner.include_default.unwrap_or(true), + ) .await?; Ok(tonic::Response::new(proto::GetFileHashesResponse { file_hashes, diff --git a/crates/turborepo-lib/src/run/cache.rs b/crates/turborepo-lib/src/run/cache.rs index 4fbbf3ff0697e..8289be226e601 100644 --- a/crates/turborepo-lib/src/run/cache.rs +++ b/crates/turborepo-lib/src/run/cache.rs @@ -506,11 +506,11 @@ impl ConfigCache { // empty inputs to get all files let inputs: Vec = vec![]; - let hash_object = match scm.get_package_file_hashes(repo_root, anchored_root, &inputs, None) - { - Ok(hash_object) => hash_object, - Err(_) => return Err(CacheError::ConfigCacheError), - }; + let hash_object = + match scm.get_package_file_hashes(repo_root, anchored_root, &inputs, false, None) { + Ok(hash_object) => hash_object, + Err(_) => return Err(CacheError::ConfigCacheError), + }; // return the hash Ok(FileHashes(hash_object).hash()) diff --git a/crates/turborepo-lib/src/run/summary/task.rs b/crates/turborepo-lib/src/run/summary/task.rs index e68ea05bd772f..6c57b97db23a8 100644 --- a/crates/turborepo-lib/src/run/summary/task.rs +++ b/crates/turborepo-lib/src/run/summary/task.rs @@ -311,13 +311,13 @@ impl From for TaskSummaryTaskDefinition { depends_on.sort(); outputs.sort(); env.sort(); - inputs.sort(); + inputs.globs.sort(); Self { outputs, cache, depends_on, - inputs, + inputs: inputs.globs, output_logs, persistent, interruptible, diff --git a/crates/turborepo-lib/src/task_graph/mod.rs b/crates/turborepo-lib/src/task_graph/mod.rs index d535d90f23470..0f3b361b8c621 100644 --- a/crates/turborepo-lib/src/task_graph/mod.rs +++ b/crates/turborepo-lib/src/task_graph/mod.rs @@ -11,40 +11,8 @@ pub use visitor::{Error as VisitorError, Visitor}; use crate::{ cli::{EnvMode, OutputLogsMode}, run::task_id::{TaskId, TaskName}, - turbo_json::RawTaskDefinition, }; -// TaskOutputs represents the patterns for including and excluding files from -// outputs -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct TaskOutputs { - pub inclusions: Vec, - pub exclusions: Vec, -} - -impl TaskOutputs { - // We consider an empty outputs to be a log output and nothing else - pub fn is_empty(&self) -> bool { - self.inclusions.len() == 1 - && self.inclusions[0].ends_with(".log") - && self.exclusions.is_empty() - } - - pub fn validated_inclusions(&self) -> Result, GlobError> { - self.inclusions - .iter() - .map(|i| ValidatedGlob::from_str(i)) - .collect() - } - - pub fn validated_exclusions(&self) -> Result, GlobError> { - self.exclusions - .iter() - .map(|e| ValidatedGlob::from_str(e)) - .collect() - } -} - // Constructed from a RawTaskDefinition #[derive(Debug, PartialEq, Clone, Eq)] pub struct TaskDefinition { @@ -70,10 +38,10 @@ pub struct TaskDefinition { // Inputs indicate the list of files this Task depends on. If any of those files change // we can conclude that any cached outputs or logs for this Task should be invalidated. - pub(crate) inputs: Vec, + pub inputs: TaskInputs, // OutputMode determines how we should log the output. - pub(crate) output_logs: OutputLogsMode, + pub output_logs: OutputLogsMode, // Persistent indicates whether the Task is expected to exit or not // Tasks marked Persistent do not exit (e.g. watch mode or dev servers) @@ -98,6 +66,22 @@ pub struct TaskDefinition { pub with: Option>>>, } +// TaskOutputs represents the patterns for including and excluding files from +// outputs +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct TaskOutputs { + pub inclusions: Vec, + pub exclusions: Vec, +} + +// Structure for holding the inputs for a task +#[derive(Debug, PartialEq, Clone, Eq, Default)] +pub struct TaskInputs { + pub globs: Vec, + // Set when $TURBO_DEFAULT$ is in inputs + pub default: bool, +} + impl Default for TaskDefinition { fn default() -> Self { Self { @@ -118,16 +102,6 @@ impl Default for TaskDefinition { } } -impl FromIterator for RawTaskDefinition { - fn from_iter>(iter: T) -> Self { - iter.into_iter() - .fold(RawTaskDefinition::default(), |mut def, other| { - def.merge(other); - def - }) - } -} - const LOG_DIR: &str = ".turbo"; impl TaskDefinition { @@ -190,6 +164,43 @@ impl TaskDefinition { } } +impl TaskInputs { + pub fn new(globs: Vec) -> Self { + Self { + globs, + default: false, + } + } + + pub fn with_default(mut self, default: bool) -> Self { + self.default = default; + self + } +} + +impl TaskOutputs { + // We consider an empty outputs to be a log output and nothing else + pub fn is_empty(&self) -> bool { + self.inclusions.len() == 1 + && self.inclusions[0].ends_with(".log") + && self.exclusions.is_empty() + } + + pub fn validated_inclusions(&self) -> Result, GlobError> { + self.inclusions + .iter() + .map(|i| ValidatedGlob::from_str(i)) + .collect() + } + + pub fn validated_exclusions(&self) -> Result, GlobError> { + self.exclusions + .iter() + .map(|e| ValidatedGlob::from_str(e)) + .collect() + } +} + fn task_log_filename(task_name: &str) -> String { format!("turbo-{}.log", task_name.replace(':', "$colon$")) } diff --git a/crates/turborepo-lib/src/task_hash.rs b/crates/turborepo-lib/src/task_hash.rs index 3f894719a5a06..30c179a3dea95 100644 --- a/crates/turborepo-lib/src/task_hash.rs +++ b/crates/turborepo-lib/src/task_hash.rs @@ -190,7 +190,8 @@ impl PackageInputsHashes { let local_hash_result = scm.get_package_file_hashes( repo_root, package_path, - &task_definition.inputs, + &task_definition.inputs.globs, + task_definition.inputs.default, Some(scm_telemetry), ); match local_hash_result { @@ -541,7 +542,7 @@ pub fn get_internal_deps_hash( let file_hashes = package_dirs .into_par_iter() - .map(|package_dir| scm.get_package_file_hashes::<&str>(root, package_dir, &[], None)) + .map(|package_dir| scm.get_package_file_hashes::<&str>(root, package_dir, &[], false, None)) .reduce( || Ok(HashMap::new()), |acc, hashes| { diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index f92b6dea976b1..c4bed46446a02 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -23,7 +23,7 @@ use crate::{ task_access::TaskAccessTraceFile, task_id::{TaskId, TaskName}, }, - task_graph::{TaskDefinition, TaskOutputs}, + task_graph::{TaskDefinition, TaskInputs, TaskOutputs}, }; mod loader; @@ -35,6 +35,7 @@ use crate::{boundaries::BoundariesConfig, config::UnnecessaryPackageTaskSyntaxEr const TURBO_ROOT: &str = "$TURBO_ROOT$"; const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; +pub const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; pub const CONFIG_FILE: &str = "turbo.json"; pub const CONFIG_FILE_JSONC: &str = "turbo.jsonc"; @@ -400,6 +401,46 @@ impl TryFrom>> for TaskOutputs { } } +impl FromIterator for RawTaskDefinition { + fn from_iter>(iter: T) -> Self { + iter.into_iter() + .fold(RawTaskDefinition::default(), |mut def, other| { + def.merge(other); + def + }) + } +} + +impl TryFrom>>> for TaskInputs { + type Error = Error; + + fn try_from(inputs: Option>>) -> Result { + let mut globs = Vec::with_capacity(inputs.as_ref().map_or(0, |inputs| inputs.len())); + + let mut default = false; + for input in inputs.into_iter().flatten() { + // If the inputs contain "$TURBO_DEFAULT$", we need to include the "default" + // file hashes as well. NOTE: we intentionally don't remove + // "$TURBO_DEFAULT$" from the inputs if it exists in the off chance that + // the user has a file named "$TURBO_DEFAULT$" in their package (pls + // no). + if input.as_str() == TURBO_DEFAULT { + default = true; + } else if Utf8Path::new(input.as_str()).is_absolute() { + let (span, text) = input.span_and_text("turbo.json"); + return Err(Error::AbsolutePathInConfig { + field: "inputs", + span, + text, + }); + } + globs.push(input.to_string()); + } + + Ok(TaskInputs { globs, default }) + } +} + impl TaskDefinition { pub fn from_raw( mut raw_task: RawTaskDefinition, @@ -468,23 +509,7 @@ impl TaskDefinition { .transpose()? .unwrap_or_default(); - let inputs = raw_task - .inputs - .unwrap_or_default() - .into_iter() - .map(|input| { - if Utf8Path::new(&input.value).is_absolute() { - let (span, text) = input.span_and_text("turbo.json"); - Err(Error::AbsolutePathInConfig { - field: "inputs", - span, - text, - }) - } else { - Ok(input.to_string()) - } - }) - .collect::, _>>()?; + let inputs = TaskInputs::try_from(raw_task.inputs)?; let pass_through_env = raw_task .pass_through_env @@ -916,7 +941,7 @@ fn replace_turbo_root_token( #[cfg(test)] mod tests { - use std::sync::Arc; + use std::{assert_matches::assert_matches, sync::Arc}; use anyhow::Result; use biome_deserialize::json::deserialize_from_json_str; @@ -929,7 +954,7 @@ mod tests { use super::{ replace_turbo_root_token_in_string, validate_with_has_no_topo, FutureFlags, Pipeline, - RawTurboJson, SpacesJson, Spanned, TurboJson, UIMode, + RawTurboJson, SpacesJson, Spanned, TurboJson, UIMode, *, }; use crate::{ boundaries::BoundariesConfig, @@ -1058,7 +1083,7 @@ mod tests { exclusions: vec![], }, cache: false, - inputs: vec!["package/a/src/**".to_string()], + inputs: TaskInputs::new(vec!["package/a/src/**".to_string()]), output_logs: OutputLogsMode::Full, pass_through_env: Some(vec!["AWS_SECRET_KEY".to_string()]), task_dependencies: vec![Spanned::>::new("cli#build".into()).with_range(26..37)], @@ -1104,7 +1129,7 @@ mod tests { exclusions: vec![], }, cache: false, - inputs: vec!["package\\a\\src\\**".to_string()], + inputs: TaskInputs::new(vec!["package\\a\\src\\**".to_string()]), output_logs: OutputLogsMode::Full, pass_through_env: Some(vec!["AWS_SECRET_KEY".to_string()]), task_dependencies: vec![Spanned::>::new("cli#build".into()).with_range(30..41)], @@ -1131,7 +1156,7 @@ mod tests { ..RawTaskDefinition::default() }, TaskDefinition { - inputs: vec!["../../config.txt".to_owned()], + inputs: TaskInputs::new(vec!["../../config.txt".to_owned()]), outputs: TaskOutputs { inclusions: vec!["../../coverage/**".to_owned()], exclusions: vec!["../../coverage/index.html".to_owned()], @@ -1505,4 +1530,36 @@ mod tests { "`with` cannot use dependency relationships." ); } + + #[test] + fn test_absolute_paths_error_in_inputs() { + assert_matches!( + TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( + if cfg!(windows) { + "C:\\win32" + } else { + "/dev/null" + } + ))])), + Err(Error::AbsolutePathInConfig { .. }) + ); + } + + #[test] + fn test_detects_turbo_default() { + let inputs = TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( + TURBO_DEFAULT, + ))])) + .unwrap(); + assert!(inputs.default); + } + + #[test] + fn test_keeps_turbo_default() { + let inputs = TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( + TURBO_DEFAULT, + ))])) + .unwrap(); + assert_eq!(inputs.globs, vec![TURBO_DEFAULT.to_string()]); + } } diff --git a/crates/turborepo-scm/src/package_deps.rs b/crates/turborepo-scm/src/package_deps.rs index d49fd385a8e14..2cf7cb504e7da 100644 --- a/crates/turborepo-scm/src/package_deps.rs +++ b/crates/turborepo-scm/src/package_deps.rs @@ -10,8 +10,6 @@ use turborepo_telemetry::events::task::{FileHashMethod, PackageTaskEventBuilder} use crate::hash_object::hash_objects; use crate::{Error, GitHashes, GitRepo, SCM}; -pub const INPUT_INCLUDE_DEFAULT_FILES: &str = "$TURBO_DEFAULT$"; - impl SCM { pub fn get_hashes_for_files( &self, @@ -32,17 +30,9 @@ impl SCM { turbo_root: &AbsoluteSystemPath, package_path: &AnchoredSystemPath, inputs: &[S], + include_default_files: bool, telemetry: Option, ) -> Result { - // If the inputs contain "$TURBO_DEFAULT$", we need to include the "default" - // file hashes as well. NOTE: we intentionally don't remove - // "$TURBO_DEFAULT$" from the inputs if it exists in the off chance that - // the user has a file named "$TURBO_DEFAULT$" in their package (pls - // no). - let include_default_files = inputs - .iter() - .any(|input| input.as_ref() == INPUT_INCLUDE_DEFAULT_FILES); - match self { SCM::Manual => { if let Some(telemetry) = telemetry { @@ -379,6 +369,7 @@ mod tests { &repo_root, &pkg_path, &[], + false, Some(PackageTaskEventBuilder::new("my-pkg", "test")), ) .unwrap(); @@ -508,13 +499,15 @@ mod tests { "2f26c7b914476b3c519e4f0fbc0d16c52a60d178".to_string(), ); - let input_tests: &[(&[&str], &[&str])] = &[ + let input_tests: &[(&[&str], bool, &[&str])] = &[ ( &["uncommitted-file"], + false, &["package.json", "turbo.json", "uncommitted-file"], ), ( &["**/*-file"], + false, &[ "committed-file", "uncommitted-file", @@ -526,6 +519,7 @@ mod tests { ), ( &["../**/*-file"], + false, &[ "committed-file", "uncommitted-file", @@ -538,6 +532,7 @@ mod tests { ), ( &["**/{uncommitted,committed}-file"], + false, &[ "committed-file", "uncommitted-file", @@ -547,6 +542,7 @@ mod tests { ), ( &["../**/{new-root,uncommitted,committed}-file"], + false, &[ "committed-file", "uncommitted-file", @@ -557,6 +553,7 @@ mod tests { ), ( &["$TURBO_DEFAULT$"], + true, &[ "committed-file", "uncommitted-file", @@ -568,6 +565,7 @@ mod tests { ), ( &["$TURBO_DEFAULT$", "!dir/*"], + true, &[ "committed-file", "uncommitted-file", @@ -578,6 +576,7 @@ mod tests { ), ( &["$TURBO_DEFAULT$", "!committed-file", "dir/ignored-file"], + true, &[ "uncommitted-file", "package.json", @@ -589,6 +588,7 @@ mod tests { ), ( &["!committed-file", "$TURBO_DEFAULT$", "dir/ignored-file"], + true, &[ "uncommitted-file", "package.json", @@ -599,18 +599,15 @@ mod tests { ], ), ]; - for (inputs, expected_files) in input_tests { + for (inputs, include_default_files, expected_files) in input_tests { let expected: GitHashes = HashMap::from_iter(expected_files.iter().map(|key| { let key = RelativeUnixPathBuf::new(*key).unwrap(); let value = all_expected.get(&key).unwrap().clone(); (key, value) })); - let include_default_files = inputs - .iter() - .any(|input| input == &INPUT_INCLUDE_DEFAULT_FILES); let hashes = git - .get_package_file_hashes(&repo_root, &package_path, inputs, include_default_files) + .get_package_file_hashes(&repo_root, &package_path, inputs, *include_default_files) .unwrap(); assert_eq!(hashes, expected); }