diff --git a/crates/turborepo-lib/src/config/env.rs b/crates/turborepo-lib/src/config/env.rs index 28afe8aba1909..26fb21395e2c1 100644 --- a/crates/turborepo-lib/src/config/env.rs +++ b/crates/turborepo-lib/src/config/env.rs @@ -255,6 +255,8 @@ impl ResolvedConfigurationOptions for EnvVars { root_turbo_json_path, log_order, sso_login_callback_port, + // Do not allow future flags to be set by env var + future_flags: None, }; Ok(output) diff --git a/crates/turborepo-lib/src/config/mod.rs b/crates/turborepo-lib/src/config/mod.rs index f7e9f497d03f2..b3a4cbc93c1d8 100644 --- a/crates/turborepo-lib/src/config/mod.rs +++ b/crates/turborepo-lib/src/config/mod.rs @@ -23,8 +23,11 @@ use turborepo_cache::CacheConfig; use turborepo_errors::TURBO_SITE; use turborepo_repository::package_graph::PackageName; -use crate::cli::{EnvMode, LogOrder}; pub use crate::turbo_json::{RawTurboJson, UIMode}; +use crate::{ + cli::{EnvMode, LogOrder}, + turbo_json::FutureFlags, +}; pub const CONFIG_FILE: &str = "turbo.json"; pub const CONFIG_FILE_JSONC: &str = "turbo.jsonc"; @@ -312,6 +315,8 @@ pub struct ConfigurationOptions { pub(crate) concurrency: Option, pub(crate) no_update_notifier: Option, pub(crate) sso_login_callback_port: Option, + #[serde(skip)] + future_flags: Option, } #[derive(Default)] @@ -471,6 +476,10 @@ impl ConfigurationOptions { pub fn sso_login_callback_port(&self) -> Option { self.sso_login_callback_port } + + pub fn future_flags(&self) -> FutureFlags { + self.future_flags.unwrap_or_default() + } } // Maps Some("") to None to emulate how Go handles empty strings diff --git a/crates/turborepo-lib/src/config/turbo_json.rs b/crates/turborepo-lib/src/config/turbo_json.rs index 269474a75fbcb..ba5e81bd78810 100644 --- a/crates/turborepo-lib/src/config/turbo_json.rs +++ b/crates/turborepo-lib/src/config/turbo_json.rs @@ -78,6 +78,7 @@ impl<'a> TurboJsonReader<'a> { opts.env_mode = turbo_json.env_mode.map(|mode| *mode.as_inner()); opts.cache_dir = cache_dir; opts.concurrency = turbo_json.concurrency.map(|c| c.as_inner().clone()); + opts.future_flags = turbo_json.future_flags.map(|f| *f.as_inner()); Ok(opts) } } diff --git a/crates/turborepo-lib/src/engine/builder.rs b/crates/turborepo-lib/src/engine/builder.rs index 1cf95f1cbcdc5..82cd8f4abd45f 100644 --- a/crates/turborepo-lib/src/engine/builder.rs +++ b/crates/turborepo-lib/src/engine/builder.rs @@ -15,7 +15,7 @@ use crate::{ task_graph::TaskDefinition, turbo_json::{ validate_extends, validate_no_package_task_syntax, validate_with_has_no_topo, - RawTaskDefinition, TurboJsonLoader, + ProcessedTaskDefinition, TurboJsonLoader, }, }; @@ -564,14 +564,12 @@ impl<'a> EngineBuilder<'a> { task_id: &Spanned, task_name: &TaskName, ) -> Result { - let raw_task_definition = RawTaskDefinition::from_iter(self.task_definition_chain( - turbo_json_loader, - task_id, - task_name, - )?); + let processed_task_definition = ProcessedTaskDefinition::from_iter( + self.task_definition_chain(turbo_json_loader, task_id, task_name)?, + ); let path_to_root = self.path_to_root(task_id.as_inner())?; - Ok(TaskDefinition::from_raw( - raw_task_definition, + Ok(TaskDefinition::from_processed( + processed_task_definition, &path_to_root, )?) } @@ -581,13 +579,13 @@ impl<'a> EngineBuilder<'a> { turbo_json_loader: &TurboJsonLoader, task_id: &Spanned, task_name: &TaskName, - ) -> Result, Error> { + ) -> Result, Error> { let mut task_definitions = Vec::new(); let root_turbo_json = turbo_json_loader.load(&PackageName::Root)?; Error::from_validation(root_turbo_json.validate(&[validate_with_has_no_topo]))?; - if let Some(root_definition) = root_turbo_json.task(task_id, task_name) { + if let Some(root_definition) = root_turbo_json.task(task_id, task_name)? { task_definitions.push(root_definition) } @@ -616,8 +614,8 @@ impl<'a> EngineBuilder<'a> { validate_with_has_no_topo, ]))?; - if let Some(workspace_def) = workspace_json.tasks.get(task_name) { - task_definitions.push(workspace_def.value.clone()); + if let Some(workspace_def) = workspace_json.task(task_id, task_name)? { + task_definitions.push(workspace_def); } } Err(config::Error::NoTurboJSON) => (), diff --git a/crates/turborepo-lib/src/opts.rs b/crates/turborepo-lib/src/opts.rs index 3025fa41f7d8f..173fc6960d272 100644 --- a/crates/turborepo-lib/src/opts.rs +++ b/crates/turborepo-lib/src/opts.rs @@ -14,7 +14,7 @@ use crate::{ OutputLogsMode, RunArgs, }, config::{ConfigurationOptions, CONFIG_FILE}, - turbo_json::UIMode, + turbo_json::{FutureFlags, UIMode}, Args, }; @@ -77,6 +77,7 @@ pub struct Opts { pub runcache_opts: RunCacheOpts, pub scope_opts: ScopeOpts, pub tui_opts: TuiOpts, + pub future_flags: FutureFlags, } impl Opts { @@ -180,6 +181,7 @@ impl Opts { let api_client_opts = APIClientOpts::from(inputs); let repo_opts = RepoOpts::from(inputs); let tui_opts = TuiOpts::from(inputs); + let future_flags = config.future_flags(); Ok(Self { repo_opts, @@ -189,6 +191,7 @@ impl Opts { runcache_opts, api_client_opts, tui_opts, + future_flags, }) } } @@ -747,6 +750,7 @@ mod test { cache_opts, runcache_opts, tui_opts, + future_flags: Default::default(), }; let synthesized = opts.synthesize_command(); assert_eq!(synthesized, expected); diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index 7dcd7b4678cb0..12c8f71a2af4f 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -25,7 +25,7 @@ use turborepo_scm::GitHashes; use crate::{ config::{resolve_turbo_config_path, CONFIG_FILE, CONFIG_FILE_JSONC}, - turbo_json::{TurboJson, TurboJsonLoader}, + turbo_json::{TurboJson, TurboJsonLoader, TurboJsonReader}, }; #[derive(Clone)] @@ -232,7 +232,7 @@ impl Subscriber { }; let root_turbo_json = TurboJsonLoader::workspace( - self.repo_root.clone(), + TurboJsonReader::new(self.repo_root.clone()), config_path, pkg_dep_graph.packages(), ) diff --git a/crates/turborepo-lib/src/run/builder.rs b/crates/turborepo-lib/src/run/builder.rs index 08b9e0e8e50fb..52c2801f33f8d 100644 --- a/crates/turborepo-lib/src/run/builder.rs +++ b/crates/turborepo-lib/src/run/builder.rs @@ -49,7 +49,7 @@ use crate::{ opts::Opts, run::{scope, task_access::TaskAccess, Error, Run, RunCache}, shim::TurboState, - turbo_json::{TurboJson, TurboJsonLoader, UIMode}, + turbo_json::{TurboJson, TurboJsonLoader, TurboJsonReader, UIMode}, DaemonConnector, }; @@ -408,16 +408,19 @@ impl RunBuilder { task_access.restore_config().await; let root_turbo_json_path = self.opts.repo_opts.root_turbo_json_path.clone(); + let future_flags = self.opts.future_flags; + + let reader = TurboJsonReader::new(self.repo_root.clone()).with_future_flags(future_flags); let turbo_json_loader = if task_access.is_enabled() { TurboJsonLoader::task_access( - self.repo_root.clone(), + reader, root_turbo_json_path.clone(), root_package_json.clone(), ) } else if is_single_package { TurboJsonLoader::single_package( - self.repo_root.clone(), + reader, root_turbo_json_path.clone(), root_package_json.clone(), ) @@ -426,20 +429,20 @@ impl RunBuilder { (self.opts.repo_opts.allow_no_turbo_json || micro_frontend_configs.is_some()) { TurboJsonLoader::workspace_no_turbo_json( - self.repo_root.clone(), + reader, pkg_dep_graph.packages(), micro_frontend_configs.clone(), ) } else if let Some(micro_frontends) = µ_frontend_configs { TurboJsonLoader::workspace_with_microfrontends( - self.repo_root.clone(), + reader, root_turbo_json_path.clone(), pkg_dep_graph.packages(), micro_frontends.clone(), ) } else { TurboJsonLoader::workspace( - self.repo_root.clone(), + reader, root_turbo_json_path.clone(), pkg_dep_graph.packages(), ) diff --git a/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md b/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md index 9728055952ae4..b920e8b76a8ba 100644 --- a/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md +++ b/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md @@ -9,9 +9,10 @@ The loading and resolving of task definitions is driven during task graph constr The configuration and task loading process follows this high-level flow: 1. **Configuration Resolution**: Collect and merge configuration from multiple sources -2. **TurboJson Loading**: Resolve `turbo.json` files, these are usually files on disk, but can be synthesized -3. **Task Definition Resolution**: Convert raw task definitions into validated structures. `extends` is handled in this step -4. **Task Graph Construction**: Build the executable task graph from resolved definitions +2. **TurboJson Loading**: Parse `turbo.json` files into raw structures +3. **Task Processing**: Convert raw definitions to processed intermediate representation with DSL token handling +4. **Task Definition Resolution**: Transform processed definitions into final validated structures +5. **Task Graph Construction**: Build the executable task graph from resolved definitions ## Phase 1: Configuration Resolution @@ -39,30 +40,66 @@ Configuration is collected from multiple sources with the following priority (hi - **`TurboJsonLoader`** (`crates/turborepo-lib/src/turbo_json/loader.rs`): Loads and resolves turbo.json files - **`RawTurboJson`**: Raw deserialized structure from JSON files -- **`TurboJson`**: Resolved and validated structure, all DSL magic strings have been handled +- **`TurboJson`**: Validated structure containing raw task definitions ### Process 1. **File Discovery**: Locate `turbo.json` or `turbo.jsonc` files 2. **Parsing**: Deserialize JSON into `RawTurboJson` structures -3. **Validation**: Convert to `TurboJson` with validation rules +3. **Basic Validation**: Convert to `TurboJson` with structural validation 4. **Workspace Resolution**: Apply workspace-specific overrides -## Phase 3: Task Definition Resolution +## Phase 3: Processed Task Definition (Intermediate Representation) + +### Key Components + +- **`ProcessedTaskDefinition`** (`crates/turborepo-lib/src/turbo_json/processed.rs`): Intermediate representation with DSL token processing +- **`ProcessedGlob`**: Parsed glob patterns with separated components (base pattern, negation flag, turbo_root flag) +- **`ProcessedInputs`/`ProcessedOutputs`**: Collections of processed globs with resolution methods + +### Processing Steps + +1. **DSL Token Detection**: Identify and separate `$TURBO_ROOT$` and `!` prefixes from glob patterns +2. **Early Validation**: Single-field validations at parse time with span information: + - Absolute paths in inputs/outputs + - Invalid `$TURBO_ROOT$` usage + - Environment variable prefixes (`$` not allowed) + - Dependency prefixes (`$` not allowed in `dependsOn`) + - Topological references (`^` not allowed in `with`) +3. **Prefix Stripping**: Store clean glob patterns without DSL prefixes +4. **Component Separation**: Track negation and turbo_root requirements as separate boolean flags + +## Phase 4: Task Definition Resolution ### Key Components - **`RawTaskDefinition`**: Raw task configuration from JSON -- **`TaskDefinition`**: Validated and processed task configuration +- **`ProcessedTaskDefinition`**: Intermediate representation with parsed DSL tokens +- **`TaskDefinition`**: Final validated and resolved task configuration - **`TaskId`** and **`TaskName`** (from `turborepo-task-id` crate): Task identification types ### Transformation Process -Raw task definitions undergo several transformations: +The resolution now follows a three-stage pipeline: + +1. **Raw → Processed** (`ProcessedTaskDefinition::from_raw`): + + - Parse glob patterns and extract DSL tokens + - Validate single-field constraints with span information: + - Absolute paths in inputs/outputs (`ProcessedGlob::from_spanned_*`) + - Invalid environment variable prefixes (`ProcessedEnv::new`, `ProcessedPassThroughEnv::new`) + - Invalid dependency syntax (`ProcessedDependsOn::new`) + - Invalid sibling task references (`ProcessedWith::new`) + - Strip prefixes and store components separately + +2. **Processed → Resolved** (`TaskDefinition::from_processed`): + + - Apply `$TURBO_ROOT$` token replacement using `resolve()` methods + - Parse `dependsOn` into topological and task dependencies + - Transform environment variables into sorted lists + - Transform outputs into inclusion/exclusion patterns + - Validate multi-field constraints: + - Interactive tasks cannot be cached (requires `cache` and `interactive` fields) + - Interruptible tasks must be persistent (requires `interruptible` and `persistent` fields) -1. **Path Resolution**: Convert relative paths and handle `$TURBO_ROOT$` tokens -2. **Dependency Parsing**: Parse `dependsOn` into topological and task dependencies -3. **Environment Variable Collection**: Extract `env` and `passThroughEnv` variables -4. **Output Processing**: Handle inclusion/exclusion patterns in outputs -5. **Inheritance**: Handle merging multiple `RawTaskDefinition`s into a single usable task definition -6. **Validation**: Ensure configuration consistency (e.g., interactive tasks can't be cached) +3. **Inheritance**: The `extend` module handles merging multiple `ProcessedTaskDefinition`s before final resolution diff --git a/crates/turborepo-lib/src/turbo_json/extend.rs b/crates/turborepo-lib/src/turbo_json/extend.rs index 4783aff452a14..df46416bb052a 100644 --- a/crates/turborepo-lib/src/turbo_json/extend.rs +++ b/crates/turborepo-lib/src/turbo_json/extend.rs @@ -1,11 +1,93 @@ //! Module for code related to "extends" behavior for task definitions -use super::RawTaskDefinition; +use super::processed::{ + ProcessedDependsOn, ProcessedEnv, ProcessedInputs, ProcessedOutputs, ProcessedPassThroughEnv, + ProcessedTaskDefinition, ProcessedWith, +}; + +/// Trait for types that can be merged with extends behavior +trait Extendable { + /// Merges another instance into self. + /// If the other instance has `extends: true`, it extends the current value. + /// Otherwise, it replaces the current value. + fn extend(&mut self, other: Self); +} + +/// Macro to handle the extend/replace logic for a field +macro_rules! merge_field_vec { + ($self:ident, $other:ident, $field:ident) => { + if $other.extends { + $self.$field.extend($other.$field); + } else { + $self.$field = $other.$field; + } + }; +} + +impl Extendable for ProcessedDependsOn { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, deps); + self.extends = other.extends; + } +} + +impl Extendable for ProcessedEnv { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, vars); + // Sort and dedup for env vars + if other.extends { + self.vars.sort(); + self.vars.dedup(); + } + self.extends = other.extends; + } +} + +impl Extendable for ProcessedOutputs { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, globs); + self.extends = other.extends; + } +} -impl FromIterator for RawTaskDefinition { - fn from_iter>(iter: T) -> Self { +impl Extendable for ProcessedPassThroughEnv { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, vars); + // Sort and dedup for env vars + if other.extends { + self.vars.sort(); + self.vars.dedup(); + } + self.extends = other.extends; + } +} + +impl Extendable for ProcessedWith { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, tasks); + self.extends = other.extends; + } +} + +impl Extendable for ProcessedInputs { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, globs); + // Handle the default flag specially + if other.extends { + // When extending, OR the default flags + self.default = self.default || other.default; + } else { + // When replacing, use the other's default + self.default = other.default; + } + self.extends = other.extends; + } +} + +impl FromIterator for ProcessedTaskDefinition { + fn from_iter>(iter: T) -> Self { iter.into_iter() - .fold(RawTaskDefinition::default(), |mut def, other| { + .fold(ProcessedTaskDefinition::default(), |mut def, other| { def.merge(other); def }) @@ -20,12 +102,36 @@ macro_rules! set_field { }}; } -impl RawTaskDefinition { - // Merges another RawTaskDefinition into this one - // By default any fields present on `other` will override present fields. - pub fn merge(&mut self, other: RawTaskDefinition) { - set_field!(self, other, outputs); +macro_rules! merge_field { + ($this:ident, $other:ident, $field:ident) => {{ + if let Some(other_field) = $other.$field { + match &mut $this.$field { + Some(self_field) => { + // Merge using the Mergeable trait + self_field.extend(other_field); + } + None => { + // No existing value, just set it + $this.$field = Some(other_field); + } + } + } + }}; +} +impl ProcessedTaskDefinition { + // Merges another ProcessedTaskDefinition into this one + // Array fields use the Mergeable trait to handle extends behavior + pub fn merge(&mut self, other: ProcessedTaskDefinition) { + // Array fields that support extends behavior + merge_field!(self, other, outputs); + merge_field!(self, other, depends_on); + merge_field!(self, other, inputs); + merge_field!(self, other, env); + merge_field!(self, other, pass_through_env); + merge_field!(self, other, with); + + // Non-array fields that are simply replaced let other_has_range = other.cache.as_ref().is_some_and(|c| c.range.is_some()); let self_does_not_have_range = self.cache.as_ref().is_some_and(|c| c.range.is_none()); @@ -35,16 +141,11 @@ impl RawTaskDefinition { { self.cache = other.cache; } - set_field!(self, other, depends_on); - set_field!(self, other, inputs); set_field!(self, other, output_logs); set_field!(self, other, persistent); set_field!(self, other, interruptible); - set_field!(self, other, env); - set_field!(self, other, pass_through_env); set_field!(self, other, interactive); set_field!(self, other, env_mode); - set_field!(self, other, with); } } @@ -54,38 +155,99 @@ mod test { use turborepo_unescape::UnescapedString; use super::*; - use crate::cli::OutputLogsMode; + use crate::{ + cli::OutputLogsMode, + turbo_json::{ + processed::{ProcessedEnv, ProcessedInputs, ProcessedOutputs}, + FutureFlags, + }, + }; // Shared test fixtures - fn create_base_task() -> RawTaskDefinition { - RawTaskDefinition { + fn create_base_task() -> ProcessedTaskDefinition { + ProcessedTaskDefinition { cache: Some(Spanned::new(true)), persistent: Some(Spanned::new(false)), - outputs: Some(vec![Spanned::new(UnescapedString::from("dist/**"))]), - inputs: Some(vec![Spanned::new(UnescapedString::from("src/**"))]), - env: Some(vec![Spanned::new(UnescapedString::from("NODE_ENV"))]), - ..Default::default() + outputs: Some( + ProcessedOutputs::new( + vec![Spanned::new(UnescapedString::from("dist/**"))], + &FutureFlags::default(), + ) + .unwrap(), + ), + inputs: Some( + ProcessedInputs::new( + vec![Spanned::new(UnescapedString::from("src/**"))], + &FutureFlags::default(), + ) + .unwrap(), + ), + env: Some( + ProcessedEnv::new( + vec![Spanned::new(UnescapedString::from("NODE_ENV"))], + &FutureFlags::default(), + ) + .unwrap(), + ), + depends_on: None, + pass_through_env: None, + output_logs: None, + interruptible: None, + interactive: None, + env_mode: None, + with: None, } } - fn create_override_task() -> RawTaskDefinition { - RawTaskDefinition { + fn create_override_task() -> ProcessedTaskDefinition { + ProcessedTaskDefinition { cache: Some(Spanned::new(false)), persistent: Some(Spanned::new(true)), - outputs: Some(vec![Spanned::new(UnescapedString::from("build/**"))]), - inputs: Some(vec![Spanned::new(UnescapedString::from("lib/**"))]), - env: Some(vec![Spanned::new(UnescapedString::from("PROD_ENV"))]), + outputs: Some( + ProcessedOutputs::new( + vec![Spanned::new(UnescapedString::from("build/**"))], + &FutureFlags::default(), + ) + .unwrap(), + ), + inputs: Some( + ProcessedInputs::new( + vec![Spanned::new(UnescapedString::from("lib/**"))], + &FutureFlags::default(), + ) + .unwrap(), + ), + env: Some( + ProcessedEnv::new( + vec![Spanned::new(UnescapedString::from("PROD_ENV"))], + &FutureFlags::default(), + ) + .unwrap(), + ), output_logs: Some(Spanned::new(OutputLogsMode::Full)), interruptible: Some(Spanned::new(true)), - ..Default::default() + depends_on: None, + pass_through_env: None, + interactive: None, + env_mode: None, + with: None, } } - fn create_partial_task() -> RawTaskDefinition { - RawTaskDefinition { + fn create_partial_task() -> ProcessedTaskDefinition { + ProcessedTaskDefinition { persistent: Some(Spanned::new(true)), output_logs: Some(Spanned::new(OutputLogsMode::HashOnly)), - ..Default::default() + cache: None, + outputs: None, + inputs: None, + env: None, + depends_on: None, + pass_through_env: None, + interruptible: None, + interactive: None, + env_mode: None, + with: None, } } @@ -153,7 +315,7 @@ mod test { let third = create_override_task(); let tasks = vec![first.clone(), second.clone(), third.clone()]; - let result: RawTaskDefinition = tasks.into_iter().collect(); + let result: ProcessedTaskDefinition = tasks.into_iter().collect(); // Fields present in the last task (third) should take priority assert_eq!(result.cache, third.cache); @@ -171,20 +333,38 @@ mod test { #[test] fn test_from_iter_combines_across_multiple_tasks() { - let first = RawTaskDefinition { + let first = ProcessedTaskDefinition { cache: Some(Spanned::new(true)), - outputs: Some(vec![Spanned::new(UnescapedString::from("dist/**"))]), + outputs: Some( + ProcessedOutputs::new( + vec![Spanned::new(UnescapedString::from("dist/**"))], + &FutureFlags::default(), + ) + .unwrap(), + ), ..Default::default() }; - let second = RawTaskDefinition { + let second = ProcessedTaskDefinition { persistent: Some(Spanned::new(false)), - inputs: Some(vec![Spanned::new(UnescapedString::from("src/**"))]), + inputs: Some( + ProcessedInputs::new( + vec![Spanned::new(UnescapedString::from("src/**"))], + &FutureFlags::default(), + ) + .unwrap(), + ), ..Default::default() }; - let third = RawTaskDefinition { - env: Some(vec![Spanned::new(UnescapedString::from("NODE_ENV"))]), + let third = ProcessedTaskDefinition { + env: Some( + ProcessedEnv::new( + vec![Spanned::new(UnescapedString::from("NODE_ENV"))], + &FutureFlags::default(), + ) + .unwrap(), + ), output_logs: Some(Spanned::new(OutputLogsMode::Full)), // Override cache from first task cache: Some(Spanned::new(false)), @@ -192,7 +372,7 @@ mod test { }; let tasks = vec![first.clone(), second.clone(), third.clone()]; - let result: RawTaskDefinition = tasks.into_iter().collect(); + let result: ProcessedTaskDefinition = tasks.into_iter().collect(); // Last task's cache should override first task's cache assert_eq!(result.cache, third.cache); @@ -211,19 +391,216 @@ mod test { #[test] fn test_from_iter_empty_iterator() { - let empty_vec: Vec = vec![]; - let result: RawTaskDefinition = empty_vec.into_iter().collect(); + let empty_vec: Vec = vec![]; + let result: ProcessedTaskDefinition = empty_vec.into_iter().collect(); // Should be equivalent to default - assert_eq!(result, RawTaskDefinition::default()); + assert_eq!(result, ProcessedTaskDefinition::default()); } #[test] fn test_from_iter_single_task() { let single_task = create_base_task(); let tasks = vec![single_task.clone()]; - let result: RawTaskDefinition = tasks.into_iter().collect(); + let result: ProcessedTaskDefinition = tasks.into_iter().collect(); assert_eq!(result, single_task); } + + // Reusable fixtures for array fields + fn env_base() -> ProcessedEnv { + ProcessedEnv { + vars: vec!["BASE_ENV".to_string()], + extends: false, + } + } + + fn env_override() -> ProcessedEnv { + ProcessedEnv { + vars: vec!["OVERRIDE_ENV".to_string()], + extends: false, + } + } + + fn env_extending() -> ProcessedEnv { + ProcessedEnv { + vars: vec!["OVERRIDE_ENV".to_string()], + extends: true, + } + } + + fn deps_base() -> ProcessedDependsOn { + ProcessedDependsOn { + deps: vec![Spanned::new(UnescapedString::from("build"))], + extends: false, + } + } + + fn deps_test() -> ProcessedDependsOn { + ProcessedDependsOn { + deps: vec![Spanned::new(UnescapedString::from("test"))], + extends: false, + } + } + + fn deps_extending() -> ProcessedDependsOn { + ProcessedDependsOn { + deps: vec![Spanned::new(UnescapedString::from("test"))], + extends: true, + } + } + + fn with_task1() -> ProcessedWith { + use turborepo_task_id::TaskName; + ProcessedWith { + tasks: vec![Spanned::new(TaskName::from("task1"))], + extends: false, + } + } + + fn with_task2_extending() -> ProcessedWith { + use turborepo_task_id::TaskName; + ProcessedWith { + tasks: vec![Spanned::new(TaskName::from("task2"))], + extends: true, + } + } + + fn with_task3() -> ProcessedWith { + use turborepo_task_id::TaskName; + ProcessedWith { + tasks: vec![Spanned::new(TaskName::from("task3"))], + extends: false, + } + } + + fn inputs_base() -> ProcessedInputs { + ProcessedInputs { + globs: vec![], + default: false, + extends: false, + } + } + + fn inputs_extending_with_default() -> ProcessedInputs { + ProcessedInputs { + globs: vec![], + default: true, + extends: true, + } + } + + fn outputs_base() -> ProcessedOutputs { + ProcessedOutputs { + globs: vec![], + extends: false, + } + } + + fn outputs_extending() -> ProcessedOutputs { + ProcessedOutputs { + globs: vec![], + extends: true, + } + } + + #[test] + fn test_merge_with_extends_true() { + let mut base = ProcessedTaskDefinition { + inputs: Some(inputs_base()), + outputs: Some(outputs_base()), + env: Some(env_base()), + depends_on: Some(deps_base()), + with: Some(with_task1()), + ..Default::default() + }; + + let extending = ProcessedTaskDefinition { + inputs: Some(inputs_extending_with_default()), + outputs: Some(outputs_extending()), + env: Some(env_extending()), + depends_on: Some(deps_extending()), + with: Some(with_task2_extending()), + ..Default::default() + }; + + base.merge(extending); + + // Verify extends behavior + assert!(base.inputs.as_ref().unwrap().default); // OR'd + assert!(base.inputs.as_ref().unwrap().extends); + assert_eq!( + base.env.as_ref().unwrap().vars, + vec!["BASE_ENV".to_string(), "OVERRIDE_ENV".to_string()] + ); + assert_eq!(base.depends_on.as_ref().unwrap().deps.len(), 2); + assert_eq!(base.with.as_ref().unwrap().tasks.len(), 2); + } + + #[test] + fn test_merge_with_extends_false() { + let mut base = ProcessedTaskDefinition { + env: Some(env_base()), + depends_on: Some(deps_base()), + with: Some(with_task1()), + ..Default::default() + }; + + let replacing = ProcessedTaskDefinition { + env: Some(env_override()), + depends_on: Some(deps_test()), + with: Some(with_task3()), + ..Default::default() + }; + + base.merge(replacing); + + // Verify replace behavior + assert_eq!(base.env, Some(env_override())); + assert_eq!(base.depends_on, Some(deps_test())); + assert_eq!(base.with, Some(with_task3())); + } + + #[test] + fn test_merge_chain_with_extends_then_replace() { + // Test that when chaining: base -> extending -> replacing + // The final replacing task overrides everything, not extends + + let base = ProcessedTaskDefinition { + depends_on: Some(deps_base()), // has "build" + env: Some(env_base()), // has "BASE_ENV" + ..Default::default() + }; + + // Middle task extends the base + let extending = ProcessedTaskDefinition { + depends_on: Some(deps_extending()), // has "test" with extends: true + env: Some(env_extending()), // has "OVERRIDE_ENV" with extends: true + ..Default::default() + }; + + // Final task replaces (no extends) + let replacing = ProcessedTaskDefinition { + depends_on: Some(ProcessedDependsOn { + deps: vec![Spanned::new(UnescapedString::from("lint"))], + extends: false, // This should replace, not extend + }), + env: Some(ProcessedEnv { + vars: vec!["FINAL_ENV".to_string()], + extends: false, // This should replace, not extend + }), + ..Default::default() + }; + + // Apply the chain of merges + let result = ProcessedTaskDefinition::from_iter(vec![base, extending, replacing]); + + assert_eq!( + result.depends_on.as_ref().unwrap().deps, + vec![Spanned::new(UnescapedString::from("lint"))] + ); + + // Verify the extends flags are now false + assert!(!result.depends_on.as_ref().unwrap().extends); + } } diff --git a/crates/turborepo-lib/src/turbo_json/future_flags.rs b/crates/turborepo-lib/src/turbo_json/future_flags.rs new file mode 100644 index 0000000000000..b865d0b4f3a1b --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/future_flags.rs @@ -0,0 +1,45 @@ +//! Future flags for enabling experimental or upcoming features +//! +//! This module contains the `FutureFlags` structure which allows users to +//! opt-in to experimental features before they become the default behavior. +//! +//! ## Usage +//! +//! Future flags can be configured in the root `turbo.json`: +//! +//! ```json +//! { +//! "futureFlags": { +//! "turboExtends": true +//! } +//! } +//! ``` +//! +//! Note: Future flags are only allowed in the root `turbo.json` and will cause +//! an error if specified in workspace packages. + +use biome_deserialize_macros::Deserializable; +use serde::Serialize; +use struct_iterable::Iterable; + +/// Future flags configuration for experimental features +/// +/// Each flag represents an experimental feature that can be enabled +/// before it becomes the default behavior in a future version. +#[derive(Serialize, Default, Debug, Copy, Clone, Iterable, Deserializable, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct FutureFlags { + /// Enable `$TURBO_EXTENDS$` + /// + /// When enabled, allows using `$TURBO_EXTENDS$` in array fields. + /// This will change the default behavior of overriding the field to instead + /// append. + pub turbo_extends_keyword: bool, +} + +impl FutureFlags { + /// Create a new FutureFlags with all features disabled + pub fn new() -> Self { + Self::default() + } +} diff --git a/crates/turborepo-lib/src/turbo_json/loader.rs b/crates/turborepo-lib/src/turbo_json/loader.rs index decd21140f76f..c271b62f5535f 100644 --- a/crates/turborepo-lib/src/turbo_json/loader.rs +++ b/crates/turborepo-lib/src/turbo_json/loader.rs @@ -16,6 +16,7 @@ use crate::{ config::{Error, CONFIG_FILE, CONFIG_FILE_JSONC}, microfrontends::MicrofrontendsConfigs, run::task_access::TASK_ACCESS_CONFIG_PATH, + turbo_json::FutureFlags, }; /// Structure for loading TurboJson structures. @@ -23,7 +24,7 @@ use crate::{ /// `turbo.json` file. #[derive(Debug, Clone)] pub struct TurboJsonLoader { - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, cache: FixedMap, strategy: Strategy, } @@ -51,16 +52,25 @@ enum Strategy { Noop, } +// A helper structure configured with all settings related to reading a +// `turbo.json` file from disk. +#[derive(Debug, Clone)] +pub struct TurboJsonReader { + repo_root: AbsoluteSystemPathBuf, + future_flags: FutureFlags, +} + impl TurboJsonLoader { /// Create a loader that will load turbo.json files throughout the workspace pub fn workspace<'a>( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, root_turbo_json_path: AbsoluteSystemPathBuf, packages: impl Iterator, ) -> Self { - let packages = package_turbo_json_dirs(&repo_root, root_turbo_json_path, packages); + let repo_root = reader.repo_root(); + let packages = package_turbo_json_dirs(repo_root, root_turbo_json_path, packages); Self { - repo_root, + reader, cache: FixedMap::new(packages.keys().cloned()), strategy: Strategy::Workspace { packages, @@ -71,14 +81,15 @@ impl TurboJsonLoader { /// Create a loader that will load turbo.json files throughout the workspace pub fn workspace_with_microfrontends<'a>( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, root_turbo_json_path: AbsoluteSystemPathBuf, packages: impl Iterator, micro_frontends_configs: MicrofrontendsConfigs, ) -> Self { - let packages = package_turbo_json_dirs(&repo_root, root_turbo_json_path, packages); + let repo_root = reader.repo_root(); + let packages = package_turbo_json_dirs(repo_root, root_turbo_json_path, packages); Self { - repo_root, + reader, cache: FixedMap::new(packages.keys().cloned()), strategy: Strategy::Workspace { packages, @@ -90,13 +101,13 @@ impl TurboJsonLoader { /// Create a loader that will construct turbo.json structures based on /// workspace `package.json`s. pub fn workspace_no_turbo_json<'a>( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, packages: impl Iterator, microfrontends_configs: Option, ) -> Self { let packages = workspace_package_scripts(packages); Self { - repo_root, + reader, cache: FixedMap::new(packages.keys().cloned()), strategy: Strategy::WorkspaceNoTurboJson { packages, @@ -108,12 +119,12 @@ impl TurboJsonLoader { /// Create a loader that will load a root turbo.json or synthesize one if /// the file doesn't exist pub fn single_package( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, root_turbo_json: AbsoluteSystemPathBuf, package_json: PackageJson, ) -> Self { Self { - repo_root, + reader, cache: FixedMap::new(Some(PackageName::Root).into_iter()), strategy: Strategy::SinglePackage { root_turbo_json, @@ -125,12 +136,12 @@ impl TurboJsonLoader { /// Create a loader that will load a root turbo.json or synthesize one if /// the file doesn't exist pub fn task_access( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, root_turbo_json: AbsoluteSystemPathBuf, package_json: PackageJson, ) -> Self { Self { - repo_root, + reader, cache: FixedMap::new(Some(PackageName::Root).into_iter()), strategy: Strategy::TaskAccess { root_turbo_json, @@ -148,11 +159,12 @@ impl TurboJsonLoader { .into_iter() .map(|(key, value)| (key, Some(value))), ); + // This never gets read from so we populate it with root + let repo_root = AbsoluteSystemPath::new(if cfg!(windows) { "C:\\" } else { "/" }) + .expect("wasn't able to create absolute system path") + .to_owned(); Self { - // This never gets read from so we populate it with - repo_root: AbsoluteSystemPath::new(if cfg!(windows) { "C:\\" } else { "/" }) - .expect("wasn't able to create absolute system path") - .to_owned(), + reader: TurboJsonReader::new(repo_root), cache, strategy: Strategy::Noop, } @@ -170,6 +182,7 @@ impl TurboJsonLoader { } fn uncached_load(&self, package: &PackageName) -> Result { + let reader = &self.reader; match &self.strategy { Strategy::SinglePackage { package_json, @@ -178,7 +191,7 @@ impl TurboJsonLoader { if !matches!(package, PackageName::Root) { Err(Error::InvalidTurboJsonLoad(package.clone())) } else { - load_from_root_package_json(&self.repo_root, root_turbo_json, package_json) + load_from_root_package_json(reader, root_turbo_json, package_json) } } Strategy::Workspace { @@ -187,7 +200,7 @@ impl TurboJsonLoader { } => { let turbo_json_path = packages.get(package).ok_or_else(|| Error::NoTurboJSON)?; let turbo_json = load_from_file( - &self.repo_root, + reader, if package == &PackageName::Root { LoadTurboJsonPath::File(turbo_json_path) } else { @@ -223,11 +236,7 @@ impl TurboJsonLoader { if !matches!(package, PackageName::Root) { Err(Error::InvalidTurboJsonLoad(package.clone())) } else { - load_task_access_trace_turbo_json( - &self.repo_root, - root_turbo_json, - package_json, - ) + load_task_access_trace_turbo_json(reader, root_turbo_json, package_json) } } Strategy::Noop => Err(Error::NoTurboJSON), @@ -277,7 +286,7 @@ enum LoadTurboJsonPath<'a> { } fn load_from_file( - repo_root: &AbsoluteSystemPath, + reader: &TurboJsonReader, turbo_json_path: LoadTurboJsonPath, ) -> Result { let result = match turbo_json_path { @@ -286,12 +295,12 @@ fn load_from_file( let turbo_jsonc_path = turbo_json_dir_path.join_component(CONFIG_FILE_JSONC); // Load both turbo.json and turbo.jsonc - let turbo_json = TurboJson::read(repo_root, &turbo_json_path); - let turbo_jsonc = TurboJson::read(repo_root, &turbo_jsonc_path); + let turbo_json = reader.read(&turbo_json_path); + let turbo_jsonc = reader.read(&turbo_jsonc_path); select_turbo_json(turbo_json_dir_path, turbo_json, turbo_jsonc) } - LoadTurboJsonPath::File(turbo_json_path) => TurboJson::read(repo_root, turbo_json_path), + LoadTurboJsonPath::File(turbo_json_path) => reader.read(turbo_json_path), }; // Handle errors or success @@ -305,11 +314,11 @@ fn load_from_file( } fn load_from_root_package_json( - repo_root: &AbsoluteSystemPath, + reader: &TurboJsonReader, turbo_json_path: &AbsoluteSystemPath, root_package_json: &PackageJson, ) -> Result { - let mut turbo_json = match TurboJson::read(repo_root, turbo_json_path) { + let mut turbo_json = match reader.read(turbo_json_path) { // we're synthesizing, but we have a starting point // Note: this will have to change to support task inference in a monorepo // for now, we're going to error on any "root" tasks and turn non-root tasks into root @@ -401,12 +410,12 @@ fn workspace_turbo_json_from_scripts(scripts: &[String]) -> Result Result { - let trace_json_path = repo_root.join_components(&TASK_ACCESS_CONFIG_PATH); - let turbo_from_trace = TurboJson::read(repo_root, &trace_json_path); + let trace_json_path = reader.repo_root().join_components(&TASK_ACCESS_CONFIG_PATH); + let turbo_from_trace = reader.read(&trace_json_path); // check the zero config case (turbo trace file, but no turbo.json file) if let Ok(Some(turbo_from_trace)) = turbo_from_trace { @@ -415,7 +424,7 @@ fn load_task_access_trace_turbo_json( return Ok(turbo_from_trace); } } - load_from_root_package_json(repo_root, turbo_json_path, root_package_json) + load_from_root_package_json(reader, turbo_json_path, root_package_json) } // Helper for selecting the correct turbo.json read result @@ -450,6 +459,28 @@ fn select_turbo_json( } } +impl TurboJsonReader { + pub fn new(repo_root: AbsoluteSystemPathBuf) -> Self { + Self { + repo_root, + future_flags: Default::default(), + } + } + + pub fn with_future_flags(mut self, future_flags: FutureFlags) -> Self { + self.future_flags = future_flags; + self + } + + pub fn read(&self, path: &AbsoluteSystemPath) -> Result, Error> { + TurboJson::read(&self.repo_root, path, self.future_flags) + } + + pub fn repo_root(&self) -> &AbsoluteSystemPath { + &self.repo_root + } +} + #[cfg(test)] mod test { use std::{ @@ -490,8 +521,9 @@ mod test { let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path())?; let root_turbo_json = repo_root.join_component("turbo.json"); fs::write(&root_turbo_json, turbo_json_content)?; + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(Some(PackageName::Root).into_iter()), strategy: Strategy::Workspace { packages: vec![(PackageName::Root, root_turbo_json.to_owned())] @@ -571,11 +603,8 @@ mod test { fs::write(&root_turbo_json, content)?; } - let loader = TurboJsonLoader::single_package( - repo_root.to_owned(), - root_turbo_json, - root_package_json, - ); + let reader = TurboJsonReader::new(repo_root.to_owned()); + let loader = TurboJsonLoader::single_package(reader, root_turbo_json, root_package_json); let mut turbo_json = loader.load(&PackageName::Root)?.clone(); turbo_json.text = None; turbo_json.path = None; @@ -633,8 +662,8 @@ mod test { ..Default::default() }; - let loader = - TurboJsonLoader::task_access(repo_root.to_owned(), root_turbo_json, root_package_json); + let reader = TurboJsonReader::new(repo_root.to_owned()); + let loader = TurboJsonLoader::task_access(reader, root_turbo_json, root_package_json); let turbo_json = loader.load(&PackageName::Root)?; let root_build = turbo_json .tasks @@ -659,16 +688,14 @@ mod test { }) .unwrap(); let non_root = PackageName::from("some-pkg"); + let reader = TurboJsonReader::new(junk_path.to_owned()); let single_loader = TurboJsonLoader::single_package( - junk_path.to_owned(), - junk_path.to_owned(), - PackageJson::default(), - ); - let task_access_loader = TurboJsonLoader::task_access( - junk_path.to_owned(), + reader.clone(), junk_path.to_owned(), PackageJson::default(), ); + let task_access_loader = + TurboJsonLoader::task_access(reader, junk_path.to_owned(), PackageJson::default()); for loader in [single_loader, task_access_loader] { let result = loader.load(&non_root); @@ -694,8 +721,9 @@ mod test { .into_iter() .collect(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(vec![PackageName::Root, PackageName::from("a")].into_iter()), strategy: Strategy::Workspace { packages, @@ -729,8 +757,9 @@ mod test { .into_iter() .collect(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(vec![PackageName::Root, PackageName::from("a")].into_iter()), strategy: Strategy::Workspace { packages, @@ -764,8 +793,9 @@ mod test { .into_iter() .collect(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(vec![PackageName::Root, PackageName::from("pkg-a")].into_iter()), strategy: Strategy::WorkspaceNoTurboJson { packages, @@ -845,8 +875,9 @@ mod test { ) .unwrap(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new( vec![ PackageName::Root, @@ -905,6 +936,7 @@ mod test { fn test_load_from_file_with_both_files() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; + let reader = TurboJsonReader::new(repo_root.to_owned()); // Create both turbo.json and turbo.jsonc let turbo_json_path = repo_root.join_component(CONFIG_FILE); @@ -914,7 +946,7 @@ mod test { turbo_jsonc_path.create_with_contents("{}")?; // Test load_from_file with turbo.json path - let result = load_from_file(repo_root, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); // The function should return an error when both files exist assert!(result.is_err()); @@ -935,13 +967,14 @@ mod test { fn test_load_from_file_with_only_turbo_json() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; + let reader = TurboJsonReader::new(repo_root.to_owned()); // Create only turbo.json let turbo_json_path = repo_root.join_component(CONFIG_FILE); turbo_json_path.create_with_contents("{}")?; // Test load_from_file - let result = load_from_file(repo_root, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); assert!(result.is_ok()); @@ -952,13 +985,14 @@ mod test { fn test_load_from_file_with_only_turbo_jsonc() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; + let reader = TurboJsonReader::new(repo_root.to_owned()); // Create only turbo.jsonc let turbo_jsonc_path = repo_root.join_component(CONFIG_FILE_JSONC); turbo_jsonc_path.create_with_contents("{}")?; // Test load_from_file - let result = load_from_file(repo_root, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); assert!(result.is_ok()); @@ -982,8 +1016,9 @@ mod test { .create_with_contents(r#"{"tasks": {"build": {"lol": true}}}"#) .unwrap(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(vec![PackageName::Root, PackageName::from("a")].into_iter()), strategy: Strategy::Workspace { packages, diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 88576494ccb4b..568235a6b8a72 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -25,17 +25,17 @@ use crate::{ }; mod extend; +pub mod future_flags; mod loader; pub mod parser; +mod processed; -pub use loader::TurboJsonLoader; +pub use future_flags::FutureFlags; +pub use loader::{TurboJsonLoader, TurboJsonReader}; +pub use processed::ProcessedTaskDefinition; use crate::{boundaries::BoundariesConfig, config::UnnecessaryPackageTaskSyntaxError}; -const TURBO_ROOT: &str = "$TURBO_ROOT$"; -const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; -pub const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; - const ENV_PIPELINE_DELIMITER: &str = "$"; const TOPOLOGICAL_PIPELINE_DELIMITER: &str = "^"; @@ -66,6 +66,7 @@ pub struct TurboJson { pub(crate) global_env: Vec, pub(crate) global_pass_through_env: Option>, pub(crate) tasks: Pipeline, + pub(crate) future_flags: FutureFlags, } // Iterable is required to enumerate allowed keys @@ -158,10 +159,6 @@ pub struct RawTurboJson { _comment: Option, } -#[derive(Serialize, Default, Debug, Clone, Iterable, Deserializable, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct FutureFlags {} - #[derive(Serialize, Default, Debug, PartialEq, Clone)] #[serde(transparent)] pub struct Pipeline(BTreeMap, Spanned>); @@ -262,35 +259,24 @@ pub struct RawTaskDefinition { with: Option>>, } -impl TryFrom>> for TaskOutputs { - type Error = Error; - fn try_from(outputs: Vec>) -> Result { +impl TaskOutputs { + /// Creates TaskOutputs from ProcessedOutputs with resolved paths + fn from_processed( + outputs: processed::ProcessedOutputs, + turbo_root_path: &RelativeUnixPath, + ) -> Result { let mut inclusions = Vec::new(); let mut exclusions = Vec::new(); - for glob in outputs { - if let Some(stripped_glob) = glob.value.strip_prefix('!') { - if Utf8Path::new(stripped_glob).is_absolute() { - let (span, text) = glob.span_and_text("turbo.json"); - return Err(Error::AbsolutePathInConfig { - field: "outputs", - span, - text, - }); - } + // Resolve all globs with the turbo_root path + // Absolute path validation was already done during ProcessedGlob creation + let resolved = outputs.resolve(turbo_root_path); + for glob_str in resolved { + if let Some(stripped_glob) = glob_str.strip_prefix('!') { exclusions.push(stripped_glob.to_string()); } else { - if Utf8Path::new(&glob.value).is_absolute() { - let (span, text) = glob.span_and_text("turbo.json"); - return Err(Error::AbsolutePathInConfig { - field: "outputs", - span, - text, - }); - } - - inclusions.push(glob.into_inner().into()); + inclusions.push(glob_str); } } @@ -304,80 +290,62 @@ impl TryFrom>> for TaskOutputs { } } -impl TryFrom>>> for TaskInputs { - type Error = Error; - - fn try_from(inputs: Option>>) -> Result { - let mut globs = Vec::with_capacity(inputs.as_ref().map_or(0, |inputs| inputs.len())); - - let mut default = false; - for input in inputs.into_iter().flatten() { - // If the inputs contain "$TURBO_DEFAULT$", we need to include the "default" - // file hashes as well. NOTE: we intentionally don't remove - // "$TURBO_DEFAULT$" from the inputs if it exists in the off chance that - // the user has a file named "$TURBO_DEFAULT$" in their package (pls - // no). - if input.as_str() == TURBO_DEFAULT { - default = true; - } else if Utf8Path::new(input.as_str()).is_absolute() { - let (span, text) = input.span_and_text("turbo.json"); - return Err(Error::AbsolutePathInConfig { - field: "inputs", - span, - text, - }); - } - globs.push(input.to_string()); - } - - Ok(TaskInputs { globs, default }) +impl TaskInputs { + /// Creates TaskInputs from ProcessedInputs with resolved paths + fn from_processed( + inputs: processed::ProcessedInputs, + turbo_root_path: &RelativeUnixPath, + ) -> Result { + // Resolve all globs with the turbo_root path + // Absolute path validation was already done during ProcessedGlob creation + Ok(TaskInputs { + globs: inputs.resolve(turbo_root_path), + default: inputs.default, + }) } } impl TaskDefinition { - pub fn from_raw( - mut raw_task: RawTaskDefinition, + /// Creates a TaskDefinition from a ProcessedTaskDefinition + pub fn from_processed( + processed: ProcessedTaskDefinition, path_to_repo_root: &RelativeUnixPath, ) -> Result { - replace_turbo_root_token(&mut raw_task, path_to_repo_root)?; - let outputs = raw_task.outputs.unwrap_or_default().try_into()?; + // Convert outputs with turbo_root resolution + let outputs = processed + .outputs + .map(|outputs| TaskOutputs::from_processed(outputs, path_to_repo_root)) + .transpose()? + .unwrap_or_default(); - let cache = raw_task.cache.is_none_or(|c| c.into_inner()); - let interactive = raw_task + let cache = processed.cache.is_none_or(|c| c.into_inner()); + let interactive = processed .interactive .as_ref() .map(|value| value.value) .unwrap_or_default(); - if let Some(interactive) = raw_task.interactive { + if let Some(interactive) = &processed.interactive { let (span, text) = interactive.span_and_text("turbo.json"); if cache && interactive.value { return Err(Error::InteractiveNoCacheable { span, text }); } } - let persistent = *raw_task.persistent.unwrap_or_default(); - let interruptible = raw_task.interruptible.unwrap_or_default(); + let persistent = *processed.persistent.unwrap_or_default(); + let interruptible = processed.interruptible.unwrap_or_default(); if *interruptible && !persistent { let (span, text) = interruptible.span_and_text("turbo.json"); return Err(Error::InterruptibleButNotPersistent { span, text }); } - let mut env_var_dependencies = HashSet::new(); let mut topological_dependencies: Vec> = Vec::new(); let mut task_dependencies: Vec> = Vec::new(); - if let Some(depends_on) = raw_task.depends_on { - for dependency in depends_on.into_inner() { - let (span, text) = dependency.span_and_text("turbo.json"); + if let Some(depends_on) = processed.depends_on { + for dependency in depends_on.deps { let (dependency, depspan) = dependency.split(); let dependency: String = dependency.into(); - if dependency.strip_prefix(ENV_PIPELINE_DELIMITER).is_some() { - return Err(Error::InvalidDependsOnValue { - field: "dependsOn", - span, - text, - }); - } else if let Some(topo_dependency) = + if let Some(topo_dependency) = dependency.strip_prefix(TOPOLOGICAL_PIPELINE_DELIMITER) { topological_dependencies.push(depspan.to(topo_dependency.to_string().into())); @@ -390,40 +358,18 @@ impl TaskDefinition { task_dependencies.sort_by(|a, b| a.value.cmp(&b.value)); topological_dependencies.sort_by(|a, b| a.value.cmp(&b.value)); - let env = raw_task - .env - .map(|env| -> Result, Error> { - gather_env_vars(env, "env", &mut env_var_dependencies)?; - let mut env_var_dependencies: Vec = - env_var_dependencies.into_iter().collect(); - env_var_dependencies.sort(); - Ok(env_var_dependencies) - }) + let env = processed.env.map(|env| env.vars).unwrap_or_default(); + + // Convert inputs with turbo_root resolution + let inputs = processed + .inputs + .map(|inputs| TaskInputs::from_processed(inputs, path_to_repo_root)) .transpose()? .unwrap_or_default(); - let inputs = TaskInputs::try_from(raw_task.inputs)?; - - let pass_through_env = raw_task - .pass_through_env - .map(|env| -> Result, Error> { - let mut pass_through_env = HashSet::new(); - gather_env_vars(env, "passThroughEnv", &mut pass_through_env)?; - let mut pass_through_env: Vec = pass_through_env.into_iter().collect(); - pass_through_env.sort(); - Ok(pass_through_env) - }) - .transpose()?; + let pass_through_env = processed.pass_through_env.map(|env| env.vars); - let with = raw_task.with.map(|with_tasks| { - with_tasks - .into_iter() - .map(|sibling| { - let (sibling, span) = sibling.split(); - span.to(TaskName::from(String::from(sibling))) - }) - .collect() - }); + let with = processed.with.map(|with_tasks| with_tasks.tasks); Ok(TaskDefinition { outputs, @@ -433,14 +379,25 @@ impl TaskDefinition { env, inputs, pass_through_env, - output_logs: *raw_task.output_logs.unwrap_or_default(), + output_logs: *processed.output_logs.unwrap_or_default(), persistent, interruptible: *interruptible, interactive, - env_mode: raw_task.env_mode.map(|mode| *mode.as_inner()), + env_mode: processed.env_mode.map(|mode| *mode.as_inner()), with, }) } + + /// Helper method for tests that still use RawTaskDefinition + #[cfg(test)] + fn from_raw( + raw_task: RawTaskDefinition, + path_to_repo_root: &RelativeUnixPath, + ) -> Result { + // Use default FutureFlags for backward compatibility + let processed = ProcessedTaskDefinition::from_raw(raw_task, &FutureFlags::default())?; + Self::from_processed(processed, path_to_repo_root) + } } impl RawTurboJson { @@ -595,6 +552,10 @@ impl TryFrom for TurboJson { .unwrap_or_default() .map(|s| s.into_iter().map(|s| s.into()).collect()), boundaries: raw_turbo.boundaries, + future_flags: raw_turbo + .future_flags + .map(|f| f.into_inner()) + .unwrap_or_default(), // Remote Cache config is handled through layered config }) } @@ -614,20 +575,40 @@ impl TurboJson { /// Reads a `RawTurboJson` from the given path /// and then converts it into `TurboJson` - pub(crate) fn read( + /// + /// Should never be called directly outside of this module. + /// `TurboJsonReader` should be used instead. + fn read( repo_root: &AbsoluteSystemPath, path: &AbsoluteSystemPath, + future_flags: FutureFlags, ) -> Result, Error> { let Some(raw_turbo_json) = RawTurboJson::read(repo_root, path)? else { return Ok(None); }; - TurboJson::try_from(raw_turbo_json).map(Some) + + let mut turbo_json = TurboJson::try_from(raw_turbo_json)?; + // Override with root's future flags (only root turbo.json can define them) + turbo_json.future_flags = future_flags; + Ok(Some(turbo_json)) } - pub fn task(&self, task_id: &TaskId, task_name: &TaskName) -> Option { + pub fn task( + &self, + task_id: &TaskId, + task_name: &TaskName, + ) -> Result, Error> { match self.tasks.get(&task_id.as_task_name()) { - Some(entry) => Some(entry.value.clone()), - None => self.tasks.get(task_name).map(|entry| entry.value.clone()), + Some(entry) => { + ProcessedTaskDefinition::from_raw(entry.value.clone(), &self.future_flags).map(Some) + } + None => self + .tasks + .get(task_name) + .map(|entry| { + ProcessedTaskDefinition::from_raw(entry.value.clone(), &self.future_flags) + }) + .transpose(), } } @@ -773,69 +754,8 @@ fn gather_env_vars( } // Takes an input/output glob that might start with TURBO_ROOT_PREFIX -// and swap it with the relative path to the turbo root. -fn replace_turbo_root_token_in_string( - input: &mut Spanned, - path_to_repo_root: &RelativeUnixPath, -) -> Result<(), Error> { - let turbo_root_index = input.find(TURBO_ROOT); - if let Some(index) = turbo_root_index { - if !input.as_inner()[index..].starts_with(TURBO_ROOT_SLASH) { - let (span, text) = input.span_and_text("turbo.json"); - return Err(Error::InvalidTurboRootNeedsSlash { span, text }); - } - } - match turbo_root_index { - Some(0) => { - // Replace - input - .as_inner_mut() - .replace_range(..TURBO_ROOT.len(), path_to_repo_root.as_str()); - Ok(()) - } - // Handle negations - Some(1) if input.starts_with('!') => { - input - .as_inner_mut() - .replace_range(1..TURBO_ROOT.len() + 1, path_to_repo_root.as_str()); - Ok(()) - } - // We do not allow for TURBO_ROOT to be used in the middle of a glob - Some(_) => { - let (span, text) = input.span_and_text("turbo.json"); - Err(Error::InvalidTurboRootUse { span, text }) - } - None => Ok(()), - } -} - -fn replace_turbo_root_token( - task_definition: &mut RawTaskDefinition, - path_to_repo_root: &RelativeUnixPath, -) -> Result<(), Error> { - for input in task_definition - .inputs - .iter_mut() - .flat_map(|inputs| inputs.iter_mut()) - { - replace_turbo_root_token_in_string(input, path_to_repo_root)?; - } - - for output in task_definition - .outputs - .iter_mut() - .flat_map(|outputs| outputs.iter_mut()) - { - replace_turbo_root_token_in_string(output, path_to_repo_root)?; - } - - Ok(()) -} - #[cfg(test)] mod tests { - use std::{assert_matches::assert_matches, sync::Arc}; - use anyhow::Result; use biome_deserialize::json::deserialize_from_json_str; use biome_json_parser::JsonParserOptions; @@ -846,10 +766,7 @@ mod tests { use turborepo_task_id::TaskName; use turborepo_unescape::UnescapedString; - use super::{ - replace_turbo_root_token_in_string, validate_with_has_no_topo, FutureFlags, Pipeline, - RawTurboJson, SpacesJson, Spanned, TurboJson, UIMode, *, - }; + use super::{processed::*, *}; use crate::{ boundaries::BoundariesConfig, cli::OutputLogsMode, @@ -1118,11 +1035,12 @@ mod tests { expected_task_outputs: TaskOutputs, ) -> Result<()> { let raw_task_outputs: Vec = serde_json::from_str(task_outputs_str)?; - let raw_task_outputs = raw_task_outputs - .into_iter() - .map(Spanned::new) - .collect::>(); - let task_outputs: TaskOutputs = raw_task_outputs.try_into()?; + let turbo_root = RelativeUnixPath::new("../..")?; + let processed_outputs = ProcessedOutputs::new( + raw_task_outputs.into_iter().map(Spanned::new).collect(), + &FutureFlags::default(), + )?; + let task_outputs = TaskOutputs::from_processed(processed_outputs, turbo_root)?; assert_eq!(task_outputs, expected_task_outputs); Ok(()) @@ -1318,27 +1236,6 @@ mod tests { ); } - #[test_case("index.ts", Ok("index.ts") ; "no token")] - #[test_case("$TURBO_ROOT$/config.txt", Ok("../../config.txt") ; "valid token")] - #[test_case("!$TURBO_ROOT$/README.md", Ok("!../../README.md") ; "negation")] - #[test_case("../$TURBO_ROOT$/config.txt", Err("\"$TURBO_ROOT$\" must be used at the start of glob.") ; "invalid token")] - #[test_case("$TURBO_ROOT$config.txt", Err("\"$TURBO_ROOT$\" must be followed by a '/'.") ; "trailing slash")] - fn test_replace_turbo_root(input: &'static str, expected: Result<&str, &str>) { - let mut spanned_string = Spanned::new(UnescapedString::from(input)) - .with_path(Arc::from("turbo.json")) - .with_text(format!("\"{input}\"")) - .with_range(1..(input.len())); - let result = replace_turbo_root_token_in_string( - &mut spanned_string, - RelativeUnixPath::new("../..").unwrap(), - ); - let actual = match result { - Ok(()) => Ok(spanned_string.as_inner().as_ref()), - Err(e) => Err(e.to_string()), - }; - assert_eq!(actual, expected.map_err(|s| s.to_owned())); - } - #[test] fn test_future_flags_not_allowed_in_workspace() { let json = r#"{ @@ -1376,7 +1273,7 @@ mod tests { "build": {} }, "futureFlags": { - "bestFeature": true + "turboExtendsKeyword": true } }"#; @@ -1390,7 +1287,12 @@ mod tests { // Verify that futureFlags is parsed correctly assert!(raw_turbo_json.future_flags.is_some()); let future_flags = raw_turbo_json.future_flags.as_ref().unwrap(); - assert_eq!(future_flags.as_inner(), &FutureFlags {}); + assert_eq!( + future_flags.as_inner(), + &FutureFlags { + turbo_extends_keyword: true + } + ); // Verify that the futureFlags field doesn't cause errors during conversion to // TurboJson @@ -1423,36 +1325,4 @@ mod tests { "`with` cannot use dependency relationships." ); } - - #[test] - fn test_absolute_paths_error_in_inputs() { - assert_matches!( - TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( - if cfg!(windows) { - "C:\\win32" - } else { - "/dev/null" - } - ))])), - Err(Error::AbsolutePathInConfig { .. }) - ); - } - - #[test] - fn test_detects_turbo_default() { - let inputs = TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( - TURBO_DEFAULT, - ))])) - .unwrap(); - assert!(inputs.default); - } - - #[test] - fn test_keeps_turbo_default() { - let inputs = TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( - TURBO_DEFAULT, - ))])) - .unwrap(); - assert_eq!(inputs.globs, vec![TURBO_DEFAULT.to_string()]); - } } diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs new file mode 100644 index 0000000000000..0853086498a6c --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -0,0 +1,642 @@ +//! Processed task definition types with DSL token handling + +use camino::Utf8Path; +use turbopath::RelativeUnixPath; +use turborepo_errors::Spanned; +use turborepo_task_id::TaskName; +use turborepo_unescape::UnescapedString; + +use super::{FutureFlags, RawTaskDefinition}; +use crate::{ + cli::{EnvMode, OutputLogsMode}, + config::Error, +}; + +const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; +const TURBO_ROOT: &str = "$TURBO_ROOT$"; +const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; +const TURBO_EXTENDS: &str = "$TURBO_EXTENDS$"; +const ENV_PIPELINE_DELIMITER: &str = "$"; +const TOPOLOGICAL_PIPELINE_DELIMITER: &str = "^"; + +/// Helper function to check for and remove $TURBO_EXTENDS$ from an array +/// Returns (processed_array, extends_found) +fn extract_turbo_extends( + mut items: Vec>, + future_flags: &FutureFlags, +) -> (Vec>, bool) { + if !future_flags.turbo_extends_keyword { + return (items, false); + } + + if let Some(pos) = items.iter().position(|item| item.as_str() == TURBO_EXTENDS) { + items.remove(pos); + (items, true) + } else { + (items, false) + } +} + +/// A processed glob with separated components +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedGlob { + /// The glob pattern without $TURBO_ROOT$ prefix + glob: String, + /// Whether the glob was negated (started with !) + negated: bool, + /// Whether the glob needs turbo_root prefix (had $TURBO_ROOT$/) + turbo_root: bool, +} + +impl ProcessedGlob { + /// Creates a ProcessedGlob from a raw glob string, stripping prefixes + fn from_spanned_internal( + value: Spanned, + field: &'static str, + ) -> Result { + let mut negated = false; + let mut turbo_root = false; + + let without_negation = if let Some(value) = value.strip_prefix('!') { + negated = true; + value + } else { + value.as_str() + }; + + let glob = if let Some(stripped) = without_negation.strip_prefix(TURBO_ROOT_SLASH) { + turbo_root = true; + stripped + } else if without_negation.starts_with(TURBO_ROOT) { + // Leading $TURBO_ROOT$ without slash + let (span, text) = value.span_and_text("turbo.json"); + return Err(Error::InvalidTurboRootNeedsSlash { span, text }); + } else if without_negation.contains(TURBO_ROOT) { + // non leading $TURBO_ROOT$ + let (span, text) = value.span_and_text("turbo.json"); + return Err(Error::InvalidTurboRootUse { span, text }); + } else { + without_negation + }; + + // Check for absolute paths (after stripping prefixes) + if Utf8Path::new(glob).is_absolute() { + let (span, text) = value.span_and_text("turbo.json"); + return Err(Error::AbsolutePathInConfig { field, span, text }); + } + + Ok(ProcessedGlob { + glob: glob.to_owned(), + negated, + turbo_root, + }) + } + + /// Creates a ProcessedGlob for outputs (validates as output field) + pub fn from_spanned_output( + value: Spanned, + ) -> Result { + Self::from_spanned_internal(value, "outputs") + } + + /// Creates a ProcessedGlob for inputs (validates as input field) + pub fn from_spanned_input( + value: Spanned, + ) -> Result { + Self::from_spanned_internal(value, "inputs") + } + + /// Creates a resolved glob string with the actual path + pub fn resolve(&self, turbo_root_path: &RelativeUnixPath) -> String { + let prefix = if self.negated { "!" } else { "" }; + + let glob = &self.glob; + if self.turbo_root { + format!("{prefix}{turbo_root_path}/{glob}") + } else { + format!("{prefix}{glob}") + } + } +} + +/// Processed depends_on field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedDependsOn { + pub deps: Vec>, + pub extends: bool, +} + +impl ProcessedDependsOn { + /// Creates a ProcessedDependsOn, validating that dependencies don't use env + /// prefix and handling TURBO_EXTENDS if enabled + pub fn new( + raw_deps: Spanned>>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_deps, extends) = extract_turbo_extends(raw_deps.into_inner(), future_flags); + + // Validate that no dependency starts with ENV_PIPELINE_DELIMITER ($) + for dep in processed_deps.iter() { + if dep.starts_with(ENV_PIPELINE_DELIMITER) { + let (span, text) = dep.span_and_text("turbo.json"); + return Err(Error::InvalidDependsOnValue { + field: "dependsOn", + span, + text, + }); + } + } + Ok(ProcessedDependsOn { + deps: processed_deps, + extends, + }) + } +} + +/// Processed env field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedEnv { + pub vars: Vec, + pub extends: bool, +} + +impl ProcessedEnv { + /// Creates a ProcessedEnv, validating that env vars don't use invalid + /// prefixes and handling TURBO_EXTENDS if enabled + pub fn new( + raw_env: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_env, extends) = extract_turbo_extends(raw_env, future_flags); + + Ok(ProcessedEnv { + vars: extract_env_vars(processed_env, "env")?, + extends, + }) + } +} + +/// Processed inputs field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedInputs { + pub globs: Vec, + pub default: bool, + pub extends: bool, +} + +impl ProcessedInputs { + pub fn new( + raw_globs: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_globs, extends) = extract_turbo_extends(raw_globs, future_flags); + + let mut globs = Vec::with_capacity(processed_globs.len()); + let mut default = false; + for raw_glob in processed_globs { + if raw_glob.as_str() == TURBO_DEFAULT { + default = true; + } + globs.push(ProcessedGlob::from_spanned_input(raw_glob)?); + } + + Ok(ProcessedInputs { + globs, + default, + extends, + }) + } + + /// Resolves all globs with the given turbo_root path + pub fn resolve(&self, turbo_root_path: &RelativeUnixPath) -> Vec { + self.globs + .iter() + .map(|glob| glob.resolve(turbo_root_path)) + .collect() + } +} + +/// Processed pass_through_env field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedPassThroughEnv { + pub vars: Vec, + pub extends: bool, +} + +impl ProcessedPassThroughEnv { + /// Creates a ProcessedPassThroughEnv, validating that env vars don't use + /// invalid prefixes and handling TURBO_EXTENDS if enabled + pub fn new( + raw_env: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_env, extends) = extract_turbo_extends(raw_env, future_flags); + + Ok(ProcessedPassThroughEnv { + vars: extract_env_vars(processed_env, "passThroughEnv")?, + extends, + }) + } +} + +fn extract_env_vars( + raw_env: Vec>, + field_name: &str, +) -> Result, Error> { + use crate::config::InvalidEnvPrefixError; + + let mut env_vars = Vec::with_capacity(raw_env.len()); + // Validate that no env var starts with ENV_PIPELINE_DELIMITER ($) + for var in raw_env { + if var.starts_with(ENV_PIPELINE_DELIMITER) { + let (span, text) = var.span_and_text("turbo.json"); + return Err(Error::InvalidEnvPrefix(Box::new(InvalidEnvPrefixError { + key: field_name.to_string(), + value: var.as_str().to_string(), + span, + text, + env_pipeline_delimiter: ENV_PIPELINE_DELIMITER, + }))); + } + + env_vars.push(String::from(var.into_inner())); + } + env_vars.sort(); + Ok(env_vars) +} + +/// Processed outputs field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedOutputs { + pub globs: Vec, + pub extends: bool, +} + +impl ProcessedOutputs { + pub fn new( + raw_globs: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_globs, extends) = extract_turbo_extends(raw_globs, future_flags); + + let globs = processed_globs + .into_iter() + .map(ProcessedGlob::from_spanned_output) + .collect::, _>>()?; + + Ok(ProcessedOutputs { globs, extends }) + } + + /// Resolves all globs with the given turbo_root path + pub fn resolve(&self, turbo_root_path: &RelativeUnixPath) -> Vec { + self.globs + .iter() + .map(|glob| glob.resolve(turbo_root_path)) + .collect() + } +} + +/// Processed with field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedWith { + pub tasks: Vec>>, + pub extends: bool, +} + +impl ProcessedWith { + /// Creates a ProcessedWith, validating that siblings don't use topological + /// prefix and handling TURBO_EXTENDS if enabled + pub fn new( + raw_with: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_with, extends) = extract_turbo_extends(raw_with, future_flags); + + // Validate that no sibling starts with TOPOLOGICAL_PIPELINE_DELIMITER (^) + let mut tasks = Vec::with_capacity(processed_with.len()); + for sibling in processed_with { + if sibling.starts_with(TOPOLOGICAL_PIPELINE_DELIMITER) { + let (span, text) = sibling.span_and_text("turbo.json"); + return Err(Error::InvalidTaskWith { span, text }); + } + let (sibling, span) = sibling.split(); + tasks.push(span.to(TaskName::from(String::from(sibling)))); + } + Ok(ProcessedWith { tasks, extends }) + } +} + +/// Intermediate representation for task definitions with DSL processing +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ProcessedTaskDefinition { + pub cache: Option>, + pub depends_on: Option, + pub env: Option, + pub inputs: Option, + pub pass_through_env: Option, + pub persistent: Option>, + pub interruptible: Option>, + pub outputs: Option, + pub output_logs: Option>, + pub interactive: Option>, + pub env_mode: Option>, + pub with: Option, +} + +impl ProcessedTaskDefinition { + /// Creates a processed task definition from raw task + pub fn from_raw( + raw_task: RawTaskDefinition, + future_flags: &FutureFlags, + ) -> Result { + Ok(ProcessedTaskDefinition { + cache: raw_task.cache, + depends_on: raw_task + .depends_on + .map(|deps| ProcessedDependsOn::new(deps, future_flags)) + .transpose()?, + env: raw_task + .env + .map(|env| ProcessedEnv::new(env, future_flags)) + .transpose()?, + inputs: raw_task + .inputs + .map(|inputs| ProcessedInputs::new(inputs, future_flags)) + .transpose()?, + pass_through_env: raw_task + .pass_through_env + .map(|env| ProcessedPassThroughEnv::new(env, future_flags)) + .transpose()?, + persistent: raw_task.persistent, + interruptible: raw_task.interruptible, + outputs: raw_task + .outputs + .map(|outputs| ProcessedOutputs::new(outputs, future_flags)) + .transpose()?, + output_logs: raw_task.output_logs, + interactive: raw_task.interactive, + env_mode: raw_task.env_mode, + with: raw_task + .with + .map(|with| ProcessedWith::new(with, future_flags)) + .transpose()?, + }) + } +} + +#[cfg(test)] +mod tests { + use std::{assert_matches::assert_matches, sync::Arc}; + + use test_case::test_case; + use turborepo_errors::Spanned; + use turborepo_unescape::UnescapedString; + + use super::*; + use crate::turbo_json::FutureFlags; + + #[test] + fn test_extract_turbo_extends_with_flag_enabled() { + let items = vec![ + Spanned::new(UnescapedString::from("item1")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("item2")), + ]; + + let (processed, extends) = extract_turbo_extends( + items, + &FutureFlags { + turbo_extends_keyword: true, + }, + ); + + assert!(extends); + assert_eq!(processed.len(), 2); + assert_eq!(processed[0].as_str(), "item1"); + assert_eq!(processed[1].as_str(), "item2"); + } + + #[test] + fn test_extract_turbo_extends_with_flag_disabled() { + let items = vec![ + Spanned::new(UnescapedString::from("item1")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("item2")), + ]; + + let (processed, extends) = extract_turbo_extends( + items, + &FutureFlags { + turbo_extends_keyword: false, + }, + ); + + assert!(!extends); + assert_eq!(processed.len(), 3); + assert_eq!(processed[1].as_str(), "$TURBO_EXTENDS$"); + } + + #[test] + fn test_extract_turbo_extends_no_marker() { + let items = vec![ + Spanned::new(UnescapedString::from("item1")), + Spanned::new(UnescapedString::from("item2")), + ]; + + let (processed, extends) = extract_turbo_extends( + items, + &FutureFlags { + turbo_extends_keyword: true, + }, + ); + + assert!(!extends); + assert_eq!(processed.len(), 2); + } + + #[test_case("$TURBO_ROOT$/config.txt", Ok((true, false)) ; "detects turbo root")] + #[test_case("!$TURBO_ROOT$/README.md", Ok((true, true)) ; "detects negated turbo root")] + #[test_case("src/**/*.ts", Ok((false, false)) ; "no turbo root")] + fn test_processed_glob_detection(input: &str, expected: Result<(bool, bool), &str>) { + // Test with input variant + let result = ProcessedGlob::from_spanned_input(Spanned::new(UnescapedString::from( + input.to_string(), + ))); + + match expected { + Ok((turbo_root, negated)) => { + let glob = result.unwrap(); + assert_eq!(glob.turbo_root, turbo_root); + assert_eq!(glob.negated, negated); + } + Err(_) => { + assert!(result.is_err()); + } + } + } + + #[test_case("$TURBO_ROOT$config.txt", "must be followed by a '/'" ; "missing slash")] + #[test_case("../$TURBO_ROOT$/config.txt", "must be used at the start of glob" ; "middle turbo root")] + fn test_processed_glob_validation_errors(input: &str, expected_error: &str) { + // Test with input variant + let result = ProcessedGlob::from_spanned_input( + Spanned::new(UnescapedString::from(input.to_string())) + .with_path(Arc::from("turbo.json")) + .with_text(format!("\"{}\"", input)) + .with_range(1..input.len() + 1), + ); + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.to_string().contains(expected_error)); + } + + #[test_case("$TURBO_ROOT$/config.txt", "../..", "../../config.txt" ; "replace turbo root")] + #[test_case("!$TURBO_ROOT$/README.md", "../..", "!../../README.md" ; "replace negated turbo root")] + #[test_case("src/**/*.ts", "../..", "src/**/*.ts" ; "no replacement needed")] + fn test_processed_glob_resolution(input: &str, replacement: &str, expected: &str) { + let replacement = RelativeUnixPath::new(replacement).unwrap(); + // Test with output variant + let glob = ProcessedGlob::from_spanned_output(Spanned::new(UnescapedString::from( + input.to_string(), + ))) + .unwrap(); + + let resolved = glob.resolve(replacement); + assert_eq!(resolved, expected); + } + + #[test] + fn test_processed_task_definition_resolve() { + // Create a raw task definition with TURBO_ROOT tokens + let raw_task = RawTaskDefinition { + inputs: Some(vec![ + Spanned::new(UnescapedString::from("$TURBO_ROOT$/config.txt")), + Spanned::new(UnescapedString::from("src/**/*.ts")), + ]), + outputs: Some(vec![ + Spanned::new(UnescapedString::from("!$TURBO_ROOT$/README.md")), + Spanned::new(UnescapedString::from("dist/**")), + ]), + ..Default::default() + }; + + // Convert to processed task definition + let processed = + ProcessedTaskDefinition::from_raw(raw_task, &FutureFlags::default()).unwrap(); + let turbo_root = RelativeUnixPath::new("../..").unwrap(); + + // Verify TURBO_ROOT detection + let inputs = processed.inputs.as_ref().unwrap(); + assert!(inputs.globs[0].turbo_root); + assert!(!inputs.globs[0].negated); + assert!(!inputs.globs[1].turbo_root); + + let outputs = processed.outputs.as_ref().unwrap(); + assert!(outputs.globs[0].turbo_root); + assert!(outputs.globs[0].negated); + assert!(!outputs.globs[1].turbo_root); + + // Resolve with turbo_root path + let resolved_inputs = inputs.resolve(turbo_root); + assert_eq!(resolved_inputs[0], "../../config.txt"); + assert_eq!(resolved_inputs[1], "src/**/*.ts"); + + let resolved_outputs = outputs.resolve(turbo_root); + assert_eq!(resolved_outputs[0], "!../../README.md"); + assert_eq!(resolved_outputs[1], "dist/**"); + } + + #[test] + fn test_detects_turbo_default() { + let raw_globs = vec![Spanned::new(UnescapedString::from(TURBO_DEFAULT))]; + + let inputs = ProcessedInputs::new(raw_globs, &FutureFlags::default()).unwrap(); + assert!(inputs.default); + assert_eq!( + inputs.globs, + vec![ProcessedGlob { + glob: TURBO_DEFAULT.to_string(), + negated: false, + turbo_root: false + }] + ); + } + + #[test] + fn test_absolute_paths_error_in_inputs() { + let absolute_path = if cfg!(windows) { + "C:\\win32" + } else { + "/dev/null" + }; + + // The error should be caught when creating the ProcessedGlob + let result = + ProcessedGlob::from_spanned_input(Spanned::new(UnescapedString::from(absolute_path))); + + assert_matches!(result, Err(Error::AbsolutePathInConfig { .. })); + } + + // Test that demonstrates the extends field is properly set when the helper is + // used + #[test] + fn test_processed_inputs_with_turbo_extends() { + let raw_globs: Vec> = vec![ + Spanned::new(UnescapedString::from("src/**")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("lib/**")), + ]; + + let inputs = ProcessedInputs::new( + raw_globs, + &FutureFlags { + turbo_extends_keyword: true, + }, + ) + .unwrap(); + + assert!(inputs.extends); + assert_eq!(inputs.globs.len(), 2); + assert_eq!(inputs.globs[0].glob, "src/**"); + assert_eq!(inputs.globs[1].glob, "lib/**"); + } + + #[test] + fn test_processed_env_turbo_extends_disabled_errors() { + // When turbo_extends is disabled, $TURBO_EXTENDS$ triggers validation error + let raw_env: Vec> = vec![ + Spanned::new(UnescapedString::from("NODE_ENV")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("API_KEY")), + ]; + + let result = ProcessedEnv::new( + raw_env, + &FutureFlags { + turbo_extends_keyword: false, + }, + ); + assert!(result.is_err()); + assert_matches!(result, Err(Error::InvalidEnvPrefix(_))); + } + + #[test] + fn test_processed_depends_on_turbo_extends_disabled_errors() { + // When turbo_extends is disabled, $TURBO_EXTENDS$ triggers validation error + let raw_deps: Vec> = vec![ + Spanned::new(UnescapedString::from("build")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("test")), + ]; + + let result = ProcessedDependsOn::new( + Spanned::new(raw_deps), + &FutureFlags { + turbo_extends_keyword: false, + }, + ); + assert!(result.is_err()); + assert_matches!(result, Err(Error::InvalidDependsOnValue { .. })); + } +} diff --git a/turborepo-tests/integration/fixtures/turbo-configs/interruptible-but-not-persistent.json b/turborepo-tests/integration/fixtures/turbo-configs/interruptible-but-not-persistent.json index 4759216b2e0f5..db1d409a8d1e5 100644 --- a/turborepo-tests/integration/fixtures/turbo-configs/interruptible-but-not-persistent.json +++ b/turborepo-tests/integration/fixtures/turbo-configs/interruptible-but-not-persistent.json @@ -10,7 +10,7 @@ "build": { "env": [ "NODE_ENV", - "$FOOBAR" + "FOOBAR" ], "interruptible": true, "outputs": []