From e7d9d27bdb1321ea3f2eb7411d919e0bc66b752a Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 30 Jul 2025 13:40:23 -0400 Subject: [PATCH 01/19] chore(turbo_json): delegate actual TurboJson reading to helper --- crates/turborepo-lib/src/turbo_json/loader.rs | 62 +++++++++++++------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/loader.rs b/crates/turborepo-lib/src/turbo_json/loader.rs index decd21140f76f..5a9b2eefe6a0b 100644 --- a/crates/turborepo-lib/src/turbo_json/loader.rs +++ b/crates/turborepo-lib/src/turbo_json/loader.rs @@ -51,6 +51,12 @@ enum Strategy { Noop, } +// A helper structure configured with all settings related to reading a +// `turbo.json` file from disk. +struct TurboJsonReader<'a> { + repo_root: &'a AbsoluteSystemPath, +} + impl TurboJsonLoader { /// Create a loader that will load turbo.json files throughout the workspace pub fn workspace<'a>( @@ -170,6 +176,7 @@ impl TurboJsonLoader { } fn uncached_load(&self, package: &PackageName) -> Result { + let reader = self.reader(); match &self.strategy { Strategy::SinglePackage { package_json, @@ -178,7 +185,7 @@ impl TurboJsonLoader { if !matches!(package, PackageName::Root) { Err(Error::InvalidTurboJsonLoad(package.clone())) } else { - load_from_root_package_json(&self.repo_root, root_turbo_json, package_json) + load_from_root_package_json(reader, root_turbo_json, package_json) } } Strategy::Workspace { @@ -187,7 +194,7 @@ impl TurboJsonLoader { } => { let turbo_json_path = packages.get(package).ok_or_else(|| Error::NoTurboJSON)?; let turbo_json = load_from_file( - &self.repo_root, + reader, if package == &PackageName::Root { LoadTurboJsonPath::File(turbo_json_path) } else { @@ -223,16 +230,18 @@ impl TurboJsonLoader { if !matches!(package, PackageName::Root) { Err(Error::InvalidTurboJsonLoad(package.clone())) } else { - load_task_access_trace_turbo_json( - &self.repo_root, - root_turbo_json, - package_json, - ) + load_task_access_trace_turbo_json(reader, root_turbo_json, package_json) } } Strategy::Noop => Err(Error::NoTurboJSON), } } + + fn reader<'a>(&'a self) -> TurboJsonReader<'a> { + TurboJsonReader { + repo_root: &self.repo_root, + } + } } /// Map all packages in the package graph to their dirs that contain a @@ -277,7 +286,7 @@ enum LoadTurboJsonPath<'a> { } fn load_from_file( - repo_root: &AbsoluteSystemPath, + reader: TurboJsonReader, turbo_json_path: LoadTurboJsonPath, ) -> Result { let result = match turbo_json_path { @@ -286,12 +295,12 @@ fn load_from_file( let turbo_jsonc_path = turbo_json_dir_path.join_component(CONFIG_FILE_JSONC); // Load both turbo.json and turbo.jsonc - let turbo_json = TurboJson::read(repo_root, &turbo_json_path); - let turbo_jsonc = TurboJson::read(repo_root, &turbo_jsonc_path); + let turbo_json = reader.read(&turbo_json_path); + let turbo_jsonc = reader.read(&turbo_jsonc_path); select_turbo_json(turbo_json_dir_path, turbo_json, turbo_jsonc) } - LoadTurboJsonPath::File(turbo_json_path) => TurboJson::read(repo_root, turbo_json_path), + LoadTurboJsonPath::File(turbo_json_path) => reader.read(turbo_json_path), }; // Handle errors or success @@ -305,11 +314,11 @@ fn load_from_file( } fn load_from_root_package_json( - repo_root: &AbsoluteSystemPath, + reader: TurboJsonReader, turbo_json_path: &AbsoluteSystemPath, root_package_json: &PackageJson, ) -> Result { - let mut turbo_json = match TurboJson::read(repo_root, turbo_json_path) { + let mut turbo_json = match reader.read(turbo_json_path) { // we're synthesizing, but we have a starting point // Note: this will have to change to support task inference in a monorepo // for now, we're going to error on any "root" tasks and turn non-root tasks into root @@ -401,12 +410,12 @@ fn workspace_turbo_json_from_scripts(scripts: &[String]) -> Result Result { - let trace_json_path = repo_root.join_components(&TASK_ACCESS_CONFIG_PATH); - let turbo_from_trace = TurboJson::read(repo_root, &trace_json_path); + let trace_json_path = reader.repo_root().join_components(&TASK_ACCESS_CONFIG_PATH); + let turbo_from_trace = reader.read(&trace_json_path); // check the zero config case (turbo trace file, but no turbo.json file) if let Ok(Some(turbo_from_trace)) = turbo_from_trace { @@ -415,7 +424,7 @@ fn load_task_access_trace_turbo_json( return Ok(turbo_from_trace); } } - load_from_root_package_json(repo_root, turbo_json_path, root_package_json) + load_from_root_package_json(reader, turbo_json_path, root_package_json) } // Helper for selecting the correct turbo.json read result @@ -450,6 +459,16 @@ fn select_turbo_json( } } +impl TurboJsonReader<'_> { + pub fn read(&self, path: &AbsoluteSystemPath) -> Result, Error> { + TurboJson::read(self.repo_root, path) + } + + pub fn repo_root(&self) -> &AbsoluteSystemPath { + self.repo_root + } +} + #[cfg(test)] mod test { use std::{ @@ -905,6 +924,7 @@ mod test { fn test_load_from_file_with_both_files() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; + let reader = TurboJsonReader { repo_root }; // Create both turbo.json and turbo.jsonc let turbo_json_path = repo_root.join_component(CONFIG_FILE); @@ -914,7 +934,7 @@ mod test { turbo_jsonc_path.create_with_contents("{}")?; // Test load_from_file with turbo.json path - let result = load_from_file(repo_root, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(reader, LoadTurboJsonPath::Dir(repo_root)); // The function should return an error when both files exist assert!(result.is_err()); @@ -935,13 +955,14 @@ mod test { fn test_load_from_file_with_only_turbo_json() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; + let reader = TurboJsonReader { repo_root }; // Create only turbo.json let turbo_json_path = repo_root.join_component(CONFIG_FILE); turbo_json_path.create_with_contents("{}")?; // Test load_from_file - let result = load_from_file(repo_root, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(reader, LoadTurboJsonPath::Dir(repo_root)); assert!(result.is_ok()); @@ -952,13 +973,14 @@ mod test { fn test_load_from_file_with_only_turbo_jsonc() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; + let reader = TurboJsonReader { repo_root }; // Create only turbo.jsonc let turbo_jsonc_path = repo_root.join_component(CONFIG_FILE_JSONC); turbo_jsonc_path.create_with_contents("{}")?; // Test load_from_file - let result = load_from_file(repo_root, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(reader, LoadTurboJsonPath::Dir(repo_root)); assert!(result.is_ok()); From fdf8d50124eb45a9ad8739e2f0d34c24137c7347 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 30 Jul 2025 14:42:03 -0400 Subject: [PATCH 02/19] chore(future_flags): allow future flags to alter behavior of turbo.json parsing --- crates/turborepo-lib/src/config/env.rs | 2 + crates/turborepo-lib/src/config/mod.rs | 11 +- crates/turborepo-lib/src/config/turbo_json.rs | 1 + crates/turborepo-lib/src/opts.rs | 6 +- .../src/package_changes_watcher.rs | 4 +- crates/turborepo-lib/src/run/builder.rs | 15 ++- crates/turborepo-lib/src/turbo_json/loader.rs | 127 ++++++++++-------- crates/turborepo-lib/src/turbo_json/mod.rs | 10 +- 8 files changed, 106 insertions(+), 70 deletions(-) diff --git a/crates/turborepo-lib/src/config/env.rs b/crates/turborepo-lib/src/config/env.rs index 28afe8aba1909..26fb21395e2c1 100644 --- a/crates/turborepo-lib/src/config/env.rs +++ b/crates/turborepo-lib/src/config/env.rs @@ -255,6 +255,8 @@ impl ResolvedConfigurationOptions for EnvVars { root_turbo_json_path, log_order, sso_login_callback_port, + // Do not allow future flags to be set by env var + future_flags: None, }; Ok(output) diff --git a/crates/turborepo-lib/src/config/mod.rs b/crates/turborepo-lib/src/config/mod.rs index f7e9f497d03f2..b3a4cbc93c1d8 100644 --- a/crates/turborepo-lib/src/config/mod.rs +++ b/crates/turborepo-lib/src/config/mod.rs @@ -23,8 +23,11 @@ use turborepo_cache::CacheConfig; use turborepo_errors::TURBO_SITE; use turborepo_repository::package_graph::PackageName; -use crate::cli::{EnvMode, LogOrder}; pub use crate::turbo_json::{RawTurboJson, UIMode}; +use crate::{ + cli::{EnvMode, LogOrder}, + turbo_json::FutureFlags, +}; pub const CONFIG_FILE: &str = "turbo.json"; pub const CONFIG_FILE_JSONC: &str = "turbo.jsonc"; @@ -312,6 +315,8 @@ pub struct ConfigurationOptions { pub(crate) concurrency: Option, pub(crate) no_update_notifier: Option, pub(crate) sso_login_callback_port: Option, + #[serde(skip)] + future_flags: Option, } #[derive(Default)] @@ -471,6 +476,10 @@ impl ConfigurationOptions { pub fn sso_login_callback_port(&self) -> Option { self.sso_login_callback_port } + + pub fn future_flags(&self) -> FutureFlags { + self.future_flags.unwrap_or_default() + } } // Maps Some("") to None to emulate how Go handles empty strings diff --git a/crates/turborepo-lib/src/config/turbo_json.rs b/crates/turborepo-lib/src/config/turbo_json.rs index 269474a75fbcb..ae13a4a740380 100644 --- a/crates/turborepo-lib/src/config/turbo_json.rs +++ b/crates/turborepo-lib/src/config/turbo_json.rs @@ -78,6 +78,7 @@ impl<'a> TurboJsonReader<'a> { opts.env_mode = turbo_json.env_mode.map(|mode| *mode.as_inner()); opts.cache_dir = cache_dir; opts.concurrency = turbo_json.concurrency.map(|c| c.as_inner().clone()); + opts.future_flags = turbo_json.future_flags.map(|f| f.as_inner().clone()); Ok(opts) } } diff --git a/crates/turborepo-lib/src/opts.rs b/crates/turborepo-lib/src/opts.rs index 3025fa41f7d8f..173fc6960d272 100644 --- a/crates/turborepo-lib/src/opts.rs +++ b/crates/turborepo-lib/src/opts.rs @@ -14,7 +14,7 @@ use crate::{ OutputLogsMode, RunArgs, }, config::{ConfigurationOptions, CONFIG_FILE}, - turbo_json::UIMode, + turbo_json::{FutureFlags, UIMode}, Args, }; @@ -77,6 +77,7 @@ pub struct Opts { pub runcache_opts: RunCacheOpts, pub scope_opts: ScopeOpts, pub tui_opts: TuiOpts, + pub future_flags: FutureFlags, } impl Opts { @@ -180,6 +181,7 @@ impl Opts { let api_client_opts = APIClientOpts::from(inputs); let repo_opts = RepoOpts::from(inputs); let tui_opts = TuiOpts::from(inputs); + let future_flags = config.future_flags(); Ok(Self { repo_opts, @@ -189,6 +191,7 @@ impl Opts { runcache_opts, api_client_opts, tui_opts, + future_flags, }) } } @@ -747,6 +750,7 @@ mod test { cache_opts, runcache_opts, tui_opts, + future_flags: Default::default(), }; let synthesized = opts.synthesize_command(); assert_eq!(synthesized, expected); diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index 7dcd7b4678cb0..12c8f71a2af4f 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -25,7 +25,7 @@ use turborepo_scm::GitHashes; use crate::{ config::{resolve_turbo_config_path, CONFIG_FILE, CONFIG_FILE_JSONC}, - turbo_json::{TurboJson, TurboJsonLoader}, + turbo_json::{TurboJson, TurboJsonLoader, TurboJsonReader}, }; #[derive(Clone)] @@ -232,7 +232,7 @@ impl Subscriber { }; let root_turbo_json = TurboJsonLoader::workspace( - self.repo_root.clone(), + TurboJsonReader::new(self.repo_root.clone()), config_path, pkg_dep_graph.packages(), ) diff --git a/crates/turborepo-lib/src/run/builder.rs b/crates/turborepo-lib/src/run/builder.rs index 08b9e0e8e50fb..52c2801f33f8d 100644 --- a/crates/turborepo-lib/src/run/builder.rs +++ b/crates/turborepo-lib/src/run/builder.rs @@ -49,7 +49,7 @@ use crate::{ opts::Opts, run::{scope, task_access::TaskAccess, Error, Run, RunCache}, shim::TurboState, - turbo_json::{TurboJson, TurboJsonLoader, UIMode}, + turbo_json::{TurboJson, TurboJsonLoader, TurboJsonReader, UIMode}, DaemonConnector, }; @@ -408,16 +408,19 @@ impl RunBuilder { task_access.restore_config().await; let root_turbo_json_path = self.opts.repo_opts.root_turbo_json_path.clone(); + let future_flags = self.opts.future_flags; + + let reader = TurboJsonReader::new(self.repo_root.clone()).with_future_flags(future_flags); let turbo_json_loader = if task_access.is_enabled() { TurboJsonLoader::task_access( - self.repo_root.clone(), + reader, root_turbo_json_path.clone(), root_package_json.clone(), ) } else if is_single_package { TurboJsonLoader::single_package( - self.repo_root.clone(), + reader, root_turbo_json_path.clone(), root_package_json.clone(), ) @@ -426,20 +429,20 @@ impl RunBuilder { (self.opts.repo_opts.allow_no_turbo_json || micro_frontend_configs.is_some()) { TurboJsonLoader::workspace_no_turbo_json( - self.repo_root.clone(), + reader, pkg_dep_graph.packages(), micro_frontend_configs.clone(), ) } else if let Some(micro_frontends) = µ_frontend_configs { TurboJsonLoader::workspace_with_microfrontends( - self.repo_root.clone(), + reader, root_turbo_json_path.clone(), pkg_dep_graph.packages(), micro_frontends.clone(), ) } else { TurboJsonLoader::workspace( - self.repo_root.clone(), + reader, root_turbo_json_path.clone(), pkg_dep_graph.packages(), ) diff --git a/crates/turborepo-lib/src/turbo_json/loader.rs b/crates/turborepo-lib/src/turbo_json/loader.rs index 5a9b2eefe6a0b..c271b62f5535f 100644 --- a/crates/turborepo-lib/src/turbo_json/loader.rs +++ b/crates/turborepo-lib/src/turbo_json/loader.rs @@ -16,6 +16,7 @@ use crate::{ config::{Error, CONFIG_FILE, CONFIG_FILE_JSONC}, microfrontends::MicrofrontendsConfigs, run::task_access::TASK_ACCESS_CONFIG_PATH, + turbo_json::FutureFlags, }; /// Structure for loading TurboJson structures. @@ -23,7 +24,7 @@ use crate::{ /// `turbo.json` file. #[derive(Debug, Clone)] pub struct TurboJsonLoader { - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, cache: FixedMap, strategy: Strategy, } @@ -53,20 +54,23 @@ enum Strategy { // A helper structure configured with all settings related to reading a // `turbo.json` file from disk. -struct TurboJsonReader<'a> { - repo_root: &'a AbsoluteSystemPath, +#[derive(Debug, Clone)] +pub struct TurboJsonReader { + repo_root: AbsoluteSystemPathBuf, + future_flags: FutureFlags, } impl TurboJsonLoader { /// Create a loader that will load turbo.json files throughout the workspace pub fn workspace<'a>( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, root_turbo_json_path: AbsoluteSystemPathBuf, packages: impl Iterator, ) -> Self { - let packages = package_turbo_json_dirs(&repo_root, root_turbo_json_path, packages); + let repo_root = reader.repo_root(); + let packages = package_turbo_json_dirs(repo_root, root_turbo_json_path, packages); Self { - repo_root, + reader, cache: FixedMap::new(packages.keys().cloned()), strategy: Strategy::Workspace { packages, @@ -77,14 +81,15 @@ impl TurboJsonLoader { /// Create a loader that will load turbo.json files throughout the workspace pub fn workspace_with_microfrontends<'a>( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, root_turbo_json_path: AbsoluteSystemPathBuf, packages: impl Iterator, micro_frontends_configs: MicrofrontendsConfigs, ) -> Self { - let packages = package_turbo_json_dirs(&repo_root, root_turbo_json_path, packages); + let repo_root = reader.repo_root(); + let packages = package_turbo_json_dirs(repo_root, root_turbo_json_path, packages); Self { - repo_root, + reader, cache: FixedMap::new(packages.keys().cloned()), strategy: Strategy::Workspace { packages, @@ -96,13 +101,13 @@ impl TurboJsonLoader { /// Create a loader that will construct turbo.json structures based on /// workspace `package.json`s. pub fn workspace_no_turbo_json<'a>( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, packages: impl Iterator, microfrontends_configs: Option, ) -> Self { let packages = workspace_package_scripts(packages); Self { - repo_root, + reader, cache: FixedMap::new(packages.keys().cloned()), strategy: Strategy::WorkspaceNoTurboJson { packages, @@ -114,12 +119,12 @@ impl TurboJsonLoader { /// Create a loader that will load a root turbo.json or synthesize one if /// the file doesn't exist pub fn single_package( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, root_turbo_json: AbsoluteSystemPathBuf, package_json: PackageJson, ) -> Self { Self { - repo_root, + reader, cache: FixedMap::new(Some(PackageName::Root).into_iter()), strategy: Strategy::SinglePackage { root_turbo_json, @@ -131,12 +136,12 @@ impl TurboJsonLoader { /// Create a loader that will load a root turbo.json or synthesize one if /// the file doesn't exist pub fn task_access( - repo_root: AbsoluteSystemPathBuf, + reader: TurboJsonReader, root_turbo_json: AbsoluteSystemPathBuf, package_json: PackageJson, ) -> Self { Self { - repo_root, + reader, cache: FixedMap::new(Some(PackageName::Root).into_iter()), strategy: Strategy::TaskAccess { root_turbo_json, @@ -154,11 +159,12 @@ impl TurboJsonLoader { .into_iter() .map(|(key, value)| (key, Some(value))), ); + // This never gets read from so we populate it with root + let repo_root = AbsoluteSystemPath::new(if cfg!(windows) { "C:\\" } else { "/" }) + .expect("wasn't able to create absolute system path") + .to_owned(); Self { - // This never gets read from so we populate it with - repo_root: AbsoluteSystemPath::new(if cfg!(windows) { "C:\\" } else { "/" }) - .expect("wasn't able to create absolute system path") - .to_owned(), + reader: TurboJsonReader::new(repo_root), cache, strategy: Strategy::Noop, } @@ -176,7 +182,7 @@ impl TurboJsonLoader { } fn uncached_load(&self, package: &PackageName) -> Result { - let reader = self.reader(); + let reader = &self.reader; match &self.strategy { Strategy::SinglePackage { package_json, @@ -236,12 +242,6 @@ impl TurboJsonLoader { Strategy::Noop => Err(Error::NoTurboJSON), } } - - fn reader<'a>(&'a self) -> TurboJsonReader<'a> { - TurboJsonReader { - repo_root: &self.repo_root, - } - } } /// Map all packages in the package graph to their dirs that contain a @@ -286,7 +286,7 @@ enum LoadTurboJsonPath<'a> { } fn load_from_file( - reader: TurboJsonReader, + reader: &TurboJsonReader, turbo_json_path: LoadTurboJsonPath, ) -> Result { let result = match turbo_json_path { @@ -314,7 +314,7 @@ fn load_from_file( } fn load_from_root_package_json( - reader: TurboJsonReader, + reader: &TurboJsonReader, turbo_json_path: &AbsoluteSystemPath, root_package_json: &PackageJson, ) -> Result { @@ -410,7 +410,7 @@ fn workspace_turbo_json_from_scripts(scripts: &[String]) -> Result Result { @@ -459,13 +459,25 @@ fn select_turbo_json( } } -impl TurboJsonReader<'_> { +impl TurboJsonReader { + pub fn new(repo_root: AbsoluteSystemPathBuf) -> Self { + Self { + repo_root, + future_flags: Default::default(), + } + } + + pub fn with_future_flags(mut self, future_flags: FutureFlags) -> Self { + self.future_flags = future_flags; + self + } + pub fn read(&self, path: &AbsoluteSystemPath) -> Result, Error> { - TurboJson::read(self.repo_root, path) + TurboJson::read(&self.repo_root, path, self.future_flags) } pub fn repo_root(&self) -> &AbsoluteSystemPath { - self.repo_root + &self.repo_root } } @@ -509,8 +521,9 @@ mod test { let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path())?; let root_turbo_json = repo_root.join_component("turbo.json"); fs::write(&root_turbo_json, turbo_json_content)?; + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(Some(PackageName::Root).into_iter()), strategy: Strategy::Workspace { packages: vec![(PackageName::Root, root_turbo_json.to_owned())] @@ -590,11 +603,8 @@ mod test { fs::write(&root_turbo_json, content)?; } - let loader = TurboJsonLoader::single_package( - repo_root.to_owned(), - root_turbo_json, - root_package_json, - ); + let reader = TurboJsonReader::new(repo_root.to_owned()); + let loader = TurboJsonLoader::single_package(reader, root_turbo_json, root_package_json); let mut turbo_json = loader.load(&PackageName::Root)?.clone(); turbo_json.text = None; turbo_json.path = None; @@ -652,8 +662,8 @@ mod test { ..Default::default() }; - let loader = - TurboJsonLoader::task_access(repo_root.to_owned(), root_turbo_json, root_package_json); + let reader = TurboJsonReader::new(repo_root.to_owned()); + let loader = TurboJsonLoader::task_access(reader, root_turbo_json, root_package_json); let turbo_json = loader.load(&PackageName::Root)?; let root_build = turbo_json .tasks @@ -678,16 +688,14 @@ mod test { }) .unwrap(); let non_root = PackageName::from("some-pkg"); + let reader = TurboJsonReader::new(junk_path.to_owned()); let single_loader = TurboJsonLoader::single_package( - junk_path.to_owned(), - junk_path.to_owned(), - PackageJson::default(), - ); - let task_access_loader = TurboJsonLoader::task_access( - junk_path.to_owned(), + reader.clone(), junk_path.to_owned(), PackageJson::default(), ); + let task_access_loader = + TurboJsonLoader::task_access(reader, junk_path.to_owned(), PackageJson::default()); for loader in [single_loader, task_access_loader] { let result = loader.load(&non_root); @@ -713,8 +721,9 @@ mod test { .into_iter() .collect(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(vec![PackageName::Root, PackageName::from("a")].into_iter()), strategy: Strategy::Workspace { packages, @@ -748,8 +757,9 @@ mod test { .into_iter() .collect(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(vec![PackageName::Root, PackageName::from("a")].into_iter()), strategy: Strategy::Workspace { packages, @@ -783,8 +793,9 @@ mod test { .into_iter() .collect(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(vec![PackageName::Root, PackageName::from("pkg-a")].into_iter()), strategy: Strategy::WorkspaceNoTurboJson { packages, @@ -864,8 +875,9 @@ mod test { ) .unwrap(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new( vec![ PackageName::Root, @@ -924,7 +936,7 @@ mod test { fn test_load_from_file_with_both_files() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; - let reader = TurboJsonReader { repo_root }; + let reader = TurboJsonReader::new(repo_root.to_owned()); // Create both turbo.json and turbo.jsonc let turbo_json_path = repo_root.join_component(CONFIG_FILE); @@ -934,7 +946,7 @@ mod test { turbo_jsonc_path.create_with_contents("{}")?; // Test load_from_file with turbo.json path - let result = load_from_file(reader, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); // The function should return an error when both files exist assert!(result.is_err()); @@ -955,14 +967,14 @@ mod test { fn test_load_from_file_with_only_turbo_json() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; - let reader = TurboJsonReader { repo_root }; + let reader = TurboJsonReader::new(repo_root.to_owned()); // Create only turbo.json let turbo_json_path = repo_root.join_component(CONFIG_FILE); turbo_json_path.create_with_contents("{}")?; // Test load_from_file - let result = load_from_file(reader, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); assert!(result.is_ok()); @@ -973,14 +985,14 @@ mod test { fn test_load_from_file_with_only_turbo_jsonc() -> Result<()> { let tmp_dir = tempdir()?; let repo_root = AbsoluteSystemPath::from_std_path(tmp_dir.path())?; - let reader = TurboJsonReader { repo_root }; + let reader = TurboJsonReader::new(repo_root.to_owned()); // Create only turbo.jsonc let turbo_jsonc_path = repo_root.join_component(CONFIG_FILE_JSONC); turbo_jsonc_path.create_with_contents("{}")?; // Test load_from_file - let result = load_from_file(reader, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); assert!(result.is_ok()); @@ -1004,8 +1016,9 @@ mod test { .create_with_contents(r#"{"tasks": {"build": {"lol": true}}}"#) .unwrap(); + let reader = TurboJsonReader::new(repo_root.to_owned()); let loader = TurboJsonLoader { - repo_root: repo_root.to_owned(), + reader, cache: FixedMap::new(vec![PackageName::Root, PackageName::from("a")].into_iter()), strategy: Strategy::Workspace { packages, diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 88576494ccb4b..55d12e61d6906 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -28,7 +28,7 @@ mod extend; mod loader; pub mod parser; -pub use loader::TurboJsonLoader; +pub use loader::{TurboJsonLoader, TurboJsonReader}; use crate::{boundaries::BoundariesConfig, config::UnnecessaryPackageTaskSyntaxError}; @@ -158,7 +158,7 @@ pub struct RawTurboJson { _comment: Option, } -#[derive(Serialize, Default, Debug, Clone, Iterable, Deserializable, PartialEq, Eq)] +#[derive(Serialize, Default, Debug, Copy, Clone, Iterable, Deserializable, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct FutureFlags {} @@ -614,9 +614,13 @@ impl TurboJson { /// Reads a `RawTurboJson` from the given path /// and then converts it into `TurboJson` - pub(crate) fn read( + /// + /// Should never be called directly outside of this module. + /// `TurboJsonReader` should be used instead. + fn read( repo_root: &AbsoluteSystemPath, path: &AbsoluteSystemPath, + _future_flags: FutureFlags, ) -> Result, Error> { let Some(raw_turbo_json) = RawTurboJson::read(repo_root, path)? else { return Ok(None); From a57cf49f2162dc3442a5cae42101625def8f2331 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 12 Aug 2025 09:34:48 -0400 Subject: [PATCH 03/19] chore(turbo_json): add turbo_extends feature flag --- crates/turborepo-lib/src/turbo_json/mod.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 55d12e61d6906..1ff3b501504df 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -160,7 +160,10 @@ pub struct RawTurboJson { #[derive(Serialize, Default, Debug, Copy, Clone, Iterable, Deserializable, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct FutureFlags {} +pub struct FutureFlags { + #[deserializable(rename = "turbo_extends")] + turbo_extends: bool, +} #[derive(Serialize, Default, Debug, PartialEq, Clone)] #[serde(transparent)] @@ -1380,7 +1383,7 @@ mod tests { "build": {} }, "futureFlags": { - "bestFeature": true + "turboExtends": true } }"#; @@ -1394,7 +1397,12 @@ mod tests { // Verify that futureFlags is parsed correctly assert!(raw_turbo_json.future_flags.is_some()); let future_flags = raw_turbo_json.future_flags.as_ref().unwrap(); - assert_eq!(future_flags.as_inner(), &FutureFlags {}); + assert_eq!( + future_flags.as_inner(), + &FutureFlags { + turbo_extends: true + } + ); // Verify that the futureFlags field doesn't cause errors during conversion to // TurboJson From 7338c539ac4418099043b4f61b5aec99e66b3612 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 12 Aug 2025 11:57:52 -0400 Subject: [PATCH 04/19] chore(turbo_json): add processed task definition --- crates/turborepo-lib/src/engine/builder.rs | 20 ++-- crates/turborepo-lib/src/turbo_json/extend.rs | 110 ++++++++++++------ crates/turborepo-lib/src/turbo_json/mod.rs | 102 +++++++++++++++- 3 files changed, 180 insertions(+), 52 deletions(-) diff --git a/crates/turborepo-lib/src/engine/builder.rs b/crates/turborepo-lib/src/engine/builder.rs index 1cf95f1cbcdc5..8625918fc2ea5 100644 --- a/crates/turborepo-lib/src/engine/builder.rs +++ b/crates/turborepo-lib/src/engine/builder.rs @@ -15,7 +15,7 @@ use crate::{ task_graph::TaskDefinition, turbo_json::{ validate_extends, validate_no_package_task_syntax, validate_with_has_no_topo, - RawTaskDefinition, TurboJsonLoader, + ProcessedTaskDefinition, TurboJsonLoader, }, }; @@ -564,14 +564,12 @@ impl<'a> EngineBuilder<'a> { task_id: &Spanned, task_name: &TaskName, ) -> Result { - let raw_task_definition = RawTaskDefinition::from_iter(self.task_definition_chain( - turbo_json_loader, - task_id, - task_name, - )?); + let processed_task_definition = ProcessedTaskDefinition::from_iter( + self.task_definition_chain(turbo_json_loader, task_id, task_name)?, + ); let path_to_root = self.path_to_root(task_id.as_inner())?; - Ok(TaskDefinition::from_raw( - raw_task_definition, + Ok(TaskDefinition::from_processed( + processed_task_definition, &path_to_root, )?) } @@ -581,7 +579,7 @@ impl<'a> EngineBuilder<'a> { turbo_json_loader: &TurboJsonLoader, task_id: &Spanned, task_name: &TaskName, - ) -> Result, Error> { + ) -> Result, Error> { let mut task_definitions = Vec::new(); let root_turbo_json = turbo_json_loader.load(&PackageName::Root)?; @@ -616,8 +614,8 @@ impl<'a> EngineBuilder<'a> { validate_with_has_no_topo, ]))?; - if let Some(workspace_def) = workspace_json.tasks.get(task_name) { - task_definitions.push(workspace_def.value.clone()); + if let Some(workspace_def) = workspace_json.task(task_id, task_name) { + task_definitions.push(workspace_def); } } Err(config::Error::NoTurboJSON) => (), diff --git a/crates/turborepo-lib/src/turbo_json/extend.rs b/crates/turborepo-lib/src/turbo_json/extend.rs index 4783aff452a14..58479e0b739d1 100644 --- a/crates/turborepo-lib/src/turbo_json/extend.rs +++ b/crates/turborepo-lib/src/turbo_json/extend.rs @@ -1,11 +1,11 @@ //! Module for code related to "extends" behavior for task definitions -use super::RawTaskDefinition; +use super::ProcessedTaskDefinition; -impl FromIterator for RawTaskDefinition { - fn from_iter>(iter: T) -> Self { +impl FromIterator for ProcessedTaskDefinition { + fn from_iter>(iter: T) -> Self { iter.into_iter() - .fold(RawTaskDefinition::default(), |mut def, other| { + .fold(ProcessedTaskDefinition::default(), |mut def, other| { def.merge(other); def }) @@ -20,10 +20,10 @@ macro_rules! set_field { }}; } -impl RawTaskDefinition { - // Merges another RawTaskDefinition into this one +impl ProcessedTaskDefinition { + // Merges another ProcessedTaskDefinition into this one // By default any fields present on `other` will override present fields. - pub fn merge(&mut self, other: RawTaskDefinition) { + pub fn merge(&mut self, other: ProcessedTaskDefinition) { set_field!(self, other, outputs); let other_has_range = other.cache.as_ref().is_some_and(|c| c.range.is_some()); @@ -54,38 +54,72 @@ mod test { use turborepo_unescape::UnescapedString; use super::*; - use crate::cli::OutputLogsMode; + use crate::{ + cli::OutputLogsMode, + turbo_json::{ProcessedEnv, ProcessedInputs, ProcessedOutputs}, + }; // Shared test fixtures - fn create_base_task() -> RawTaskDefinition { - RawTaskDefinition { + fn create_base_task() -> ProcessedTaskDefinition { + ProcessedTaskDefinition { cache: Some(Spanned::new(true)), persistent: Some(Spanned::new(false)), - outputs: Some(vec![Spanned::new(UnescapedString::from("dist/**"))]), - inputs: Some(vec![Spanned::new(UnescapedString::from("src/**"))]), - env: Some(vec![Spanned::new(UnescapedString::from("NODE_ENV"))]), - ..Default::default() + outputs: Some(ProcessedOutputs(vec![Spanned::new(UnescapedString::from( + "dist/**", + ))])), + inputs: Some(ProcessedInputs(vec![Spanned::new(UnescapedString::from( + "src/**", + ))])), + env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( + "NODE_ENV", + ))])), + depends_on: None, + pass_through_env: None, + output_logs: None, + interruptible: None, + interactive: None, + env_mode: None, + with: None, } } - fn create_override_task() -> RawTaskDefinition { - RawTaskDefinition { + fn create_override_task() -> ProcessedTaskDefinition { + ProcessedTaskDefinition { cache: Some(Spanned::new(false)), persistent: Some(Spanned::new(true)), - outputs: Some(vec![Spanned::new(UnescapedString::from("build/**"))]), - inputs: Some(vec![Spanned::new(UnescapedString::from("lib/**"))]), - env: Some(vec![Spanned::new(UnescapedString::from("PROD_ENV"))]), + outputs: Some(ProcessedOutputs(vec![Spanned::new(UnescapedString::from( + "build/**", + ))])), + inputs: Some(ProcessedInputs(vec![Spanned::new(UnescapedString::from( + "lib/**", + ))])), + env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( + "PROD_ENV", + ))])), output_logs: Some(Spanned::new(OutputLogsMode::Full)), interruptible: Some(Spanned::new(true)), - ..Default::default() + depends_on: None, + pass_through_env: None, + interactive: None, + env_mode: None, + with: None, } } - fn create_partial_task() -> RawTaskDefinition { - RawTaskDefinition { + fn create_partial_task() -> ProcessedTaskDefinition { + ProcessedTaskDefinition { persistent: Some(Spanned::new(true)), output_logs: Some(Spanned::new(OutputLogsMode::HashOnly)), - ..Default::default() + cache: None, + outputs: None, + inputs: None, + env: None, + depends_on: None, + pass_through_env: None, + interruptible: None, + interactive: None, + env_mode: None, + with: None, } } @@ -153,7 +187,7 @@ mod test { let third = create_override_task(); let tasks = vec![first.clone(), second.clone(), third.clone()]; - let result: RawTaskDefinition = tasks.into_iter().collect(); + let result: ProcessedTaskDefinition = tasks.into_iter().collect(); // Fields present in the last task (third) should take priority assert_eq!(result.cache, third.cache); @@ -171,20 +205,26 @@ mod test { #[test] fn test_from_iter_combines_across_multiple_tasks() { - let first = RawTaskDefinition { + let first = ProcessedTaskDefinition { cache: Some(Spanned::new(true)), - outputs: Some(vec![Spanned::new(UnescapedString::from("dist/**"))]), + outputs: Some(ProcessedOutputs(vec![Spanned::new(UnescapedString::from( + "dist/**", + ))])), ..Default::default() }; - let second = RawTaskDefinition { + let second = ProcessedTaskDefinition { persistent: Some(Spanned::new(false)), - inputs: Some(vec![Spanned::new(UnescapedString::from("src/**"))]), + inputs: Some(ProcessedInputs(vec![Spanned::new(UnescapedString::from( + "src/**", + ))])), ..Default::default() }; - let third = RawTaskDefinition { - env: Some(vec![Spanned::new(UnescapedString::from("NODE_ENV"))]), + let third = ProcessedTaskDefinition { + env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( + "NODE_ENV", + ))])), output_logs: Some(Spanned::new(OutputLogsMode::Full)), // Override cache from first task cache: Some(Spanned::new(false)), @@ -192,7 +232,7 @@ mod test { }; let tasks = vec![first.clone(), second.clone(), third.clone()]; - let result: RawTaskDefinition = tasks.into_iter().collect(); + let result: ProcessedTaskDefinition = tasks.into_iter().collect(); // Last task's cache should override first task's cache assert_eq!(result.cache, third.cache); @@ -211,18 +251,18 @@ mod test { #[test] fn test_from_iter_empty_iterator() { - let empty_vec: Vec = vec![]; - let result: RawTaskDefinition = empty_vec.into_iter().collect(); + let empty_vec: Vec = vec![]; + let result: ProcessedTaskDefinition = empty_vec.into_iter().collect(); // Should be equivalent to default - assert_eq!(result, RawTaskDefinition::default()); + assert_eq!(result, ProcessedTaskDefinition::default()); } #[test] fn test_from_iter_single_task() { let single_task = create_base_task(); let tasks = vec![single_task.clone()]; - let result: RawTaskDefinition = tasks.into_iter().collect(); + let result: ProcessedTaskDefinition = tasks.into_iter().collect(); assert_eq!(result, single_task); } diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 1ff3b501504df..21bd33e7103eb 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -161,7 +161,6 @@ pub struct RawTurboJson { #[derive(Serialize, Default, Debug, Copy, Clone, Iterable, Deserializable, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct FutureFlags { - #[deserializable(rename = "turbo_extends")] turbo_extends: bool, } @@ -337,11 +336,89 @@ impl TryFrom>>> for TaskInputs { } } +/// Processed depends_on field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedDependsOn(pub Spanned>>); + +/// Processed env field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedEnv(pub Vec>); + +/// Processed inputs field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedInputs(pub Vec>); + +/// Processed pass_through_env field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedPassThroughEnv(pub Vec>); + +/// Processed outputs field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedOutputs(pub Vec>); + +/// Intermediate representation for task definitions with DSL processing +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ProcessedTaskDefinition { + pub cache: Option>, + pub depends_on: Option, + pub env: Option, + pub inputs: Option, + pub pass_through_env: Option, + pub persistent: Option>, + pub interruptible: Option>, + pub outputs: Option, + pub output_logs: Option>, + pub interactive: Option>, + pub env_mode: Option>, + pub with: Option>>, +} + +impl ProcessedTaskDefinition { + /// Creates a processed task definition from raw task + pub fn from_raw(raw_task: RawTaskDefinition) -> Self { + ProcessedTaskDefinition { + cache: raw_task.cache, + depends_on: raw_task.depends_on.map(ProcessedDependsOn), + env: raw_task.env.map(ProcessedEnv), + inputs: raw_task.inputs.map(ProcessedInputs), + pass_through_env: raw_task.pass_through_env.map(ProcessedPassThroughEnv), + persistent: raw_task.persistent, + interruptible: raw_task.interruptible, + outputs: raw_task.outputs.map(ProcessedOutputs), + output_logs: raw_task.output_logs, + interactive: raw_task.interactive, + env_mode: raw_task.env_mode, + with: raw_task.with, + } + } + + /// Converts back to RawTaskDefinition + pub fn into_raw(self) -> RawTaskDefinition { + RawTaskDefinition { + cache: self.cache, + depends_on: self.depends_on.map(|d| d.0), + env: self.env.map(|e| e.0), + inputs: self.inputs.map(|i| i.0), + pass_through_env: self.pass_through_env.map(|p| p.0), + persistent: self.persistent, + interruptible: self.interruptible, + outputs: self.outputs.map(|o| o.0), + output_logs: self.output_logs, + interactive: self.interactive, + env_mode: self.env_mode, + with: self.with, + } + } +} + impl TaskDefinition { - pub fn from_raw( - mut raw_task: RawTaskDefinition, + /// Creates a TaskDefinition from a ProcessedTaskDefinition + pub fn from_processed( + processed: ProcessedTaskDefinition, path_to_repo_root: &RelativeUnixPath, ) -> Result { + // Convert back to raw and apply token replacement + let mut raw_task = processed.into_raw(); replace_turbo_root_token(&mut raw_task, path_to_repo_root)?; let outputs = raw_task.outputs.unwrap_or_default().try_into()?; @@ -444,6 +521,16 @@ impl TaskDefinition { with, }) } + + /// Helper method for tests that still use RawTaskDefinition + #[cfg(test)] + fn from_raw( + raw_task: RawTaskDefinition, + path_to_repo_root: &RelativeUnixPath, + ) -> Result { + let processed = ProcessedTaskDefinition::from_raw(raw_task); + Self::from_processed(processed, path_to_repo_root) + } } impl RawTurboJson { @@ -631,10 +718,13 @@ impl TurboJson { TurboJson::try_from(raw_turbo_json).map(Some) } - pub fn task(&self, task_id: &TaskId, task_name: &TaskName) -> Option { + pub fn task(&self, task_id: &TaskId, task_name: &TaskName) -> Option { match self.tasks.get(&task_id.as_task_name()) { - Some(entry) => Some(entry.value.clone()), - None => self.tasks.get(task_name).map(|entry| entry.value.clone()), + Some(entry) => Some(ProcessedTaskDefinition::from_raw(entry.value.clone())), + None => self + .tasks + .get(task_name) + .map(|entry| ProcessedTaskDefinition::from_raw(entry.value.clone())), } } From 6e4f36fec824e4e2cc3cefbdb219fb544d799fe5 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 12 Aug 2025 12:34:11 -0400 Subject: [PATCH 05/19] chore(turbo_json): move processed task definition to own module --- crates/turborepo-lib/src/turbo_json/extend.rs | 4 +- crates/turborepo-lib/src/turbo_json/mod.rs | 77 +---------------- .../turborepo-lib/src/turbo_json/processed.rs | 82 +++++++++++++++++++ 3 files changed, 86 insertions(+), 77 deletions(-) create mode 100644 crates/turborepo-lib/src/turbo_json/processed.rs diff --git a/crates/turborepo-lib/src/turbo_json/extend.rs b/crates/turborepo-lib/src/turbo_json/extend.rs index 58479e0b739d1..7cfa0fb2b461e 100644 --- a/crates/turborepo-lib/src/turbo_json/extend.rs +++ b/crates/turborepo-lib/src/turbo_json/extend.rs @@ -1,6 +1,6 @@ //! Module for code related to "extends" behavior for task definitions -use super::ProcessedTaskDefinition; +use super::processed::ProcessedTaskDefinition; impl FromIterator for ProcessedTaskDefinition { fn from_iter>(iter: T) -> Self { @@ -56,7 +56,7 @@ mod test { use super::*; use crate::{ cli::OutputLogsMode, - turbo_json::{ProcessedEnv, ProcessedInputs, ProcessedOutputs}, + turbo_json::processed::{ProcessedEnv, ProcessedInputs, ProcessedOutputs}, }; // Shared test fixtures diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 21bd33e7103eb..6eec4aba97df0 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -27,8 +27,10 @@ use crate::{ mod extend; mod loader; pub mod parser; +mod processed; pub use loader::{TurboJsonLoader, TurboJsonReader}; +pub use processed::ProcessedTaskDefinition; use crate::{boundaries::BoundariesConfig, config::UnnecessaryPackageTaskSyntaxError}; @@ -336,81 +338,6 @@ impl TryFrom>>> for TaskInputs { } } -/// Processed depends_on field with DSL detection -#[derive(Debug, Clone, PartialEq)] -pub struct ProcessedDependsOn(pub Spanned>>); - -/// Processed env field with DSL detection -#[derive(Debug, Clone, PartialEq)] -pub struct ProcessedEnv(pub Vec>); - -/// Processed inputs field with DSL detection -#[derive(Debug, Clone, PartialEq)] -pub struct ProcessedInputs(pub Vec>); - -/// Processed pass_through_env field with DSL detection -#[derive(Debug, Clone, PartialEq)] -pub struct ProcessedPassThroughEnv(pub Vec>); - -/// Processed outputs field with DSL detection -#[derive(Debug, Clone, PartialEq)] -pub struct ProcessedOutputs(pub Vec>); - -/// Intermediate representation for task definitions with DSL processing -#[derive(Debug, Clone, PartialEq, Default)] -pub struct ProcessedTaskDefinition { - pub cache: Option>, - pub depends_on: Option, - pub env: Option, - pub inputs: Option, - pub pass_through_env: Option, - pub persistent: Option>, - pub interruptible: Option>, - pub outputs: Option, - pub output_logs: Option>, - pub interactive: Option>, - pub env_mode: Option>, - pub with: Option>>, -} - -impl ProcessedTaskDefinition { - /// Creates a processed task definition from raw task - pub fn from_raw(raw_task: RawTaskDefinition) -> Self { - ProcessedTaskDefinition { - cache: raw_task.cache, - depends_on: raw_task.depends_on.map(ProcessedDependsOn), - env: raw_task.env.map(ProcessedEnv), - inputs: raw_task.inputs.map(ProcessedInputs), - pass_through_env: raw_task.pass_through_env.map(ProcessedPassThroughEnv), - persistent: raw_task.persistent, - interruptible: raw_task.interruptible, - outputs: raw_task.outputs.map(ProcessedOutputs), - output_logs: raw_task.output_logs, - interactive: raw_task.interactive, - env_mode: raw_task.env_mode, - with: raw_task.with, - } - } - - /// Converts back to RawTaskDefinition - pub fn into_raw(self) -> RawTaskDefinition { - RawTaskDefinition { - cache: self.cache, - depends_on: self.depends_on.map(|d| d.0), - env: self.env.map(|e| e.0), - inputs: self.inputs.map(|i| i.0), - pass_through_env: self.pass_through_env.map(|p| p.0), - persistent: self.persistent, - interruptible: self.interruptible, - outputs: self.outputs.map(|o| o.0), - output_logs: self.output_logs, - interactive: self.interactive, - env_mode: self.env_mode, - with: self.with, - } - } -} - impl TaskDefinition { /// Creates a TaskDefinition from a ProcessedTaskDefinition pub fn from_processed( diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs new file mode 100644 index 0000000000000..f5df4a5f6dab2 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -0,0 +1,82 @@ +//! Processed task definition types with DSL token handling + +use turborepo_errors::Spanned; +use turborepo_unescape::UnescapedString; + +use super::RawTaskDefinition; +use crate::cli::{EnvMode, OutputLogsMode}; + +/// Processed depends_on field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedDependsOn(pub Spanned>>); + +/// Processed env field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedEnv(pub Vec>); + +/// Processed inputs field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedInputs(pub Vec>); + +/// Processed pass_through_env field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedPassThroughEnv(pub Vec>); + +/// Processed outputs field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedOutputs(pub Vec>); + +/// Intermediate representation for task definitions with DSL processing +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ProcessedTaskDefinition { + pub cache: Option>, + pub depends_on: Option, + pub env: Option, + pub inputs: Option, + pub pass_through_env: Option, + pub persistent: Option>, + pub interruptible: Option>, + pub outputs: Option, + pub output_logs: Option>, + pub interactive: Option>, + pub env_mode: Option>, + pub with: Option>>, +} + +impl ProcessedTaskDefinition { + /// Creates a processed task definition from raw task + pub fn from_raw(raw_task: RawTaskDefinition) -> Self { + ProcessedTaskDefinition { + cache: raw_task.cache, + depends_on: raw_task.depends_on.map(ProcessedDependsOn), + env: raw_task.env.map(ProcessedEnv), + inputs: raw_task.inputs.map(ProcessedInputs), + pass_through_env: raw_task.pass_through_env.map(ProcessedPassThroughEnv), + persistent: raw_task.persistent, + interruptible: raw_task.interruptible, + outputs: raw_task.outputs.map(ProcessedOutputs), + output_logs: raw_task.output_logs, + interactive: raw_task.interactive, + env_mode: raw_task.env_mode, + with: raw_task.with, + } + } + + /// Converts back to RawTaskDefinition + pub fn into_raw(self) -> RawTaskDefinition { + RawTaskDefinition { + cache: self.cache, + depends_on: self.depends_on.map(|d| d.0), + env: self.env.map(|e| e.0), + inputs: self.inputs.map(|i| i.0), + pass_through_env: self.pass_through_env.map(|p| p.0), + persistent: self.persistent, + interruptible: self.interruptible, + outputs: self.outputs.map(|o| o.0), + output_logs: self.output_logs, + interactive: self.interactive, + env_mode: self.env_mode, + with: self.with, + } + } +} From 033b10a512302bd730d486ea4801c1f3b727ff17 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 12 Aug 2025 14:45:30 -0400 Subject: [PATCH 06/19] chore(turbo_json): fully use processed task definition --- crates/turborepo-lib/src/engine/builder.rs | 4 +- crates/turborepo-lib/src/turbo_json/extend.rs | 44 +-- crates/turborepo-lib/src/turbo_json/mod.rs | 269 ++++++----------- .../turborepo-lib/src/turbo_json/processed.rs | 274 ++++++++++++++++-- 4 files changed, 370 insertions(+), 221 deletions(-) diff --git a/crates/turborepo-lib/src/engine/builder.rs b/crates/turborepo-lib/src/engine/builder.rs index 8625918fc2ea5..82cd8f4abd45f 100644 --- a/crates/turborepo-lib/src/engine/builder.rs +++ b/crates/turborepo-lib/src/engine/builder.rs @@ -585,7 +585,7 @@ impl<'a> EngineBuilder<'a> { let root_turbo_json = turbo_json_loader.load(&PackageName::Root)?; Error::from_validation(root_turbo_json.validate(&[validate_with_has_no_topo]))?; - if let Some(root_definition) = root_turbo_json.task(task_id, task_name) { + if let Some(root_definition) = root_turbo_json.task(task_id, task_name)? { task_definitions.push(root_definition) } @@ -614,7 +614,7 @@ impl<'a> EngineBuilder<'a> { validate_with_has_no_topo, ]))?; - if let Some(workspace_def) = workspace_json.task(task_id, task_name) { + if let Some(workspace_def) = workspace_json.task(task_id, task_name)? { task_definitions.push(workspace_def); } } diff --git a/crates/turborepo-lib/src/turbo_json/extend.rs b/crates/turborepo-lib/src/turbo_json/extend.rs index 7cfa0fb2b461e..3ba5edc8005ba 100644 --- a/crates/turborepo-lib/src/turbo_json/extend.rs +++ b/crates/turborepo-lib/src/turbo_json/extend.rs @@ -56,7 +56,7 @@ mod test { use super::*; use crate::{ cli::OutputLogsMode, - turbo_json::processed::{ProcessedEnv, ProcessedInputs, ProcessedOutputs}, + turbo_json::processed::{ProcessedEnv, ProcessedGlob, ProcessedInputs, ProcessedOutputs}, }; // Shared test fixtures @@ -64,12 +64,14 @@ mod test { ProcessedTaskDefinition { cache: Some(Spanned::new(true)), persistent: Some(Spanned::new(false)), - outputs: Some(ProcessedOutputs(vec![Spanned::new(UnescapedString::from( - "dist/**", - ))])), - inputs: Some(ProcessedInputs(vec![Spanned::new(UnescapedString::from( - "src/**", - ))])), + outputs: Some(ProcessedOutputs(vec![ProcessedGlob::from_spanned_output( + Spanned::new(UnescapedString::from("dist/**")), + ) + .unwrap()])), + inputs: Some(ProcessedInputs(vec![ProcessedGlob::from_spanned_input( + Spanned::new(UnescapedString::from("src/**")), + ) + .unwrap()])), env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( "NODE_ENV", ))])), @@ -87,12 +89,14 @@ mod test { ProcessedTaskDefinition { cache: Some(Spanned::new(false)), persistent: Some(Spanned::new(true)), - outputs: Some(ProcessedOutputs(vec![Spanned::new(UnescapedString::from( - "build/**", - ))])), - inputs: Some(ProcessedInputs(vec![Spanned::new(UnescapedString::from( - "lib/**", - ))])), + outputs: Some(ProcessedOutputs(vec![ProcessedGlob::from_spanned_output( + Spanned::new(UnescapedString::from("build/**")), + ) + .unwrap()])), + inputs: Some(ProcessedInputs(vec![ProcessedGlob::from_spanned_input( + Spanned::new(UnescapedString::from("lib/**")), + ) + .unwrap()])), env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( "PROD_ENV", ))])), @@ -207,17 +211,19 @@ mod test { fn test_from_iter_combines_across_multiple_tasks() { let first = ProcessedTaskDefinition { cache: Some(Spanned::new(true)), - outputs: Some(ProcessedOutputs(vec![Spanned::new(UnescapedString::from( - "dist/**", - ))])), + outputs: Some(ProcessedOutputs(vec![ProcessedGlob::from_spanned_output( + Spanned::new(UnescapedString::from("dist/**")), + ) + .unwrap()])), ..Default::default() }; let second = ProcessedTaskDefinition { persistent: Some(Spanned::new(false)), - inputs: Some(ProcessedInputs(vec![Spanned::new(UnescapedString::from( - "src/**", - ))])), + inputs: Some(ProcessedInputs(vec![ProcessedGlob::from_spanned_input( + Spanned::new(UnescapedString::from("src/**")), + ) + .unwrap()])), ..Default::default() }; diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 6eec4aba97df0..dd73e3a8dd545 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -266,35 +266,24 @@ pub struct RawTaskDefinition { with: Option>>, } -impl TryFrom>> for TaskOutputs { - type Error = Error; - fn try_from(outputs: Vec>) -> Result { +impl TaskOutputs { + /// Creates TaskOutputs from ProcessedOutputs with resolved paths + fn from_processed( + outputs: processed::ProcessedOutputs, + turbo_root_path: &str, + ) -> Result { let mut inclusions = Vec::new(); let mut exclusions = Vec::new(); - for glob in outputs { - if let Some(stripped_glob) = glob.value.strip_prefix('!') { - if Utf8Path::new(stripped_glob).is_absolute() { - let (span, text) = glob.span_and_text("turbo.json"); - return Err(Error::AbsolutePathInConfig { - field: "outputs", - span, - text, - }); - } + // Resolve all globs with the turbo_root path + // Absolute path validation was already done during ProcessedGlob creation + let resolved = outputs.resolve(turbo_root_path); + for glob_str in resolved { + if let Some(stripped_glob) = glob_str.strip_prefix('!') { exclusions.push(stripped_glob.to_string()); } else { - if Utf8Path::new(&glob.value).is_absolute() { - let (span, text) = glob.span_and_text("turbo.json"); - return Err(Error::AbsolutePathInConfig { - field: "outputs", - span, - text, - }); - } - - inclusions.push(glob.into_inner().into()); + inclusions.push(glob_str); } } @@ -308,30 +297,25 @@ impl TryFrom>> for TaskOutputs { } } -impl TryFrom>>> for TaskInputs { - type Error = Error; +impl TaskInputs { + /// Creates TaskInputs from ProcessedInputs with resolved paths + fn from_processed( + inputs: processed::ProcessedInputs, + turbo_root_path: &str, + ) -> Result { + let mut globs = Vec::new(); + let mut default = false; - fn try_from(inputs: Option>>) -> Result { - let mut globs = Vec::with_capacity(inputs.as_ref().map_or(0, |inputs| inputs.len())); + // Resolve all globs with the turbo_root path + // Absolute path validation was already done during ProcessedGlob creation + let resolved = inputs.resolve(turbo_root_path); - let mut default = false; - for input in inputs.into_iter().flatten() { - // If the inputs contain "$TURBO_DEFAULT$", we need to include the "default" - // file hashes as well. NOTE: we intentionally don't remove - // "$TURBO_DEFAULT$" from the inputs if it exists in the off chance that - // the user has a file named "$TURBO_DEFAULT$" in their package (pls - // no). - if input.as_str() == TURBO_DEFAULT { + for glob_str in resolved { + // Check for $TURBO_DEFAULT$ + if glob_str == TURBO_DEFAULT { default = true; - } else if Utf8Path::new(input.as_str()).is_absolute() { - let (span, text) = input.span_and_text("turbo.json"); - return Err(Error::AbsolutePathInConfig { - field: "inputs", - span, - text, - }); } - globs.push(input.to_string()); + globs.push(glob_str); } Ok(TaskInputs { globs, default }) @@ -344,27 +328,29 @@ impl TaskDefinition { processed: ProcessedTaskDefinition, path_to_repo_root: &RelativeUnixPath, ) -> Result { - // Convert back to raw and apply token replacement - let mut raw_task = processed.into_raw(); - replace_turbo_root_token(&mut raw_task, path_to_repo_root)?; - let outputs = raw_task.outputs.unwrap_or_default().try_into()?; + // Convert outputs with turbo_root resolution + let outputs = processed + .outputs + .map(|outputs| TaskOutputs::from_processed(outputs, path_to_repo_root.as_str())) + .transpose()? + .unwrap_or_default(); - let cache = raw_task.cache.is_none_or(|c| c.into_inner()); - let interactive = raw_task + let cache = processed.cache.is_none_or(|c| c.into_inner()); + let interactive = processed .interactive .as_ref() .map(|value| value.value) .unwrap_or_default(); - if let Some(interactive) = raw_task.interactive { + if let Some(interactive) = &processed.interactive { let (span, text) = interactive.span_and_text("turbo.json"); if cache && interactive.value { return Err(Error::InteractiveNoCacheable { span, text }); } } - let persistent = *raw_task.persistent.unwrap_or_default(); - let interruptible = raw_task.interruptible.unwrap_or_default(); + let persistent = *processed.persistent.unwrap_or_default(); + let interruptible = processed.interruptible.unwrap_or_default(); if *interruptible && !persistent { let (span, text) = interruptible.span_and_text("turbo.json"); return Err(Error::InterruptibleButNotPersistent { span, text }); @@ -373,8 +359,8 @@ impl TaskDefinition { let mut env_var_dependencies = HashSet::new(); let mut topological_dependencies: Vec> = Vec::new(); let mut task_dependencies: Vec> = Vec::new(); - if let Some(depends_on) = raw_task.depends_on { - for dependency in depends_on.into_inner() { + if let Some(depends_on) = processed.depends_on { + for dependency in depends_on.0.into_inner() { let (span, text) = dependency.span_and_text("turbo.json"); let (dependency, depspan) = dependency.split(); let dependency: String = dependency.into(); @@ -397,10 +383,10 @@ impl TaskDefinition { task_dependencies.sort_by(|a, b| a.value.cmp(&b.value)); topological_dependencies.sort_by(|a, b| a.value.cmp(&b.value)); - let env = raw_task + let env = processed .env .map(|env| -> Result, Error> { - gather_env_vars(env, "env", &mut env_var_dependencies)?; + gather_env_vars(env.0, "env", &mut env_var_dependencies)?; let mut env_var_dependencies: Vec = env_var_dependencies.into_iter().collect(); env_var_dependencies.sort(); @@ -409,20 +395,25 @@ impl TaskDefinition { .transpose()? .unwrap_or_default(); - let inputs = TaskInputs::try_from(raw_task.inputs)?; + // Convert inputs with turbo_root resolution + let inputs = processed + .inputs + .map(|inputs| TaskInputs::from_processed(inputs, path_to_repo_root.as_str())) + .transpose()? + .unwrap_or_default(); - let pass_through_env = raw_task + let pass_through_env = processed .pass_through_env .map(|env| -> Result, Error> { let mut pass_through_env = HashSet::new(); - gather_env_vars(env, "passThroughEnv", &mut pass_through_env)?; + gather_env_vars(env.0, "passThroughEnv", &mut pass_through_env)?; let mut pass_through_env: Vec = pass_through_env.into_iter().collect(); pass_through_env.sort(); Ok(pass_through_env) }) .transpose()?; - let with = raw_task.with.map(|with_tasks| { + let with = processed.with.map(|with_tasks| { with_tasks .into_iter() .map(|sibling| { @@ -440,11 +431,11 @@ impl TaskDefinition { env, inputs, pass_through_env, - output_logs: *raw_task.output_logs.unwrap_or_default(), + output_logs: *processed.output_logs.unwrap_or_default(), persistent, interruptible: *interruptible, interactive, - env_mode: raw_task.env_mode.map(|mode| *mode.as_inner()), + env_mode: processed.env_mode.map(|mode| *mode.as_inner()), with, }) } @@ -455,7 +446,7 @@ impl TaskDefinition { raw_task: RawTaskDefinition, path_to_repo_root: &RelativeUnixPath, ) -> Result { - let processed = ProcessedTaskDefinition::from_raw(raw_task); + let processed = ProcessedTaskDefinition::from_raw(raw_task)?; Self::from_processed(processed, path_to_repo_root) } } @@ -645,13 +636,18 @@ impl TurboJson { TurboJson::try_from(raw_turbo_json).map(Some) } - pub fn task(&self, task_id: &TaskId, task_name: &TaskName) -> Option { + pub fn task( + &self, + task_id: &TaskId, + task_name: &TaskName, + ) -> Result, Error> { match self.tasks.get(&task_id.as_task_name()) { - Some(entry) => Some(ProcessedTaskDefinition::from_raw(entry.value.clone())), + Some(entry) => ProcessedTaskDefinition::from_raw(entry.value.clone()).map(Some), None => self .tasks .get(task_name) - .map(|entry| ProcessedTaskDefinition::from_raw(entry.value.clone())), + .map(|entry| ProcessedTaskDefinition::from_raw(entry.value.clone())) + .transpose(), } } @@ -797,68 +793,9 @@ fn gather_env_vars( } // Takes an input/output glob that might start with TURBO_ROOT_PREFIX -// and swap it with the relative path to the turbo root. -fn replace_turbo_root_token_in_string( - input: &mut Spanned, - path_to_repo_root: &RelativeUnixPath, -) -> Result<(), Error> { - let turbo_root_index = input.find(TURBO_ROOT); - if let Some(index) = turbo_root_index { - if !input.as_inner()[index..].starts_with(TURBO_ROOT_SLASH) { - let (span, text) = input.span_and_text("turbo.json"); - return Err(Error::InvalidTurboRootNeedsSlash { span, text }); - } - } - match turbo_root_index { - Some(0) => { - // Replace - input - .as_inner_mut() - .replace_range(..TURBO_ROOT.len(), path_to_repo_root.as_str()); - Ok(()) - } - // Handle negations - Some(1) if input.starts_with('!') => { - input - .as_inner_mut() - .replace_range(1..TURBO_ROOT.len() + 1, path_to_repo_root.as_str()); - Ok(()) - } - // We do not allow for TURBO_ROOT to be used in the middle of a glob - Some(_) => { - let (span, text) = input.span_and_text("turbo.json"); - Err(Error::InvalidTurboRootUse { span, text }) - } - None => Ok(()), - } -} - -fn replace_turbo_root_token( - task_definition: &mut RawTaskDefinition, - path_to_repo_root: &RelativeUnixPath, -) -> Result<(), Error> { - for input in task_definition - .inputs - .iter_mut() - .flat_map(|inputs| inputs.iter_mut()) - { - replace_turbo_root_token_in_string(input, path_to_repo_root)?; - } - - for output in task_definition - .outputs - .iter_mut() - .flat_map(|outputs| outputs.iter_mut()) - { - replace_turbo_root_token_in_string(output, path_to_repo_root)?; - } - - Ok(()) -} - #[cfg(test)] mod tests { - use std::{assert_matches::assert_matches, sync::Arc}; + use std::assert_matches::assert_matches; use anyhow::Result; use biome_deserialize::json::deserialize_from_json_str; @@ -870,10 +807,7 @@ mod tests { use turborepo_task_id::TaskName; use turborepo_unescape::UnescapedString; - use super::{ - replace_turbo_root_token_in_string, validate_with_has_no_topo, FutureFlags, Pipeline, - RawTurboJson, SpacesJson, Spanned, TurboJson, UIMode, *, - }; + use super::{processed::*, *}; use crate::{ boundaries::BoundariesConfig, cli::OutputLogsMode, @@ -1142,11 +1076,15 @@ mod tests { expected_task_outputs: TaskOutputs, ) -> Result<()> { let raw_task_outputs: Vec = serde_json::from_str(task_outputs_str)?; - let raw_task_outputs = raw_task_outputs - .into_iter() - .map(Spanned::new) - .collect::>(); - let task_outputs: TaskOutputs = raw_task_outputs.try_into()?; + let processed_outputs = ProcessedOutputs( + raw_task_outputs + .into_iter() + .map(|s| ProcessedGlob::from_spanned_output(Spanned::new(s))) + .collect::, _>>() + .map_err(|e| anyhow::anyhow!("{}", e))?, + ); + // Use "../.." as a dummy turbo_root_path for tests + let task_outputs = TaskOutputs::from_processed(processed_outputs, "../..")?; assert_eq!(task_outputs, expected_task_outputs); Ok(()) @@ -1342,27 +1280,6 @@ mod tests { ); } - #[test_case("index.ts", Ok("index.ts") ; "no token")] - #[test_case("$TURBO_ROOT$/config.txt", Ok("../../config.txt") ; "valid token")] - #[test_case("!$TURBO_ROOT$/README.md", Ok("!../../README.md") ; "negation")] - #[test_case("../$TURBO_ROOT$/config.txt", Err("\"$TURBO_ROOT$\" must be used at the start of glob.") ; "invalid token")] - #[test_case("$TURBO_ROOT$config.txt", Err("\"$TURBO_ROOT$\" must be followed by a '/'.") ; "trailing slash")] - fn test_replace_turbo_root(input: &'static str, expected: Result<&str, &str>) { - let mut spanned_string = Spanned::new(UnescapedString::from(input)) - .with_path(Arc::from("turbo.json")) - .with_text(format!("\"{input}\"")) - .with_range(1..(input.len())); - let result = replace_turbo_root_token_in_string( - &mut spanned_string, - RelativeUnixPath::new("../..").unwrap(), - ); - let actual = match result { - Ok(()) => Ok(spanned_string.as_inner().as_ref()), - Err(e) => Err(e.to_string()), - }; - assert_eq!(actual, expected.map_err(|s| s.to_owned())); - } - #[test] fn test_future_flags_not_allowed_in_workspace() { let json = r#"{ @@ -1455,33 +1372,29 @@ mod tests { #[test] fn test_absolute_paths_error_in_inputs() { - assert_matches!( - TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( - if cfg!(windows) { - "C:\\win32" - } else { - "/dev/null" - } - ))])), - Err(Error::AbsolutePathInConfig { .. }) - ); + let absolute_path = if cfg!(windows) { + "C:\\win32" + } else { + "/dev/null" + }; + + // The error should be caught when creating the ProcessedGlob + let result = + ProcessedGlob::from_spanned_input(Spanned::new(UnescapedString::from(absolute_path))); + + assert_matches!(result, Err(Error::AbsolutePathInConfig { .. })); } #[test] fn test_detects_turbo_default() { - let inputs = TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( - TURBO_DEFAULT, - ))])) - .unwrap(); - assert!(inputs.default); - } + let processed_inputs = ProcessedInputs(vec![ProcessedGlob::from_spanned_input( + Spanned::new(UnescapedString::from(TURBO_DEFAULT)), + ) + .unwrap()]); - #[test] - fn test_keeps_turbo_default() { - let inputs = TaskInputs::try_from(Some(vec![Spanned::new(UnescapedString::from( - TURBO_DEFAULT, - ))])) - .unwrap(); + // Use "../.." as a dummy turbo_root_path for tests + let inputs = TaskInputs::from_processed(processed_inputs, "../..").unwrap(); + assert!(inputs.default); assert_eq!(inputs.globs, vec![TURBO_DEFAULT.to_string()]); } } diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index f5df4a5f6dab2..c9083754def3f 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -6,6 +6,109 @@ use turborepo_unescape::UnescapedString; use super::RawTaskDefinition; use crate::cli::{EnvMode, OutputLogsMode}; +const TURBO_ROOT: &str = "$TURBO_ROOT$"; +const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; + +/// A processed glob with separated components +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedGlob { + /// The glob pattern without $TURBO_ROOT$ prefix + pub glob: Spanned, + /// Whether the glob was negated (started with !) + pub negated: bool, + /// Whether the glob needs turbo_root prefix (had $TURBO_ROOT$/) + pub turbo_root: bool, +} + +impl ProcessedGlob { + /// Creates a ProcessedGlob from a raw glob string, stripping prefixes + fn from_spanned_internal( + mut value: Spanned, + field: &'static str, + ) -> Result { + use camino::Utf8Path; + + use crate::config::Error; + + let original_value = value.clone(); + let mut negated = false; + let mut turbo_root = false; + let mut start_idx = 0; + + // Check for negation + if value.as_str().starts_with("!") { + negated = true; + start_idx = 1; + } + + // Check for TURBO_ROOT at the appropriate position + if value.as_str()[start_idx..].starts_with(TURBO_ROOT) { + // Validate it has the required slash + if !value.as_str()[start_idx..].starts_with(TURBO_ROOT_SLASH) { + let (span, text) = original_value.span_and_text("turbo.json"); + return Err(Error::InvalidTurboRootNeedsSlash { span, text }); + } + turbo_root = true; + // Strip the $TURBO_ROOT$/ prefix (keeping the content after it) + let new_value = value.as_str()[start_idx + TURBO_ROOT_SLASH.len()..].to_string(); + *value.as_inner_mut() = UnescapedString::from(new_value); + } else if value.as_str().contains(TURBO_ROOT) { + // TURBO_ROOT found in the middle is not allowed + let (span, text) = original_value.span_and_text("turbo.json"); + return Err(Error::InvalidTurboRootUse { span, text }); + } else if negated { + // If negated but no TURBO_ROOT, strip the negation prefix + let new_value = value.as_str()[1..].to_string(); + *value.as_inner_mut() = UnescapedString::from(new_value); + } + + // Check for absolute paths (after stripping prefixes) + if Utf8Path::new(value.as_str()).is_absolute() { + let (span, text) = original_value.span_and_text("turbo.json"); + return Err(Error::AbsolutePathInConfig { field, span, text }); + } + + Ok(ProcessedGlob { + glob: value, + negated, + turbo_root, + }) + } + + /// Creates a ProcessedGlob for outputs (validates as output field) + pub fn from_spanned_output( + value: Spanned, + ) -> Result { + Self::from_spanned_internal(value, "outputs") + } + + /// Creates a ProcessedGlob for inputs (validates as input field) + pub fn from_spanned_input( + value: Spanned, + ) -> Result { + Self::from_spanned_internal(value, "inputs") + } + + /// Creates a resolved glob string with the actual path + pub fn resolve(&self, turbo_root_path: &str) -> String { + let mut result = String::new(); + + if self.negated { + result.push('!'); + } + + if self.turbo_root { + result.push_str(turbo_root_path); + if !turbo_root_path.ends_with('/') && !self.glob.as_str().is_empty() { + result.push('/'); + } + } + + result.push_str(self.glob.as_str()); + result + } +} + /// Processed depends_on field with DSL detection #[derive(Debug, Clone, PartialEq)] pub struct ProcessedDependsOn(pub Spanned>>); @@ -16,7 +119,17 @@ pub struct ProcessedEnv(pub Vec>); /// Processed inputs field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedInputs(pub Vec>); +pub struct ProcessedInputs(pub Vec); + +impl ProcessedInputs { + /// Resolves all globs with the given turbo_root path + pub fn resolve(&self, turbo_root_path: &str) -> Vec { + self.0 + .iter() + .map(|glob| glob.resolve(turbo_root_path)) + .collect() + } +} /// Processed pass_through_env field with DSL detection #[derive(Debug, Clone, PartialEq)] @@ -24,7 +137,17 @@ pub struct ProcessedPassThroughEnv(pub Vec>); /// Processed outputs field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedOutputs(pub Vec>); +pub struct ProcessedOutputs(pub Vec); + +impl ProcessedOutputs { + /// Resolves all globs with the given turbo_root path + pub fn resolve(&self, turbo_root_path: &str) -> Vec { + self.0 + .iter() + .map(|glob| glob.resolve(turbo_root_path)) + .collect() + } +} /// Intermediate representation for task definitions with DSL processing #[derive(Debug, Clone, PartialEq, Default)] @@ -45,38 +168,145 @@ pub struct ProcessedTaskDefinition { impl ProcessedTaskDefinition { /// Creates a processed task definition from raw task - pub fn from_raw(raw_task: RawTaskDefinition) -> Self { - ProcessedTaskDefinition { + pub fn from_raw(raw_task: RawTaskDefinition) -> Result { + let inputs = raw_task + .inputs + .map(|inputs| -> Result { + let globs = inputs + .into_iter() + .map(ProcessedGlob::from_spanned_input) + .collect::, _>>()?; + Ok(ProcessedInputs(globs)) + }) + .transpose()?; + + let outputs = raw_task + .outputs + .map( + |outputs| -> Result { + let globs = outputs + .into_iter() + .map(ProcessedGlob::from_spanned_output) + .collect::, _>>()?; + Ok(ProcessedOutputs(globs)) + }, + ) + .transpose()?; + + Ok(ProcessedTaskDefinition { cache: raw_task.cache, depends_on: raw_task.depends_on.map(ProcessedDependsOn), env: raw_task.env.map(ProcessedEnv), - inputs: raw_task.inputs.map(ProcessedInputs), + inputs, pass_through_env: raw_task.pass_through_env.map(ProcessedPassThroughEnv), persistent: raw_task.persistent, interruptible: raw_task.interruptible, - outputs: raw_task.outputs.map(ProcessedOutputs), + outputs, output_logs: raw_task.output_logs, interactive: raw_task.interactive, env_mode: raw_task.env_mode, with: raw_task.with, - } + }) } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use test_case::test_case; + use turborepo_errors::Spanned; + use turborepo_unescape::UnescapedString; + + use super::*; + + #[test_case("$TURBO_ROOT$/config.txt", Ok((true, false)) ; "detects turbo root")] + #[test_case("!$TURBO_ROOT$/README.md", Ok((true, true)) ; "detects negated turbo root")] + #[test_case("src/**/*.ts", Ok((false, false)) ; "no turbo root")] + fn test_processed_glob_detection(input: &str, expected: Result<(bool, bool), &str>) { + // Test with input variant + let result = ProcessedGlob::from_spanned_input(Spanned::new(UnescapedString::from( + input.to_string(), + ))); - /// Converts back to RawTaskDefinition - pub fn into_raw(self) -> RawTaskDefinition { - RawTaskDefinition { - cache: self.cache, - depends_on: self.depends_on.map(|d| d.0), - env: self.env.map(|e| e.0), - inputs: self.inputs.map(|i| i.0), - pass_through_env: self.pass_through_env.map(|p| p.0), - persistent: self.persistent, - interruptible: self.interruptible, - outputs: self.outputs.map(|o| o.0), - output_logs: self.output_logs, - interactive: self.interactive, - env_mode: self.env_mode, - with: self.with, + match expected { + Ok((turbo_root, negated)) => { + let glob = result.unwrap(); + assert_eq!(glob.turbo_root, turbo_root); + assert_eq!(glob.negated, negated); + } + Err(_) => { + assert!(result.is_err()); + } } } + + #[test_case("$TURBO_ROOT$config.txt", "must be followed by a '/'" ; "missing slash")] + #[test_case("../$TURBO_ROOT$/config.txt", "must be used at the start of glob" ; "middle turbo root")] + fn test_processed_glob_validation_errors(input: &str, expected_error: &str) { + // Test with input variant + let result = ProcessedGlob::from_spanned_input( + Spanned::new(UnescapedString::from(input.to_string())) + .with_path(Arc::from("turbo.json")) + .with_text(format!("\"{}\"", input)) + .with_range(1..input.len() + 1), + ); + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.to_string().contains(expected_error)); + } + + #[test_case("$TURBO_ROOT$/config.txt", "../..", "../../config.txt" ; "replace turbo root")] + #[test_case("!$TURBO_ROOT$/README.md", "../..", "!../../README.md" ; "replace negated turbo root")] + #[test_case("src/**/*.ts", "../..", "src/**/*.ts" ; "no replacement needed")] + fn test_processed_glob_resolution(input: &str, replacement: &str, expected: &str) { + // Test with output variant + let glob = ProcessedGlob::from_spanned_output(Spanned::new(UnescapedString::from( + input.to_string(), + ))) + .unwrap(); + + let resolved = glob.resolve(replacement); + assert_eq!(resolved, expected); + } + + #[test] + fn test_processed_task_definition_resolve() { + // Create a raw task definition with TURBO_ROOT tokens + let raw_task = super::RawTaskDefinition { + inputs: Some(vec![ + Spanned::new(UnescapedString::from("$TURBO_ROOT$/config.txt")), + Spanned::new(UnescapedString::from("src/**/*.ts")), + ]), + outputs: Some(vec![ + Spanned::new(UnescapedString::from("!$TURBO_ROOT$/README.md")), + Spanned::new(UnescapedString::from("dist/**")), + ]), + ..Default::default() + }; + + // Convert to processed task definition + let processed = ProcessedTaskDefinition::from_raw(raw_task).unwrap(); + + // Verify TURBO_ROOT detection + let inputs = processed.inputs.as_ref().unwrap(); + assert!(inputs.0[0].turbo_root); + assert!(!inputs.0[0].negated); + assert!(!inputs.0[1].turbo_root); + + let outputs = processed.outputs.as_ref().unwrap(); + assert!(outputs.0[0].turbo_root); + assert!(outputs.0[0].negated); + assert!(!outputs.0[1].turbo_root); + + // Resolve with turbo_root path + let resolved_inputs = inputs.resolve("../.."); + assert_eq!(resolved_inputs[0], "../../config.txt"); + assert_eq!(resolved_inputs[1], "src/**/*.ts"); + + let resolved_outputs = outputs.resolve("../.."); + assert_eq!(resolved_outputs[0], "!../../README.md"); + assert_eq!(resolved_outputs[1], "dist/**"); + } } From 9c4c3a1bd792b1674a52889f40430c52e7f58e66 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 09:47:39 -0400 Subject: [PATCH 07/19] chore(turbo_json): clean up code --- crates/turborepo-lib/src/turbo_json/mod.rs | 17 ++-- .../turborepo-lib/src/turbo_json/processed.rs | 97 ++++++++----------- 2 files changed, 52 insertions(+), 62 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index dd73e3a8dd545..809eedc904d1d 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -270,7 +270,7 @@ impl TaskOutputs { /// Creates TaskOutputs from ProcessedOutputs with resolved paths fn from_processed( outputs: processed::ProcessedOutputs, - turbo_root_path: &str, + turbo_root_path: &RelativeUnixPath, ) -> Result { let mut inclusions = Vec::new(); let mut exclusions = Vec::new(); @@ -301,7 +301,7 @@ impl TaskInputs { /// Creates TaskInputs from ProcessedInputs with resolved paths fn from_processed( inputs: processed::ProcessedInputs, - turbo_root_path: &str, + turbo_root_path: &RelativeUnixPath, ) -> Result { let mut globs = Vec::new(); let mut default = false; @@ -331,7 +331,7 @@ impl TaskDefinition { // Convert outputs with turbo_root resolution let outputs = processed .outputs - .map(|outputs| TaskOutputs::from_processed(outputs, path_to_repo_root.as_str())) + .map(|outputs| TaskOutputs::from_processed(outputs, path_to_repo_root)) .transpose()? .unwrap_or_default(); @@ -398,7 +398,7 @@ impl TaskDefinition { // Convert inputs with turbo_root resolution let inputs = processed .inputs - .map(|inputs| TaskInputs::from_processed(inputs, path_to_repo_root.as_str())) + .map(|inputs| TaskInputs::from_processed(inputs, path_to_repo_root)) .transpose()? .unwrap_or_default(); @@ -1076,6 +1076,7 @@ mod tests { expected_task_outputs: TaskOutputs, ) -> Result<()> { let raw_task_outputs: Vec = serde_json::from_str(task_outputs_str)?; + let turbo_root = RelativeUnixPath::new("../..")?; let processed_outputs = ProcessedOutputs( raw_task_outputs .into_iter() @@ -1083,8 +1084,7 @@ mod tests { .collect::, _>>() .map_err(|e| anyhow::anyhow!("{}", e))?, ); - // Use "../.." as a dummy turbo_root_path for tests - let task_outputs = TaskOutputs::from_processed(processed_outputs, "../..")?; + let task_outputs = TaskOutputs::from_processed(processed_outputs, turbo_root)?; assert_eq!(task_outputs, expected_task_outputs); Ok(()) @@ -1392,8 +1392,9 @@ mod tests { ) .unwrap()]); - // Use "../.." as a dummy turbo_root_path for tests - let inputs = TaskInputs::from_processed(processed_inputs, "../..").unwrap(); + let inputs = + TaskInputs::from_processed(processed_inputs, RelativeUnixPath::new("../..").unwrap()) + .unwrap(); assert!(inputs.default); assert_eq!(inputs.globs, vec![TURBO_DEFAULT.to_string()]); } diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index c9083754def3f..616a2222fc3bb 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -1,10 +1,15 @@ //! Processed task definition types with DSL token handling +use camino::Utf8Path; +use turbopath::RelativeUnixPath; use turborepo_errors::Spanned; use turborepo_unescape::UnescapedString; use super::RawTaskDefinition; -use crate::cli::{EnvMode, OutputLogsMode}; +use crate::{ + cli::{EnvMode, OutputLogsMode}, + config::Error, +}; const TURBO_ROOT: &str = "$TURBO_ROOT$"; const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; @@ -13,63 +18,52 @@ const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; #[derive(Debug, Clone, PartialEq)] pub struct ProcessedGlob { /// The glob pattern without $TURBO_ROOT$ prefix - pub glob: Spanned, + glob: String, /// Whether the glob was negated (started with !) - pub negated: bool, + negated: bool, /// Whether the glob needs turbo_root prefix (had $TURBO_ROOT$/) - pub turbo_root: bool, + turbo_root: bool, } impl ProcessedGlob { /// Creates a ProcessedGlob from a raw glob string, stripping prefixes fn from_spanned_internal( - mut value: Spanned, + value: Spanned, field: &'static str, ) -> Result { - use camino::Utf8Path; - - use crate::config::Error; - - let original_value = value.clone(); let mut negated = false; let mut turbo_root = false; - let mut start_idx = 0; - // Check for negation - if value.as_str().starts_with("!") { + let without_negation = if let Some(value) = value.strip_prefix('!') { negated = true; - start_idx = 1; - } + value + } else { + value.as_str() + }; - // Check for TURBO_ROOT at the appropriate position - if value.as_str()[start_idx..].starts_with(TURBO_ROOT) { - // Validate it has the required slash - if !value.as_str()[start_idx..].starts_with(TURBO_ROOT_SLASH) { - let (span, text) = original_value.span_and_text("turbo.json"); - return Err(Error::InvalidTurboRootNeedsSlash { span, text }); - } + let glob = if let Some(stripped) = without_negation.strip_prefix(TURBO_ROOT_SLASH) { turbo_root = true; - // Strip the $TURBO_ROOT$/ prefix (keeping the content after it) - let new_value = value.as_str()[start_idx + TURBO_ROOT_SLASH.len()..].to_string(); - *value.as_inner_mut() = UnescapedString::from(new_value); - } else if value.as_str().contains(TURBO_ROOT) { - // TURBO_ROOT found in the middle is not allowed - let (span, text) = original_value.span_and_text("turbo.json"); + stripped + } else if without_negation.starts_with(TURBO_ROOT) { + // Leading $TURBO_ROOT$ without slash + let (span, text) = value.span_and_text("turbo.json"); + return Err(Error::InvalidTurboRootNeedsSlash { span, text }); + } else if without_negation.contains(TURBO_ROOT) { + // non leading $TURBO_ROOT$ + let (span, text) = value.span_and_text("turbo.json"); return Err(Error::InvalidTurboRootUse { span, text }); - } else if negated { - // If negated but no TURBO_ROOT, strip the negation prefix - let new_value = value.as_str()[1..].to_string(); - *value.as_inner_mut() = UnescapedString::from(new_value); - } + } else { + without_negation + }; // Check for absolute paths (after stripping prefixes) - if Utf8Path::new(value.as_str()).is_absolute() { - let (span, text) = original_value.span_and_text("turbo.json"); + if Utf8Path::new(glob).is_absolute() { + let (span, text) = value.span_and_text("turbo.json"); return Err(Error::AbsolutePathInConfig { field, span, text }); } Ok(ProcessedGlob { - glob: value, + glob: glob.to_owned(), negated, turbo_root, }) @@ -90,22 +84,15 @@ impl ProcessedGlob { } /// Creates a resolved glob string with the actual path - pub fn resolve(&self, turbo_root_path: &str) -> String { - let mut result = String::new(); - - if self.negated { - result.push('!'); - } + pub fn resolve(&self, turbo_root_path: &RelativeUnixPath) -> String { + let prefix = if self.negated { "!" } else { "" }; + let glob = &self.glob; if self.turbo_root { - result.push_str(turbo_root_path); - if !turbo_root_path.ends_with('/') && !self.glob.as_str().is_empty() { - result.push('/'); - } + format!("{prefix}{turbo_root_path}/{glob}") + } else { + format!("{prefix}{glob}") } - - result.push_str(self.glob.as_str()); - result } } @@ -123,7 +110,7 @@ pub struct ProcessedInputs(pub Vec); impl ProcessedInputs { /// Resolves all globs with the given turbo_root path - pub fn resolve(&self, turbo_root_path: &str) -> Vec { + pub fn resolve(&self, turbo_root_path: &RelativeUnixPath) -> Vec { self.0 .iter() .map(|glob| glob.resolve(turbo_root_path)) @@ -141,7 +128,7 @@ pub struct ProcessedOutputs(pub Vec); impl ProcessedOutputs { /// Resolves all globs with the given turbo_root path - pub fn resolve(&self, turbo_root_path: &str) -> Vec { + pub fn resolve(&self, turbo_root_path: &RelativeUnixPath) -> Vec { self.0 .iter() .map(|glob| glob.resolve(turbo_root_path)) @@ -261,6 +248,7 @@ mod tests { #[test_case("!$TURBO_ROOT$/README.md", "../..", "!../../README.md" ; "replace negated turbo root")] #[test_case("src/**/*.ts", "../..", "src/**/*.ts" ; "no replacement needed")] fn test_processed_glob_resolution(input: &str, replacement: &str, expected: &str) { + let replacement = RelativeUnixPath::new(replacement).unwrap(); // Test with output variant let glob = ProcessedGlob::from_spanned_output(Spanned::new(UnescapedString::from( input.to_string(), @@ -274,7 +262,7 @@ mod tests { #[test] fn test_processed_task_definition_resolve() { // Create a raw task definition with TURBO_ROOT tokens - let raw_task = super::RawTaskDefinition { + let raw_task = RawTaskDefinition { inputs: Some(vec![ Spanned::new(UnescapedString::from("$TURBO_ROOT$/config.txt")), Spanned::new(UnescapedString::from("src/**/*.ts")), @@ -288,6 +276,7 @@ mod tests { // Convert to processed task definition let processed = ProcessedTaskDefinition::from_raw(raw_task).unwrap(); + let turbo_root = RelativeUnixPath::new("../..").unwrap(); // Verify TURBO_ROOT detection let inputs = processed.inputs.as_ref().unwrap(); @@ -301,11 +290,11 @@ mod tests { assert!(!outputs.0[1].turbo_root); // Resolve with turbo_root path - let resolved_inputs = inputs.resolve("../.."); + let resolved_inputs = inputs.resolve(turbo_root); assert_eq!(resolved_inputs[0], "../../config.txt"); assert_eq!(resolved_inputs[1], "src/**/*.ts"); - let resolved_outputs = outputs.resolve("../.."); + let resolved_outputs = outputs.resolve(turbo_root); assert_eq!(resolved_outputs[0], "!../../README.md"); assert_eq!(resolved_outputs[1], "dist/**"); } From 5084eea2dcbd44117b40154f2c2b43812e3778fe Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 09:54:37 -0400 Subject: [PATCH 08/19] chore(turbo_json): add processed with type --- crates/turborepo-lib/src/turbo_json/mod.rs | 1 + crates/turborepo-lib/src/turbo_json/processed.rs | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 809eedc904d1d..0c4274a9893da 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -415,6 +415,7 @@ impl TaskDefinition { let with = processed.with.map(|with_tasks| { with_tasks + .0 .into_iter() .map(|sibling| { let (sibling, span) = sibling.split(); diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index 616a2222fc3bb..d8aec085fe3fa 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -136,6 +136,10 @@ impl ProcessedOutputs { } } +/// Processed with field with DSL detection +#[derive(Debug, Clone, PartialEq)] +pub struct ProcessedWith(pub Vec>); + /// Intermediate representation for task definitions with DSL processing #[derive(Debug, Clone, PartialEq, Default)] pub struct ProcessedTaskDefinition { @@ -150,7 +154,7 @@ pub struct ProcessedTaskDefinition { pub output_logs: Option>, pub interactive: Option>, pub env_mode: Option>, - pub with: Option>>, + pub with: Option, } impl ProcessedTaskDefinition { @@ -192,7 +196,7 @@ impl ProcessedTaskDefinition { output_logs: raw_task.output_logs, interactive: raw_task.interactive, env_mode: raw_task.env_mode, - with: raw_task.with, + with: raw_task.with.map(ProcessedWith), }) } } From c2f3d65a73b1ebaa99429eb207061f18616d41b7 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 10:00:23 -0400 Subject: [PATCH 09/19] update architecture.md --- .../src/turbo_json/ARCHITECTURE.md | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md b/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md index 9728055952ae4..d60f2ac198e0b 100644 --- a/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md +++ b/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md @@ -9,9 +9,10 @@ The loading and resolving of task definitions is driven during task graph constr The configuration and task loading process follows this high-level flow: 1. **Configuration Resolution**: Collect and merge configuration from multiple sources -2. **TurboJson Loading**: Resolve `turbo.json` files, these are usually files on disk, but can be synthesized -3. **Task Definition Resolution**: Convert raw task definitions into validated structures. `extends` is handled in this step -4. **Task Graph Construction**: Build the executable task graph from resolved definitions +2. **TurboJson Loading**: Parse `turbo.json` files into raw structures +3. **Task Processing**: Convert raw definitions to processed intermediate representation with DSL token handling +4. **Task Definition Resolution**: Transform processed definitions into final validated structures +5. **Task Graph Construction**: Build the executable task graph from resolved definitions ## Phase 1: Configuration Resolution @@ -39,30 +40,55 @@ Configuration is collected from multiple sources with the following priority (hi - **`TurboJsonLoader`** (`crates/turborepo-lib/src/turbo_json/loader.rs`): Loads and resolves turbo.json files - **`RawTurboJson`**: Raw deserialized structure from JSON files -- **`TurboJson`**: Resolved and validated structure, all DSL magic strings have been handled +- **`TurboJson`**: Validated structure containing raw task definitions ### Process 1. **File Discovery**: Locate `turbo.json` or `turbo.jsonc` files 2. **Parsing**: Deserialize JSON into `RawTurboJson` structures -3. **Validation**: Convert to `TurboJson` with validation rules +3. **Basic Validation**: Convert to `TurboJson` with structural validation 4. **Workspace Resolution**: Apply workspace-specific overrides -## Phase 3: Task Definition Resolution +## Phase 3: Processed Task Definition (Intermediate Representation) + +### Key Components + +- **`ProcessedTaskDefinition`** (`crates/turborepo-lib/src/turbo_json/processed.rs`): Intermediate representation with DSL token processing +- **`ProcessedGlob`**: Parsed glob patterns with separated components (base pattern, negation flag, turbo_root flag) +- **`ProcessedInputs`/`ProcessedOutputs`**: Collections of processed globs with resolution methods + +### Processing Steps + +1. **DSL Token Detection**: Identify and separate `$TURBO_ROOT$` and `!` prefixes from glob patterns +2. **Early Validation**: Validate absolute paths and invalid token usage at parse time with span information +3. **Prefix Stripping**: Store clean glob patterns without DSL prefixes +4. **Component Separation**: Track negation and turbo_root requirements as separate boolean flags + +## Phase 4: Task Definition Resolution ### Key Components - **`RawTaskDefinition`**: Raw task configuration from JSON -- **`TaskDefinition`**: Validated and processed task configuration +- **`ProcessedTaskDefinition`**: Intermediate representation with parsed DSL tokens +- **`TaskDefinition`**: Final validated and resolved task configuration - **`TaskId`** and **`TaskName`** (from `turborepo-task-id` crate): Task identification types ### Transformation Process -Raw task definitions undergo several transformations: +The resolution now follows a three-stage pipeline: + +1. **Raw → Processed** (`ProcessedTaskDefinition::from_raw`): + + - Parse glob patterns and extract DSL tokens + - Validate absolute paths with span information + - Strip prefixes and store components separately + +2. **Processed → Resolved** (`TaskDefinition::from_processed`): + + - Apply `$TURBO_ROOT$` token replacement using `resolve()` methods + - Parse `dependsOn` into topological and task dependencies + - Extract environment variables from `env` and `passThroughEnv` + - Transform outputs into inclusion/exclusion patterns + - Validate configuration consistency (e.g., interactive tasks can't be cached) -1. **Path Resolution**: Convert relative paths and handle `$TURBO_ROOT$` tokens -2. **Dependency Parsing**: Parse `dependsOn` into topological and task dependencies -3. **Environment Variable Collection**: Extract `env` and `passThroughEnv` variables -4. **Output Processing**: Handle inclusion/exclusion patterns in outputs -5. **Inheritance**: Handle merging multiple `RawTaskDefinition`s into a single usable task definition -6. **Validation**: Ensure configuration consistency (e.g., interactive tasks can't be cached) +3. **Inheritance**: The `extend` module handles merging multiple `ProcessedTaskDefinition`s before final resolution From 77b6aaf690c391ad78e823a0715ea931cc5ed31f Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 10:34:50 -0400 Subject: [PATCH 10/19] make turbo_default private --- crates/turborepo-lib/src/turbo_json/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 0c4274a9893da..4f9f92f3d83b5 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -36,7 +36,7 @@ use crate::{boundaries::BoundariesConfig, config::UnnecessaryPackageTaskSyntaxEr const TURBO_ROOT: &str = "$TURBO_ROOT$"; const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; -pub const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; +const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; const ENV_PIPELINE_DELIMITER: &str = "$"; const TOPOLOGICAL_PIPELINE_DELIMITER: &str = "^"; From 7d0fb7fc1967a0581589d89d821319508cf7e6aa Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 10:40:54 -0400 Subject: [PATCH 11/19] chore(turbo_json): remove unused constants --- crates/turborepo-lib/src/turbo_json/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 4f9f92f3d83b5..16bb6f5e318d4 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -34,8 +34,6 @@ pub use processed::ProcessedTaskDefinition; use crate::{boundaries::BoundariesConfig, config::UnnecessaryPackageTaskSyntaxError}; -const TURBO_ROOT: &str = "$TURBO_ROOT$"; -const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; const ENV_PIPELINE_DELIMITER: &str = "$"; From e947d86112fa40026e4f0b6d7599d68d1b3cc057 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 10:57:39 -0400 Subject: [PATCH 12/19] chore(turbo_json): move TURBO_DEFAULT handling to processing step --- crates/turborepo-lib/src/turbo_json/extend.rs | 21 ++++---- crates/turborepo-lib/src/turbo_json/mod.rs | 34 ++---------- .../turborepo-lib/src/turbo_json/processed.rs | 54 +++++++++++++------ 3 files changed, 52 insertions(+), 57 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/extend.rs b/crates/turborepo-lib/src/turbo_json/extend.rs index 3ba5edc8005ba..1150dca1a65fa 100644 --- a/crates/turborepo-lib/src/turbo_json/extend.rs +++ b/crates/turborepo-lib/src/turbo_json/extend.rs @@ -68,10 +68,9 @@ mod test { Spanned::new(UnescapedString::from("dist/**")), ) .unwrap()])), - inputs: Some(ProcessedInputs(vec![ProcessedGlob::from_spanned_input( - Spanned::new(UnescapedString::from("src/**")), - ) - .unwrap()])), + inputs: Some( + ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("src/**"))]).unwrap(), + ), env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( "NODE_ENV", ))])), @@ -93,10 +92,9 @@ mod test { Spanned::new(UnescapedString::from("build/**")), ) .unwrap()])), - inputs: Some(ProcessedInputs(vec![ProcessedGlob::from_spanned_input( - Spanned::new(UnescapedString::from("lib/**")), - ) - .unwrap()])), + inputs: Some( + ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("lib/**"))]).unwrap(), + ), env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( "PROD_ENV", ))])), @@ -220,10 +218,9 @@ mod test { let second = ProcessedTaskDefinition { persistent: Some(Spanned::new(false)), - inputs: Some(ProcessedInputs(vec![ProcessedGlob::from_spanned_input( - Spanned::new(UnescapedString::from("src/**")), - ) - .unwrap()])), + inputs: Some( + ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("src/**"))]).unwrap(), + ), ..Default::default() }; diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 16bb6f5e318d4..e2729f2a8167a 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -34,8 +34,6 @@ pub use processed::ProcessedTaskDefinition; use crate::{boundaries::BoundariesConfig, config::UnnecessaryPackageTaskSyntaxError}; -const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; - const ENV_PIPELINE_DELIMITER: &str = "$"; const TOPOLOGICAL_PIPELINE_DELIMITER: &str = "^"; @@ -301,22 +299,12 @@ impl TaskInputs { inputs: processed::ProcessedInputs, turbo_root_path: &RelativeUnixPath, ) -> Result { - let mut globs = Vec::new(); - let mut default = false; - // Resolve all globs with the turbo_root path // Absolute path validation was already done during ProcessedGlob creation - let resolved = inputs.resolve(turbo_root_path); - - for glob_str in resolved { - // Check for $TURBO_DEFAULT$ - if glob_str == TURBO_DEFAULT { - default = true; - } - globs.push(glob_str); - } - - Ok(TaskInputs { globs, default }) + Ok(TaskInputs { + globs: inputs.resolve(turbo_root_path), + default: inputs.default, + }) } } @@ -1383,18 +1371,4 @@ mod tests { assert_matches!(result, Err(Error::AbsolutePathInConfig { .. })); } - - #[test] - fn test_detects_turbo_default() { - let processed_inputs = ProcessedInputs(vec![ProcessedGlob::from_spanned_input( - Spanned::new(UnescapedString::from(TURBO_DEFAULT)), - ) - .unwrap()]); - - let inputs = - TaskInputs::from_processed(processed_inputs, RelativeUnixPath::new("../..").unwrap()) - .unwrap(); - assert!(inputs.default); - assert_eq!(inputs.globs, vec![TURBO_DEFAULT.to_string()]); - } } diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index d8aec085fe3fa..f8e62758521a1 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -11,6 +11,7 @@ use crate::{ config::Error, }; +const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; const TURBO_ROOT: &str = "$TURBO_ROOT$"; const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; @@ -106,12 +107,28 @@ pub struct ProcessedEnv(pub Vec>); /// Processed inputs field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedInputs(pub Vec); +pub struct ProcessedInputs { + globs: Vec, + pub default: bool, +} impl ProcessedInputs { + pub fn new(raw_globs: Vec>) -> Result { + let mut globs = Vec::with_capacity(raw_globs.len()); + let mut default = false; + for raw_glob in raw_globs { + if raw_glob.as_str() == TURBO_DEFAULT { + default = true; + } + globs.push(ProcessedGlob::from_spanned_input(raw_glob)?); + } + + Ok(ProcessedInputs { globs, default }) + } + /// Resolves all globs with the given turbo_root path pub fn resolve(&self, turbo_root_path: &RelativeUnixPath) -> Vec { - self.0 + self.globs .iter() .map(|glob| glob.resolve(turbo_root_path)) .collect() @@ -160,16 +177,7 @@ pub struct ProcessedTaskDefinition { impl ProcessedTaskDefinition { /// Creates a processed task definition from raw task pub fn from_raw(raw_task: RawTaskDefinition) -> Result { - let inputs = raw_task - .inputs - .map(|inputs| -> Result { - let globs = inputs - .into_iter() - .map(ProcessedGlob::from_spanned_input) - .collect::, _>>()?; - Ok(ProcessedInputs(globs)) - }) - .transpose()?; + let inputs = raw_task.inputs.map(ProcessedInputs::new).transpose()?; let outputs = raw_task .outputs @@ -284,9 +292,9 @@ mod tests { // Verify TURBO_ROOT detection let inputs = processed.inputs.as_ref().unwrap(); - assert!(inputs.0[0].turbo_root); - assert!(!inputs.0[0].negated); - assert!(!inputs.0[1].turbo_root); + assert!(inputs.globs[0].turbo_root); + assert!(!inputs.globs[0].negated); + assert!(!inputs.globs[1].turbo_root); let outputs = processed.outputs.as_ref().unwrap(); assert!(outputs.0[0].turbo_root); @@ -302,4 +310,20 @@ mod tests { assert_eq!(resolved_outputs[0], "!../../README.md"); assert_eq!(resolved_outputs[1], "dist/**"); } + + #[test] + fn test_detects_turbo_default() { + let raw_globs = vec![Spanned::new(UnescapedString::from(TURBO_DEFAULT))]; + + let inputs = ProcessedInputs::new(raw_globs).unwrap(); + assert!(inputs.default); + assert_eq!( + inputs.globs, + vec![ProcessedGlob { + glob: TURBO_DEFAULT.to_string(), + negated: false, + turbo_root: false + }] + ); + } } From 4651646a5adf9a9af7589014550130bae32ee9cb Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 11:16:34 -0400 Subject: [PATCH 13/19] chore(turbo_json): move processed outputs to constructor --- crates/turborepo-lib/src/turbo_json/extend.rs | 26 +++++----- crates/turborepo-lib/src/turbo_json/mod.rs | 26 +--------- .../turborepo-lib/src/turbo_json/processed.rs | 51 ++++++++++++------- 3 files changed, 48 insertions(+), 55 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/extend.rs b/crates/turborepo-lib/src/turbo_json/extend.rs index 1150dca1a65fa..6a91711b9b514 100644 --- a/crates/turborepo-lib/src/turbo_json/extend.rs +++ b/crates/turborepo-lib/src/turbo_json/extend.rs @@ -56,7 +56,7 @@ mod test { use super::*; use crate::{ cli::OutputLogsMode, - turbo_json::processed::{ProcessedEnv, ProcessedGlob, ProcessedInputs, ProcessedOutputs}, + turbo_json::processed::{ProcessedEnv, ProcessedInputs, ProcessedOutputs}, }; // Shared test fixtures @@ -64,10 +64,10 @@ mod test { ProcessedTaskDefinition { cache: Some(Spanned::new(true)), persistent: Some(Spanned::new(false)), - outputs: Some(ProcessedOutputs(vec![ProcessedGlob::from_spanned_output( - Spanned::new(UnescapedString::from("dist/**")), - ) - .unwrap()])), + outputs: Some( + ProcessedOutputs::new(vec![Spanned::new(UnescapedString::from("dist/**"))]) + .unwrap(), + ), inputs: Some( ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("src/**"))]).unwrap(), ), @@ -88,10 +88,10 @@ mod test { ProcessedTaskDefinition { cache: Some(Spanned::new(false)), persistent: Some(Spanned::new(true)), - outputs: Some(ProcessedOutputs(vec![ProcessedGlob::from_spanned_output( - Spanned::new(UnescapedString::from("build/**")), - ) - .unwrap()])), + outputs: Some( + ProcessedOutputs::new(vec![Spanned::new(UnescapedString::from("build/**"))]) + .unwrap(), + ), inputs: Some( ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("lib/**"))]).unwrap(), ), @@ -209,10 +209,10 @@ mod test { fn test_from_iter_combines_across_multiple_tasks() { let first = ProcessedTaskDefinition { cache: Some(Spanned::new(true)), - outputs: Some(ProcessedOutputs(vec![ProcessedGlob::from_spanned_output( - Spanned::new(UnescapedString::from("dist/**")), - ) - .unwrap()])), + outputs: Some( + ProcessedOutputs::new(vec![Spanned::new(UnescapedString::from("dist/**"))]) + .unwrap(), + ), ..Default::default() }; diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index e2729f2a8167a..386195745a09c 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -782,8 +782,6 @@ fn gather_env_vars( // Takes an input/output glob that might start with TURBO_ROOT_PREFIX #[cfg(test)] mod tests { - use std::assert_matches::assert_matches; - use anyhow::Result; use biome_deserialize::json::deserialize_from_json_str; use biome_json_parser::JsonParserOptions; @@ -1064,13 +1062,8 @@ mod tests { ) -> Result<()> { let raw_task_outputs: Vec = serde_json::from_str(task_outputs_str)?; let turbo_root = RelativeUnixPath::new("../..")?; - let processed_outputs = ProcessedOutputs( - raw_task_outputs - .into_iter() - .map(|s| ProcessedGlob::from_spanned_output(Spanned::new(s))) - .collect::, _>>() - .map_err(|e| anyhow::anyhow!("{}", e))?, - ); + let processed_outputs = + ProcessedOutputs::new(raw_task_outputs.into_iter().map(Spanned::new).collect())?; let task_outputs = TaskOutputs::from_processed(processed_outputs, turbo_root)?; assert_eq!(task_outputs, expected_task_outputs); @@ -1356,19 +1349,4 @@ mod tests { "`with` cannot use dependency relationships." ); } - - #[test] - fn test_absolute_paths_error_in_inputs() { - let absolute_path = if cfg!(windows) { - "C:\\win32" - } else { - "/dev/null" - }; - - // The error should be caught when creating the ProcessedGlob - let result = - ProcessedGlob::from_spanned_input(Spanned::new(UnescapedString::from(absolute_path))); - - assert_matches!(result, Err(Error::AbsolutePathInConfig { .. })); - } } diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index f8e62758521a1..0adb86c557e7b 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -141,12 +141,23 @@ pub struct ProcessedPassThroughEnv(pub Vec>); /// Processed outputs field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedOutputs(pub Vec); +pub struct ProcessedOutputs { + globs: Vec, +} impl ProcessedOutputs { + pub fn new(raw_globs: Vec>) -> Result { + let globs = raw_globs + .into_iter() + .map(ProcessedGlob::from_spanned_input) + .collect::, _>>()?; + + Ok(ProcessedOutputs { globs }) + } + /// Resolves all globs with the given turbo_root path pub fn resolve(&self, turbo_root_path: &RelativeUnixPath) -> Vec { - self.0 + self.globs .iter() .map(|glob| glob.resolve(turbo_root_path)) .collect() @@ -179,18 +190,7 @@ impl ProcessedTaskDefinition { pub fn from_raw(raw_task: RawTaskDefinition) -> Result { let inputs = raw_task.inputs.map(ProcessedInputs::new).transpose()?; - let outputs = raw_task - .outputs - .map( - |outputs| -> Result { - let globs = outputs - .into_iter() - .map(ProcessedGlob::from_spanned_output) - .collect::, _>>()?; - Ok(ProcessedOutputs(globs)) - }, - ) - .transpose()?; + let outputs = raw_task.outputs.map(ProcessedOutputs::new).transpose()?; Ok(ProcessedTaskDefinition { cache: raw_task.cache, @@ -211,7 +211,7 @@ impl ProcessedTaskDefinition { #[cfg(test)] mod tests { - use std::sync::Arc; + use std::{assert_matches::assert_matches, sync::Arc}; use test_case::test_case; use turborepo_errors::Spanned; @@ -297,9 +297,9 @@ mod tests { assert!(!inputs.globs[1].turbo_root); let outputs = processed.outputs.as_ref().unwrap(); - assert!(outputs.0[0].turbo_root); - assert!(outputs.0[0].negated); - assert!(!outputs.0[1].turbo_root); + assert!(outputs.globs[0].turbo_root); + assert!(outputs.globs[0].negated); + assert!(!outputs.globs[1].turbo_root); // Resolve with turbo_root path let resolved_inputs = inputs.resolve(turbo_root); @@ -326,4 +326,19 @@ mod tests { }] ); } + + #[test] + fn test_absolute_paths_error_in_inputs() { + let absolute_path = if cfg!(windows) { + "C:\\win32" + } else { + "/dev/null" + }; + + // The error should be caught when creating the ProcessedGlob + let result = + ProcessedGlob::from_spanned_input(Spanned::new(UnescapedString::from(absolute_path))); + + assert_matches!(result, Err(Error::AbsolutePathInConfig { .. })); + } } From c40b0f7a515cffc3f14da42bb84938ff23922d8d Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 13:00:35 -0400 Subject: [PATCH 14/19] chore(turbo_json): move additional validation to processing step --- .../src/turbo_json/ARCHITECTURE.md | 19 +++- crates/turborepo-lib/src/turbo_json/extend.rs | 18 +-- crates/turborepo-lib/src/turbo_json/mod.rs | 44 +------ .../turborepo-lib/src/turbo_json/processed.rs | 107 +++++++++++++++--- 4 files changed, 122 insertions(+), 66 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md b/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md index d60f2ac198e0b..b920e8b76a8ba 100644 --- a/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md +++ b/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md @@ -60,7 +60,12 @@ Configuration is collected from multiple sources with the following priority (hi ### Processing Steps 1. **DSL Token Detection**: Identify and separate `$TURBO_ROOT$` and `!` prefixes from glob patterns -2. **Early Validation**: Validate absolute paths and invalid token usage at parse time with span information +2. **Early Validation**: Single-field validations at parse time with span information: + - Absolute paths in inputs/outputs + - Invalid `$TURBO_ROOT$` usage + - Environment variable prefixes (`$` not allowed) + - Dependency prefixes (`$` not allowed in `dependsOn`) + - Topological references (`^` not allowed in `with`) 3. **Prefix Stripping**: Store clean glob patterns without DSL prefixes 4. **Component Separation**: Track negation and turbo_root requirements as separate boolean flags @@ -80,15 +85,21 @@ The resolution now follows a three-stage pipeline: 1. **Raw → Processed** (`ProcessedTaskDefinition::from_raw`): - Parse glob patterns and extract DSL tokens - - Validate absolute paths with span information + - Validate single-field constraints with span information: + - Absolute paths in inputs/outputs (`ProcessedGlob::from_spanned_*`) + - Invalid environment variable prefixes (`ProcessedEnv::new`, `ProcessedPassThroughEnv::new`) + - Invalid dependency syntax (`ProcessedDependsOn::new`) + - Invalid sibling task references (`ProcessedWith::new`) - Strip prefixes and store components separately 2. **Processed → Resolved** (`TaskDefinition::from_processed`): - Apply `$TURBO_ROOT$` token replacement using `resolve()` methods - Parse `dependsOn` into topological and task dependencies - - Extract environment variables from `env` and `passThroughEnv` + - Transform environment variables into sorted lists - Transform outputs into inclusion/exclusion patterns - - Validate configuration consistency (e.g., interactive tasks can't be cached) + - Validate multi-field constraints: + - Interactive tasks cannot be cached (requires `cache` and `interactive` fields) + - Interruptible tasks must be persistent (requires `interruptible` and `persistent` fields) 3. **Inheritance**: The `extend` module handles merging multiple `ProcessedTaskDefinition`s before final resolution diff --git a/crates/turborepo-lib/src/turbo_json/extend.rs b/crates/turborepo-lib/src/turbo_json/extend.rs index 6a91711b9b514..cf373dc9596ec 100644 --- a/crates/turborepo-lib/src/turbo_json/extend.rs +++ b/crates/turborepo-lib/src/turbo_json/extend.rs @@ -71,9 +71,9 @@ mod test { inputs: Some( ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("src/**"))]).unwrap(), ), - env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( - "NODE_ENV", - ))])), + env: Some( + ProcessedEnv::new(vec![Spanned::new(UnescapedString::from("NODE_ENV"))]).unwrap(), + ), depends_on: None, pass_through_env: None, output_logs: None, @@ -95,9 +95,9 @@ mod test { inputs: Some( ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("lib/**"))]).unwrap(), ), - env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( - "PROD_ENV", - ))])), + env: Some( + ProcessedEnv::new(vec![Spanned::new(UnescapedString::from("PROD_ENV"))]).unwrap(), + ), output_logs: Some(Spanned::new(OutputLogsMode::Full)), interruptible: Some(Spanned::new(true)), depends_on: None, @@ -225,9 +225,9 @@ mod test { }; let third = ProcessedTaskDefinition { - env: Some(ProcessedEnv(vec![Spanned::new(UnescapedString::from( - "NODE_ENV", - ))])), + env: Some( + ProcessedEnv::new(vec![Spanned::new(UnescapedString::from("NODE_ENV"))]).unwrap(), + ), output_logs: Some(Spanned::new(OutputLogsMode::Full)), // Override cache from first task cache: Some(Spanned::new(false)), diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 386195745a09c..208578f8a4d65 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -342,21 +342,13 @@ impl TaskDefinition { return Err(Error::InterruptibleButNotPersistent { span, text }); } - let mut env_var_dependencies = HashSet::new(); let mut topological_dependencies: Vec> = Vec::new(); let mut task_dependencies: Vec> = Vec::new(); if let Some(depends_on) = processed.depends_on { for dependency in depends_on.0.into_inner() { - let (span, text) = dependency.span_and_text("turbo.json"); let (dependency, depspan) = dependency.split(); let dependency: String = dependency.into(); - if dependency.strip_prefix(ENV_PIPELINE_DELIMITER).is_some() { - return Err(Error::InvalidDependsOnValue { - field: "dependsOn", - span, - text, - }); - } else if let Some(topo_dependency) = + if let Some(topo_dependency) = dependency.strip_prefix(TOPOLOGICAL_PIPELINE_DELIMITER) { topological_dependencies.push(depspan.to(topo_dependency.to_string().into())); @@ -369,17 +361,7 @@ impl TaskDefinition { task_dependencies.sort_by(|a, b| a.value.cmp(&b.value)); topological_dependencies.sort_by(|a, b| a.value.cmp(&b.value)); - let env = processed - .env - .map(|env| -> Result, Error> { - gather_env_vars(env.0, "env", &mut env_var_dependencies)?; - let mut env_var_dependencies: Vec = - env_var_dependencies.into_iter().collect(); - env_var_dependencies.sort(); - Ok(env_var_dependencies) - }) - .transpose()? - .unwrap_or_default(); + let env = processed.env.map(|env| env.0).unwrap_or_default(); // Convert inputs with turbo_root resolution let inputs = processed @@ -388,27 +370,9 @@ impl TaskDefinition { .transpose()? .unwrap_or_default(); - let pass_through_env = processed - .pass_through_env - .map(|env| -> Result, Error> { - let mut pass_through_env = HashSet::new(); - gather_env_vars(env.0, "passThroughEnv", &mut pass_through_env)?; - let mut pass_through_env: Vec = pass_through_env.into_iter().collect(); - pass_through_env.sort(); - Ok(pass_through_env) - }) - .transpose()?; + let pass_through_env = processed.pass_through_env.map(|env| env.0); - let with = processed.with.map(|with_tasks| { - with_tasks - .0 - .into_iter() - .map(|sibling| { - let (sibling, span) = sibling.split(); - span.to(TaskName::from(String::from(sibling))) - }) - .collect() - }); + let with = processed.with.map(|with_tasks| with_tasks.0); Ok(TaskDefinition { outputs, diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index 0adb86c557e7b..7d2fb112c2b45 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -3,6 +3,7 @@ use camino::Utf8Path; use turbopath::RelativeUnixPath; use turborepo_errors::Spanned; +use turborepo_task_id::TaskName; use turborepo_unescape::UnescapedString; use super::RawTaskDefinition; @@ -14,6 +15,8 @@ use crate::{ const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; const TURBO_ROOT: &str = "$TURBO_ROOT$"; const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; +const ENV_PIPELINE_DELIMITER: &str = "$"; +const TOPOLOGICAL_PIPELINE_DELIMITER: &str = "^"; /// A processed glob with separated components #[derive(Debug, Clone, PartialEq)] @@ -101,9 +104,36 @@ impl ProcessedGlob { #[derive(Debug, Clone, PartialEq)] pub struct ProcessedDependsOn(pub Spanned>>); +impl ProcessedDependsOn { + /// Creates a ProcessedDependsOn, validating that dependencies don't use env + /// prefix + pub fn new(raw_deps: Spanned>>) -> Result { + // Validate that no dependency starts with ENV_PIPELINE_DELIMITER ($) + for dep in raw_deps.value.iter() { + if dep.starts_with(ENV_PIPELINE_DELIMITER) { + let (span, text) = dep.span_and_text("turbo.json"); + return Err(Error::InvalidDependsOnValue { + field: "dependsOn", + span, + text, + }); + } + } + Ok(ProcessedDependsOn(raw_deps)) + } +} + /// Processed env field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedEnv(pub Vec>); +pub struct ProcessedEnv(pub Vec); + +impl ProcessedEnv { + /// Creates a ProcessedEnv, validating that env vars don't use invalid + /// prefixes + pub fn new(raw_env: Vec>) -> Result { + Ok(ProcessedEnv(extract_env_vars(raw_env)?)) + } +} /// Processed inputs field with DSL detection #[derive(Debug, Clone, PartialEq)] @@ -137,7 +167,38 @@ impl ProcessedInputs { /// Processed pass_through_env field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedPassThroughEnv(pub Vec>); +pub struct ProcessedPassThroughEnv(pub Vec); + +impl ProcessedPassThroughEnv { + /// Creates a ProcessedPassThroughEnv, validating that env vars don't use + /// invalid prefixes + pub fn new(raw_env: Vec>) -> Result { + Ok(ProcessedPassThroughEnv(extract_env_vars(raw_env)?)) + } +} + +fn extract_env_vars(raw_env: Vec>) -> Result, Error> { + use crate::config::InvalidEnvPrefixError; + + let mut env_vars = Vec::with_capacity(raw_env.len()); + // Validate that no env var starts with ENV_PIPELINE_DELIMITER ($) + for var in raw_env { + if var.starts_with(ENV_PIPELINE_DELIMITER) { + let (span, text) = var.span_and_text("turbo.json"); + return Err(Error::InvalidEnvPrefix(Box::new(InvalidEnvPrefixError { + key: "passThroughEnv".to_string(), + value: var.as_str().to_string(), + span, + text, + env_pipeline_delimiter: ENV_PIPELINE_DELIMITER, + }))); + } + + env_vars.push(String::from(var.into_inner())); + } + env_vars.sort(); + Ok(env_vars) +} /// Processed outputs field with DSL detection #[derive(Debug, Clone, PartialEq)] @@ -166,7 +227,25 @@ impl ProcessedOutputs { /// Processed with field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedWith(pub Vec>); +pub struct ProcessedWith(pub Vec>>); + +impl ProcessedWith { + /// Creates a ProcessedWith, validating that siblings don't use topological + /// prefix + pub fn new(raw_with: Vec>) -> Result { + // Validate that no sibling starts with TOPOLOGICAL_PIPELINE_DELIMITER (^) + let mut with = Vec::with_capacity(raw_with.len()); + for sibling in raw_with { + if sibling.starts_with(TOPOLOGICAL_PIPELINE_DELIMITER) { + let (span, text) = sibling.span_and_text("turbo.json"); + return Err(Error::InvalidTaskWith { span, text }); + } + let (sibling, span) = sibling.split(); + with.push(span.to(TaskName::from(String::from(sibling)))); + } + Ok(ProcessedWith(with)) + } +} /// Intermediate representation for task definitions with DSL processing #[derive(Debug, Clone, PartialEq, Default)] @@ -188,23 +267,25 @@ pub struct ProcessedTaskDefinition { impl ProcessedTaskDefinition { /// Creates a processed task definition from raw task pub fn from_raw(raw_task: RawTaskDefinition) -> Result { - let inputs = raw_task.inputs.map(ProcessedInputs::new).transpose()?; - - let outputs = raw_task.outputs.map(ProcessedOutputs::new).transpose()?; - Ok(ProcessedTaskDefinition { cache: raw_task.cache, - depends_on: raw_task.depends_on.map(ProcessedDependsOn), - env: raw_task.env.map(ProcessedEnv), - inputs, - pass_through_env: raw_task.pass_through_env.map(ProcessedPassThroughEnv), + depends_on: raw_task + .depends_on + .map(ProcessedDependsOn::new) + .transpose()?, + env: raw_task.env.map(ProcessedEnv::new).transpose()?, + inputs: raw_task.inputs.map(ProcessedInputs::new).transpose()?, + pass_through_env: raw_task + .pass_through_env + .map(ProcessedPassThroughEnv::new) + .transpose()?, persistent: raw_task.persistent, interruptible: raw_task.interruptible, - outputs, + outputs: raw_task.outputs.map(ProcessedOutputs::new).transpose()?, output_logs: raw_task.output_logs, interactive: raw_task.interactive, env_mode: raw_task.env_mode, - with: raw_task.with.map(ProcessedWith), + with: raw_task.with.map(ProcessedWith::new).transpose()?, }) } } From 2571af796f4bc198274ed2c5c53e6cf53431ed51 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 13:17:44 -0400 Subject: [PATCH 15/19] chore(turbo_json): move future flags to own module --- .../src/turbo_json/future_flags.rs | 45 +++++++++++++++++++ crates/turborepo-lib/src/turbo_json/mod.rs | 8 +--- 2 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 crates/turborepo-lib/src/turbo_json/future_flags.rs diff --git a/crates/turborepo-lib/src/turbo_json/future_flags.rs b/crates/turborepo-lib/src/turbo_json/future_flags.rs new file mode 100644 index 0000000000000..9879dbb1b57f5 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/future_flags.rs @@ -0,0 +1,45 @@ +//! Future flags for enabling experimental or upcoming features +//! +//! This module contains the `FutureFlags` structure which allows users to +//! opt-in to experimental features before they become the default behavior. +//! +//! ## Usage +//! +//! Future flags can be configured in the root `turbo.json`: +//! +//! ```json +//! { +//! "futureFlags": { +//! "turboExtends": true +//! } +//! } +//! ``` +//! +//! Note: Future flags are only allowed in the root `turbo.json` and will cause +//! an error if specified in workspace packages. + +use biome_deserialize_macros::Deserializable; +use serde::Serialize; +use struct_iterable::Iterable; + +/// Future flags configuration for experimental features +/// +/// Each flag represents an experimental feature that can be enabled +/// before it becomes the default behavior in a future version. +#[derive(Serialize, Default, Debug, Copy, Clone, Iterable, Deserializable, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct FutureFlags { + /// Enable `$TURBO_EXTENDS$` + /// + /// When enabled, allows using `$TURBO_EXTENDS$` in array fields. + /// This will change the default behavior of overriding the field to instead + /// append. + pub turbo_extends: bool, +} + +impl FutureFlags { + /// Create a new FutureFlags with all features disabled + pub fn new() -> Self { + Self::default() + } +} diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 208578f8a4d65..9d47638cdc01c 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -25,10 +25,12 @@ use crate::{ }; mod extend; +pub mod future_flags; mod loader; pub mod parser; mod processed; +pub use future_flags::FutureFlags; pub use loader::{TurboJsonLoader, TurboJsonReader}; pub use processed::ProcessedTaskDefinition; @@ -156,12 +158,6 @@ pub struct RawTurboJson { _comment: Option, } -#[derive(Serialize, Default, Debug, Copy, Clone, Iterable, Deserializable, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct FutureFlags { - turbo_extends: bool, -} - #[derive(Serialize, Default, Debug, PartialEq, Clone)] #[serde(transparent)] pub struct Pipeline(BTreeMap, Spanned>); From 172ce87ef1ec450911d9aa4a793afffde31a7e09 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 15:00:23 -0400 Subject: [PATCH 16/19] feat(turbo_json): implement TURBO_EXTENDS --- crates/turborepo-lib/src/turbo_json/extend.rs | 378 +++++++++++++++++- crates/turborepo-lib/src/turbo_json/mod.rs | 38 +- .../turborepo-lib/src/turbo_json/processed.rs | 301 ++++++++++++-- 3 files changed, 642 insertions(+), 75 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/extend.rs b/crates/turborepo-lib/src/turbo_json/extend.rs index cf373dc9596ec..df46416bb052a 100644 --- a/crates/turborepo-lib/src/turbo_json/extend.rs +++ b/crates/turborepo-lib/src/turbo_json/extend.rs @@ -1,6 +1,88 @@ //! Module for code related to "extends" behavior for task definitions -use super::processed::ProcessedTaskDefinition; +use super::processed::{ + ProcessedDependsOn, ProcessedEnv, ProcessedInputs, ProcessedOutputs, ProcessedPassThroughEnv, + ProcessedTaskDefinition, ProcessedWith, +}; + +/// Trait for types that can be merged with extends behavior +trait Extendable { + /// Merges another instance into self. + /// If the other instance has `extends: true`, it extends the current value. + /// Otherwise, it replaces the current value. + fn extend(&mut self, other: Self); +} + +/// Macro to handle the extend/replace logic for a field +macro_rules! merge_field_vec { + ($self:ident, $other:ident, $field:ident) => { + if $other.extends { + $self.$field.extend($other.$field); + } else { + $self.$field = $other.$field; + } + }; +} + +impl Extendable for ProcessedDependsOn { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, deps); + self.extends = other.extends; + } +} + +impl Extendable for ProcessedEnv { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, vars); + // Sort and dedup for env vars + if other.extends { + self.vars.sort(); + self.vars.dedup(); + } + self.extends = other.extends; + } +} + +impl Extendable for ProcessedOutputs { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, globs); + self.extends = other.extends; + } +} + +impl Extendable for ProcessedPassThroughEnv { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, vars); + // Sort and dedup for env vars + if other.extends { + self.vars.sort(); + self.vars.dedup(); + } + self.extends = other.extends; + } +} + +impl Extendable for ProcessedWith { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, tasks); + self.extends = other.extends; + } +} + +impl Extendable for ProcessedInputs { + fn extend(&mut self, other: Self) { + merge_field_vec!(self, other, globs); + // Handle the default flag specially + if other.extends { + // When extending, OR the default flags + self.default = self.default || other.default; + } else { + // When replacing, use the other's default + self.default = other.default; + } + self.extends = other.extends; + } +} impl FromIterator for ProcessedTaskDefinition { fn from_iter>(iter: T) -> Self { @@ -20,12 +102,36 @@ macro_rules! set_field { }}; } +macro_rules! merge_field { + ($this:ident, $other:ident, $field:ident) => {{ + if let Some(other_field) = $other.$field { + match &mut $this.$field { + Some(self_field) => { + // Merge using the Mergeable trait + self_field.extend(other_field); + } + None => { + // No existing value, just set it + $this.$field = Some(other_field); + } + } + } + }}; +} + impl ProcessedTaskDefinition { // Merges another ProcessedTaskDefinition into this one - // By default any fields present on `other` will override present fields. + // Array fields use the Mergeable trait to handle extends behavior pub fn merge(&mut self, other: ProcessedTaskDefinition) { - set_field!(self, other, outputs); - + // Array fields that support extends behavior + merge_field!(self, other, outputs); + merge_field!(self, other, depends_on); + merge_field!(self, other, inputs); + merge_field!(self, other, env); + merge_field!(self, other, pass_through_env); + merge_field!(self, other, with); + + // Non-array fields that are simply replaced let other_has_range = other.cache.as_ref().is_some_and(|c| c.range.is_some()); let self_does_not_have_range = self.cache.as_ref().is_some_and(|c| c.range.is_none()); @@ -35,16 +141,11 @@ impl ProcessedTaskDefinition { { self.cache = other.cache; } - set_field!(self, other, depends_on); - set_field!(self, other, inputs); set_field!(self, other, output_logs); set_field!(self, other, persistent); set_field!(self, other, interruptible); - set_field!(self, other, env); - set_field!(self, other, pass_through_env); set_field!(self, other, interactive); set_field!(self, other, env_mode); - set_field!(self, other, with); } } @@ -56,7 +157,10 @@ mod test { use super::*; use crate::{ cli::OutputLogsMode, - turbo_json::processed::{ProcessedEnv, ProcessedInputs, ProcessedOutputs}, + turbo_json::{ + processed::{ProcessedEnv, ProcessedInputs, ProcessedOutputs}, + FutureFlags, + }, }; // Shared test fixtures @@ -65,14 +169,25 @@ mod test { cache: Some(Spanned::new(true)), persistent: Some(Spanned::new(false)), outputs: Some( - ProcessedOutputs::new(vec![Spanned::new(UnescapedString::from("dist/**"))]) - .unwrap(), + ProcessedOutputs::new( + vec![Spanned::new(UnescapedString::from("dist/**"))], + &FutureFlags::default(), + ) + .unwrap(), ), inputs: Some( - ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("src/**"))]).unwrap(), + ProcessedInputs::new( + vec![Spanned::new(UnescapedString::from("src/**"))], + &FutureFlags::default(), + ) + .unwrap(), ), env: Some( - ProcessedEnv::new(vec![Spanned::new(UnescapedString::from("NODE_ENV"))]).unwrap(), + ProcessedEnv::new( + vec![Spanned::new(UnescapedString::from("NODE_ENV"))], + &FutureFlags::default(), + ) + .unwrap(), ), depends_on: None, pass_through_env: None, @@ -89,14 +204,25 @@ mod test { cache: Some(Spanned::new(false)), persistent: Some(Spanned::new(true)), outputs: Some( - ProcessedOutputs::new(vec![Spanned::new(UnescapedString::from("build/**"))]) - .unwrap(), + ProcessedOutputs::new( + vec![Spanned::new(UnescapedString::from("build/**"))], + &FutureFlags::default(), + ) + .unwrap(), ), inputs: Some( - ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("lib/**"))]).unwrap(), + ProcessedInputs::new( + vec![Spanned::new(UnescapedString::from("lib/**"))], + &FutureFlags::default(), + ) + .unwrap(), ), env: Some( - ProcessedEnv::new(vec![Spanned::new(UnescapedString::from("PROD_ENV"))]).unwrap(), + ProcessedEnv::new( + vec![Spanned::new(UnescapedString::from("PROD_ENV"))], + &FutureFlags::default(), + ) + .unwrap(), ), output_logs: Some(Spanned::new(OutputLogsMode::Full)), interruptible: Some(Spanned::new(true)), @@ -210,8 +336,11 @@ mod test { let first = ProcessedTaskDefinition { cache: Some(Spanned::new(true)), outputs: Some( - ProcessedOutputs::new(vec![Spanned::new(UnescapedString::from("dist/**"))]) - .unwrap(), + ProcessedOutputs::new( + vec![Spanned::new(UnescapedString::from("dist/**"))], + &FutureFlags::default(), + ) + .unwrap(), ), ..Default::default() }; @@ -219,14 +348,22 @@ mod test { let second = ProcessedTaskDefinition { persistent: Some(Spanned::new(false)), inputs: Some( - ProcessedInputs::new(vec![Spanned::new(UnescapedString::from("src/**"))]).unwrap(), + ProcessedInputs::new( + vec![Spanned::new(UnescapedString::from("src/**"))], + &FutureFlags::default(), + ) + .unwrap(), ), ..Default::default() }; let third = ProcessedTaskDefinition { env: Some( - ProcessedEnv::new(vec![Spanned::new(UnescapedString::from("NODE_ENV"))]).unwrap(), + ProcessedEnv::new( + vec![Spanned::new(UnescapedString::from("NODE_ENV"))], + &FutureFlags::default(), + ) + .unwrap(), ), output_logs: Some(Spanned::new(OutputLogsMode::Full)), // Override cache from first task @@ -269,4 +406,201 @@ mod test { assert_eq!(result, single_task); } + + // Reusable fixtures for array fields + fn env_base() -> ProcessedEnv { + ProcessedEnv { + vars: vec!["BASE_ENV".to_string()], + extends: false, + } + } + + fn env_override() -> ProcessedEnv { + ProcessedEnv { + vars: vec!["OVERRIDE_ENV".to_string()], + extends: false, + } + } + + fn env_extending() -> ProcessedEnv { + ProcessedEnv { + vars: vec!["OVERRIDE_ENV".to_string()], + extends: true, + } + } + + fn deps_base() -> ProcessedDependsOn { + ProcessedDependsOn { + deps: vec![Spanned::new(UnescapedString::from("build"))], + extends: false, + } + } + + fn deps_test() -> ProcessedDependsOn { + ProcessedDependsOn { + deps: vec![Spanned::new(UnescapedString::from("test"))], + extends: false, + } + } + + fn deps_extending() -> ProcessedDependsOn { + ProcessedDependsOn { + deps: vec![Spanned::new(UnescapedString::from("test"))], + extends: true, + } + } + + fn with_task1() -> ProcessedWith { + use turborepo_task_id::TaskName; + ProcessedWith { + tasks: vec![Spanned::new(TaskName::from("task1"))], + extends: false, + } + } + + fn with_task2_extending() -> ProcessedWith { + use turborepo_task_id::TaskName; + ProcessedWith { + tasks: vec![Spanned::new(TaskName::from("task2"))], + extends: true, + } + } + + fn with_task3() -> ProcessedWith { + use turborepo_task_id::TaskName; + ProcessedWith { + tasks: vec![Spanned::new(TaskName::from("task3"))], + extends: false, + } + } + + fn inputs_base() -> ProcessedInputs { + ProcessedInputs { + globs: vec![], + default: false, + extends: false, + } + } + + fn inputs_extending_with_default() -> ProcessedInputs { + ProcessedInputs { + globs: vec![], + default: true, + extends: true, + } + } + + fn outputs_base() -> ProcessedOutputs { + ProcessedOutputs { + globs: vec![], + extends: false, + } + } + + fn outputs_extending() -> ProcessedOutputs { + ProcessedOutputs { + globs: vec![], + extends: true, + } + } + + #[test] + fn test_merge_with_extends_true() { + let mut base = ProcessedTaskDefinition { + inputs: Some(inputs_base()), + outputs: Some(outputs_base()), + env: Some(env_base()), + depends_on: Some(deps_base()), + with: Some(with_task1()), + ..Default::default() + }; + + let extending = ProcessedTaskDefinition { + inputs: Some(inputs_extending_with_default()), + outputs: Some(outputs_extending()), + env: Some(env_extending()), + depends_on: Some(deps_extending()), + with: Some(with_task2_extending()), + ..Default::default() + }; + + base.merge(extending); + + // Verify extends behavior + assert!(base.inputs.as_ref().unwrap().default); // OR'd + assert!(base.inputs.as_ref().unwrap().extends); + assert_eq!( + base.env.as_ref().unwrap().vars, + vec!["BASE_ENV".to_string(), "OVERRIDE_ENV".to_string()] + ); + assert_eq!(base.depends_on.as_ref().unwrap().deps.len(), 2); + assert_eq!(base.with.as_ref().unwrap().tasks.len(), 2); + } + + #[test] + fn test_merge_with_extends_false() { + let mut base = ProcessedTaskDefinition { + env: Some(env_base()), + depends_on: Some(deps_base()), + with: Some(with_task1()), + ..Default::default() + }; + + let replacing = ProcessedTaskDefinition { + env: Some(env_override()), + depends_on: Some(deps_test()), + with: Some(with_task3()), + ..Default::default() + }; + + base.merge(replacing); + + // Verify replace behavior + assert_eq!(base.env, Some(env_override())); + assert_eq!(base.depends_on, Some(deps_test())); + assert_eq!(base.with, Some(with_task3())); + } + + #[test] + fn test_merge_chain_with_extends_then_replace() { + // Test that when chaining: base -> extending -> replacing + // The final replacing task overrides everything, not extends + + let base = ProcessedTaskDefinition { + depends_on: Some(deps_base()), // has "build" + env: Some(env_base()), // has "BASE_ENV" + ..Default::default() + }; + + // Middle task extends the base + let extending = ProcessedTaskDefinition { + depends_on: Some(deps_extending()), // has "test" with extends: true + env: Some(env_extending()), // has "OVERRIDE_ENV" with extends: true + ..Default::default() + }; + + // Final task replaces (no extends) + let replacing = ProcessedTaskDefinition { + depends_on: Some(ProcessedDependsOn { + deps: vec![Spanned::new(UnescapedString::from("lint"))], + extends: false, // This should replace, not extend + }), + env: Some(ProcessedEnv { + vars: vec!["FINAL_ENV".to_string()], + extends: false, // This should replace, not extend + }), + ..Default::default() + }; + + // Apply the chain of merges + let result = ProcessedTaskDefinition::from_iter(vec![base, extending, replacing]); + + assert_eq!( + result.depends_on.as_ref().unwrap().deps, + vec![Spanned::new(UnescapedString::from("lint"))] + ); + + // Verify the extends flags are now false + assert!(!result.depends_on.as_ref().unwrap().extends); + } } diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 9d47638cdc01c..36332edee0e42 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -66,6 +66,7 @@ pub struct TurboJson { pub(crate) global_env: Vec, pub(crate) global_pass_through_env: Option>, pub(crate) tasks: Pipeline, + pub(crate) future_flags: FutureFlags, } // Iterable is required to enumerate allowed keys @@ -341,7 +342,7 @@ impl TaskDefinition { let mut topological_dependencies: Vec> = Vec::new(); let mut task_dependencies: Vec> = Vec::new(); if let Some(depends_on) = processed.depends_on { - for dependency in depends_on.0.into_inner() { + for dependency in depends_on.deps { let (dependency, depspan) = dependency.split(); let dependency: String = dependency.into(); if let Some(topo_dependency) = @@ -357,7 +358,7 @@ impl TaskDefinition { task_dependencies.sort_by(|a, b| a.value.cmp(&b.value)); topological_dependencies.sort_by(|a, b| a.value.cmp(&b.value)); - let env = processed.env.map(|env| env.0).unwrap_or_default(); + let env = processed.env.map(|env| env.vars).unwrap_or_default(); // Convert inputs with turbo_root resolution let inputs = processed @@ -366,9 +367,9 @@ impl TaskDefinition { .transpose()? .unwrap_or_default(); - let pass_through_env = processed.pass_through_env.map(|env| env.0); + let pass_through_env = processed.pass_through_env.map(|env| env.vars); - let with = processed.with.map(|with_tasks| with_tasks.0); + let with = processed.with.map(|with_tasks| with_tasks.tasks); Ok(TaskDefinition { outputs, @@ -393,7 +394,8 @@ impl TaskDefinition { raw_task: RawTaskDefinition, path_to_repo_root: &RelativeUnixPath, ) -> Result { - let processed = ProcessedTaskDefinition::from_raw(raw_task)?; + // Use default FutureFlags for backward compatibility + let processed = ProcessedTaskDefinition::from_raw(raw_task, &FutureFlags::default())?; Self::from_processed(processed, path_to_repo_root) } } @@ -550,6 +552,10 @@ impl TryFrom for TurboJson { .unwrap_or_default() .map(|s| s.into_iter().map(|s| s.into()).collect()), boundaries: raw_turbo.boundaries, + future_flags: raw_turbo + .future_flags + .map(|f| f.into_inner()) + .unwrap_or_default(), // Remote Cache config is handled through layered config }) } @@ -575,12 +581,16 @@ impl TurboJson { fn read( repo_root: &AbsoluteSystemPath, path: &AbsoluteSystemPath, - _future_flags: FutureFlags, + future_flags: FutureFlags, ) -> Result, Error> { let Some(raw_turbo_json) = RawTurboJson::read(repo_root, path)? else { return Ok(None); }; - TurboJson::try_from(raw_turbo_json).map(Some) + + let mut turbo_json = TurboJson::try_from(raw_turbo_json)?; + // Override with root's future flags (only root turbo.json can define them) + turbo_json.future_flags = future_flags; + Ok(Some(turbo_json)) } pub fn task( @@ -589,11 +599,15 @@ impl TurboJson { task_name: &TaskName, ) -> Result, Error> { match self.tasks.get(&task_id.as_task_name()) { - Some(entry) => ProcessedTaskDefinition::from_raw(entry.value.clone()).map(Some), + Some(entry) => { + ProcessedTaskDefinition::from_raw(entry.value.clone(), &self.future_flags).map(Some) + } None => self .tasks .get(task_name) - .map(|entry| ProcessedTaskDefinition::from_raw(entry.value.clone())) + .map(|entry| { + ProcessedTaskDefinition::from_raw(entry.value.clone(), &self.future_flags) + }) .transpose(), } } @@ -1022,8 +1036,10 @@ mod tests { ) -> Result<()> { let raw_task_outputs: Vec = serde_json::from_str(task_outputs_str)?; let turbo_root = RelativeUnixPath::new("../..")?; - let processed_outputs = - ProcessedOutputs::new(raw_task_outputs.into_iter().map(Spanned::new).collect())?; + let processed_outputs = ProcessedOutputs::new( + raw_task_outputs.into_iter().map(Spanned::new).collect(), + &FutureFlags::default(), + )?; let task_outputs = TaskOutputs::from_processed(processed_outputs, turbo_root)?; assert_eq!(task_outputs, expected_task_outputs); diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index 7d2fb112c2b45..c53d64c7141a8 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -6,7 +6,7 @@ use turborepo_errors::Spanned; use turborepo_task_id::TaskName; use turborepo_unescape::UnescapedString; -use super::RawTaskDefinition; +use super::{FutureFlags, RawTaskDefinition}; use crate::{ cli::{EnvMode, OutputLogsMode}, config::Error, @@ -15,9 +15,28 @@ use crate::{ const TURBO_DEFAULT: &str = "$TURBO_DEFAULT$"; const TURBO_ROOT: &str = "$TURBO_ROOT$"; const TURBO_ROOT_SLASH: &str = "$TURBO_ROOT$/"; +const TURBO_EXTENDS: &str = "$TURBO_EXTENDS$"; const ENV_PIPELINE_DELIMITER: &str = "$"; const TOPOLOGICAL_PIPELINE_DELIMITER: &str = "^"; +/// Helper function to check for and remove $TURBO_EXTENDS$ from an array +/// Returns (processed_array, extends_found) +fn extract_turbo_extends( + mut items: Vec>, + future_flags: &FutureFlags, +) -> (Vec>, bool) { + if !future_flags.turbo_extends { + return (items, false); + } + + if let Some(pos) = items.iter().position(|item| item.as_str() == TURBO_EXTENDS) { + items.remove(pos); + (items, true) + } else { + (items, false) + } +} + /// A processed glob with separated components #[derive(Debug, Clone, PartialEq)] pub struct ProcessedGlob { @@ -102,14 +121,22 @@ impl ProcessedGlob { /// Processed depends_on field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedDependsOn(pub Spanned>>); +pub struct ProcessedDependsOn { + pub deps: Vec>, + pub extends: bool, +} impl ProcessedDependsOn { /// Creates a ProcessedDependsOn, validating that dependencies don't use env - /// prefix - pub fn new(raw_deps: Spanned>>) -> Result { + /// prefix and handling TURBO_EXTENDS if enabled + pub fn new( + raw_deps: Spanned>>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_deps, extends) = extract_turbo_extends(raw_deps.into_inner(), future_flags); + // Validate that no dependency starts with ENV_PIPELINE_DELIMITER ($) - for dep in raw_deps.value.iter() { + for dep in processed_deps.iter() { if dep.starts_with(ENV_PIPELINE_DELIMITER) { let (span, text) = dep.span_and_text("turbo.json"); return Err(Error::InvalidDependsOnValue { @@ -119,41 +146,65 @@ impl ProcessedDependsOn { }); } } - Ok(ProcessedDependsOn(raw_deps)) + Ok(ProcessedDependsOn { + deps: processed_deps, + extends, + }) } } /// Processed env field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedEnv(pub Vec); +pub struct ProcessedEnv { + pub vars: Vec, + pub extends: bool, +} impl ProcessedEnv { /// Creates a ProcessedEnv, validating that env vars don't use invalid - /// prefixes - pub fn new(raw_env: Vec>) -> Result { - Ok(ProcessedEnv(extract_env_vars(raw_env)?)) + /// prefixes and handling TURBO_EXTENDS if enabled + pub fn new( + raw_env: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_env, extends) = extract_turbo_extends(raw_env, future_flags); + + Ok(ProcessedEnv { + vars: extract_env_vars(processed_env, "env")?, + extends, + }) } } /// Processed inputs field with DSL detection #[derive(Debug, Clone, PartialEq)] pub struct ProcessedInputs { - globs: Vec, + pub globs: Vec, pub default: bool, + pub extends: bool, } impl ProcessedInputs { - pub fn new(raw_globs: Vec>) -> Result { - let mut globs = Vec::with_capacity(raw_globs.len()); + pub fn new( + raw_globs: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_globs, extends) = extract_turbo_extends(raw_globs, future_flags); + + let mut globs = Vec::with_capacity(processed_globs.len()); let mut default = false; - for raw_glob in raw_globs { + for raw_glob in processed_globs { if raw_glob.as_str() == TURBO_DEFAULT { default = true; } globs.push(ProcessedGlob::from_spanned_input(raw_glob)?); } - Ok(ProcessedInputs { globs, default }) + Ok(ProcessedInputs { + globs, + default, + extends, + }) } /// Resolves all globs with the given turbo_root path @@ -167,17 +218,31 @@ impl ProcessedInputs { /// Processed pass_through_env field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedPassThroughEnv(pub Vec); +pub struct ProcessedPassThroughEnv { + pub vars: Vec, + pub extends: bool, +} impl ProcessedPassThroughEnv { /// Creates a ProcessedPassThroughEnv, validating that env vars don't use - /// invalid prefixes - pub fn new(raw_env: Vec>) -> Result { - Ok(ProcessedPassThroughEnv(extract_env_vars(raw_env)?)) + /// invalid prefixes and handling TURBO_EXTENDS if enabled + pub fn new( + raw_env: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_env, extends) = extract_turbo_extends(raw_env, future_flags); + + Ok(ProcessedPassThroughEnv { + vars: extract_env_vars(processed_env, "passThroughEnv")?, + extends, + }) } } -fn extract_env_vars(raw_env: Vec>) -> Result, Error> { +fn extract_env_vars( + raw_env: Vec>, + field_name: &str, +) -> Result, Error> { use crate::config::InvalidEnvPrefixError; let mut env_vars = Vec::with_capacity(raw_env.len()); @@ -186,7 +251,7 @@ fn extract_env_vars(raw_env: Vec>) -> Result>) -> Result, + pub globs: Vec, + pub extends: bool, } impl ProcessedOutputs { - pub fn new(raw_globs: Vec>) -> Result { - let globs = raw_globs + pub fn new( + raw_globs: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_globs, extends) = extract_turbo_extends(raw_globs, future_flags); + + let globs = processed_globs .into_iter() - .map(ProcessedGlob::from_spanned_input) + .map(ProcessedGlob::from_spanned_output) .collect::, _>>()?; - Ok(ProcessedOutputs { globs }) + Ok(ProcessedOutputs { globs, extends }) } /// Resolves all globs with the given turbo_root path @@ -227,23 +298,31 @@ impl ProcessedOutputs { /// Processed with field with DSL detection #[derive(Debug, Clone, PartialEq)] -pub struct ProcessedWith(pub Vec>>); +pub struct ProcessedWith { + pub tasks: Vec>>, + pub extends: bool, +} impl ProcessedWith { /// Creates a ProcessedWith, validating that siblings don't use topological - /// prefix - pub fn new(raw_with: Vec>) -> Result { + /// prefix and handling TURBO_EXTENDS if enabled + pub fn new( + raw_with: Vec>, + future_flags: &FutureFlags, + ) -> Result { + let (processed_with, extends) = extract_turbo_extends(raw_with, future_flags); + // Validate that no sibling starts with TOPOLOGICAL_PIPELINE_DELIMITER (^) - let mut with = Vec::with_capacity(raw_with.len()); - for sibling in raw_with { + let mut tasks = Vec::with_capacity(processed_with.len()); + for sibling in processed_with { if sibling.starts_with(TOPOLOGICAL_PIPELINE_DELIMITER) { let (span, text) = sibling.span_and_text("turbo.json"); return Err(Error::InvalidTaskWith { span, text }); } let (sibling, span) = sibling.split(); - with.push(span.to(TaskName::from(String::from(sibling)))); + tasks.push(span.to(TaskName::from(String::from(sibling)))); } - Ok(ProcessedWith(with)) + Ok(ProcessedWith { tasks, extends }) } } @@ -266,26 +345,41 @@ pub struct ProcessedTaskDefinition { impl ProcessedTaskDefinition { /// Creates a processed task definition from raw task - pub fn from_raw(raw_task: RawTaskDefinition) -> Result { + pub fn from_raw( + raw_task: RawTaskDefinition, + future_flags: &FutureFlags, + ) -> Result { Ok(ProcessedTaskDefinition { cache: raw_task.cache, depends_on: raw_task .depends_on - .map(ProcessedDependsOn::new) + .map(|deps| ProcessedDependsOn::new(deps, future_flags)) + .transpose()?, + env: raw_task + .env + .map(|env| ProcessedEnv::new(env, future_flags)) + .transpose()?, + inputs: raw_task + .inputs + .map(|inputs| ProcessedInputs::new(inputs, future_flags)) .transpose()?, - env: raw_task.env.map(ProcessedEnv::new).transpose()?, - inputs: raw_task.inputs.map(ProcessedInputs::new).transpose()?, pass_through_env: raw_task .pass_through_env - .map(ProcessedPassThroughEnv::new) + .map(|env| ProcessedPassThroughEnv::new(env, future_flags)) .transpose()?, persistent: raw_task.persistent, interruptible: raw_task.interruptible, - outputs: raw_task.outputs.map(ProcessedOutputs::new).transpose()?, + outputs: raw_task + .outputs + .map(|outputs| ProcessedOutputs::new(outputs, future_flags)) + .transpose()?, output_logs: raw_task.output_logs, interactive: raw_task.interactive, env_mode: raw_task.env_mode, - with: raw_task.with.map(ProcessedWith::new).transpose()?, + with: raw_task + .with + .map(|with| ProcessedWith::new(with, future_flags)) + .transpose()?, }) } } @@ -299,6 +393,66 @@ mod tests { use turborepo_unescape::UnescapedString; use super::*; + use crate::turbo_json::FutureFlags; + + #[test] + fn test_extract_turbo_extends_with_flag_enabled() { + let items = vec![ + Spanned::new(UnescapedString::from("item1")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("item2")), + ]; + + let (processed, extends) = extract_turbo_extends( + items, + &FutureFlags { + turbo_extends: true, + }, + ); + + assert!(extends); + assert_eq!(processed.len(), 2); + assert_eq!(processed[0].as_str(), "item1"); + assert_eq!(processed[1].as_str(), "item2"); + } + + #[test] + fn test_extract_turbo_extends_with_flag_disabled() { + let items = vec![ + Spanned::new(UnescapedString::from("item1")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("item2")), + ]; + + let (processed, extends) = extract_turbo_extends( + items, + &FutureFlags { + turbo_extends: false, + }, + ); + + assert!(!extends); + assert_eq!(processed.len(), 3); + assert_eq!(processed[1].as_str(), "$TURBO_EXTENDS$"); + } + + #[test] + fn test_extract_turbo_extends_no_marker() { + let items = vec![ + Spanned::new(UnescapedString::from("item1")), + Spanned::new(UnescapedString::from("item2")), + ]; + + let (processed, extends) = extract_turbo_extends( + items, + &FutureFlags { + turbo_extends: true, + }, + ); + + assert!(!extends); + assert_eq!(processed.len(), 2); + } #[test_case("$TURBO_ROOT$/config.txt", Ok((true, false)) ; "detects turbo root")] #[test_case("!$TURBO_ROOT$/README.md", Ok((true, true)) ; "detects negated turbo root")] @@ -368,7 +522,8 @@ mod tests { }; // Convert to processed task definition - let processed = ProcessedTaskDefinition::from_raw(raw_task).unwrap(); + let processed = + ProcessedTaskDefinition::from_raw(raw_task, &FutureFlags::default()).unwrap(); let turbo_root = RelativeUnixPath::new("../..").unwrap(); // Verify TURBO_ROOT detection @@ -396,7 +551,7 @@ mod tests { fn test_detects_turbo_default() { let raw_globs = vec![Spanned::new(UnescapedString::from(TURBO_DEFAULT))]; - let inputs = ProcessedInputs::new(raw_globs).unwrap(); + let inputs = ProcessedInputs::new(raw_globs, &FutureFlags::default()).unwrap(); assert!(inputs.default); assert_eq!( inputs.globs, @@ -422,4 +577,66 @@ mod tests { assert_matches!(result, Err(Error::AbsolutePathInConfig { .. })); } + + // Test that demonstrates the extends field is properly set when the helper is + // used + #[test] + fn test_processed_inputs_with_turbo_extends() { + let raw_globs: Vec> = vec![ + Spanned::new(UnescapedString::from("src/**")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("lib/**")), + ]; + + let inputs = ProcessedInputs::new( + raw_globs, + &FutureFlags { + turbo_extends: true, + }, + ) + .unwrap(); + + assert!(inputs.extends); + assert_eq!(inputs.globs.len(), 2); + assert_eq!(inputs.globs[0].glob, "src/**"); + assert_eq!(inputs.globs[1].glob, "lib/**"); + } + + #[test] + fn test_processed_env_turbo_extends_disabled_errors() { + // When turbo_extends is disabled, $TURBO_EXTENDS$ triggers validation error + let raw_env: Vec> = vec![ + Spanned::new(UnescapedString::from("NODE_ENV")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("API_KEY")), + ]; + + let result = ProcessedEnv::new( + raw_env, + &FutureFlags { + turbo_extends: false, + }, + ); + assert!(result.is_err()); + assert_matches!(result, Err(Error::InvalidEnvPrefix(_))); + } + + #[test] + fn test_processed_depends_on_turbo_extends_disabled_errors() { + // When turbo_extends is disabled, $TURBO_EXTENDS$ triggers validation error + let raw_deps: Vec> = vec![ + Spanned::new(UnescapedString::from("build")), + Spanned::new(UnescapedString::from("$TURBO_EXTENDS$")), + Spanned::new(UnescapedString::from("test")), + ]; + + let result = ProcessedDependsOn::new( + Spanned::new(raw_deps), + &FutureFlags { + turbo_extends: false, + }, + ); + assert!(result.is_err()); + assert_matches!(result, Err(Error::InvalidDependsOnValue { .. })); + } } From bbec95dfafcdc2f96b1aca438d22ece26aaf13e8 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 13 Aug 2025 16:50:00 -0400 Subject: [PATCH 17/19] chore(config): fix clippy lint --- crates/turborepo-lib/src/config/turbo_json.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/turborepo-lib/src/config/turbo_json.rs b/crates/turborepo-lib/src/config/turbo_json.rs index ae13a4a740380..ba5e81bd78810 100644 --- a/crates/turborepo-lib/src/config/turbo_json.rs +++ b/crates/turborepo-lib/src/config/turbo_json.rs @@ -78,7 +78,7 @@ impl<'a> TurboJsonReader<'a> { opts.env_mode = turbo_json.env_mode.map(|mode| *mode.as_inner()); opts.cache_dir = cache_dir; opts.concurrency = turbo_json.concurrency.map(|c| c.as_inner().clone()); - opts.future_flags = turbo_json.future_flags.map(|f| f.as_inner().clone()); + opts.future_flags = turbo_json.future_flags.map(|f| *f.as_inner()); Ok(opts) } } From 2123e6f8a91a30d2f52c100843863eff3edaa51c Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 14 Aug 2025 09:31:03 -0400 Subject: [PATCH 18/19] chore(turbo_json): update test fixture to have one failure --- .../turbo-configs/interruptible-but-not-persistent.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turborepo-tests/integration/fixtures/turbo-configs/interruptible-but-not-persistent.json b/turborepo-tests/integration/fixtures/turbo-configs/interruptible-but-not-persistent.json index 4759216b2e0f5..db1d409a8d1e5 100644 --- a/turborepo-tests/integration/fixtures/turbo-configs/interruptible-but-not-persistent.json +++ b/turborepo-tests/integration/fixtures/turbo-configs/interruptible-but-not-persistent.json @@ -10,7 +10,7 @@ "build": { "env": [ "NODE_ENV", - "$FOOBAR" + "FOOBAR" ], "interruptible": true, "outputs": [] From e66c7c7f3e9b06258f3690671886df26d944ce69 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 21 Aug 2025 13:48:50 -0400 Subject: [PATCH 19/19] chore(turbo_json): change to more specific turboExtendsKeyword flag name --- .../turborepo-lib/src/turbo_json/future_flags.rs | 2 +- crates/turborepo-lib/src/turbo_json/mod.rs | 4 ++-- crates/turborepo-lib/src/turbo_json/processed.rs | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/future_flags.rs b/crates/turborepo-lib/src/turbo_json/future_flags.rs index 9879dbb1b57f5..b865d0b4f3a1b 100644 --- a/crates/turborepo-lib/src/turbo_json/future_flags.rs +++ b/crates/turborepo-lib/src/turbo_json/future_flags.rs @@ -34,7 +34,7 @@ pub struct FutureFlags { /// When enabled, allows using `$TURBO_EXTENDS$` in array fields. /// This will change the default behavior of overriding the field to instead /// append. - pub turbo_extends: bool, + pub turbo_extends_keyword: bool, } impl FutureFlags { diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 36332edee0e42..568235a6b8a72 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -1273,7 +1273,7 @@ mod tests { "build": {} }, "futureFlags": { - "turboExtends": true + "turboExtendsKeyword": true } }"#; @@ -1290,7 +1290,7 @@ mod tests { assert_eq!( future_flags.as_inner(), &FutureFlags { - turbo_extends: true + turbo_extends_keyword: true } ); diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index c53d64c7141a8..0853086498a6c 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -25,7 +25,7 @@ fn extract_turbo_extends( mut items: Vec>, future_flags: &FutureFlags, ) -> (Vec>, bool) { - if !future_flags.turbo_extends { + if !future_flags.turbo_extends_keyword { return (items, false); } @@ -406,7 +406,7 @@ mod tests { let (processed, extends) = extract_turbo_extends( items, &FutureFlags { - turbo_extends: true, + turbo_extends_keyword: true, }, ); @@ -427,7 +427,7 @@ mod tests { let (processed, extends) = extract_turbo_extends( items, &FutureFlags { - turbo_extends: false, + turbo_extends_keyword: false, }, ); @@ -446,7 +446,7 @@ mod tests { let (processed, extends) = extract_turbo_extends( items, &FutureFlags { - turbo_extends: true, + turbo_extends_keyword: true, }, ); @@ -591,7 +591,7 @@ mod tests { let inputs = ProcessedInputs::new( raw_globs, &FutureFlags { - turbo_extends: true, + turbo_extends_keyword: true, }, ) .unwrap(); @@ -614,7 +614,7 @@ mod tests { let result = ProcessedEnv::new( raw_env, &FutureFlags { - turbo_extends: false, + turbo_extends_keyword: false, }, ); assert!(result.is_err()); @@ -633,7 +633,7 @@ mod tests { let result = ProcessedDependsOn::new( Spanned::new(raw_deps), &FutureFlags { - turbo_extends: false, + turbo_extends_keyword: false, }, ); assert!(result.is_err());