diff --git a/crates/turborepo-lib/src/commands/prune.rs b/crates/turborepo-lib/src/commands/prune.rs index f57add8b76844..dfdb9eadead82 100644 --- a/crates/turborepo-lib/src/commands/prune.rs +++ b/crates/turborepo-lib/src/commands/prune.rs @@ -18,7 +18,7 @@ use turborepo_ui::BOLD; use super::CommandBase; use crate::{ config::{CONFIG_FILE, CONFIG_FILE_JSONC}, - turbo_json::RawTurboJson, + turbo_json::{RawRootTurboJson, RawTurboJson}, }; pub const DEFAULT_OUTPUT_DIR: &str = "out"; @@ -473,7 +473,8 @@ impl<'a> Prune<'a> { return Ok(None); }; - let turbo_json = RawTurboJson::parse(&turbo_json_contents, turbo_json_name.as_str())?; + let turbo_json = + RawRootTurboJson::parse(&turbo_json_contents, turbo_json_name.as_str())?.into(); Ok(Some((turbo_json, turbo_json_name))) } } diff --git a/crates/turborepo-lib/src/config/mod.rs b/crates/turborepo-lib/src/config/mod.rs index b3a4cbc93c1d8..fad0c7ed21dd8 100644 --- a/crates/turborepo-lib/src/config/mod.rs +++ b/crates/turborepo-lib/src/config/mod.rs @@ -23,7 +23,7 @@ use turborepo_cache::CacheConfig; use turborepo_errors::TURBO_SITE; use turborepo_repository::package_graph::PackageName; -pub use crate::turbo_json::{RawTurboJson, UIMode}; +pub use crate::turbo_json::UIMode; use crate::{ cli::{EnvMode, LogOrder}, turbo_json::FutureFlags, @@ -140,6 +140,13 @@ pub enum Error { #[source_code] text: NamedSource, }, + #[error("You must extend from the root of the workspace first.")] + ExtendsRootFirst { + #[label("'//' should be first")] + span: Option, + #[source_code] + text: NamedSource, + }, #[error("`{field}` cannot contain an environment variable.")] InvalidDependsOnValue { field: &'static str, diff --git a/crates/turborepo-lib/src/config/turbo_json.rs b/crates/turborepo-lib/src/config/turbo_json.rs index ba5e81bd78810..1041c18b807ef 100644 --- a/crates/turborepo-lib/src/config/turbo_json.rs +++ b/crates/turborepo-lib/src/config/turbo_json.rs @@ -2,7 +2,7 @@ use camino::Utf8PathBuf; use turbopath::{AbsoluteSystemPath, RelativeUnixPath}; use super::{ConfigurationOptions, Error, ResolvedConfigurationOptions}; -use crate::turbo_json::{RawRemoteCacheOptions, RawTurboJson}; +use crate::turbo_json::{RawRemoteCacheOptions, RawRootTurboJson, RawTurboJson}; pub struct TurboJsonReader<'a> { repo_root: &'a AbsoluteSystemPath, @@ -89,8 +89,16 @@ impl<'a> ResolvedConfigurationOptions for TurboJsonReader<'a> { existing_config: &ConfigurationOptions, ) -> Result { let turbo_json_path = existing_config.root_turbo_json_path(self.repo_root)?; - let turbo_json = RawTurboJson::read(self.repo_root, &turbo_json_path) - .map(|turbo_json| turbo_json.unwrap_or_default())?; + let root_relative_turbo_json_path = self.repo_root.anchor(&turbo_json_path).map_or_else( + |_| turbo_json_path.as_str().to_owned(), + |relative| relative.to_string(), + ); + let turbo_json = match turbo_json_path.read_existing_to_string()? { + Some(contents) => { + RawRootTurboJson::parse(&contents, &root_relative_turbo_json_path)?.into() + } + None => RawTurboJson::default(), + }; Self::turbo_json_to_config_options(turbo_json) } } @@ -99,6 +107,7 @@ impl<'a> ResolvedConfigurationOptions for TurboJsonReader<'a> { mod test { use serde_json::json; use tempfile::tempdir; + use test_case::test_case; use super::*; use crate::config::CONFIG_FILE; @@ -165,7 +174,7 @@ mod test { let login_url = "localhost:3001"; let team_slug = "acme-packers"; let team_id = "id-123"; - let turbo_json = RawTurboJson::parse( + let turbo_json = RawRootTurboJson::parse( &serde_json::to_string_pretty(&json!({ "remoteCache": { "enabled": true, @@ -182,7 +191,8 @@ mod test { .unwrap(), "junk", ) - .unwrap(); + .unwrap() + .into(); let config = TurboJsonReader::turbo_json_to_config_options(turbo_json).unwrap(); assert!(config.enabled()); assert_eq!(config.timeout(), timeout); @@ -194,4 +204,27 @@ mod test { assert!(config.signature()); assert!(config.preflight()); } + + #[test_case(None, false)] + #[test_case(Some(false), false)] + #[test_case(Some(true), true)] + fn test_dangerously_disable_package_manager_check(value: Option, expected: bool) { + let turbo_json = RawRootTurboJson::parse( + &serde_json::to_string_pretty( + &(if let Some(value) = value { + json!({ + "dangerouslyDisablePackageManagerCheck": value + }) + } else { + json!({}) + }), + ) + .unwrap(), + "turbo.json", + ) + .unwrap() + .into(); + let config = TurboJsonReader::turbo_json_to_config_options(turbo_json).unwrap(); + assert_eq!(config.allow_no_package_manager(), expected); + } } diff --git a/crates/turborepo-lib/src/engine/builder.rs b/crates/turborepo-lib/src/engine/builder.rs index 82cd8f4abd45f..9bbda4115682d 100644 --- a/crates/turborepo-lib/src/engine/builder.rs +++ b/crates/turborepo-lib/src/engine/builder.rs @@ -14,8 +14,7 @@ use crate::{ config, task_graph::TaskDefinition, turbo_json::{ - validate_extends, validate_no_package_task_syntax, validate_with_has_no_topo, - ProcessedTaskDefinition, TurboJsonLoader, + validator::Validator, FutureFlags, ProcessedTaskDefinition, TurboJson, TurboJsonLoader, }, }; @@ -86,6 +85,26 @@ pub struct MissingRootTaskInTurboJsonError { text: NamedSource, } +#[derive(Debug, thiserror::Error, Diagnostic)] +#[error("Cannot extend from '{package_name}' without a package 'turbo.json'.")] +pub struct MissingTurboJsonExtends { + package_name: String, + #[label("Extended from here")] + span: Option, + #[source_code] + text: NamedSource, +} + +#[derive(Debug, thiserror::Error, Diagnostic)] +#[error("Cyclic extends detected: {}", cycle.join(" -> "))] +pub struct CyclicExtends { + cycle: Vec, + #[label("Cycle detected here")] + span: Option, + #[source_code] + text: NamedSource, +} + #[derive(Debug, thiserror::Error, Diagnostic)] pub enum Error { #[error("Missing tasks in project")] @@ -103,6 +122,12 @@ pub enum Error { MissingPackageTask(Box), #[error(transparent)] #[diagnostic(transparent)] + MissingTurboJsonExtends(Box), + #[error(transparent)] + #[diagnostic(transparent)] + CyclicExtends(Box), + #[error(transparent)] + #[diagnostic(transparent)] Config(#[from] crate::config::Error), #[error("Invalid turbo.json configuration")] Validation { @@ -127,6 +152,7 @@ pub struct EngineBuilder<'a> { tasks_only: bool, add_all_tasks: bool, should_validate_engine: bool, + validator: Validator, } impl<'a> EngineBuilder<'a> { @@ -147,9 +173,15 @@ impl<'a> EngineBuilder<'a> { tasks_only: false, add_all_tasks: false, should_validate_engine: true, + validator: Validator::new(), } } + pub fn with_future_flags(mut self, future_flags: FutureFlags) -> Self { + self.validator = self.validator.with_future_flags(future_flags); + self + } + pub fn with_tasks_only(mut self, tasks_only: bool) -> Self { self.tasks_only = tasks_only; self @@ -580,12 +612,17 @@ impl<'a> EngineBuilder<'a> { task_id: &Spanned, task_name: &TaskName, ) -> Result, Error> { + let package_name = PackageName::from(task_id.package()); + let mut turbo_json_chain = self + .turbo_json_chain(turbo_json_loader, &package_name)? + .into_iter(); let mut task_definitions = Vec::new(); - let root_turbo_json = turbo_json_loader.load(&PackageName::Root)?; - Error::from_validation(root_turbo_json.validate(&[validate_with_has_no_topo]))?; - - if let Some(root_definition) = root_turbo_json.task(task_id, task_name)? { + if let Some(root_definition) = turbo_json_chain + .next() + .expect("root turbo.json is always in chain") + .task(task_id, task_name)? + { task_definitions.push(root_definition) } @@ -605,23 +642,9 @@ impl<'a> EngineBuilder<'a> { }; } - if task_id.package() != ROOT_PKG_NAME { - match turbo_json_loader.load(&PackageName::from(task_id.package())) { - Ok(workspace_json) => { - Error::from_validation(workspace_json.validate(&[ - validate_no_package_task_syntax, - validate_extends, - validate_with_has_no_topo, - ]))?; - - if let Some(workspace_def) = workspace_json.task(task_id, task_name)? { - task_definitions.push(workspace_def); - } - } - Err(config::Error::NoTurboJSON) => (), - Err(e) => { - return Err(e.into()); - } + for turbo_json in turbo_json_chain { + if let Some(workspace_def) = turbo_json.task(task_id, task_name)? { + task_definitions.push(workspace_def); } } @@ -640,6 +663,117 @@ impl<'a> EngineBuilder<'a> { Ok(task_definitions) } + // Provide the chain of turbo.json's to load to fully resolve all extends for a + // package turbo.json. + fn turbo_json_chain<'b>( + &self, + turbo_json_loader: &'b TurboJsonLoader, + package_name: &PackageName, + ) -> Result, Error> { + let validator = &self.validator; + let mut turbo_jsons = Vec::with_capacity(2); + + enum ReadReq { + // An inferred check we perform for each package to see if there is a package specific + // turbo.json + Infer(PackageName), + // A specifically requested read from a package name being present in `extends` + Request(Spanned), + } + + impl ReadReq { + fn package_name(&self) -> &PackageName { + match self { + ReadReq::Infer(package_name) => package_name, + ReadReq::Request(package_name) => package_name.as_inner(), + } + } + + fn required(&self) -> Option<(Option, NamedSource)> { + match self { + ReadReq::Infer(_) => None, + ReadReq::Request(spanned) => Some(spanned.span_and_text("turbo.json")), + } + } + } + + let mut read_stack = vec![(ReadReq::Infer(package_name.clone()), vec![])]; + let mut visited = std::collections::HashSet::new(); + + while let Some((read_req, mut path)) = read_stack.pop() { + let package_name = read_req.package_name(); + + // Check for cycle by seeing if this package is already in the current path + if let Some(cycle_index) = path.iter().position(|p: &PackageName| p == package_name) { + // Found a cycle - build the cycle portion for error + let mut cycle = path[cycle_index..] + .iter() + .map(|p| p.to_string()) + .collect::>(); + cycle.push(package_name.to_string()); + + let (span, text) = read_req + .required() + .unwrap_or_else(|| (None, NamedSource::new("turbo.json", String::new()))); + + return Err(Error::CyclicExtends(Box::new(CyclicExtends { + cycle, + span, + text, + }))); + } + + // Skip if we've already fully processed this package + if visited.contains(package_name) { + continue; + } + + let turbo_json = turbo_json_loader + .load(package_name) + .map(Some) + .or_else(|err| { + if let Some((span, text)) = read_req.required() + && matches!(err, config::Error::NoTurboJSON) + { + Err(Error::MissingTurboJsonExtends(Box::new( + MissingTurboJsonExtends { + package_name: read_req.package_name().to_string(), + span, + text, + }, + ))) + } else if matches!(err, config::Error::NoTurboJSON) { + Ok(None) + } else { + Err(err.into()) + } + })?; + if let Some(turbo_json) = turbo_json { + Error::from_validation(validator.validate_turbo_json(package_name, turbo_json))?; + turbo_jsons.push(turbo_json); + visited.insert(package_name.clone()); + + // Add current package to path for cycle detection + path.push(package_name.clone()); + + // Add the new turbo.json we are extending from + let (extends, span) = turbo_json.extends.clone().split(); + for extend_package in extends { + let extend_package_name = PackageName::from(extend_package); + read_stack.push(( + ReadReq::Request(span.clone().to(extend_package_name)), + path.clone(), + )); + } + } else if turbo_jsons.is_empty() { + // If there is no package turbo.json extend from root by default + read_stack.push((ReadReq::Infer(PackageName::Root), path)); + } + } + + Ok(turbo_jsons.into_iter().rev().collect()) + } + // Returns that path from a task's package directory to the repo root fn path_to_root(&self, task_id: &TaskId) -> Result { let package_name = PackageName::from(task_id.package()); @@ -709,7 +843,7 @@ mod test { use super::*; use crate::{ engine::TaskNode, - turbo_json::{RawTurboJson, TurboJson}, + turbo_json::{RawPackageTurboJson, RawRootTurboJson, RawTurboJson, TurboJson}, }; // Only used to prevent package graph construction from attempting to read @@ -813,8 +947,13 @@ mod test { } fn turbo_json(value: serde_json::Value) -> TurboJson { + let is_package = value.as_object().unwrap().contains_key("extends"); let json_text = serde_json::to_string(&value).unwrap(); - let raw = RawTurboJson::parse(&json_text, "").unwrap(); + let raw: RawTurboJson = if is_package { + RawPackageTurboJson::parse(&json_text, "").unwrap().into() + } else { + RawRootTurboJson::parse(&json_text, "").unwrap().into() + }; TurboJson::try_from(raw).unwrap() } @@ -1461,14 +1600,12 @@ mod test { }, ); let turbo_jsons = vec![(PackageName::Root, { - let mut t_json = turbo_json(json!({ + turbo_json(json!({ "tasks": { - "web#dev": { "persistent": true }, + "web#dev": { "persistent": true, "with": ["api#serve"] }, "api#serve": { "persistent": true } } - })); - t_json.with_task(TaskName::from("web#dev"), &TaskName::from("api#serve")); - t_json + })) })] .into_iter() .collect(); @@ -1661,4 +1798,63 @@ mod test { "../.." ); } + + #[test] + fn test_cyclic_extends() { + let repo_root_dir = TempDir::with_prefix("repo").unwrap(); + let repo_root = AbsoluteSystemPathBuf::new(repo_root_dir.path().to_str().unwrap()).unwrap(); + let package_graph = mock_package_graph( + &repo_root, + package_jsons! { + repo_root, + "app1" => [], + "app2" => [] + }, + ); + + // Create a self-referencing cycle: Root extends itself + let turbo_jsons = vec![ + ( + PackageName::Root, + turbo_json(json!({ + "extends": ["//"], // Root extending itself creates a cycle + "tasks": { + "build": {} + } + })), + ), + ( + PackageName::from("app1"), + turbo_json(json!({ + "extends": ["//"], + "tasks": {} + })), + ), + ( + PackageName::from("app2"), + turbo_json(json!({ + "extends": ["//"], + "tasks": {} + })), + ), + ] + .into_iter() + .collect(); + + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine_result = EngineBuilder::new(&repo_root, &package_graph, &loader, false) + .with_tasks(Some(Spanned::new(TaskName::from("build")))) + .with_workspaces(vec![PackageName::from("app1")]) + .build(); + + assert!(engine_result.is_err()); + if let Err(Error::CyclicExtends(box CyclicExtends { cycle, .. })) = engine_result { + // The cycle should contain root (//) since it's a self-reference + assert!(cycle.contains(&"//".to_string())); + // Should have at least 2 entries to show the cycle (// -> //) + assert!(cycle.len() >= 2); + } else { + panic!("Expected CyclicExtends error, got {:?}", engine_result); + } + } } diff --git a/crates/turborepo-lib/src/lib.rs b/crates/turborepo-lib/src/lib.rs index 65b8ed29b9819..5fcc430c94766 100644 --- a/crates/turborepo-lib/src/lib.rs +++ b/crates/turborepo-lib/src/lib.rs @@ -4,6 +4,7 @@ #![feature(once_cell_try)] #![feature(try_blocks)] #![feature(impl_trait_in_assoc_type)] +#![feature(let_chains)] #![deny(clippy::all)] // Clippy's needless mut lint is buggy: https://github.com/rust-lang/rust-clippy/issues/11299 #![allow(clippy::needless_pass_by_ref_mut)] diff --git a/crates/turborepo-lib/src/run/builder.rs b/crates/turborepo-lib/src/run/builder.rs index 52c2801f33f8d..eacc02b6aec12 100644 --- a/crates/turborepo-lib/src/run/builder.rs +++ b/crates/turborepo-lib/src/run/builder.rs @@ -541,6 +541,7 @@ impl RunBuilder { .with_root_tasks(root_turbo_json.tasks.keys().cloned()) .with_tasks_only(self.opts.run_opts.only) .with_workspaces(filtered_pkgs.cloned().collect()) + .with_future_flags(self.opts.future_flags) .with_tasks(tasks); if self.add_all_tasks { diff --git a/crates/turborepo-lib/src/run/task_access.rs b/crates/turborepo-lib/src/run/task_access.rs index 7c7a846d570ac..840aab5f22b49 100644 --- a/crates/turborepo-lib/src/run/task_access.rs +++ b/crates/turborepo-lib/src/run/task_access.rs @@ -13,7 +13,7 @@ use turborepo_scm::SCM; use turborepo_unescape::UnescapedString; use super::ConfigCache; -use crate::{config::RawTurboJson, gitignore::ensure_turbo_is_gitignored}; +use crate::{gitignore::ensure_turbo_is_gitignored, turbo_json::RawTurboJson}; // Environment variable key that will be used to enable, and set the expected // trace location diff --git a/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md b/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md index b920e8b76a8ba..21b8c50caa4bf 100644 --- a/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md +++ b/crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md @@ -36,18 +36,39 @@ Configuration is collected from multiple sources with the following priority (hi ## Phase 2: TurboJson Loading +### Schema Differentiation + +Turborepo uses two different schemas for `turbo.json` files depending on their location: + +- **Root `turbo.json`** (`RawRootTurboJson`): Located at the repository root + + - Can define global configuration options (`globalEnv`, `globalDependencies`, `globalPassThroughEnv`) + - Can set repository-wide settings (`remoteCache`, `ui`, `daemon`, `envMode`, etc.) + - Can define `futureFlags` for experimental features + - Cannot use `extends` field + +- **Package `turbo.json`** (`RawPackageTurboJson`): Located in workspace packages + - Limited to task definitions and workspace-specific configuration + - Must use `extends: ["//"]` to inherit from root configuration + - Can define workspace-specific `tags` and `boundaries` + - Cannot define global settings or `futureFlags` + ### Key Components - **`TurboJsonLoader`** (`crates/turborepo-lib/src/turbo_json/loader.rs`): Loads and resolves turbo.json files -- **`RawTurboJson`**: Raw deserialized structure from JSON files +- **`RawRootTurboJson`** (`crates/turborepo-lib/src/turbo_json/raw.rs`): Root turbo.json schema +- **`RawPackageTurboJson`** (`crates/turborepo-lib/src/turbo_json/raw.rs`): Package turbo.json schema +- **`RawTurboJson`** (`crates/turborepo-lib/src/turbo_json/raw.rs`): Unified raw structure that can represent either type - **`TurboJson`**: Validated structure containing raw task definitions ### Process 1. **File Discovery**: Locate `turbo.json` or `turbo.jsonc` files -2. **Parsing**: Deserialize JSON into `RawTurboJson` structures -3. **Basic Validation**: Convert to `TurboJson` with structural validation -4. **Workspace Resolution**: Apply workspace-specific overrides +2. **Schema Detection**: Determine if file is at repository root or in a package +3. **Parsing**: Deserialize JSON into appropriate schema (`RawRootTurboJson` or `RawPackageTurboJson`) +4. **Unification**: Convert to unified `RawTurboJson` structure +5. **Basic Validation**: Convert to `TurboJson` with structural validation +6. **Workspace Resolution**: Apply workspace-specific overrides ## Phase 3: Processed Task Definition (Intermediate Representation) diff --git a/crates/turborepo-lib/src/turbo_json/future_flags.rs b/crates/turborepo-lib/src/turbo_json/future_flags.rs index b865d0b4f3a1b..8d613c9f0455d 100644 --- a/crates/turborepo-lib/src/turbo_json/future_flags.rs +++ b/crates/turborepo-lib/src/turbo_json/future_flags.rs @@ -28,6 +28,7 @@ use struct_iterable::Iterable; /// before it becomes the default behavior in a future version. #[derive(Serialize, Default, Debug, Copy, Clone, Iterable, Deserializable, PartialEq, Eq)] #[serde(rename_all = "camelCase")] +#[deserializable()] pub struct FutureFlags { /// Enable `$TURBO_EXTENDS$` /// @@ -35,6 +36,12 @@ pub struct FutureFlags { /// This will change the default behavior of overriding the field to instead /// append. pub turbo_extends_keyword: bool, + /// Enable extending from a non-root `turbo.json` + /// + /// When enabled, allows using extends targeting `turbo.json`s other than + /// root. All `turbo.json` must still extend from the root `turbo.json` + /// first. + pub non_root_extends: bool, } impl FutureFlags { diff --git a/crates/turborepo-lib/src/turbo_json/loader.rs b/crates/turborepo-lib/src/turbo_json/loader.rs index c271b62f5535f..4135f52259e0c 100644 --- a/crates/turborepo-lib/src/turbo_json/loader.rs +++ b/crates/turborepo-lib/src/turbo_json/loader.rs @@ -199,13 +199,15 @@ impl TurboJsonLoader { micro_frontends_configs, } => { let turbo_json_path = packages.get(package).ok_or_else(|| Error::NoTurboJSON)?; + let is_root = package == &PackageName::Root; let turbo_json = load_from_file( reader, - if package == &PackageName::Root { + if is_root { LoadTurboJsonPath::File(turbo_json_path) } else { LoadTurboJsonPath::Dir(turbo_json_path) }, + is_root, ); if let Some(mfe_configs) = micro_frontends_configs { mfe_configs.update_turbo_json(package, turbo_json) @@ -288,6 +290,7 @@ enum LoadTurboJsonPath<'a> { fn load_from_file( reader: &TurboJsonReader, turbo_json_path: LoadTurboJsonPath, + is_root: bool, ) -> Result { let result = match turbo_json_path { LoadTurboJsonPath::Dir(turbo_json_dir_path) => { @@ -295,12 +298,12 @@ fn load_from_file( let turbo_jsonc_path = turbo_json_dir_path.join_component(CONFIG_FILE_JSONC); // Load both turbo.json and turbo.jsonc - let turbo_json = reader.read(&turbo_json_path); - let turbo_jsonc = reader.read(&turbo_jsonc_path); + let turbo_json = reader.read(&turbo_json_path, is_root); + let turbo_jsonc = reader.read(&turbo_jsonc_path, is_root); select_turbo_json(turbo_json_dir_path, turbo_json, turbo_jsonc) } - LoadTurboJsonPath::File(turbo_json_path) => reader.read(turbo_json_path), + LoadTurboJsonPath::File(turbo_json_path) => reader.read(turbo_json_path, is_root), }; // Handle errors or success @@ -318,7 +321,7 @@ fn load_from_root_package_json( turbo_json_path: &AbsoluteSystemPath, root_package_json: &PackageJson, ) -> Result { - let mut turbo_json = match reader.read(turbo_json_path) { + let mut turbo_json = match reader.read(turbo_json_path, true) { // we're synthesizing, but we have a starting point // Note: this will have to change to support task inference in a monorepo // for now, we're going to error on any "root" tasks and turn non-root tasks into root @@ -415,7 +418,7 @@ fn load_task_access_trace_turbo_json( root_package_json: &PackageJson, ) -> Result { let trace_json_path = reader.repo_root().join_components(&TASK_ACCESS_CONFIG_PATH); - let turbo_from_trace = reader.read(&trace_json_path); + let turbo_from_trace = reader.read(&trace_json_path, true); // check the zero config case (turbo trace file, but no turbo.json file) if let Ok(Some(turbo_from_trace)) = turbo_from_trace { @@ -472,8 +475,12 @@ impl TurboJsonReader { self } - pub fn read(&self, path: &AbsoluteSystemPath) -> Result, Error> { - TurboJson::read(&self.repo_root, path, self.future_flags) + pub fn read( + &self, + path: &AbsoluteSystemPath, + is_root: bool, + ) -> Result, Error> { + TurboJson::read(&self.repo_root, path, is_root, self.future_flags) } pub fn repo_root(&self) -> &AbsoluteSystemPath { @@ -946,7 +953,7 @@ mod test { turbo_jsonc_path.create_with_contents("{}")?; // Test load_from_file with turbo.json path - let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root), true); // The function should return an error when both files exist assert!(result.is_err()); @@ -974,7 +981,7 @@ mod test { turbo_json_path.create_with_contents("{}")?; // Test load_from_file - let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root), true); assert!(result.is_ok()); @@ -992,13 +999,83 @@ mod test { turbo_jsonc_path.create_with_contents("{}")?; // Test load_from_file - let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root)); + let result = load_from_file(&reader, LoadTurboJsonPath::Dir(repo_root), true); assert!(result.is_ok()); Ok(()) } + #[test] + fn test_context_aware_parsing() { + // Test that the reader correctly determines root vs package contexts + let root_dir = tempdir().unwrap(); + let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path()).unwrap(); + + // Create a root turbo.json with root-only fields + let root_turbo_json = repo_root.join_component("turbo.json"); + root_turbo_json + .create_with_contents( + r#"{ + "globalEnv": ["NODE_ENV"], + "tasks": {"build": {}} + }"#, + ) + .unwrap(); + + // Create a package turbo.json with extends + let pkg_dir = repo_root.join_components(&["packages", "foo"]); + pkg_dir.create_dir_all().unwrap(); + let pkg_turbo_json = pkg_dir.join_component("turbo.json"); + pkg_turbo_json + .create_with_contents( + r#"{ + "extends": ["//"], + "tasks": {"test": {}} + }"#, + ) + .unwrap(); + + let reader = TurboJsonReader::new(repo_root.to_owned()); + + // Reading root turbo.json should work with globalEnv + let root_result = reader.read(&root_turbo_json, true); + assert!(root_result.is_ok()); + let root_json = root_result.unwrap().unwrap(); + assert!(!root_json.global_env.is_empty()); + + // Reading package turbo.json should work with extends + let pkg_result = reader.read(&pkg_turbo_json, false); + assert!(pkg_result.is_ok()); + let pkg_json = pkg_result.unwrap().unwrap(); + assert!(!pkg_json.extends.is_empty()); + + // Now test invalid cases + // Root turbo.json with extends should fail + root_turbo_json + .create_with_contents( + r#"{ + "extends": ["//"], + "tasks": {"build": {}} + }"#, + ) + .unwrap(); + let invalid_root = reader.read(&root_turbo_json, true); + assert!(invalid_root.is_err()); + + // Package turbo.json with globalEnv should fail + pkg_turbo_json + .create_with_contents( + r#"{ + "globalEnv": ["NODE_ENV"], + "tasks": {"test": {}} + }"#, + ) + .unwrap(); + let invalid_pkg = reader.read(&pkg_turbo_json, false); + assert!(invalid_pkg.is_err()); + } + #[test] fn test_invalid_workspace_turbo_json() { let root_dir = tempdir().unwrap(); diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 568235a6b8a72..f72f19d71587b 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashSet}, fmt::Display, ops::{Deref, DerefMut}, sync::Arc, @@ -8,9 +8,7 @@ use std::{ use biome_deserialize_macros::Deserializable; use camino::Utf8Path; use clap::ValueEnum; -use miette::{NamedSource, SourceSpan}; use serde::{Deserialize, Serialize}; -use struct_iterable::Iterable; use turbopath::{AbsoluteSystemPath, RelativeUnixPath}; use turborepo_errors::Spanned; use turborepo_repository::package_graph::ROOT_PKG_NAME; @@ -18,9 +16,8 @@ use turborepo_task_id::{TaskId, TaskName}; use turborepo_unescape::UnescapedString; use crate::{ - cli::{EnvMode, OutputLogsMode}, + cli::EnvMode, config::{Error, InvalidEnvPrefixError}, - run::task_access::TaskAccessTraceFile, task_graph::{TaskDefinition, TaskInputs, TaskOutputs}, }; @@ -29,12 +26,17 @@ pub mod future_flags; mod loader; pub mod parser; mod processed; +mod raw; +pub mod validator; pub use future_flags::FutureFlags; pub use loader::{TurboJsonLoader, TurboJsonReader}; pub use processed::ProcessedTaskDefinition; +pub use raw::{ + RawPackageTurboJson, RawRemoteCacheOptions, RawRootTurboJson, RawTaskDefinition, RawTurboJson, +}; -use crate::{boundaries::BoundariesConfig, config::UnnecessaryPackageTaskSyntaxError}; +use crate::boundaries::BoundariesConfig; const ENV_PIPELINE_DELIMITER: &str = "$"; const TOPOLOGICAL_PIPELINE_DELIMITER: &str = "^"; @@ -69,96 +71,6 @@ pub struct TurboJson { pub(crate) future_flags: FutureFlags, } -// Iterable is required to enumerate allowed keys -#[derive(Clone, Debug, Default, Iterable, Serialize, Deserializable)] -#[serde(rename_all = "camelCase")] -pub struct RawRemoteCacheOptions { - #[serde(skip_serializing_if = "Option::is_none")] - pub api_url: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub login_url: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub team_slug: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub team_id: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub signature: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub preflight: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub enabled: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub upload_timeout: Option>, -} - -#[derive(Serialize, Default, Debug, Clone, Iterable, Deserializable)] -#[serde(rename_all = "camelCase")] -// The raw deserialized turbo.json file. -pub struct RawTurboJson { - #[serde(skip)] - span: Spanned<()>, - - #[serde(rename = "$schema", skip_serializing_if = "Option::is_none")] - schema: Option, - - #[serde(skip_serializing)] - pub experimental_spaces: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - extends: Option>>, - // Global root filesystem dependencies - #[serde(skip_serializing_if = "Option::is_none")] - global_dependencies: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - global_env: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - global_pass_through_env: Option>>, - // Tasks is a map of task entries which define the task graph - // and cache behavior on a per task or per package-task basis. - #[serde(skip_serializing_if = "Option::is_none")] - pub tasks: Option, - - #[serde(skip_serializing)] - pub pipeline: Option>, - // Configuration options when interfacing with the remote cache - #[serde(skip_serializing_if = "Option::is_none")] - pub remote_cache: Option, - #[serde(skip_serializing_if = "Option::is_none", rename = "ui")] - pub ui: Option>, - #[serde( - skip_serializing_if = "Option::is_none", - rename = "dangerouslyDisablePackageManagerCheck" - )] - pub allow_no_package_manager: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub daemon: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub env_mode: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_dir: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub no_update_notifier: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub tags: Option>>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub boundaries: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub future_flags: Option>, - - #[deserializable(rename = "//")] - #[serde(skip)] - _comment: Option, -} - #[derive(Serialize, Default, Debug, PartialEq, Clone)] #[serde(transparent)] pub struct Pipeline(BTreeMap, Spanned>); @@ -226,39 +138,6 @@ impl UIMode { } } -#[derive(Serialize, Default, Debug, PartialEq, Clone, Iterable, Deserializable)] -#[serde(rename_all = "camelCase")] -#[deserializable(unknown_fields = "deny")] -pub struct RawTaskDefinition { - #[serde(skip_serializing_if = "Option::is_none")] - cache: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - depends_on: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - env: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - inputs: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pass_through_env: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - persistent: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - interruptible: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - outputs: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - output_logs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - interactive: Option>, - // TODO: Remove this once we have the ability to load task definitions directly - // instead of deriving them from a TurboJson - #[serde(skip)] - env_mode: Option>, - // This can currently only be set internally and isn't a part of turbo.json - #[serde(skip_serializing_if = "Option::is_none")] - with: Option>>, -} - impl TaskOutputs { /// Creates TaskOutputs from ProcessedOutputs with resolved paths fn from_processed( @@ -400,79 +279,6 @@ impl TaskDefinition { } } -impl RawTurboJson { - pub(crate) fn read( - repo_root: &AbsoluteSystemPath, - path: &AbsoluteSystemPath, - ) -> Result, Error> { - let Some(contents) = path.read_existing_to_string()? else { - return Ok(None); - }; - // Anchoring the path can fail if the path resides outside of the repository - // Just display absolute path in that case. - let root_relative_path = repo_root.anchor(path).map_or_else( - |_| path.as_str().to_owned(), - |relative| relative.to_string(), - ); - let raw_turbo_json = RawTurboJson::parse(&contents, &root_relative_path)?; - - Ok(Some(raw_turbo_json)) - } - - /// Produces a new turbo.json without any tasks that reference non-existent - /// workspaces - pub fn prune_tasks>(&self, workspaces: &[S]) -> Self { - let mut this = self.clone(); - if let Some(pipeline) = &mut this.tasks { - pipeline.0.retain(|task_name, _| { - task_name.in_workspace(ROOT_PKG_NAME) - || workspaces - .iter() - .any(|workspace| task_name.in_workspace(workspace.as_ref())) - }) - } - - this - } - - pub fn from_task_access_trace(trace: &HashMap) -> Option { - if trace.is_empty() { - return None; - } - - let mut pipeline = Pipeline::default(); - - for (task_name, trace_file) in trace { - let spanned_outputs: Vec> = trace_file - .outputs - .iter() - .map(|output| Spanned::new(output.clone())) - .collect(); - let task_definition = RawTaskDefinition { - outputs: Some(spanned_outputs), - env: Some( - trace_file - .accessed - .env_var_keys - .iter() - .map(|unescaped_string| Spanned::new(unescaped_string.clone())) - .collect(), - ), - ..Default::default() - }; - - let name = TaskName::from(task_name.as_str()); - let root_task = name.into_root_task(); - pipeline.insert(root_task, Spanned::new(task_definition.clone())); - } - - Some(RawTurboJson { - tasks: Some(pipeline), - ..RawTurboJson::default() - }) - } -} - impl TryFrom for TurboJson { type Error = Error; @@ -581,9 +387,10 @@ impl TurboJson { fn read( repo_root: &AbsoluteSystemPath, path: &AbsoluteSystemPath, + is_root: bool, future_flags: FutureFlags, ) -> Result, Error> { - let Some(raw_turbo_json) = RawTurboJson::read(repo_root, path)? else { + let Some(raw_turbo_json) = RawTurboJson::read(repo_root, path, is_root)? else { return Ok(None); }; @@ -612,13 +419,6 @@ impl TurboJson { } } - pub fn validate(&self, validations: &[TurboJSONValidation]) -> Vec { - validations - .iter() - .flat_map(|validation| validation(self)) - .collect() - } - pub fn has_root_tasks(&self) -> bool { self.tasks .iter() @@ -661,72 +461,6 @@ impl TurboJson { } } -type TurboJSONValidation = fn(&TurboJson) -> Vec; - -pub fn validate_no_package_task_syntax(turbo_json: &TurboJson) -> Vec { - turbo_json - .tasks - .iter() - .filter(|(task_name, _)| task_name.is_package_task()) - .map(|(task_name, entry)| { - let (span, text) = entry.span_and_text("turbo.json"); - Error::UnnecessaryPackageTaskSyntax(Box::new(UnnecessaryPackageTaskSyntaxError { - actual: task_name.to_string(), - wanted: task_name.task().to_string(), - span, - text, - })) - }) - .collect() -} - -pub fn validate_extends(turbo_json: &TurboJson) -> Vec { - match turbo_json.extends.first() { - Some(package_name) if package_name != ROOT_PKG_NAME || turbo_json.extends.len() > 1 => { - let (span, text) = turbo_json.extends.span_and_text("turbo.json"); - vec![Error::ExtendFromNonRoot { span, text }] - } - None => { - let path = turbo_json - .path - .as_ref() - .map_or("turbo.json", |p| p.as_ref()); - - let (span, text) = match turbo_json.text { - Some(ref text) => { - let len = text.len(); - let span: SourceSpan = (0, len - 1).into(); - (Some(span), text.to_string()) - } - None => (None, String::new()), - }; - - vec![Error::NoExtends { - span, - text: NamedSource::new(path, text), - }] - } - _ => vec![], - } -} - -pub fn validate_with_has_no_topo(turbo_json: &TurboJson) -> Vec { - turbo_json - .tasks - .iter() - .flat_map(|(_, definition)| { - definition.with.iter().flatten().filter_map(|with_task| { - if with_task.starts_with(TOPOLOGICAL_PIPELINE_DELIMITER) { - let (span, text) = with_task.span_and_text("turbo.json"); - Some(Error::InvalidTaskWith { span, text }) - } else { - None - } - }) - }) - .collect() -} - fn gather_env_vars( vars: Vec>>, key: &str, @@ -753,7 +487,6 @@ fn gather_env_vars( Ok(()) } -// Takes an input/output glob that might start with TURBO_ROOT_PREFIX #[cfg(test)] mod tests { use anyhow::Result; @@ -1111,7 +844,7 @@ mod tests { #[test_case(r#"{ "tags": ["my-tag"] }"#, "one tag")] #[test_case(r#"{ "tags": ["my-tag", "my-other-tag"] }"#, "two tags")] fn test_tags(json: &str, name: &str) { - let json = RawTurboJson::parse(json, "").unwrap(); + let json = RawRootTurboJson::parse(json, "").unwrap(); insta::assert_json_snapshot!(name.replace(' ', "_"), json.tags); } @@ -1119,7 +852,7 @@ mod tests { #[test_case(r#"{ "ui": "stream" }"#, Some(UIMode::Stream) ; "stream")] #[test_case(r#"{}"#, None ; "missing")] fn test_ui(json: &str, expected: Option) { - let json = RawTurboJson::parse(json, "").unwrap(); + let json = RawRootTurboJson::parse(json, "").unwrap(); assert_eq!(json.ui.as_ref().map(|ui| *ui.as_inner()), expected); } @@ -1127,14 +860,14 @@ mod tests { #[test_case(r#"{ "experimentalSpaces": {} }"#, Some(SpacesJson { id: None }))] #[test_case(r#"{}"#, None)] fn test_spaces(json: &str, expected: Option) { - let json = RawTurboJson::parse(json, "").unwrap(); + let json = RawRootTurboJson::parse(json, "").unwrap(); assert_eq!(json.experimental_spaces, expected); } #[test_case(r#"{ "daemon": true }"#, r#"{"daemon":true}"# ; "daemon_on")] #[test_case(r#"{ "daemon": false }"#, r#"{"daemon":false}"# ; "daemon_off")] fn test_daemon(json: &str, expected: &str) { - let parsed = RawTurboJson::parse(json, "").unwrap(); + let parsed: RawTurboJson = RawRootTurboJson::parse(json, "").unwrap().into(); let actual = serde_json::to_string(&parsed).unwrap(); assert_eq!(actual, expected); } @@ -1142,7 +875,7 @@ mod tests { #[test_case(r#"{ "ui": "tui" }"#, r#"{"ui":"tui"}"# ; "tui")] #[test_case(r#"{ "ui": "stream" }"#, r#"{"ui":"stream"}"# ; "stream")] fn test_ui_serialization(input: &str, expected: &str) { - let parsed = RawTurboJson::parse(input, "").unwrap(); + let parsed: RawTurboJson = RawRootTurboJson::parse(input, "").unwrap().into(); let actual = serde_json::to_string(&parsed).unwrap(); assert_eq!(actual, expected); } @@ -1151,7 +884,7 @@ mod tests { #[test_case(r#"{"dangerouslyDisablePackageManagerCheck":false}"#, Some(false) ; "f")] #[test_case(r#"{}"#, None ; "missing")] fn test_allow_no_package_manager_serde(json_str: &str, expected: Option) { - let json = RawTurboJson::parse(json_str, "").unwrap(); + let json: RawTurboJson = RawRootTurboJson::parse(json_str, "").unwrap().into(); assert_eq!( json.allow_no_package_manager .as_ref() @@ -1290,7 +1023,8 @@ mod tests { assert_eq!( future_flags.as_inner(), &FutureFlags { - turbo_extends_keyword: true + turbo_extends_keyword: true, + non_root_extends: false, } ); @@ -1300,29 +1034,72 @@ mod tests { assert!(turbo_json.is_ok()); } - #[test] - fn test_validate_with_has_no_topo() { - let turbo_json = TurboJson { - tasks: Pipeline( - vec![( - TaskName::from("dev"), - Spanned::new(RawTaskDefinition { - with: Some(vec![Spanned::new(UnescapedString::from("^proxy"))]), - ..Default::default() - }), - )] - .into_iter() - .collect(), - ), - ..Default::default() - }; + #[test_case( + r#"{"extends": ["//"], "tasks": {"build": {}}}"#, + false ; "root config with extends should fail" + )] + #[test_case( + r#"{"globalEnv": ["NODE_ENV"], "globalDependencies": ["package.json"], "tasks": {"build": {}}}"#, + true ; "root config with global fields should succeed" + )] + #[test_case( + r#"{"futureFlags": {"turboExtendsKeyword": true}, "tasks": {"build": {}}}"#, + true ; "root config with futureFlags should succeed" + )] + #[test_case( + r#"{"remoteCache": {"enabled": true}, "tasks": {"build": {}}}"#, + true ; "root config with remoteCache should succeed" + )] + fn test_root_config_validation(json: &str, should_succeed: bool) { + let result = RawRootTurboJson::parse(json, "turbo.json"); + assert_eq!(result.is_ok(), should_succeed); - let errs = validate_with_has_no_topo(&turbo_json); - assert_eq!(errs.len(), 1); - let error = &errs[0]; - assert_eq!( - error.to_string(), - "`with` cannot use dependency relationships." - ); + if should_succeed { + let raw_config = RawTurboJson::from(result.unwrap()); + assert!(raw_config.extends.is_none()); + } + } + + #[test_case( + r#"{"extends": ["//"], "tasks": {"build": {}}, "tags": ["frontend"]}"#, + true ; "package config with extends and tags should succeed" + )] + #[test_case( + r#"{"extends": ["//"], "boundaries": {}, "tasks": {"test": {}}}"#, + true ; "package config with extends and boundaries should succeed" + )] + #[test_case( + r#"{"globalEnv": ["NODE_ENV"], "tasks": {"test": {}}}"#, + false ; "package config with globalEnv should fail" + )] + #[test_case( + r#"{"extends": ["//"], "globalDependencies": ["package.json"], "tasks": {"test": {}}}"#, + false ; "package config with globalDependencies should fail" + )] + #[test_case( + r#"{"extends": ["//"], "futureFlags": {}, "tasks": {"test": {}}}"#, + false ; "package config with futureFlags should fail" + )] + #[test_case( + r#"{"extends": ["//"], "remoteCache": {"enabled": true}, "tasks": {"test": {}}}"#, + false ; "package config with remoteCache should fail" + )] + #[test_case( + r#"{"extends": ["//"], "ui": "tui", "tasks": {"test": {}}}"#, + false ; "package config with ui should fail" + )] + fn test_package_config_validation(json: &str, should_succeed: bool) { + let result = RawPackageTurboJson::parse(json, "packages/foo/turbo.json"); + assert_eq!(result.is_ok(), should_succeed); + + if should_succeed { + let package_config = result.unwrap(); + let raw_config = RawTurboJson::from(package_config); + assert!(raw_config.extends.is_some()); + // Verify root-only fields are None + assert!(raw_config.global_env.is_none()); + assert!(raw_config.global_dependencies.is_none()); + assert!(raw_config.future_flags.is_none()); + } } } diff --git a/crates/turborepo-lib/src/turbo_json/parser.rs b/crates/turborepo-lib/src/turbo_json/parser.rs index 86bf53bc93945..e28dbd3dd6593 100644 --- a/crates/turborepo-lib/src/turbo_json/parser.rs +++ b/crates/turborepo-lib/src/turbo_json/parser.rs @@ -18,7 +18,10 @@ use turborepo_unescape::UnescapedString; use crate::{ boundaries::{BoundariesConfig, Permissions, Rule}, - turbo_json::{Pipeline, RawRemoteCacheOptions, RawTaskDefinition, RawTurboJson, Spanned}, + turbo_json::{ + Pipeline, RawPackageTurboJson, RawRemoteCacheOptions, RawRootTurboJson, RawTaskDefinition, + RawTurboJson, Spanned, + }, }; #[derive(Debug, Error, Diagnostic)] @@ -90,63 +93,6 @@ impl DeserializationVisitor for PipelineVisitor { } } -impl WithMetadata for RawTurboJson { - fn add_text(&mut self, text: Arc) { - self.span.add_text(text.clone()); - self.extends.add_text(text.clone()); - self.tags.add_text(text.clone()); - if let Some(tags) = &mut self.tags { - tags.value.add_text(text.clone()); - } - self.global_dependencies.add_text(text.clone()); - self.global_env.add_text(text.clone()); - self.global_pass_through_env.add_text(text.clone()); - self.boundaries.add_text(text.clone()); - if let Some(boundaries) = &mut self.boundaries { - boundaries.value.add_text(text.clone()); - } - - self.tasks.add_text(text.clone()); - self.cache_dir.add_text(text.clone()); - self.pipeline.add_text(text.clone()); - self.remote_cache.add_text(text.clone()); - self.ui.add_text(text.clone()); - self.allow_no_package_manager.add_text(text.clone()); - self.daemon.add_text(text.clone()); - self.env_mode.add_text(text.clone()); - self.no_update_notifier.add_text(text.clone()); - self.concurrency.add_text(text.clone()); - self.future_flags.add_text(text); - } - - fn add_path(&mut self, path: Arc) { - self.span.add_path(path.clone()); - self.extends.add_path(path.clone()); - self.tags.add_path(path.clone()); - if let Some(tags) = &mut self.tags { - tags.value.add_path(path.clone()); - } - self.global_dependencies.add_path(path.clone()); - self.global_env.add_path(path.clone()); - self.global_pass_through_env.add_path(path.clone()); - self.boundaries.add_path(path.clone()); - if let Some(boundaries) = &mut self.boundaries { - boundaries.value.add_path(path.clone()); - } - self.tasks.add_path(path.clone()); - self.cache_dir.add_path(path.clone()); - self.pipeline.add_path(path.clone()); - self.remote_cache.add_path(path.clone()); - self.ui.add_path(path.clone()); - self.allow_no_package_manager.add_path(path.clone()); - self.daemon.add_path(path.clone()); - self.env_mode.add_path(path.clone()); - self.no_update_notifier.add_path(path.clone()); - self.concurrency.add_path(path.clone()); - self.future_flags.add_path(path); - } -} - impl WithMetadata for Pipeline { fn add_text(&mut self, text: Arc) { for (_, entry) in self.0.iter_mut() { @@ -309,65 +255,178 @@ impl WithMetadata for RawRemoteCacheOptions { } } +impl WithMetadata for RawRootTurboJson { + fn add_text(&mut self, text: Arc) { + self.span.add_text(text.clone()); + self.tags.add_text(text.clone()); + if let Some(tags) = &mut self.tags { + tags.value.add_text(text.clone()); + } + self.global_dependencies.add_text(text.clone()); + self.global_env.add_text(text.clone()); + self.global_pass_through_env.add_text(text.clone()); + self.boundaries.add_text(text.clone()); + if let Some(boundaries) = &mut self.boundaries { + boundaries.value.add_text(text.clone()); + } + + self.tasks.add_text(text.clone()); + self.cache_dir.add_text(text.clone()); + self.pipeline.add_text(text.clone()); + self.remote_cache.add_text(text.clone()); + self.ui.add_text(text.clone()); + self.allow_no_package_manager.add_text(text.clone()); + self.daemon.add_text(text.clone()); + self.env_mode.add_text(text.clone()); + self.no_update_notifier.add_text(text.clone()); + self.concurrency.add_text(text.clone()); + self.future_flags.add_text(text); + } + + fn add_path(&mut self, path: Arc) { + self.span.add_path(path.clone()); + self.tags.add_path(path.clone()); + if let Some(tags) = &mut self.tags { + tags.value.add_path(path.clone()); + } + self.global_dependencies.add_path(path.clone()); + self.global_env.add_path(path.clone()); + self.global_pass_through_env.add_path(path.clone()); + self.boundaries.add_path(path.clone()); + if let Some(boundaries) = &mut self.boundaries { + boundaries.value.add_path(path.clone()); + } + self.tasks.add_path(path.clone()); + self.cache_dir.add_path(path.clone()); + self.pipeline.add_path(path.clone()); + self.remote_cache.add_path(path.clone()); + self.ui.add_path(path.clone()); + self.allow_no_package_manager.add_path(path.clone()); + self.daemon.add_path(path.clone()); + self.env_mode.add_path(path.clone()); + self.no_update_notifier.add_path(path.clone()); + self.concurrency.add_path(path.clone()); + self.future_flags.add_path(path); + } +} + +impl WithMetadata for RawPackageTurboJson { + fn add_text(&mut self, text: Arc) { + self.span.add_text(text.clone()); + self.extends.add_text(text.clone()); + self.tags.add_text(text.clone()); + if let Some(tags) = &mut self.tags { + tags.value.add_text(text.clone()); + } + self.boundaries.add_text(text.clone()); + if let Some(boundaries) = &mut self.boundaries { + boundaries.value.add_text(text.clone()); + } + self.tasks.add_text(text.clone()); + self.pipeline.add_text(text); + } + + fn add_path(&mut self, path: Arc) { + self.span.add_path(path.clone()); + self.extends.add_path(path.clone()); + self.tags.add_path(path.clone()); + if let Some(tags) = &mut self.tags { + tags.value.add_path(path.clone()); + } + self.boundaries.add_path(path.clone()); + if let Some(boundaries) = &mut self.boundaries { + boundaries.value.add_path(path.clone()); + } + self.tasks.add_path(path.clone()); + self.pipeline.add_path(path); + } +} + +impl RawRootTurboJson { + pub fn parse(text: &str, file_path: &str) -> Result { + let turbo_json = parse_turbo_json::(text, file_path)?; + + if turbo_json.experimental_spaces.is_some() { + warn!("`experimentalSpaces` key in turbo.json is deprecated and does not do anything") + } + + Ok(turbo_json) + } +} + +impl RawPackageTurboJson { + pub fn parse(text: &str, file_path: &str) -> Result { + parse_turbo_json::(text, file_path) + } +} + impl RawTurboJson { // A simple helper for tests #[cfg(test)] pub fn parse_from_serde(value: serde_json::Value) -> Result { let json_string = serde_json::to_string(&value).expect("should be able to serialize"); - Self::parse(&json_string, "turbo.json") + let raw_root = RawRootTurboJson::parse(&json_string, "turbo.json")?; + Ok(Self::from(raw_root)) } - /// Parses a turbo.json file into the raw representation with span info - /// attached. - /// - /// # Arguments - /// - /// * `text`: The text contents of the turbo.json file - /// * `file_path`: The path to the turbo.json file. Just used for error - /// display, so doesn't need to actually be a correct path. - /// - /// returns: Result - pub fn parse(text: &str, file_path: &str) -> Result { - let result = deserialize_from_json_str::( - text, - JsonParserOptions::default() - .with_allow_comments() - .with_allow_trailing_commas(), - file_path, - ); - - if !result.diagnostics().is_empty() { - let diagnostics = result - .into_diagnostics() - .into_iter() - .map(|d| { - d.with_file_source_code(text) - .with_file_path(file_path) - .as_ref() - .into() - }) - .collect(); - - return Err(Error { - diagnostics, - backtrace: backtrace::Backtrace::capture(), - }); - } +} - // It's highly unlikely that biome would fail to produce a deserialized value - // *and* not return any errors, but it's still possible. In that case, we - // just print that there is an error and return. - let mut turbo_json = result.into_deserialized().ok_or_else(|| Error { - diagnostics: vec![], +fn parse_turbo_json( + text: &str, + file_path: &str, +) -> Result { + let result = deserialize_from_json_str::( + text, + JsonParserOptions::default() + .with_allow_comments() + .with_allow_trailing_commas(), + file_path, + ); + + if !result.diagnostics().is_empty() { + let diagnostics = result + .into_diagnostics() + .into_iter() + .map(|d| { + d.with_file_source_code(text) + .with_file_path(file_path) + .as_ref() + .into() + }) + .collect(); + + return Err(Error { + diagnostics, backtrace: backtrace::Backtrace::capture(), - })?; + }); + } - if turbo_json.experimental_spaces.is_some() { - warn!("`experimentalSpaces` key in turbo.json is deprecated and does not do anything") - } + let mut turbo_json = result.into_deserialized().ok_or_else(|| Error { + diagnostics: vec![], + backtrace: backtrace::Backtrace::capture(), + })?; + turbo_json.add_text(Arc::from(text)); + turbo_json.add_path(Arc::from(file_path)); - turbo_json.add_text(Arc::from(text)); - turbo_json.add_path(Arc::from(file_path)); + Ok(turbo_json) +} - Ok(turbo_json) +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + use test_case::test_case; + + use super::*; + + #[test_case(r#"{"daemon": true}"#; "daemon in package turbo.json")] + fn test_root_only_fields_in_package_turbo_json(json: &str) { + let result = RawPackageTurboJson::parse(json, "packages/foo/turbo.json"); + assert!(result.is_err()); + + let report = miette::Report::new(result.unwrap_err()); + let mut msg = String::new(); + miette::NarratableReportHandler::new() + .render_report(&mut msg, report.as_ref()) + .unwrap(); + assert_snapshot!(msg); } } diff --git a/crates/turborepo-lib/src/turbo_json/processed.rs b/crates/turborepo-lib/src/turbo_json/processed.rs index 0853086498a6c..cd9548d245b6b 100644 --- a/crates/turborepo-lib/src/turbo_json/processed.rs +++ b/crates/turborepo-lib/src/turbo_json/processed.rs @@ -407,6 +407,7 @@ mod tests { items, &FutureFlags { turbo_extends_keyword: true, + non_root_extends: false, }, ); @@ -428,6 +429,7 @@ mod tests { items, &FutureFlags { turbo_extends_keyword: false, + non_root_extends: false, }, ); @@ -447,6 +449,7 @@ mod tests { items, &FutureFlags { turbo_extends_keyword: true, + non_root_extends: false, }, ); @@ -592,6 +595,7 @@ mod tests { raw_globs, &FutureFlags { turbo_extends_keyword: true, + non_root_extends: false, }, ) .unwrap(); @@ -615,6 +619,7 @@ mod tests { raw_env, &FutureFlags { turbo_extends_keyword: false, + non_root_extends: false, }, ); assert!(result.is_err()); @@ -634,6 +639,7 @@ mod tests { Spanned::new(raw_deps), &FutureFlags { turbo_extends_keyword: false, + non_root_extends: false, }, ); assert!(result.is_err()); diff --git a/crates/turborepo-lib/src/turbo_json/raw.rs b/crates/turborepo-lib/src/turbo_json/raw.rs new file mode 100644 index 0000000000000..38a5eaa281b41 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/raw.rs @@ -0,0 +1,303 @@ +use std::collections::HashMap; + +use biome_deserialize_macros::Deserializable; +use serde::Serialize; +use struct_iterable::Iterable; +use turbopath::AbsoluteSystemPath; +use turborepo_errors::Spanned; +use turborepo_repository::package_graph::ROOT_PKG_NAME; +use turborepo_task_id::TaskName; +use turborepo_unescape::UnescapedString; + +use super::{FutureFlags, Pipeline, SpacesJson, UIMode}; +use crate::{ + boundaries::BoundariesConfig, + cli::{EnvMode, OutputLogsMode}, + config::Error, + run::task_access::TaskAccessTraceFile, +}; + +// Iterable is required to enumerate allowed keys +#[derive(Clone, Debug, Default, Iterable, Serialize, Deserializable)] +#[serde(rename_all = "camelCase")] +pub struct RawRemoteCacheOptions { + #[serde(skip_serializing_if = "Option::is_none")] + pub api_url: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub login_url: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub team_slug: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub team_id: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub preflight: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub upload_timeout: Option>, +} + +// Root turbo.json +#[derive(Default, Debug, Clone, Iterable, Deserializable)] +pub struct RawRootTurboJson { + pub(crate) span: Spanned<()>, + + #[deserializable(rename = "$schema")] + pub(crate) schema: Option, + pub(crate) experimental_spaces: Option, + + // Global root filesystem dependencies + pub(crate) global_dependencies: Option>>, + pub(crate) global_env: Option>>, + pub(crate) global_pass_through_env: Option>>, + // Tasks is a map of task entries which define the task graph + // and cache behavior on a per task or per package-task basis. + pub(crate) tasks: Option, + pub(crate) pipeline: Option>, + // Configuration options when interfacing with the remote cache + pub(crate) remote_cache: Option, + pub(crate) ui: Option>, + #[deserializable(rename = "dangerouslyDisablePackageManagerCheck")] + pub(crate) allow_no_package_manager: Option>, + pub(crate) daemon: Option>, + pub(crate) env_mode: Option>, + pub(crate) no_update_notifier: Option>, + pub(crate) cache_dir: Option>, + pub(crate) concurrency: Option>, + pub(crate) tags: Option>>>, + pub(crate) boundaries: Option>, + + pub(crate) future_flags: Option>, + #[deserializable(rename = "//")] + pub(crate) _comment: Option, +} + +// Package turbo.json +#[derive(Default, Debug, Clone, Iterable, Deserializable)] +pub struct RawPackageTurboJson { + pub(crate) span: Spanned<()>, + #[deserializable(rename = "$schema")] + pub(crate) schema: Option, + pub(crate) extends: Option>>, + pub(crate) tasks: Option, + pub(crate) pipeline: Option>, + pub(crate) tags: Option>>>, + pub(crate) boundaries: Option>, + #[deserializable(rename = "//")] + pub(crate) _comment: Option, +} + +// Unified structure that represents either root or package turbo.json +#[derive(Serialize, Default, Debug, Clone, Iterable, Deserializable)] +#[serde(rename_all = "camelCase")] +pub struct RawTurboJson { + #[serde(skip)] + pub(crate) span: Spanned<()>, + #[serde(rename = "$schema", skip_serializing_if = "Option::is_none")] + pub(crate) schema: Option, + #[serde(skip_serializing)] + pub experimental_spaces: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) extends: Option>>, + // Global root filesystem dependencies + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) global_dependencies: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) global_env: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) global_pass_through_env: Option>>, + // Tasks is a map of task entries which define the task graph + // and cache behavior on a per task or per package-task basis. + #[serde(skip_serializing_if = "Option::is_none")] + pub tasks: Option, + #[serde(skip_serializing)] + pub pipeline: Option>, + // Configuration options when interfacing with the remote cache + #[serde(skip_serializing_if = "Option::is_none")] + pub remote_cache: Option, + #[serde(skip_serializing_if = "Option::is_none", rename = "ui")] + pub ui: Option>, + #[serde( + skip_serializing_if = "Option::is_none", + rename = "dangerouslyDisablePackageManagerCheck" + )] + pub allow_no_package_manager: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub daemon: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub env_mode: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_dir: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_update_notifier: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub tags: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub boundaries: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub future_flags: Option>, + #[deserializable(rename = "//")] + #[serde(skip)] + pub(crate) _comment: Option, +} + +#[derive(Serialize, Default, Debug, PartialEq, Clone, Iterable, Deserializable)] +#[serde(rename_all = "camelCase")] +#[deserializable(unknown_fields = "deny")] +pub struct RawTaskDefinition { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) cache: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) depends_on: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) env: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) inputs: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) pass_through_env: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) persistent: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) interruptible: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) outputs: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) output_logs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) interactive: Option>, + // TODO: Remove this once we have the ability to load task definitions directly + // instead of deriving them from a TurboJson + #[serde(skip)] + pub(crate) env_mode: Option>, + // This can currently only be set internally and isn't a part of turbo.json + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) with: Option>>, +} + +impl From for RawTurboJson { + fn from(root: RawRootTurboJson) -> Self { + RawTurboJson { + span: root.span, + schema: root.schema, + experimental_spaces: root.experimental_spaces, + global_dependencies: root.global_dependencies, + global_env: root.global_env, + global_pass_through_env: root.global_pass_through_env, + tasks: root.tasks, + pipeline: root.pipeline, + remote_cache: root.remote_cache, + ui: root.ui, + allow_no_package_manager: root.allow_no_package_manager, + daemon: root.daemon, + env_mode: root.env_mode, + cache_dir: root.cache_dir, + no_update_notifier: root.no_update_notifier, + tags: root.tags, + boundaries: root.boundaries, + concurrency: root.concurrency, + future_flags: root.future_flags, + _comment: root._comment, + extends: None, // Root configs never have extends + } + } +} + +impl From for RawTurboJson { + fn from(pkg: RawPackageTurboJson) -> Self { + RawTurboJson { + span: pkg.span, + schema: pkg.schema, + extends: pkg.extends, + tasks: pkg.tasks, + pipeline: pkg.pipeline, + boundaries: pkg.boundaries, + tags: pkg.tags, + _comment: pkg._comment, + ..Default::default() + } + } +} + +impl RawTurboJson { + pub(super) fn read( + repo_root: &AbsoluteSystemPath, + path: &AbsoluteSystemPath, + is_root: bool, + ) -> Result, Error> { + let Some(contents) = path.read_existing_to_string()? else { + return Ok(None); + }; + + // Anchoring the path can fail if the path resides outside of the repository + // Just display absolute path in that case. + let root_relative_path = repo_root.anchor(path).map_or_else( + |_| path.as_str().to_owned(), + |relative| relative.to_string(), + ); + + Ok(Some(if is_root { + RawTurboJson::from(RawRootTurboJson::parse(&contents, &root_relative_path)?) + } else { + RawTurboJson::from(RawPackageTurboJson::parse(&contents, &root_relative_path)?) + })) + } + + /// Produces a new turbo.json without any tasks that reference non-existent + /// workspaces + pub fn prune_tasks>(&self, workspaces: &[S]) -> Self { + let mut this = self.clone(); + if let Some(pipeline) = &mut this.tasks { + pipeline.0.retain(|task_name, _| { + task_name.in_workspace(ROOT_PKG_NAME) + || workspaces + .iter() + .any(|workspace| task_name.in_workspace(workspace.as_ref())) + }) + } + + this + } + + pub fn from_task_access_trace(trace: &HashMap) -> Option { + if trace.is_empty() { + return None; + } + + let mut pipeline = Pipeline::default(); + + for (task_name, trace_file) in trace { + let spanned_outputs: Vec> = trace_file + .outputs + .iter() + .map(|output| Spanned::new(output.clone())) + .collect(); + let task_definition = RawTaskDefinition { + outputs: Some(spanned_outputs), + env: Some( + trace_file + .accessed + .env_var_keys + .iter() + .map(|unescaped_string| Spanned::new(unescaped_string.clone())) + .collect(), + ), + ..Default::default() + }; + + let name = TaskName::from(task_name.as_str()); + let root_task = name.into_root_task(); + pipeline.insert(root_task, Spanned::new(task_definition.clone())); + } + + Some(RawTurboJson { + tasks: Some(pipeline), + ..RawTurboJson::default() + }) + } +} diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__parser__tests__root_only_fields_in_package_turbo_json.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__parser__tests__root_only_fields_in_package_turbo_json.snap new file mode 100644 index 0000000000000..32821b8aa6fbc --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__parser__tests__root_only_fields_in_package_turbo_json.snap @@ -0,0 +1,15 @@ +--- +source: crates/turborepo-lib/src/turbo_json/parser.rs +expression: msg +--- +Failed to parse turbo.json. + Diagnostic severity: error +diagnostic code: turbo_json_parse_error + +Error: Found an unknown key `daemon`. + Diagnostic severity: error + +Begin snippet for packages/foo/turbo.json starting at line 1, column 1 + +snippet line 1: {"daemon": true} + label at line 1, columns 2 to 9 diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package.snap new file mode 100644 index 0000000000000..8ad00ef7da721 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "You can only extend from the root of the workspace.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_then_root.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_then_root.snap new file mode 100644 index 0000000000000..8ad00ef7da721 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_then_root.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "You can only extend from the root of the workspace.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_then_root_true.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_then_root_true.snap new file mode 100644 index 0000000000000..0426033282eed --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_then_root_true.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "You must extend from the root of the workspace first.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_true.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_true.snap new file mode 100644 index 0000000000000..0426033282eed --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_extends_from_non_root_package_true.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "You must extend from the root of the workspace first.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_including_root.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_including_root.snap new file mode 100644 index 0000000000000..8ad00ef7da721 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_including_root.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "You can only extend from the root of the workspace.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_including_root_true.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_including_root_true.snap new file mode 100644 index 0000000000000..e4380722ce1a6 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_including_root_true.snap @@ -0,0 +1,5 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_not_including_root.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_not_including_root.snap new file mode 100644 index 0000000000000..8ad00ef7da721 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_not_including_root.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "You can only extend from the root of the workspace.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_not_including_root_true.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_not_including_root_true.snap new file mode 100644 index 0000000000000..0426033282eed --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_multiple_extends_not_including_root_true.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "You must extend from the root of the workspace first.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_no_extends.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_no_extends.snap new file mode 100644 index 0000000000000..f3499260c60b8 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_no_extends.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "No \"extends\" key found.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_no_extends_true.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_no_extends_true.snap new file mode 100644 index 0000000000000..f3499260c60b8 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_no_extends_true.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "No \"extends\" key found.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_valid_extends_from_root.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_valid_extends_from_root.snap new file mode 100644 index 0000000000000..e4380722ce1a6 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_valid_extends_from_root.snap @@ -0,0 +1,5 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_valid_extends_from_root_true.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_valid_extends_from_root_true.snap new file mode 100644 index 0000000000000..e4380722ce1a6 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_extends_valid_extends_from_root_true.snap @@ -0,0 +1,5 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_multiple_mixed_tasks.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_multiple_mixed_tasks.snap new file mode 100644 index 0000000000000..e0e709f509296 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_multiple_mixed_tasks.snap @@ -0,0 +1,8 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "\"pkg-a#test\". Use \"test\" instead.", + "\"pkg-b#lint\". Use \"lint\" instead.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_non_package_task.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_non_package_task.snap new file mode 100644 index 0000000000000..e4380722ce1a6 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_non_package_task.snap @@ -0,0 +1,5 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_single_package_task.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_single_package_task.snap new file mode 100644 index 0000000000000..1e5ff13b7023b --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_no_package_task_syntax_single_package_task.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "\"my-package#build\". Use \"build\" instead.", +] diff --git a/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_with_has_no_topo.snap b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_with_has_no_topo.snap new file mode 100644 index 0000000000000..3bb0959f33c4f --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/snapshots/turborepo_lib__turbo_json__validator__test__validate_with_has_no_topo.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/turbo_json/validator.rs +expression: error_messages +--- +[ + "`with` cannot use dependency relationships.", +] diff --git a/crates/turborepo-lib/src/turbo_json/validator.rs b/crates/turborepo-lib/src/turbo_json/validator.rs new file mode 100644 index 0000000000000..4ef1707ecaaf8 --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/validator.rs @@ -0,0 +1,301 @@ +use miette::{NamedSource, SourceSpan}; +use turborepo_repository::package_graph::{PackageName, ROOT_PKG_NAME}; + +use super::{Error, FutureFlags, TurboJson, TOPOLOGICAL_PIPELINE_DELIMITER}; +use crate::config::UnnecessaryPackageTaskSyntaxError; + +pub type TurboJSONValidation = fn(&Validator, &TurboJson) -> Vec; + +/// Validator for TurboJson structures with context-aware validation +pub struct Validator { + non_root_extends: bool, +} + +const ROOT_VALIDATIONS: &[TurboJSONValidation] = &[validate_with_has_no_topo]; +const PACKAGE_VALIDATIONS: &[TurboJSONValidation] = &[ + validate_with_has_no_topo, + validate_no_package_task_syntax, + validate_extends, +]; + +impl Validator { + /// Creates a new validator instance + pub fn new() -> Self { + Self { + non_root_extends: false, + } + } + + pub fn with_future_flags(mut self, future_flags: FutureFlags) -> Self { + self.non_root_extends = future_flags.non_root_extends; + self + } + + /// Validates a TurboJson based on its package context + /// + /// Root turbo.json files have different validation rules than workspace + /// turbo.json files + pub fn validate_turbo_json( + &self, + package_name: &PackageName, + turbo_json: &TurboJson, + ) -> Vec { + let validations = match package_name { + PackageName::Root => ROOT_VALIDATIONS, + PackageName::Other(_) => PACKAGE_VALIDATIONS, + }; + validations + .iter() + .flat_map(|validation| validation(self, turbo_json)) + .collect() + } +} + +impl Default for Validator { + fn default() -> Self { + Self::new() + } +} + +pub fn validate_no_package_task_syntax( + _validator: &Validator, + turbo_json: &TurboJson, +) -> Vec { + turbo_json + .tasks + .iter() + .filter(|(task_name, _)| task_name.is_package_task()) + .map(|(task_name, entry)| { + let (span, text) = entry.span_and_text("turbo.json"); + Error::UnnecessaryPackageTaskSyntax(Box::new(UnnecessaryPackageTaskSyntaxError { + actual: task_name.to_string(), + wanted: task_name.task().to_string(), + span, + text, + })) + }) + .collect() +} + +pub fn validate_extends(validator: &Validator, turbo_json: &TurboJson) -> Vec { + if turbo_json.extends.is_empty() { + let path = turbo_json + .path + .as_ref() + .map_or("turbo.json", |p| p.as_ref()); + + let (span, text) = match turbo_json.text { + Some(ref text) => { + let len = text.len(); + let span: SourceSpan = (0, len - 1).into(); + (Some(span), text.to_string()) + } + None => (None, String::new()), + }; + + return vec![Error::NoExtends { + span, + text: NamedSource::new(path, text), + }]; + } + if let Some(package_name) = turbo_json.extends.first() + && package_name != ROOT_PKG_NAME + && validator.non_root_extends + { + let path = turbo_json + .path + .as_ref() + .map_or("turbo.json", |p| p.as_ref()); + + let (span, text) = match turbo_json.text { + Some(ref text) => { + let len = text.len(); + let span: SourceSpan = (0, len - 1).into(); + (Some(span), text.to_string()) + } + None => (None, String::new()), + }; + // Root needs to be first + return vec![Error::ExtendsRootFirst { + span, + text: NamedSource::new(path, text), + }]; + } + // If we allow for non-root extends we don't need to perform this check + (!validator.non_root_extends + && turbo_json + .extends + .iter() + .any(|package_name| package_name != ROOT_PKG_NAME)) + .then(|| { + let (span, text) = turbo_json.extends.span_and_text("turbo.json"); + Error::ExtendFromNonRoot { span, text } + }) + .into_iter() + .collect() +} + +pub fn validate_with_has_no_topo(_validator: &Validator, turbo_json: &TurboJson) -> Vec { + turbo_json + .tasks + .iter() + .flat_map(|(_, definition)| { + definition.with.iter().flatten().filter_map(|with_task| { + if with_task.starts_with(TOPOLOGICAL_PIPELINE_DELIMITER) { + let (span, text) = with_task.span_and_text("turbo.json"); + Some(Error::InvalidTaskWith { span, text }) + } else { + None + } + }) + }) + .collect() +} + +#[cfg(test)] +mod test { + use std::assert_matches::assert_matches; + + use test_case::test_case; + use turborepo_errors::Spanned; + use turborepo_task_id::TaskName; + use turborepo_unescape::UnescapedString; + + use super::*; + use crate::turbo_json::{Pipeline, RawTaskDefinition}; + + #[test] + fn test_validate_with_has_no_topo() { + let turbo_json = TurboJson { + tasks: Pipeline( + vec![( + TaskName::from("dev"), + Spanned::new(RawTaskDefinition { + with: Some(vec![Spanned::new(UnescapedString::from("^proxy"))]), + ..Default::default() + }), + )] + .into_iter() + .collect(), + ), + ..Default::default() + }; + + let validator = Validator::new(); + let errs = validate_with_has_no_topo(&validator, &turbo_json); + let error_messages: Vec = errs.iter().map(|e| e.to_string()).collect(); + insta::assert_debug_snapshot!("validate_with_has_no_topo", error_messages); + } + + #[test_case( + vec!["my-package#build"], + "single_package_task" + )] + #[test_case( + vec!["build"], + "non_package_task" + )] + #[test_case( + vec!["pkg-a#test", "pkg-b#lint", "build"], + "multiple_mixed_tasks" + )] + fn test_validate_no_package_task_syntax(tasks: Vec<&str>, name: &str) { + let turbo_json = TurboJson { + tasks: Pipeline( + tasks + .into_iter() + .map(|task_name| { + ( + TaskName::from(task_name.to_string()), + Spanned::new(RawTaskDefinition::default()), + ) + }) + .collect(), + ), + ..Default::default() + }; + + let validator = Validator::new(); + let errs = validate_no_package_task_syntax(&validator, &turbo_json); + let error_messages: Vec = errs.iter().map(|e| e.to_string()).collect(); + let snapshot_name = format!("validate_no_package_task_syntax_{}", name); + insta::assert_debug_snapshot!(snapshot_name, error_messages); + } + + #[test_case( + vec![], + "no_extends" + )] + #[test_case( + vec!["//"], + "valid_extends_from_root" + )] + #[test_case( + vec!["some-package"], + "extends_from_non_root_package" + )] + #[test_case( + vec!["//", "other-package"], + "multiple_extends_including_root" + )] + #[test_case( + vec!["package-a", "package-b"], + "multiple_extends_not_including_root" + )] + #[test_case( + vec!["some-package", "//"], + "extends_from_non_root_package_then_root" + )] + fn test_validate_extends(extends: Vec<&str>, name: &str) { + let turbo_json = TurboJson { + extends: Spanned::new(extends.into_iter().map(String::from).collect()), + ..Default::default() + }; + + for non_root_extends in [false, true] { + let validator = Validator { non_root_extends }; + let errs = validate_extends(&validator, &turbo_json); + let error_messages: Vec = errs.iter().map(|e| e.to_string()).collect(); + let mut snapshot_name = format!("validate_extends_{}", name); + if non_root_extends { + snapshot_name.push_str("_true"); + } + insta::assert_debug_snapshot!(snapshot_name, error_messages); + } + } + + #[test] + fn test_validator_with_root_package() { + let validator = Validator::new(); + + // Root turbo.json can have package task syntax + let turbo_json = TurboJson { + tasks: Pipeline( + vec![(TaskName::from("app#build"), Spanned::default())] + .into_iter() + .collect(), + ), + ..Default::default() + }; + + let errs = validator.validate_turbo_json(&PackageName::Root, &turbo_json); + assert!( + errs.is_empty(), + "Root turbo.json should allow package task syntax" + ); + } + + #[test] + fn test_validator_with_missing_extends() { + let validator = Validator::new(); + + // Workspace turbo.json without extends should error + let turbo_json = TurboJson { + ..Default::default() + }; + + let errs = validator.validate_turbo_json(&PackageName::from("app"), &turbo_json); + assert_eq!(errs.len(), 1, "Workspace turbo.json should have extends"); + assert_matches!(errs[0], Error::NoExtends { .. }); + } +} diff --git a/crates/turborepo-paths/src/absolute_system_path.rs b/crates/turborepo-paths/src/absolute_system_path.rs index 6ca9aaeef4245..bbdc90cfd0774 100644 --- a/crates/turborepo-paths/src/absolute_system_path.rs +++ b/crates/turborepo-paths/src/absolute_system_path.rs @@ -466,6 +466,36 @@ impl AbsoluteSystemPath { Ok(()) } + + /// Checks if this path is the direct parent of the given path. + /// + /// Returns `true` if this path is the immediate parent directory + /// of `possible_child`, `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// # use turbopath::AbsoluteSystemPath; + /// # use std::path::Path; + /// # #[cfg(unix)] + /// # { + /// let parent = AbsoluteSystemPath::new("/foo/bar").unwrap(); + /// let child = AbsoluteSystemPath::new("/foo/bar/baz").unwrap(); + /// let grandchild = AbsoluteSystemPath::new("/foo/bar/baz/qux").unwrap(); + /// let sibling = AbsoluteSystemPath::new("/foo/qux").unwrap(); + /// + /// assert!(parent.is_parent(&child)); + /// assert!(!parent.is_parent(&grandchild)); // Not direct parent + /// assert!(!parent.is_parent(&sibling)); + /// assert!(!parent.is_parent(&parent)); // Not parent of itself + /// # } + /// ``` + pub fn is_parent(&self, possible_child: &AbsoluteSystemPath) -> bool { + let Some(parent) = possible_child.parent() else { + return false; + }; + parent == self + } } impl<'a> From<&'a AbsoluteSystemPath> for CandidatePath<'a> { @@ -683,4 +713,25 @@ mod tests { let relation = abs_path.relation_to_path(&other_path); assert_eq!(relation, expected); } + + #[test_case(&["foo", "bar"], &["foo", "bar", "baz"], true ; "parent is direct parent of child")] + #[test_case(&["foo"], &["foo", "bar"], true ; "root parent is direct parent")] + #[test_case(&["foo", "bar"], &["foo", "bar", "baz", "qux"], false ; "not direct parent of grandchild")] + #[test_case(&["foo", "bar"], &["foo", "baz"], false ; "not parent of sibling")] + #[test_case(&["foo", "bar"], &["foo"], false ; "child is not parent of parent")] + #[test_case(&["foo", "bar"], &["foo", "bar"], false ; "path is not parent of itself")] + #[test_case(&[], &["foo"], true ; "root is parent of top-level dir")] + #[test_case(&["foo", "bar"], &["completely", "different"], false ; "unrelated paths")] + fn test_is_parent(parent_components: &[&str], child_components: &[&str], expected: bool) { + let root = if cfg!(windows) { "C:\\" } else { "/" }; + + let parent_path = AbsoluteSystemPathBuf::try_from(root) + .unwrap() + .join_components(parent_components); + let child_path = AbsoluteSystemPathBuf::try_from(root) + .unwrap() + .join_components(child_components); + + assert_eq!(parent_path.is_parent(&child_path), expected); + } } diff --git a/turborepo-tests/integration/fixtures/turbo-configs/package-task.json b/turborepo-tests/integration/fixtures/turbo-configs/package-task.json index a0793b4b048cd..3ba4dff49e01c 100644 --- a/turborepo-tests/integration/fixtures/turbo-configs/package-task.json +++ b/turborepo-tests/integration/fixtures/turbo-configs/package-task.json @@ -1,7 +1,5 @@ { "$schema": "https://turborepo.com/schema.json", - "globalDependencies": ["foo.txt"], - "globalEnv": ["SOME_ENV_VAR"], "extends": ["//"], "tasks": { // this comment verifies that turbo can read .json files with comments diff --git a/turborepo-tests/integration/tests/bad-turbo-json.t b/turborepo-tests/integration/tests/bad-turbo-json.t index cf0aae71ac581..35e4da02413a1 100644 --- a/turborepo-tests/integration/tests/bad-turbo-json.t +++ b/turborepo-tests/integration/tests/bad-turbo-json.t @@ -13,15 +13,15 @@ Run build with package task in non-root turbo.json unnecessary-package-task-syntax) x "my-app#build". Use "build" instead. - ,-\(apps(\/|\\)my-app(\/|\\)turbo.json:8:21\) (re) - 7 | // this comment verifies that turbo can read .json files + ,-\(apps(\/|\\)my-app(\/|\\)turbo.json:6:21\) (re) + 5 | // this comment verifies that turbo can read .json files with comments - 8 | ,-> "my-app#build": { - 9 | | "outputs": ("banana.txt", "apple.json"), - 10 | | "inputs": ("$TURBO_DEFAULT$", ".env.local") - 11 | |-> } + 6 | ,-> "my-app#build": { + 7 | | "outputs": ("banana.txt", "apple.json"), + 8 | | "inputs": ("$TURBO_DEFAULT$", ".env.local") + 9 | |-> } : `---- unnecessary package syntax found here - 12 | } + 10 | } `----