这是indexloc提供的服务,不要输入任何密码
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
e7d9d27
chore(turbo_json): delegate actual TurboJson reading to helper
chris-olszewski Jul 30, 2025
fdf8d50
chore(future_flags): allow future flags to alter behavior of turbo.js…
chris-olszewski Jul 30, 2025
a57cf49
chore(turbo_json): add turbo_extends feature flag
chris-olszewski Aug 12, 2025
7338c53
chore(turbo_json): add processed task definition
chris-olszewski Aug 12, 2025
6e4f36f
chore(turbo_json): move processed task definition to own module
chris-olszewski Aug 12, 2025
033b10a
chore(turbo_json): fully use processed task definition
chris-olszewski Aug 12, 2025
9c4c3a1
chore(turbo_json): clean up code
chris-olszewski Aug 13, 2025
5084eea
chore(turbo_json): add processed with type
chris-olszewski Aug 13, 2025
c2f3d65
update architecture.md
chris-olszewski Aug 13, 2025
77b6aaf
make turbo_default private
chris-olszewski Aug 13, 2025
7d0fb7f
chore(turbo_json): remove unused constants
chris-olszewski Aug 13, 2025
e947d86
chore(turbo_json): move TURBO_DEFAULT handling to processing step
chris-olszewski Aug 13, 2025
4651646
chore(turbo_json): move processed outputs to constructor
chris-olszewski Aug 13, 2025
c40b0f7
chore(turbo_json): move additional validation to processing step
chris-olszewski Aug 13, 2025
2571af7
chore(turbo_json): move future flags to own module
chris-olszewski Aug 13, 2025
172ce87
feat(turbo_json): implement TURBO_EXTENDS
chris-olszewski Aug 13, 2025
bbec95d
chore(config): fix clippy lint
chris-olszewski Aug 13, 2025
2123e6f
chore(turbo_json): update test fixture to have one failure
chris-olszewski Aug 14, 2025
e66c7c7
chore(turbo_json): change to more specific turboExtendsKeyword flag name
chris-olszewski Aug 21, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions crates/turborepo-lib/src/config/env.rs
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,8 @@ impl ResolvedConfigurationOptions for EnvVars {
root_turbo_json_path,
log_order,
sso_login_callback_port,
// Do not allow future flags to be set by env var
future_flags: None,
};

Ok(output)
Expand Down
11 changes: 10 additions & 1 deletion crates/turborepo-lib/src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@ use turborepo_cache::CacheConfig;
use turborepo_errors::TURBO_SITE;
use turborepo_repository::package_graph::PackageName;

use crate::cli::{EnvMode, LogOrder};
pub use crate::turbo_json::{RawTurboJson, UIMode};
use crate::{
cli::{EnvMode, LogOrder},
turbo_json::FutureFlags,
};

pub const CONFIG_FILE: &str = "turbo.json";
pub const CONFIG_FILE_JSONC: &str = "turbo.jsonc";
Expand Down Expand Up @@ -312,6 +315,8 @@ pub struct ConfigurationOptions {
pub(crate) concurrency: Option<String>,
pub(crate) no_update_notifier: Option<bool>,
pub(crate) sso_login_callback_port: Option<u16>,
#[serde(skip)]
future_flags: Option<FutureFlags>,
}

#[derive(Default)]
Expand Down Expand Up @@ -471,6 +476,10 @@ impl ConfigurationOptions {
pub fn sso_login_callback_port(&self) -> Option<u16> {
self.sso_login_callback_port
}

pub fn future_flags(&self) -> FutureFlags {
self.future_flags.unwrap_or_default()
}
}

// Maps Some("") to None to emulate how Go handles empty strings
Expand Down
1 change: 1 addition & 0 deletions crates/turborepo-lib/src/config/turbo_json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ impl<'a> TurboJsonReader<'a> {
opts.env_mode = turbo_json.env_mode.map(|mode| *mode.as_inner());
opts.cache_dir = cache_dir;
opts.concurrency = turbo_json.concurrency.map(|c| c.as_inner().clone());
opts.future_flags = turbo_json.future_flags.map(|f| *f.as_inner());
Ok(opts)
}
}
Expand Down
22 changes: 10 additions & 12 deletions crates/turborepo-lib/src/engine/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use crate::{
task_graph::TaskDefinition,
turbo_json::{
validate_extends, validate_no_package_task_syntax, validate_with_has_no_topo,
RawTaskDefinition, TurboJsonLoader,
ProcessedTaskDefinition, TurboJsonLoader,
},
};

Expand Down Expand Up @@ -564,14 +564,12 @@ impl<'a> EngineBuilder<'a> {
task_id: &Spanned<TaskId>,
task_name: &TaskName,
) -> Result<TaskDefinition, Error> {
let raw_task_definition = RawTaskDefinition::from_iter(self.task_definition_chain(
turbo_json_loader,
task_id,
task_name,
)?);
let processed_task_definition = ProcessedTaskDefinition::from_iter(
self.task_definition_chain(turbo_json_loader, task_id, task_name)?,
);
let path_to_root = self.path_to_root(task_id.as_inner())?;
Ok(TaskDefinition::from_raw(
raw_task_definition,
Ok(TaskDefinition::from_processed(
processed_task_definition,
&path_to_root,
)?)
}
Expand All @@ -581,13 +579,13 @@ impl<'a> EngineBuilder<'a> {
turbo_json_loader: &TurboJsonLoader,
task_id: &Spanned<TaskId>,
task_name: &TaskName,
) -> Result<Vec<RawTaskDefinition>, Error> {
) -> Result<Vec<ProcessedTaskDefinition>, Error> {
let mut task_definitions = Vec::new();

let root_turbo_json = turbo_json_loader.load(&PackageName::Root)?;
Error::from_validation(root_turbo_json.validate(&[validate_with_has_no_topo]))?;

if let Some(root_definition) = root_turbo_json.task(task_id, task_name) {
if let Some(root_definition) = root_turbo_json.task(task_id, task_name)? {
task_definitions.push(root_definition)
}

Expand Down Expand Up @@ -616,8 +614,8 @@ impl<'a> EngineBuilder<'a> {
validate_with_has_no_topo,
]))?;

if let Some(workspace_def) = workspace_json.tasks.get(task_name) {
task_definitions.push(workspace_def.value.clone());
if let Some(workspace_def) = workspace_json.task(task_id, task_name)? {
task_definitions.push(workspace_def);
}
}
Err(config::Error::NoTurboJSON) => (),
Expand Down
6 changes: 5 additions & 1 deletion crates/turborepo-lib/src/opts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use crate::{
OutputLogsMode, RunArgs,
},
config::{ConfigurationOptions, CONFIG_FILE},
turbo_json::UIMode,
turbo_json::{FutureFlags, UIMode},
Args,
};

Expand Down Expand Up @@ -77,6 +77,7 @@ pub struct Opts {
pub runcache_opts: RunCacheOpts,
pub scope_opts: ScopeOpts,
pub tui_opts: TuiOpts,
pub future_flags: FutureFlags,
}

impl Opts {
Expand Down Expand Up @@ -180,6 +181,7 @@ impl Opts {
let api_client_opts = APIClientOpts::from(inputs);
let repo_opts = RepoOpts::from(inputs);
let tui_opts = TuiOpts::from(inputs);
let future_flags = config.future_flags();

Ok(Self {
repo_opts,
Expand All @@ -189,6 +191,7 @@ impl Opts {
runcache_opts,
api_client_opts,
tui_opts,
future_flags,
})
}
}
Expand Down Expand Up @@ -747,6 +750,7 @@ mod test {
cache_opts,
runcache_opts,
tui_opts,
future_flags: Default::default(),
};
let synthesized = opts.synthesize_command();
assert_eq!(synthesized, expected);
Expand Down
4 changes: 2 additions & 2 deletions crates/turborepo-lib/src/package_changes_watcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use turborepo_scm::GitHashes;

use crate::{
config::{resolve_turbo_config_path, CONFIG_FILE, CONFIG_FILE_JSONC},
turbo_json::{TurboJson, TurboJsonLoader},
turbo_json::{TurboJson, TurboJsonLoader, TurboJsonReader},
};

#[derive(Clone)]
Expand Down Expand Up @@ -232,7 +232,7 @@ impl Subscriber {
};

let root_turbo_json = TurboJsonLoader::workspace(
self.repo_root.clone(),
TurboJsonReader::new(self.repo_root.clone()),
config_path,
pkg_dep_graph.packages(),
)
Expand Down
15 changes: 9 additions & 6 deletions crates/turborepo-lib/src/run/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ use crate::{
opts::Opts,
run::{scope, task_access::TaskAccess, Error, Run, RunCache},
shim::TurboState,
turbo_json::{TurboJson, TurboJsonLoader, UIMode},
turbo_json::{TurboJson, TurboJsonLoader, TurboJsonReader, UIMode},
DaemonConnector,
};

Expand Down Expand Up @@ -408,16 +408,19 @@ impl RunBuilder {
task_access.restore_config().await;

let root_turbo_json_path = self.opts.repo_opts.root_turbo_json_path.clone();
let future_flags = self.opts.future_flags;

let reader = TurboJsonReader::new(self.repo_root.clone()).with_future_flags(future_flags);

let turbo_json_loader = if task_access.is_enabled() {
TurboJsonLoader::task_access(
self.repo_root.clone(),
reader,
root_turbo_json_path.clone(),
root_package_json.clone(),
)
} else if is_single_package {
TurboJsonLoader::single_package(
self.repo_root.clone(),
reader,
root_turbo_json_path.clone(),
root_package_json.clone(),
)
Expand All @@ -426,20 +429,20 @@ impl RunBuilder {
(self.opts.repo_opts.allow_no_turbo_json || micro_frontend_configs.is_some())
{
TurboJsonLoader::workspace_no_turbo_json(
self.repo_root.clone(),
reader,
pkg_dep_graph.packages(),
micro_frontend_configs.clone(),
)
} else if let Some(micro_frontends) = &micro_frontend_configs {
TurboJsonLoader::workspace_with_microfrontends(
self.repo_root.clone(),
reader,
root_turbo_json_path.clone(),
pkg_dep_graph.packages(),
micro_frontends.clone(),
)
} else {
TurboJsonLoader::workspace(
self.repo_root.clone(),
reader,
root_turbo_json_path.clone(),
pkg_dep_graph.packages(),
)
Expand Down
65 changes: 51 additions & 14 deletions crates/turborepo-lib/src/turbo_json/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ The loading and resolving of task definitions is driven during task graph constr
The configuration and task loading process follows this high-level flow:

1. **Configuration Resolution**: Collect and merge configuration from multiple sources
2. **TurboJson Loading**: Resolve `turbo.json` files, these are usually files on disk, but can be synthesized
3. **Task Definition Resolution**: Convert raw task definitions into validated structures. `extends` is handled in this step
4. **Task Graph Construction**: Build the executable task graph from resolved definitions
2. **TurboJson Loading**: Parse `turbo.json` files into raw structures
3. **Task Processing**: Convert raw definitions to processed intermediate representation with DSL token handling
4. **Task Definition Resolution**: Transform processed definitions into final validated structures
5. **Task Graph Construction**: Build the executable task graph from resolved definitions

## Phase 1: Configuration Resolution

Expand Down Expand Up @@ -39,30 +40,66 @@ Configuration is collected from multiple sources with the following priority (hi

- **`TurboJsonLoader`** (`crates/turborepo-lib/src/turbo_json/loader.rs`): Loads and resolves turbo.json files
- **`RawTurboJson`**: Raw deserialized structure from JSON files
- **`TurboJson`**: Resolved and validated structure, all DSL magic strings have been handled
- **`TurboJson`**: Validated structure containing raw task definitions

### Process

1. **File Discovery**: Locate `turbo.json` or `turbo.jsonc` files
2. **Parsing**: Deserialize JSON into `RawTurboJson` structures
3. **Validation**: Convert to `TurboJson` with validation rules
3. **Basic Validation**: Convert to `TurboJson` with structural validation
4. **Workspace Resolution**: Apply workspace-specific overrides

## Phase 3: Task Definition Resolution
## Phase 3: Processed Task Definition (Intermediate Representation)

### Key Components

- **`ProcessedTaskDefinition`** (`crates/turborepo-lib/src/turbo_json/processed.rs`): Intermediate representation with DSL token processing
- **`ProcessedGlob`**: Parsed glob patterns with separated components (base pattern, negation flag, turbo_root flag)
- **`ProcessedInputs`/`ProcessedOutputs`**: Collections of processed globs with resolution methods

### Processing Steps

1. **DSL Token Detection**: Identify and separate `$TURBO_ROOT$` and `!` prefixes from glob patterns
2. **Early Validation**: Single-field validations at parse time with span information:
- Absolute paths in inputs/outputs
- Invalid `$TURBO_ROOT$` usage
- Environment variable prefixes (`$` not allowed)
- Dependency prefixes (`$` not allowed in `dependsOn`)
- Topological references (`^` not allowed in `with`)
3. **Prefix Stripping**: Store clean glob patterns without DSL prefixes
4. **Component Separation**: Track negation and turbo_root requirements as separate boolean flags

## Phase 4: Task Definition Resolution

### Key Components

- **`RawTaskDefinition`**: Raw task configuration from JSON
- **`TaskDefinition`**: Validated and processed task configuration
- **`ProcessedTaskDefinition`**: Intermediate representation with parsed DSL tokens
- **`TaskDefinition`**: Final validated and resolved task configuration
- **`TaskId`** and **`TaskName`** (from `turborepo-task-id` crate): Task identification types

### Transformation Process

Raw task definitions undergo several transformations:
The resolution now follows a three-stage pipeline:

1. **Raw → Processed** (`ProcessedTaskDefinition::from_raw`):

- Parse glob patterns and extract DSL tokens
- Validate single-field constraints with span information:
- Absolute paths in inputs/outputs (`ProcessedGlob::from_spanned_*`)
- Invalid environment variable prefixes (`ProcessedEnv::new`, `ProcessedPassThroughEnv::new`)
- Invalid dependency syntax (`ProcessedDependsOn::new`)
- Invalid sibling task references (`ProcessedWith::new`)
- Strip prefixes and store components separately

2. **Processed → Resolved** (`TaskDefinition::from_processed`):

- Apply `$TURBO_ROOT$` token replacement using `resolve()` methods
- Parse `dependsOn` into topological and task dependencies
- Transform environment variables into sorted lists
- Transform outputs into inclusion/exclusion patterns
- Validate multi-field constraints:
- Interactive tasks cannot be cached (requires `cache` and `interactive` fields)
- Interruptible tasks must be persistent (requires `interruptible` and `persistent` fields)

1. **Path Resolution**: Convert relative paths and handle `$TURBO_ROOT$` tokens
2. **Dependency Parsing**: Parse `dependsOn` into topological and task dependencies
3. **Environment Variable Collection**: Extract `env` and `passThroughEnv` variables
4. **Output Processing**: Handle inclusion/exclusion patterns in outputs
5. **Inheritance**: Handle merging multiple `RawTaskDefinition`s into a single usable task definition
6. **Validation**: Ensure configuration consistency (e.g., interactive tasks can't be cached)
3. **Inheritance**: The `extend` module handles merging multiple `ProcessedTaskDefinition`s before final resolution
Loading
Loading