diff --git a/crates/turborepo-env/src/lib.rs b/crates/turborepo-env/src/lib.rs index 29425f9aa3ef0..5b5de1fb5965b 100644 --- a/crates/turborepo-env/src/lib.rs +++ b/crates/turborepo-env/src/lib.rs @@ -218,7 +218,7 @@ impl EnvironmentVariableMap { // that user exclusions have primacy over inferred inclusions. pub fn wildcard_map_from_wildcards_unresolved( &self, - wildcard_patterns: &[String], + wildcard_patterns: &[impl AsRef], ) -> Result { if wildcard_patterns.is_empty() { return Ok(WildcardMaps { @@ -229,6 +229,60 @@ impl EnvironmentVariableMap { self.wildcard_map_from_wildcards(wildcard_patterns) } + + /// Return a detailed map for which environment variables are factored into + /// the task's hash + pub fn hashable_task_env( + &self, + computed_wildcards: &[String], + task_env: &[String], + ) -> Result { + let mut explicit_env_var_map = EnvironmentVariableMap::default(); + let mut all_env_var_map = EnvironmentVariableMap::default(); + let mut matching_env_var_map = EnvironmentVariableMap::default(); + let inference_env_var_map = self.from_wildcards(computed_wildcards)?; + + let user_env_var_set = self.wildcard_map_from_wildcards_unresolved(task_env)?; + + all_env_var_map.union(&user_env_var_set.inclusions); + all_env_var_map.union(&inference_env_var_map); + all_env_var_map.difference(&user_env_var_set.exclusions); + + explicit_env_var_map.union(&user_env_var_set.inclusions); + explicit_env_var_map.difference(&user_env_var_set.exclusions); + + matching_env_var_map.union(&inference_env_var_map); + matching_env_var_map.difference(&user_env_var_set.exclusions); + + Ok(DetailedMap { + all: all_env_var_map, + by_source: BySource { + explicit: explicit_env_var_map, + matching: matching_env_var_map, + }, + }) + } + + /// Constructs an environment map that contains pass through environment + /// variables + pub fn pass_through_env( + &self, + builtins: &[&str], + global_env: &Self, + task_pass_through: &[impl AsRef], + ) -> Result { + let mut pass_through_env = EnvironmentVariableMap::default(); + let default_env_var_pass_through_map = self.from_wildcards(builtins)?; + let task_pass_through_env = + self.wildcard_map_from_wildcards_unresolved(task_pass_through)?; + + pass_through_env.union(&default_env_var_pass_through_map); + pass_through_env.union(global_env); + pass_through_env.union(&task_pass_through_env.inclusions); + pass_through_env.difference(&task_pass_through_env.exclusions); + + Ok(pass_through_env) + } } const WILDCARD: char = '*'; @@ -358,4 +412,58 @@ mod tests { actual.sort(); assert_eq!(actual, expected); } + + #[test_case(&["FOO*"], &["BAR"], &["BAR", "FOO", "FOOBAR", "FOOD"] ; "wildcard")] + #[test_case(&["FOO*", "!FOOBAR"], &["BAR"], &["BAR", "FOO", "FOOD"] ; "omit wild")] + #[test_case(&["FOO*"], &["!FOOBAR"], &["FOO", "FOOD"] ; "omit task")] + fn test_hashable_env(wildcards: &[&str], task: &[&str], expected: &[&str]) { + let env_at_start = EnvironmentVariableMap( + vec![ + ("FOO", "bar"), + ("FOOBAR", "baz"), + ("FOOD", "cheese"), + ("BAR", "nuts"), + ] + .into_iter() + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect(), + ); + let wildcards: Vec<_> = wildcards.iter().map(|s| s.to_string()).collect(); + let task: Vec<_> = task.iter().map(|s| s.to_string()).collect(); + let output = env_at_start.hashable_task_env(&wildcards, &task).unwrap(); + let mut actual: Vec<_> = output.all.keys().map(|s| s.as_str()).collect(); + actual.sort(); + assert_eq!(actual, expected); + } + + #[test_case(&["FOO*"], &["FOO", "FOOBAR", "FOOD", "PATH"] ; "folds 3 sources")] + #[test_case(&["!FOO"], &["PATH"] ; "remove global")] + #[test_case(&["!PATH"], &["FOO"] ; "remove builtin")] + #[test_case(&["FOO*", "!FOOD"], &["FOO", "FOOBAR", "PATH"] ; "mixing negations")] + fn test_pass_through_env(task: &[&str], expected: &[&str]) { + let env_at_start = EnvironmentVariableMap( + vec![ + ("PATH", "of"), + ("FOO", "bar"), + ("FOOBAR", "baz"), + ("FOOD", "cheese"), + ("BAR", "nuts"), + ] + .into_iter() + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect(), + ); + let global_env = EnvironmentVariableMap( + vec![("FOO", "bar")] + .into_iter() + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect(), + ); + let output = env_at_start + .pass_through_env(&["PATH"], &global_env, task) + .unwrap(); + let mut actual: Vec<_> = output.keys().map(|s| s.as_str()).collect(); + actual.sort(); + assert_eq!(actual, expected); + } } diff --git a/crates/turborepo-lib/src/task_graph/visitor/mod.rs b/crates/turborepo-lib/src/task_graph/visitor/mod.rs index 2797d693d3de6..b8d2b8b89c52c 100644 --- a/crates/turborepo-lib/src/task_graph/visitor/mod.rs +++ b/crates/turborepo-lib/src/task_graph/visitor/mod.rs @@ -52,7 +52,6 @@ use crate::{ pub struct Visitor<'a> { color_cache: ColorSelector, dry: bool, - global_env: EnvironmentVariableMap, global_env_mode: EnvMode, manager: ProcessManager, run_opts: &'a RunOpts, @@ -146,6 +145,7 @@ impl<'a> Visitor<'a> { run_opts, env_at_execution_start, global_hash, + global_env, ); let sink = Self::sink(run_opts); @@ -172,7 +172,6 @@ impl<'a> Visitor<'a> { sink, task_hasher, color_config, - global_env, ui_sender, is_watch, warnings: Default::default(), @@ -259,9 +258,9 @@ impl<'a> Visitor<'a> { // We do this calculation earlier than we do in Go due to the `task_hasher` // being !Send. In the future we can look at doing this right before // task execution instead. - let execution_env = - self.task_hasher - .env(&info, task_env_mode, task_definition, &self.global_env)?; + let execution_env = self + .task_hasher + .env(&info, task_env_mode, task_definition)?; let task_cache = self.run_cache.task_cache( task_definition, diff --git a/crates/turborepo-lib/src/task_hash.rs b/crates/turborepo-lib/src/task_hash.rs index 5e78ee45e79bc..18e9c2c44c878 100644 --- a/crates/turborepo-lib/src/task_hash.rs +++ b/crates/turborepo-lib/src/task_hash.rs @@ -242,6 +242,7 @@ pub struct TaskHasher<'a> { hashes: HashMap, String>, run_opts: &'a RunOpts, env_at_execution_start: &'a EnvironmentVariableMap, + global_env: EnvironmentVariableMap, global_hash: &'a str, task_hash_tracker: TaskHashTracker, } @@ -252,6 +253,7 @@ impl<'a> TaskHasher<'a> { run_opts: &'a RunOpts, env_at_execution_start: &'a EnvironmentVariableMap, global_hash: &'a str, + global_env: EnvironmentVariableMap, ) -> Self { let PackageInputsHashes { hashes, @@ -262,6 +264,7 @@ impl<'a> TaskHasher<'a> { run_opts, env_at_execution_start, global_hash, + global_env, task_hash_tracker: TaskHashTracker::new(expanded_hashes), } } @@ -283,13 +286,11 @@ impl<'a> TaskHasher<'a> { .hashes .get(task_id) .ok_or_else(|| Error::MissingPackageFileHash(task_id.to_string()))?; - let mut explicit_env_var_map = EnvironmentVariableMap::default(); - let mut all_env_var_map = EnvironmentVariableMap::default(); - let mut matching_env_var_map = EnvironmentVariableMap::default(); - - let framework_slug = if do_framework_inference { - // See if we infer a framework - if let Some(framework) = infer_framework(workspace, is_monorepo) { + // See if we infer a framework + let framework = do_framework_inference + .then(|| infer_framework(workspace, is_monorepo)) + .flatten() + .inspect(|framework| { debug!("auto detected framework for {}", task_id.package()); debug!( "framework: {}, env_prefix: {:?}", @@ -297,66 +298,39 @@ impl<'a> TaskHasher<'a> { framework.env_wildcards() ); telemetry.track_framework(framework.slug()); - let mut computed_wildcards = framework - .env_wildcards() - .iter() - .map(|s| s.to_string()) - .collect::>(); - - if let Some(exclude_prefix) = - self.env_at_execution_start.get("TURBO_CI_VENDOR_ENV_KEY") - { - if !exclude_prefix.is_empty() { - let computed_exclude = format!("!{}*", exclude_prefix); - debug!( - "excluding environment variables matching wildcard {}", - computed_exclude - ); - computed_wildcards.push(computed_exclude); - } - } - - let inference_env_var_map = self - .env_at_execution_start - .from_wildcards(&computed_wildcards)?; - - let user_env_var_set = self - .env_at_execution_start - .wildcard_map_from_wildcards_unresolved(&task_definition.env)?; - - all_env_var_map.union(&user_env_var_set.inclusions); - all_env_var_map.union(&inference_env_var_map); - all_env_var_map.difference(&user_env_var_set.exclusions); + }); + let framework_slug = framework.map(|f| f.slug().to_string()); - explicit_env_var_map.union(&user_env_var_set.inclusions); - explicit_env_var_map.difference(&user_env_var_set.exclusions); + let env_vars = if let Some(framework) = framework { + let mut computed_wildcards = framework.env_wildcards().to_vec(); - matching_env_var_map.union(&inference_env_var_map); - matching_env_var_map.difference(&user_env_var_set.exclusions); - Some(framework.slug().to_string()) - } else { - all_env_var_map = self - .env_at_execution_start - .from_wildcards(&task_definition.env)?; - - explicit_env_var_map.union(&all_env_var_map); - None + if let Some(exclude_prefix) = self + .env_at_execution_start + .get("TURBO_CI_VENDOR_ENV_KEY") + .filter(|prefix| !prefix.is_empty()) + { + let computed_exclude = format!("!{}*", exclude_prefix); + debug!( + "excluding environment variables matching wildcard {}", + computed_exclude + ); + computed_wildcards.push(computed_exclude); } + + self.env_at_execution_start + .hashable_task_env(&computed_wildcards, &task_definition.env)? } else { - all_env_var_map = self + let all_env_var_map = self .env_at_execution_start .from_wildcards(&task_definition.env)?; - explicit_env_var_map.union(&all_env_var_map); - None - }; - - let env_vars = DetailedMap { - all: all_env_var_map, - by_source: BySource { - explicit: explicit_env_var_map, - matching: matching_env_var_map, - }, + DetailedMap { + all: all_env_var_map.clone(), + by_source: BySource { + explicit: all_env_var_map, + matching: EnvironmentVariableMap::default(), + }, + } }; let hashable_env_pairs = env_vars.all.to_hashable(); @@ -458,74 +432,72 @@ impl<'a> TaskHasher<'a> { task_id: &TaskId, task_env_mode: EnvMode, task_definition: &TaskDefinition, - global_env: &EnvironmentVariableMap, ) -> Result { match task_env_mode { EnvMode::Strict => { - let mut pass_through_env = EnvironmentVariableMap::default(); - let default_env_var_pass_through_map = - self.env_at_execution_start.from_wildcards(&[ - "HOME", - "USER", - "TZ", - "LANG", - "SHELL", - "PWD", - "CI", - "NODE_OPTIONS", - "COREPACK_HOME", - "LD_LIBRARY_PATH", - "DYLD_FALLBACK_LIBRARY_PATH", - "LIBPATH", - "COLORTERM", - "TERM", - "TERM_PROGRAM", - "DISPLAY", - "TMP", - "TEMP", - // VSCode IDE - https://github.com/microsoft/vscode-js-debug/blob/5b0f41dbe845d693a541c1fae30cec04c878216f/src/targets/node/nodeLauncherBase.ts#L320 - "VSCODE_*", - "ELECTRON_RUN_AS_NODE", - // Docker - https://docs.docker.com/engine/reference/commandline/cli/#environment-variables - "DOCKER_*", - "BUILDKIT_*", - // Docker compose - https://docs.docker.com/compose/environment-variables/envvars/ - "COMPOSE_*", - // Jetbrains IDE - "JB_IDE_*", - "JB_INTERPRETER", - "_JETBRAINS_TEST_RUNNER_RUN_SCOPE_TYPE", - // Vercel specific - "VERCEL", - "VERCEL_*", - "NEXT_*", - "USE_OUTPUT_FOR_EDGE_FUNCTIONS", - "NOW_BUILDER", - // Command Prompt casing of env variables - "APPDATA", - "PATH", - "PROGRAMDATA", - "SYSTEMROOT", - "SYSTEMDRIVE", - ])?; - let tracker_env = self - .task_hash_tracker - .env_vars(task_id) - .ok_or_else(|| Error::MissingEnvVars(task_id.clone().into_owned()))?; - - pass_through_env.union(&default_env_var_pass_through_map); - pass_through_env.union(global_env); - pass_through_env.union(&tracker_env.all); - - let env_var_pass_through_map = self.env_at_execution_start.from_wildcards( + let mut full_task_env = EnvironmentVariableMap::default(); + let builtin_pass_through = &[ + "HOME", + "USER", + "TZ", + "LANG", + "SHELL", + "PWD", + "CI", + "NODE_OPTIONS", + "COREPACK_HOME", + "LD_LIBRARY_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "LIBPATH", + "COLORTERM", + "TERM", + "TERM_PROGRAM", + "DISPLAY", + "TMP", + "TEMP", + // VSCode IDE - https://github.com/microsoft/vscode-js-debug/blob/5b0f41dbe845d693a541c1fae30cec04c878216f/src/targets/node/nodeLauncherBase.ts#L320 + "VSCODE_*", + "ELECTRON_RUN_AS_NODE", + // Docker - https://docs.docker.com/engine/reference/commandline/cli/#environment-variables + "DOCKER_*", + "BUILDKIT_*", + // Docker compose - https://docs.docker.com/compose/environment-variables/envvars/ + "COMPOSE_*", + // Jetbrains IDE + "JB_IDE_*", + "JB_INTERPRETER", + "_JETBRAINS_TEST_RUNNER_RUN_SCOPE_TYPE", + // Vercel specific + "VERCEL", + "VERCEL_*", + "NEXT_*", + "USE_OUTPUT_FOR_EDGE_FUNCTIONS", + "NOW_BUILDER", + // Command Prompt casing of env variables + "APPDATA", + "PATH", + "PROGRAMDATA", + "SYSTEMROOT", + "SYSTEMDRIVE", + ]; + let pass_through_env_vars = self.env_at_execution_start.pass_through_env( + builtin_pass_through, + &self.global_env, task_definition .pass_through_env .as_deref() .unwrap_or_default(), )?; - pass_through_env.union(&env_var_pass_through_map); - Ok(pass_through_env) + let tracker_env = self + .task_hash_tracker + .env_vars(task_id) + .ok_or_else(|| Error::MissingEnvVars(task_id.clone().into_owned()))?; + + full_task_env.union(&pass_through_env_vars); + full_task_env.union(&tracker_env.all); + + Ok(full_task_env) } EnvMode::Loose => Ok(self.env_at_execution_start.clone()), }