From 796de0ea787b44906b5ac6a8891ff631ad090dea Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 21 Nov 2024 15:06:08 -0500 Subject: [PATCH 1/4] docs(cache): document TURBO_REMOTE_CACHE_UPLOAD_TIMEOUT env var --- docs/repo-docs/reference/system-environment-variables.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/repo-docs/reference/system-environment-variables.mdx b/docs/repo-docs/reference/system-environment-variables.mdx index f29a11cde5d9a..f21d1b06f0e51 100644 --- a/docs/repo-docs/reference/system-environment-variables.mdx +++ b/docs/repo-docs/reference/system-environment-variables.mdx @@ -30,6 +30,7 @@ System environment variables are always overridden by flag values provided direc | `TURBO_REMOTE_CACHE_READ_ONLY` | Prevent writing to the [Remote Cache](/repo/docs/core-concepts/remote-caching) - but still allow reading. | | `TURBO_REMOTE_CACHE_SIGNATURE_KEY` | Sign artifacts with a secret key. For more information, visit [the Artifact Integrity section](/repo/docs/core-concepts/remote-caching#artifact-integrity-and-authenticity-verification). | | `TURBO_REMOTE_CACHE_TIMEOUT` | Set a timeout in seconds for `turbo` to get artifacts from [Remote Cache](/repo/docs/core-concepts/remote-caching). | +| `TURBO_REMOTE_CACHE_UPLOAD_TIMEOUT` | Set a timeout in seconds for `turbo` to upload artifacts to [Remote Cache](/repo/docs/core-concepts/remote-caching). | | `TURBO_REMOTE_ONLY` | Always ignore the local filesystem cache for all tasks. | | `TURBO_RUN_SUMMARY` | Generate a [Run Summary](/repo/docs/reference/run#--summarize) when you run tasks. | | `TURBO_SCM_BASE` | Base used by `--affected` when calculating what has changed from `base...head` | From c4bf759b550bc80f1ffa71599504f08cd66e5367 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 21 Nov 2024 15:13:17 -0500 Subject: [PATCH 2/4] chore(config): add test for TURBO_REMOTE_CACHE_UPLOAD_TIMEOUT --- crates/turborepo-lib/src/config/env.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/turborepo-lib/src/config/env.rs b/crates/turborepo-lib/src/config/env.rs index 9d50639ca71cb..a1e26988d0b8e 100644 --- a/crates/turborepo-lib/src/config/env.rs +++ b/crates/turborepo-lib/src/config/env.rs @@ -319,6 +319,7 @@ mod test { env.insert("turbo_remote_cache_read_only".into(), "1".into()); env.insert("turbo_run_summary".into(), "true".into()); env.insert("turbo_allow_no_turbo_json".into(), "true".into()); + env.insert("turbo_remote_cache_upload_timeout".into(), "200".into()); let config = EnvVars::new(&env) .unwrap() @@ -331,6 +332,7 @@ mod test { assert!(config.remote_cache_read_only()); assert!(config.run_summary()); assert!(config.allow_no_turbo_json()); + assert_eq!(config.upload_timeout(), 200); assert_eq!(turbo_api, config.api_url.unwrap()); assert_eq!(turbo_login, config.login_url.unwrap()); assert_eq!(turbo_team, config.team_slug.unwrap()); From 4e3e99f60b29689d83e8b393a826ed3bba55d1fb Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 21 Nov 2024 16:33:51 -0500 Subject: [PATCH 3/4] feat(config): add uploadTimeout option to remoteCache config --- crates/turborepo-lib/src/config/turbo_json.rs | 77 +++++++++++++++---- crates/turborepo-lib/src/turbo_json/mod.rs | 3 + 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/crates/turborepo-lib/src/config/turbo_json.rs b/crates/turborepo-lib/src/config/turbo_json.rs index e4cf24c1182cd..ccec280d991c2 100644 --- a/crates/turborepo-lib/src/config/turbo_json.rs +++ b/crates/turborepo-lib/src/config/turbo_json.rs @@ -12,23 +12,10 @@ impl<'a> TurboJsonReader<'a> { pub fn new(repo_root: &'a AbsoluteSystemPath) -> Self { Self { repo_root } } -} -impl<'a> ResolvedConfigurationOptions for TurboJsonReader<'a> { - fn get_configuration_options( - &self, - existing_config: &ConfigurationOptions, + fn turbo_json_to_config_options( + turbo_json: RawTurboJson, ) -> Result { - let turbo_json_path = existing_config.root_turbo_json_path(self.repo_root); - let turbo_json = RawTurboJson::read(self.repo_root, &turbo_json_path).or_else(|e| { - if let Error::Io(e) = &e { - if matches!(e.kind(), std::io::ErrorKind::NotFound) { - return Ok(Default::default()); - } - } - - Err(e) - })?; let mut opts = if let Some(remote_cache_options) = &turbo_json.remote_cache { remote_cache_options.into() } else { @@ -64,8 +51,28 @@ impl<'a> ResolvedConfigurationOptions for TurboJsonReader<'a> { } } +impl<'a> ResolvedConfigurationOptions for TurboJsonReader<'a> { + fn get_configuration_options( + &self, + existing_config: &ConfigurationOptions, + ) -> Result { + let turbo_json_path = existing_config.root_turbo_json_path(self.repo_root); + let turbo_json = RawTurboJson::read(self.repo_root, &turbo_json_path).or_else(|e| { + if let Error::Io(e) = &e { + if matches!(e.kind(), std::io::ErrorKind::NotFound) { + return Ok(Default::default()); + } + } + + Err(e) + })?; + Self::turbo_json_to_config_options(turbo_json) + } +} + #[cfg(test)] mod test { + use serde_json::json; use tempfile::tempdir; use super::*; @@ -111,7 +118,7 @@ mod test { }; root_turbo_json_path .create_with_contents( - serde_json::to_string_pretty(&serde_json::json!({ + serde_json::to_string_pretty(&json!({ "daemon": false })) .unwrap(), @@ -123,4 +130,42 @@ mod test { // Make sure we read the correct turbo.json assert_eq!(config.daemon(), Some(false)); } + + #[test] + fn test_remote_cache_options() { + let timeout = 100; + let upload_timeout = 200; + let api_url = "localhost:3000"; + let login_url = "localhost:3001"; + let team_slug = "acme-packers"; + let team_id = "id-123"; + let turbo_json = RawTurboJson::parse( + &serde_json::to_string_pretty(&json!({ + "remoteCache": { + "enabled": true, + "timeout": timeout, + "uploadTimeout": upload_timeout, + "apiUrl": api_url, + "loginUrl": login_url, + "teamSlug": team_slug, + "teamId": team_id, + "signature": true, + "preflight": true + } + })) + .unwrap(), + "junk", + ) + .unwrap(); + let config = TurboJsonReader::turbo_json_to_config_options(turbo_json).unwrap(); + assert!(config.enabled()); + assert_eq!(config.timeout(), timeout); + assert_eq!(config.upload_timeout(), upload_timeout); + assert_eq!(config.api_url(), api_url); + assert_eq!(config.login_url(), login_url); + assert_eq!(config.team_slug(), Some(team_slug)); + assert_eq!(config.team_id(), Some(team_id)); + assert!(config.signature()); + assert!(config.preflight()); + } } diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index 25c5f02f84ba9..6c22dfb8c6713 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -78,6 +78,8 @@ pub(crate) struct RawRemoteCacheOptions { timeout: Option, #[serde(skip_serializing_if = "Option::is_none")] enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + upload_timeout: Option, } impl From<&RawRemoteCacheOptions> for ConfigurationOptions { @@ -90,6 +92,7 @@ impl From<&RawRemoteCacheOptions> for ConfigurationOptions { signature: remote_cache_opts.signature, preflight: remote_cache_opts.preflight, timeout: remote_cache_opts.timeout, + upload_timeout: remote_cache_opts.upload_timeout, enabled: remote_cache_opts.enabled, ..Self::default() } From fc450f07255eccf601bedcba87c95c9d8484d024 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Thu, 21 Nov 2024 16:46:44 -0500 Subject: [PATCH 4/4] chore: document remoteCache.uploadTimeout --- docs/repo-docs/reference/configuration.mdx | 12 ++++++++++++ packages/turbo-types/schemas/schema.json | 5 +++++ packages/turbo-types/schemas/schema.v2.json | 5 +++++ packages/turbo-types/src/types/config-v2.ts | 8 ++++++++ 4 files changed, 30 insertions(+) diff --git a/docs/repo-docs/reference/configuration.mdx b/docs/repo-docs/reference/configuration.mdx index 4d3a7553c00a0..828fdcac3ebff 100644 --- a/docs/repo-docs/reference/configuration.mdx +++ b/docs/repo-docs/reference/configuration.mdx @@ -4,6 +4,7 @@ description: Learn how to configure Turborepo through `turbo.json`. --- import { Callout } from '#/components/callout'; +import { InVersion } from '#/components/in-version'; import Link from 'next/link'; Configure the behavior of `turbo` by adding a `turbo.json` file in your Workspace's root directory. @@ -533,6 +534,17 @@ Sets a timeout for remote cache operations. Value is given in seconds and only whole values are accepted. If `0` is passed, then there is no timeout for any cache operations. + +### `uploadTimeout` + +Default: `60` + +Sets a timeout for remote cache uploads. +Value is given in seconds and only whole values are accepted. +If `0` is passed, then there is no timeout for any remote cache uploads. + + + ### `apiUrl` Default: `"https://vercel.com"` diff --git a/packages/turbo-types/schemas/schema.json b/packages/turbo-types/schemas/schema.json index 86fdcf0476344..4d772ca14f6ce 100644 --- a/packages/turbo-types/schemas/schema.json +++ b/packages/turbo-types/schemas/schema.json @@ -213,6 +213,11 @@ "type": "number", "description": "Sets a timeout for remote cache operations. Value is given in seconds and only whole values are accepted. If `0` is passed, then there is no timeout for any cache operations.", "default": 30 + }, + "uploadTimeout": { + "type": "number", + "description": "Sets a timeout for remote cache uploads. Value is given in seconds and only whole values are accepted. If `0` is passed, then there is no timeout for any remote cache uploads.", + "default": 60 } }, "additionalProperties": false diff --git a/packages/turbo-types/schemas/schema.v2.json b/packages/turbo-types/schemas/schema.v2.json index 86fdcf0476344..4d772ca14f6ce 100644 --- a/packages/turbo-types/schemas/schema.v2.json +++ b/packages/turbo-types/schemas/schema.v2.json @@ -213,6 +213,11 @@ "type": "number", "description": "Sets a timeout for remote cache operations. Value is given in seconds and only whole values are accepted. If `0` is passed, then there is no timeout for any cache operations.", "default": 30 + }, + "uploadTimeout": { + "type": "number", + "description": "Sets a timeout for remote cache uploads. Value is given in seconds and only whole values are accepted. If `0` is passed, then there is no timeout for any remote cache uploads.", + "default": 60 } }, "additionalProperties": false diff --git a/packages/turbo-types/src/types/config-v2.ts b/packages/turbo-types/src/types/config-v2.ts index 1be725d352404..8b6206622f5e0 100644 --- a/packages/turbo-types/src/types/config-v2.ts +++ b/packages/turbo-types/src/types/config-v2.ts @@ -336,6 +336,14 @@ export interface RemoteCache { * @defaultValue `30` */ timeout?: number; + /** + * Sets a timeout for remote cache uploads. Value is given in seconds and + * only whole values are accepted. If `0` is passed, then there is no timeout + * for any remote cache uploads. + * + * @defaultValue `60` + */ + uploadTimeout?: number; } export const isRootSchemaV2 = (schema: Schema): schema is RootSchema =>