From a0bc13386cacf67862954a7faf9fbe46a3985ee8 Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Wed, 12 Feb 2025 23:53:29 -0500 Subject: [PATCH 01/17] fix affected_packages --- Cargo.lock | 1 + .../src/package_changes_watcher.rs | 7 ++- .../src/run/scope/change_detector.rs | 23 ++++----- .../src/change_mapper/mod.rs | 50 ++++++++++++++----- .../src/change_mapper/package.rs | 7 +-- crates/turborepo-scm/src/git.rs | 1 + packages/turbo-repository/js/index.d.ts | 3 +- packages/turbo-repository/rust/Cargo.toml | 1 + packages/turbo-repository/rust/src/lib.rs | 40 ++++++++++++--- 9 files changed, 95 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38fdac8c4f000..14ce161d2b1cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6707,6 +6707,7 @@ dependencies = [ "tokio", "turbopath", "turborepo-repository", + "turborepo-scm", ] [[package]] diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index cc4cf7bfb19e2..27452a219bfcf 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -15,7 +15,9 @@ use turborepo_filewatch::{ NotifyError, OptionalWatch, }; use turborepo_repository::{ - change_mapper::{ChangeMapper, GlobalDepsPackageChangeMapper, PackageChanges}, + change_mapper::{ + ChangeMapper, GlobalDepsPackageChangeMapper, LockfileContents, PackageChanges, + }, package_graph::{PackageGraph, PackageGraphBuilder, PackageName, WorkspacePackage}, package_json::PackageJson, }; @@ -342,7 +344,8 @@ impl Subscriber { continue; } - let changed_packages = change_mapper.changed_packages(changed_files.clone(), None); + let changed_packages = change_mapper + .changed_packages(changed_files.clone(), LockfileContents::Unknown); tracing::warn!("changed_files: {:?}", changed_files); tracing::warn!("changed_packages: {:?}", changed_packages); diff --git a/crates/turborepo-lib/src/run/scope/change_detector.rs b/crates/turborepo-lib/src/run/scope/change_detector.rs index e0f5b289ac17e..670b17d491ae6 100644 --- a/crates/turborepo-lib/src/run/scope/change_detector.rs +++ b/crates/turborepo-lib/src/run/scope/change_detector.rs @@ -5,7 +5,7 @@ use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf}; use turborepo_repository::{ change_mapper::{ AllPackageChangeReason, ChangeMapper, DefaultPackageChangeMapper, Error, - GlobalDepsPackageChangeMapper, PackageChanges, PackageInclusionReason, + GlobalDepsPackageChangeMapper, LockfileContents, PackageChanges, PackageInclusionReason, }, package_graph::{PackageGraph, PackageName}, }; @@ -52,13 +52,12 @@ impl<'a> ScopeChangeDetector<'a> { } /// Gets the lockfile content from SCM if it has changed. - /// Does *not* error if cannot get content, instead just - /// returns a Some(None) - fn get_lockfile_contents( + /// Does *not* error if cannot get content. + pub fn get_lockfile_contents( &self, from_ref: Option<&str>, changed_files: &HashSet, - ) -> Option>> { + ) -> LockfileContents { let lockfile_path = self .pkg_graph .package_manager() @@ -70,23 +69,21 @@ impl<'a> ScopeChangeDetector<'a> { &lockfile_path, ) { debug!("lockfile did not change"); - return None; + return LockfileContents::Unchanged; } - let lockfile_path = self - .pkg_graph - .package_manager() - .lockfile_path(self.turbo_root); - let Ok(content) = self.scm.previous_content(from_ref, &lockfile_path) else { - return Some(None); + debug!("lockfile did change but could not get previous content"); + return LockfileContents::UnknownChange; }; - Some(Some(content)) + debug!("lockfile changed, have the previous content"); + LockfileContents::Changed(content) } } impl<'a> GitChangeDetector for ScopeChangeDetector<'a> { + /// get the actual changed packages between two git refs fn changed_packages( &self, from_ref: Option<&str>, diff --git a/crates/turborepo-repository/src/change_mapper/mod.rs b/crates/turborepo-repository/src/change_mapper/mod.rs index dce6b0325d323..65bd608c13dd3 100644 --- a/crates/turborepo-repository/src/change_mapper/mod.rs +++ b/crates/turborepo-repository/src/change_mapper/mod.rs @@ -29,6 +29,21 @@ pub enum LockfileChange { ChangedPackages(HashSet), } +/// This describes the state of a change to a lockfile. +pub enum LockfileContents { + /// We know the lockfile did not change + Unchanged, + /// We know the lockfile changed but don't have the file contents of the + /// previous lockfile (i.e. `git status`, or perhaps a lockfile that was + /// deleted or otherwise inaccessible with the information we have) + UnknownChange, + /// We are unsure whether the lockfile changed or not + Unknown, + /// We know the lockfile changed and have the contents of the previous + /// lockfile + Changed(Vec), +} + #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub enum PackageInclusionReason { /// All the packages are invalidated @@ -122,13 +137,7 @@ impl<'a, PD: PackageChangeMapper> ChangeMapper<'a, PD> { pub fn changed_packages( &self, changed_files: HashSet, - // None - we don't know if the lockfile changed - // - // Some(None) - we know the lockfile changed, but don't know exactly why (i.e. `git status` - // and the lockfile is there) - // - // Some(Some(content)) - we know the lockfile changed and have the contents - lockfile_change: Option>>, + lockfile_contents: LockfileContents, ) -> Result { if let Some(file) = Self::default_global_file_changed(&changed_files) { debug!("global file changed"); @@ -142,15 +151,16 @@ impl<'a, PD: PackageChangeMapper> ChangeMapper<'a, PD> { // get filtered files and add the packages that contain them let filtered_changed_files = self.filter_ignored_files(changed_files.iter())?; + // calculate lockfile_change here based on changed_files match self.get_changed_packages(filtered_changed_files.into_iter()) { PackageChanges::All(reason) => Ok(PackageChanges::All(reason)), PackageChanges::Some(mut changed_pkgs) => { - match lockfile_change { - Some(Some(content)) => { + match lockfile_contents { + LockfileContents::Changed(previous_lockfile_contents) => { // if we run into issues, don't error, just assume all packages have changed let Ok(lockfile_changes) = - self.get_changed_packages_from_lockfile(&content) + self.get_changed_packages_from_lockfile(&previous_lockfile_contents) else { debug!( "unable to determine lockfile changes, assuming all packages \ @@ -183,13 +193,27 @@ impl<'a, PD: PackageChangeMapper> ChangeMapper<'a, PD> { } // We don't have the actual contents, so just invalidate everything - Some(None) => { - debug!("no previous lockfile available, assuming all packages changed"); + LockfileContents::UnknownChange => { + // this can happen in a blobless checkout ((TODO check whether this is + // treeless)) + debug!( + "we know the lockfile changed but we don't have the contents so we \ + have to assume all packages changed and rebuild everything" + ); Ok(PackageChanges::All( AllPackageChangeReason::LockfileChangedWithoutDetails, )) } - None => Ok(PackageChanges::Some(changed_pkgs)), + + // We don't know if the lockfile changed or not, so we can't assume anything + LockfileContents::Unknown => { + debug!("no previous lockfile available, assuming all packages changed"); + Ok(PackageChanges::Some(changed_pkgs)) + } + LockfileContents::Unchanged => { + debug!("the lockfile did not change"); + Ok(PackageChanges::Some(changed_pkgs)) + } } } } diff --git a/crates/turborepo-repository/src/change_mapper/package.rs b/crates/turborepo-repository/src/change_mapper/package.rs index 7c6d57f96265b..d98be0a054e2b 100644 --- a/crates/turborepo-repository/src/change_mapper/package.rs +++ b/crates/turborepo-repository/src/change_mapper/package.rs @@ -160,7 +160,8 @@ mod tests { use super::{DefaultPackageChangeMapper, GlobalDepsPackageChangeMapper}; use crate::{ change_mapper::{ - AllPackageChangeReason, ChangeMapper, PackageChanges, PackageInclusionReason, + AllPackageChangeReason, ChangeMapper, LockfileContents, PackageChanges, + PackageInclusionReason, }, discovery::{self, PackageDiscovery}, package_graph::{PackageGraphBuilder, WorkspacePackage}, @@ -208,7 +209,7 @@ mod tests { [AnchoredSystemPathBuf::from_raw("README.md")?] .into_iter() .collect(), - None, + LockfileContents::Unknown, )?; // We should return All because we don't have global deps and @@ -228,7 +229,7 @@ mod tests { [AnchoredSystemPathBuf::from_raw("README.md")?] .into_iter() .collect(), - None, + LockfileContents::Unknown, )?; // We only get a root workspace change since we have global deps specified and diff --git a/crates/turborepo-scm/src/git.rs b/crates/turborepo-scm/src/git.rs index c9f57e454f4d8..9e6d51ef8b234 100644 --- a/crates/turborepo-scm/src/git.rs +++ b/crates/turborepo-scm/src/git.rs @@ -37,6 +37,7 @@ impl SCM { } } + /// get the actual changed files between two git refs pub fn changed_files( &self, turbo_root: &AbsoluteSystemPath, diff --git a/packages/turbo-repository/js/index.d.ts b/packages/turbo-repository/js/index.d.ts index 26d252b6f8741..9459bffcfb71f 100644 --- a/packages/turbo-repository/js/index.d.ts +++ b/packages/turbo-repository/js/index.d.ts @@ -57,6 +57,7 @@ export class Workspace { */ affectedPackages( files: Array, - changedLockfile?: string | undefined | null + comparison?: string | undefined | null, + optimizeGlobalInvalidations?: boolean | undefined | null ): Promise>; } diff --git a/packages/turbo-repository/rust/Cargo.toml b/packages/turbo-repository/rust/Cargo.toml index 6a97de05ced1e..4002be42fd3bd 100644 --- a/packages/turbo-repository/rust/Cargo.toml +++ b/packages/turbo-repository/rust/Cargo.toml @@ -17,6 +17,7 @@ thiserror = { workspace = true } tokio = { workspace = true } turbopath = { workspace = true } turborepo-repository = { workspace = true } +turborepo-scm = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/packages/turbo-repository/rust/src/lib.rs b/packages/turbo-repository/rust/src/lib.rs index 8460427b6c39f..de4cb68257643 100644 --- a/packages/turbo-repository/rust/src/lib.rs +++ b/packages/turbo-repository/rust/src/lib.rs @@ -5,12 +5,15 @@ use std::{ use napi::Error; use napi_derive::napi; -use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf}; +use turbopath::{AbsoluteSystemPath, AnchoredSystemPath, AnchoredSystemPathBuf}; use turborepo_repository::{ - change_mapper::{ChangeMapper, GlobalDepsPackageChangeMapper, PackageChanges}, + change_mapper::{ + ChangeMapper, GlobalDepsPackageChangeMapper, LockfileContents, PackageChanges, + }, inference::RepoState as WorkspaceState, package_graph::{PackageGraph, PackageName, PackageNode, WorkspacePackage, ROOT_PKG_NAME}, }; +use turborepo_scm::SCM; mod internal; #[napi] @@ -182,6 +185,24 @@ impl Workspace { Ok(map) } + pub fn get_lockfile_contents( + &self, + changed_files: &HashSet, + workspace_root: &AbsoluteSystemPath, + from_commit: Option<&str>, + ) -> LockfileContents { + let lockfile_name = self.graph.package_manager().lockfile_name(); + changed_files + .get(AnchoredSystemPath::new(&lockfile_name).unwrap()) + .and_then(|_| { + let git = SCM::new(workspace_root); + let anchored_path = workspace_root.join_component(&lockfile_name); + let contents = git.previous_content(from_commit, &anchored_path).unwrap(); + Some(LockfileContents::Changed(contents)) + }) + .unwrap_or(LockfileContents::Unknown) + } + /// Given a set of "changed" files, returns a set of packages that are /// "affected" by the changes. The `files` argument is expected to be a list /// of strings relative to the monorepo root and use the current system's @@ -190,13 +211,14 @@ impl Workspace { pub async fn affected_packages( &self, files: Vec, - changed_lockfile: Option, + comparison: Option<&str>, // this is required when optimize_global_invalidations is true + optimize_global_invalidations: Option, ) -> Result, Error> { let workspace_root = match AbsoluteSystemPath::new(&self.absolute_path) { Ok(path) => path, Err(e) => return Err(Error::from_reason(e.to_string())), }; - let hash_set_of_paths: HashSet = files + let changed_files: HashSet = files .into_iter() .filter_map(|path| { let path_components = path.split(std::path::MAIN_SEPARATOR).collect::>(); @@ -209,8 +231,14 @@ impl Workspace { let global_deps_package_detector = GlobalDepsPackageChangeMapper::new(&self.graph, std::iter::empty::<&str>()).unwrap(); let mapper = ChangeMapper::new(&self.graph, vec![], global_deps_package_detector); - let lockfile_change = changed_lockfile.map(|s| Some(s.into_bytes())); - let package_changes = match mapper.changed_packages(hash_set_of_paths, lockfile_change) { + + let lockfile_contents = if let Some(true) = optimize_global_invalidations { + self.get_lockfile_contents(&changed_files, &workspace_root, comparison) + } else { + LockfileContents::Unknown + }; + + let package_changes = match mapper.changed_packages(changed_files, lockfile_contents) { Ok(changes) => changes, Err(e) => return Err(Error::from_reason(e.to_string())), }; From e31882b1e6c1e8db0b063d18b9995a08b5477bf8 Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Thu, 13 Feb 2025 00:05:11 -0500 Subject: [PATCH 02/17] self review --- crates/turborepo-repository/src/change_mapper/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/turborepo-repository/src/change_mapper/mod.rs b/crates/turborepo-repository/src/change_mapper/mod.rs index 65bd608c13dd3..24a1b7f7a1077 100644 --- a/crates/turborepo-repository/src/change_mapper/mod.rs +++ b/crates/turborepo-repository/src/change_mapper/mod.rs @@ -194,8 +194,7 @@ impl<'a, PD: PackageChangeMapper> ChangeMapper<'a, PD> { // We don't have the actual contents, so just invalidate everything LockfileContents::UnknownChange => { - // this can happen in a blobless checkout ((TODO check whether this is - // treeless)) + // this can happen in a blobless checkout debug!( "we know the lockfile changed but we don't have the contents so we \ have to assume all packages changed and rebuild everything" From c764e0c1f644c06d8ef160fc0fa2878b8c4ea5c7 Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Thu, 13 Feb 2025 10:57:58 -0500 Subject: [PATCH 03/17] handle unknown state when failing --- Cargo.lock | 1 + packages/turbo-repository/rust/Cargo.toml | 1 + packages/turbo-repository/rust/src/lib.rs | 14 +++++++++----- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14ce161d2b1cf..c6ed8ac2ab00c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6705,6 +6705,7 @@ dependencies = [ "pretty_assertions", "thiserror", "tokio", + "tracing", "turbopath", "turborepo-repository", "turborepo-scm", diff --git a/packages/turbo-repository/rust/Cargo.toml b/packages/turbo-repository/rust/Cargo.toml index 4002be42fd3bd..bd2c8299252b2 100644 --- a/packages/turbo-repository/rust/Cargo.toml +++ b/packages/turbo-repository/rust/Cargo.toml @@ -18,6 +18,7 @@ tokio = { workspace = true } turbopath = { workspace = true } turborepo-repository = { workspace = true } turborepo-scm = { workspace = true } +tracing = "0.1.37" [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/packages/turbo-repository/rust/src/lib.rs b/packages/turbo-repository/rust/src/lib.rs index de4cb68257643..b655aec3ba1d5 100644 --- a/packages/turbo-repository/rust/src/lib.rs +++ b/packages/turbo-repository/rust/src/lib.rs @@ -5,6 +5,7 @@ use std::{ use napi::Error; use napi_derive::napi; +use tracing::debug; use turbopath::{AbsoluteSystemPath, AnchoredSystemPath, AnchoredSystemPathBuf}; use turborepo_repository::{ change_mapper::{ @@ -193,12 +194,15 @@ impl Workspace { ) -> LockfileContents { let lockfile_name = self.graph.package_manager().lockfile_name(); changed_files - .get(AnchoredSystemPath::new(&lockfile_name).unwrap()) - .and_then(|_| { + .contains(AnchoredSystemPath::new(&lockfile_name).unwrap()) + .then(|| { let git = SCM::new(workspace_root); - let anchored_path = workspace_root.join_component(&lockfile_name); - let contents = git.previous_content(from_commit, &anchored_path).unwrap(); - Some(LockfileContents::Changed(contents)) + let anchored_path = workspace_root.join_component(lockfile_name); + git.previous_content(from_commit, &anchored_path) + .map(LockfileContents::Changed) + .inspect_err(|e| debug!("{e}")) + .ok() + .unwrap_or(LockfileContents::UnknownChange) }) .unwrap_or(LockfileContents::Unknown) } From b6cf2c875877f203d15e1ec6255d28d94e246db3 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Mon, 24 Feb 2025 15:03:19 -0500 Subject: [PATCH 04/17] fix regression and tests --- Cargo.lock | 2 + crates/turborepo-repository/Cargo.toml | 1 + .../src/change_mapper/package.rs | 13 +++++ .../__tests__/affected-packages.test.ts | 57 +++++++------------ packages/turbo-repository/rust/Cargo.toml | 1 + packages/turbo-repository/rust/src/lib.rs | 40 ++++++++++--- 6 files changed, 69 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6ed8ac2ab00c..d06ce10e339b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6699,6 +6699,7 @@ dependencies = [ name = "turborepo-napi" version = "0.1.0" dependencies = [ + "either", "napi", "napi-build", "napi-derive", @@ -6722,6 +6723,7 @@ dependencies = [ "biome_diagnostics", "biome_json_parser", "biome_json_syntax", + "either", "globwalk", "itertools 0.10.5", "lazy-regex", diff --git a/crates/turborepo-repository/Cargo.toml b/crates/turborepo-repository/Cargo.toml index b562597e12cfe..9f95d9f39f0e6 100644 --- a/crates/turborepo-repository/Cargo.toml +++ b/crates/turborepo-repository/Cargo.toml @@ -17,6 +17,7 @@ biome_diagnostics = { workspace = true } biome_json_parser = { workspace = true } biome_json_syntax = { workspace = true } +either = { workspace = true } globwalk = { version = "0.1.0", path = "../turborepo-globwalk" } itertools = { workspace = true } lazy-regex = "2.5.0" diff --git a/crates/turborepo-repository/src/change_mapper/package.rs b/crates/turborepo-repository/src/change_mapper/package.rs index d98be0a054e2b..7ee5932c1d274 100644 --- a/crates/turborepo-repository/src/change_mapper/package.rs +++ b/crates/turborepo-repository/src/change_mapper/package.rs @@ -24,6 +24,19 @@ pub trait PackageChangeMapper { fn detect_package(&self, file: &AnchoredSystemPath) -> PackageMapping; } +impl PackageChangeMapper for either::Either +where + L: PackageChangeMapper, + R: PackageChangeMapper, +{ + fn detect_package(&self, file: &AnchoredSystemPath) -> PackageMapping { + match self { + either::Either::Left(l) => l.detect_package(file), + either::Either::Right(r) => r.detect_package(file), + } + } +} + /// Detects package by checking if the file is inside the package. /// /// Does *not* use the `globalDependencies` in turbo.json. diff --git a/packages/turbo-repository/__tests__/affected-packages.test.ts b/packages/turbo-repository/__tests__/affected-packages.test.ts index 4cb62e0b50f6b..fb14a8f6b8c53 100644 --- a/packages/turbo-repository/__tests__/affected-packages.test.ts +++ b/packages/turbo-repository/__tests__/affected-packages.test.ts @@ -1,4 +1,4 @@ -import { describe, it } from "node:test"; +import { beforeEach, describe, it } from "node:test"; import { strict as assert } from "node:assert"; import * as path from "node:path"; import { Workspace, Package, PackageManager } from "../js/dist/index.js"; @@ -8,7 +8,6 @@ type PackageReduced = Pick; interface AffectedPackagesTestParams { description: string; files: string[]; - changedLockfile?: string | undefined | null; expected: PackageReduced[]; } @@ -33,49 +32,22 @@ describe("affectedPackages", () => { ], }, { - description: - "a lockfile change will only affect packages impacted by the change", - files: [], - changedLockfile: `lockfileVersion: '6.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: {} - - apps/app: - dependencies: - microdiff: - specifier: ^1.4.0 - version: 1.5.0 - ui: - specifier: workspace:* - version: link:../../packages/ui - - packages/blank: {} - - packages/ui: {} - -packages: - - /microdiff@1.5.0: - resolution: {integrity: sha512-Drq+/THMvDdzRYrK0oxJmOKiC24ayUV8ahrt8l3oRK51PWt6gdtrIGrlIH3pT/lFh1z93FbAcidtsHcWbnRz8Q==} - dev: false -`, - expected: [{ name: "app-a", relativePath: "apps/app" }], + description: "a lockfile change will affect all packages", + files: ["pnpm-lock.yaml"], + expected: [ + { name: "app-a", relativePath: "apps/app" }, + { name: "ui", relativePath: "packages/ui" }, + ], }, ]; - for (const { description, files, expected, changedLockfile } of tests) { + for (const { description, files, expected } of tests) { it(description, async () => { const dir = path.resolve(__dirname, "./fixtures/monorepo"); const workspace = await Workspace.find(dir); const reduced: PackageReduced[] = ( - await workspace.affectedPackages(files, changedLockfile) + await workspace.affectedPackages(files) ).map((pkg) => { return { name: pkg.name, @@ -86,4 +58,15 @@ packages: assert.deepEqual(reduced, expected); }); } + + describe("optimizedLockfileUpdates", () => { + it("errors if not provided comparison ref", async () => { + const dir = path.resolve(__dirname, "./fixtures/monorepo"); + const workspace = await Workspace.find(dir); + + assert.rejects( + workspace.affectedPackages(["pnpm-lock.yaml"], null, true) + ); + }); + }); }); diff --git a/packages/turbo-repository/rust/Cargo.toml b/packages/turbo-repository/rust/Cargo.toml index bd2c8299252b2..3dd33dfd1d9ad 100644 --- a/packages/turbo-repository/rust/Cargo.toml +++ b/packages/turbo-repository/rust/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib"] workspace = true [dependencies] +either = { workspace = true } napi = { version = "2.14.0", features = ["tokio_rt"] } napi-derive = "2.14.0" thiserror = { workspace = true } diff --git a/packages/turbo-repository/rust/src/lib.rs b/packages/turbo-repository/rust/src/lib.rs index b655aec3ba1d5..deb89e8e9b3dd 100644 --- a/packages/turbo-repository/rust/src/lib.rs +++ b/packages/turbo-repository/rust/src/lib.rs @@ -3,13 +3,15 @@ use std::{ hash::Hash, }; +use either::Either; use napi::Error; use napi_derive::napi; use tracing::debug; use turbopath::{AbsoluteSystemPath, AnchoredSystemPath, AnchoredSystemPathBuf}; use turborepo_repository::{ change_mapper::{ - ChangeMapper, GlobalDepsPackageChangeMapper, LockfileContents, PackageChanges, + ChangeMapper, DefaultPackageChangeMapper, GlobalDepsPackageChangeMapper, LockfileContents, + PackageChanges, }, inference::RepoState as WorkspaceState, package_graph::{PackageGraph, PackageName, PackageNode, WorkspacePackage, ROOT_PKG_NAME}, @@ -190,7 +192,7 @@ impl Workspace { &self, changed_files: &HashSet, workspace_root: &AbsoluteSystemPath, - from_commit: Option<&str>, + from_commit: &str, ) -> LockfileContents { let lockfile_name = self.graph.package_manager().lockfile_name(); changed_files @@ -198,7 +200,7 @@ impl Workspace { .then(|| { let git = SCM::new(workspace_root); let anchored_path = workspace_root.join_component(lockfile_name); - git.previous_content(from_commit, &anchored_path) + git.previous_content(Some(from_commit), &anchored_path) .map(LockfileContents::Changed) .inspect_err(|e| debug!("{e}")) .ok() @@ -218,6 +220,16 @@ impl Workspace { comparison: Option<&str>, // this is required when optimize_global_invalidations is true optimize_global_invalidations: Option, ) -> Result, Error> { + let comparison = optimize_global_invalidations + .unwrap_or(false) + .then(|| { + comparison.ok_or_else(|| { + Error::from_reason( + "optimizeGlobalInvalidations true, but no comparison commit given", + ) + }) + }) + .transpose()?; let workspace_root = match AbsoluteSystemPath::new(&self.absolute_path) { Ok(path) => path, Err(e) => return Err(Error::from_reason(e.to_string())), @@ -232,12 +244,24 @@ impl Workspace { .collect(); // Create a ChangeMapper with no ignore patterns - let global_deps_package_detector = - GlobalDepsPackageChangeMapper::new(&self.graph, std::iter::empty::<&str>()).unwrap(); - let mapper = ChangeMapper::new(&self.graph, vec![], global_deps_package_detector); + let change_detector = comparison + .is_some() + .then(|| { + Either::Left( + GlobalDepsPackageChangeMapper::new(&self.graph, std::iter::empty::<&str>()) + .unwrap(), + ) + }) + .unwrap_or_else(|| Either::Right(DefaultPackageChangeMapper::new(&self.graph))); + let mapper = ChangeMapper::new(&self.graph, vec![], change_detector); - let lockfile_contents = if let Some(true) = optimize_global_invalidations { - self.get_lockfile_contents(&changed_files, &workspace_root, comparison) + let lockfile_contents = if let Some(comparison) = comparison { + self.get_lockfile_contents(&changed_files, workspace_root, comparison) + } else if changed_files.contains( + AnchoredSystemPath::new(self.graph.package_manager().lockfile_name()) + .expect("the lockfile name will not be an absolute path"), + ) { + LockfileContents::UnknownChange } else { LockfileContents::Unknown }; From f0aec47c17b68f81033f58823c32c896a47ed519 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 26 Feb 2025 10:00:19 -0500 Subject: [PATCH 05/17] fix(repository): still assume files not belonging to a package result in a global change (#10047) ### Description Make sure we keep as much of the default behavior as possible. We now have a nesting doll sorta structure for change mappers: ``` Most conservative > Least Conservative Default > DefaultWLockfile > Global ``` --- .../src/change_mapper/mod.rs | 4 +- .../src/change_mapper/package.rs | 62 ++++++++++++------- .../__tests__/affected-packages.test.ts | 23 +++++++ packages/turbo-repository/rust/src/lib.rs | 11 +--- 4 files changed, 69 insertions(+), 31 deletions(-) diff --git a/crates/turborepo-repository/src/change_mapper/mod.rs b/crates/turborepo-repository/src/change_mapper/mod.rs index 24a1b7f7a1077..95e3ee0ab49d8 100644 --- a/crates/turborepo-repository/src/change_mapper/mod.rs +++ b/crates/turborepo-repository/src/change_mapper/mod.rs @@ -7,8 +7,8 @@ use std::{ }; pub use package::{ - DefaultPackageChangeMapper, Error, GlobalDepsPackageChangeMapper, PackageChangeMapper, - PackageMapping, + DefaultPackageChangeMapper, DefaultPackageChangeMapperWithLockfile, Error, + GlobalDepsPackageChangeMapper, PackageChangeMapper, PackageMapping, }; use tracing::debug; use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf}; diff --git a/crates/turborepo-repository/src/change_mapper/package.rs b/crates/turborepo-repository/src/change_mapper/package.rs index 7ee5932c1d274..2fb79fc52044c 100644 --- a/crates/turborepo-repository/src/change_mapper/package.rs +++ b/crates/turborepo-repository/src/change_mapper/package.rs @@ -86,6 +86,43 @@ impl PackageChangeMapper for DefaultPackageChangeMapper<'_> { } } +pub struct DefaultPackageChangeMapperWithLockfile<'a> { + base: DefaultPackageChangeMapper<'a>, +} + +impl<'a> DefaultPackageChangeMapperWithLockfile<'a> { + pub fn new(pkg_dep_graph: &'a PackageGraph) -> Self { + Self { + base: DefaultPackageChangeMapper::new(pkg_dep_graph), + } + } +} + +impl PackageChangeMapper for DefaultPackageChangeMapperWithLockfile<'_> { + fn detect_package(&self, path: &AnchoredSystemPath) -> PackageMapping { + // If we have a lockfile change, we consider this as a root package change, + // since there's a chance that the root package uses a workspace package + // dependency (this is cursed behavior but sadly possible). There's a chance + // that we can make this more accurate by checking which package + // manager, since not all package managers may permit root pulling from + // workspace package dependencies + if PackageManager::supported_managers() + .iter() + .any(|pm| pm.lockfile_name() == path.as_str()) + { + PackageMapping::Package(( + WorkspacePackage { + name: PackageName::Root, + path: AnchoredSystemPathBuf::from_raw("").unwrap(), + }, + PackageInclusionReason::ConservativeRootLockfileChanged, + )) + } else { + self.base.detect_package(path) + } + } +} + #[derive(Error, Debug)] pub enum Error { #[error(transparent)] @@ -101,7 +138,7 @@ pub enum Error { /// changes all packages. Since we have a list of global deps, /// we can check against that and avoid invalidating in unnecessary cases. pub struct GlobalDepsPackageChangeMapper<'a> { - pkg_dep_graph: &'a PackageGraph, + base: DefaultPackageChangeMapperWithLockfile<'a>, global_deps_matcher: wax::Any<'a>, } @@ -110,10 +147,11 @@ impl<'a> GlobalDepsPackageChangeMapper<'a> { pkg_dep_graph: &'a PackageGraph, global_deps: I, ) -> Result { + let base = DefaultPackageChangeMapperWithLockfile::new(pkg_dep_graph); let global_deps_matcher = wax::any(global_deps)?; Ok(Self { - pkg_dep_graph, + base, global_deps_matcher, }) } @@ -121,25 +159,7 @@ impl<'a> GlobalDepsPackageChangeMapper<'a> { impl PackageChangeMapper for GlobalDepsPackageChangeMapper<'_> { fn detect_package(&self, path: &AnchoredSystemPath) -> PackageMapping { - // If we have a lockfile change, we consider this as a root package change, - // since there's a chance that the root package uses a workspace package - // dependency (this is cursed behavior but sadly possible). There's a chance - // that we can make this more accurate by checking which package - // manager, since not all package managers may permit root pulling from - // workspace package dependencies - if PackageManager::supported_managers() - .iter() - .any(|pm| pm.lockfile_name() == path.as_str()) - { - return PackageMapping::Package(( - WorkspacePackage { - name: PackageName::Root, - path: AnchoredSystemPathBuf::from_raw("").unwrap(), - }, - PackageInclusionReason::ConservativeRootLockfileChanged, - )); - } - match DefaultPackageChangeMapper::new(self.pkg_dep_graph).detect_package(path) { + match self.base.detect_package(path) { // Since `DefaultPackageChangeMapper` is overly conservative, we can check here if // the path is actually in globalDeps and if not, return it as // PackageDetection::Package(WorkspacePackage::root()). diff --git a/packages/turbo-repository/__tests__/affected-packages.test.ts b/packages/turbo-repository/__tests__/affected-packages.test.ts index fb14a8f6b8c53..7c4a1008fea10 100644 --- a/packages/turbo-repository/__tests__/affected-packages.test.ts +++ b/packages/turbo-repository/__tests__/affected-packages.test.ts @@ -68,5 +68,28 @@ describe("affectedPackages", () => { workspace.affectedPackages(["pnpm-lock.yaml"], null, true) ); }); + + it("still considers root file changes as global", async () => { + const dir = path.resolve(__dirname, "./fixtures/monorepo"); + const workspace = await Workspace.find(dir); + + const reduced: PackageReduced[] = ( + await workspace.affectedPackages( + ["file-we-do-not-understand.txt"], + "HEAD", + true + ) + ).map((pkg) => { + return { + name: pkg.name, + relativePath: pkg.relativePath, + }; + }); + + assert.deepEqual(reduced, [ + { name: "app-a", relativePath: "apps/app" }, + { name: "ui", relativePath: "packages/ui" }, + ]); + }); }); }); diff --git a/packages/turbo-repository/rust/src/lib.rs b/packages/turbo-repository/rust/src/lib.rs index deb89e8e9b3dd..71bd0e8728922 100644 --- a/packages/turbo-repository/rust/src/lib.rs +++ b/packages/turbo-repository/rust/src/lib.rs @@ -10,8 +10,8 @@ use tracing::debug; use turbopath::{AbsoluteSystemPath, AnchoredSystemPath, AnchoredSystemPathBuf}; use turborepo_repository::{ change_mapper::{ - ChangeMapper, DefaultPackageChangeMapper, GlobalDepsPackageChangeMapper, LockfileContents, - PackageChanges, + ChangeMapper, DefaultPackageChangeMapper, DefaultPackageChangeMapperWithLockfile, + LockfileContents, PackageChanges, }, inference::RepoState as WorkspaceState, package_graph::{PackageGraph, PackageName, PackageNode, WorkspacePackage, ROOT_PKG_NAME}, @@ -246,12 +246,7 @@ impl Workspace { // Create a ChangeMapper with no ignore patterns let change_detector = comparison .is_some() - .then(|| { - Either::Left( - GlobalDepsPackageChangeMapper::new(&self.graph, std::iter::empty::<&str>()) - .unwrap(), - ) - }) + .then(|| Either::Left(DefaultPackageChangeMapperWithLockfile::new(&self.graph))) .unwrap_or_else(|| Either::Right(DefaultPackageChangeMapper::new(&self.graph))); let mapper = ChangeMapper::new(&self.graph, vec![], change_detector); From a1ae291ff78e38d8a010ce959abc8a5640b007ad Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 28 Feb 2025 16:50:28 -0500 Subject: [PATCH 06/17] chore(@turbo/repository): no longer release glibc version (#10062) ### Description libgit2 doesn't like cross compilation so lets stop trying to do it ### Testing Instructions https://github.com/vercel/turborepo/actions/runs/13596017421 --- .../workflows/turborepo-library-release.yml | 30 -------- .../workflows/turborepo-native-lib-test.yml | 22 +++++- packages/turbo-repository/js/index.js | 69 +++---------------- packages/turbo-repository/js/package.json | 2 - packages/turbo-repository/package.json | 6 +- packages/turbo-repository/turbo.json | 4 +- 6 files changed, 36 insertions(+), 97 deletions(-) diff --git a/.github/workflows/turborepo-library-release.yml b/.github/workflows/turborepo-library-release.yml index c4d04c8e87fca..e501b964cafe3 100644 --- a/.github/workflows/turborepo-library-release.yml +++ b/.github/workflows/turborepo-library-release.yml @@ -23,30 +23,6 @@ jobs: - host: macos-latest target: x86_64-apple-darwin - - host: ubuntu-latest - target: aarch64-unknown-linux-gnu - setup: | - sudo apt update - sudo apt install -y g++-aarch64-linux-gnu libc6-dev-arm64-cross xz-utils - mkdir zig - curl --show-error --location https://ziglang.org/builds/zig-linux-x86_64-0.14.0-dev.3028+cdc9d65b0.tar.xz | tar -J -xf - -C zig --strip-components 1 - export PATH=$PATH:$(pwd)/zig - echo "$(pwd)/zig" >> $GITHUB_PATH - - - host: ubuntu-latest - target: x86_64-unknown-linux-gnu - container: amazon/aws-lambda-nodejs:20 - install: | - microdnf install -y gcc gcc-c++ git tar xz - curl https://sh.rustup.rs -sSf | bash -s -- -y - npm i -g pnpm@8.9.0 - mkdir ../zig - curl --show-error --location https://ziglang.org/builds/zig-linux-x86_64-0.14.0-dev.3028+cdc9d65b0.tar.xz | tar -J -xf - -C ../zig --strip-components 1 - export PATH=$PATH:$(pwd)/../zig - echo "$(pwd)/../zig" >> $GITHUB_PATH - setup: | - pnpm install - - host: ubuntu-latest target: x86_64-unknown-linux-musl container: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2023-09-17-alpine @@ -150,9 +126,7 @@ jobs: run: | mv native-packages/turbo-library-aarch64-apple-darwin/@turbo/repository.darwin-arm64.node packages/turbo-repository/npm/darwin-arm64/ mv native-packages/turbo-library-x86_64-apple-darwin/@turbo/repository.darwin-x64.node packages/turbo-repository/npm/darwin-x64/ - mv native-packages/turbo-library-aarch64-unknown-linux-gnu/@turbo/repository.linux-arm64-gnu.node packages/turbo-repository/npm/linux-arm64-gnu/ mv native-packages/turbo-library-aarch64-unknown-linux-musl/@turbo/repository.linux-arm64-musl.node packages/turbo-repository/npm/linux-arm64-musl/ - mv native-packages/turbo-library-x86_64-unknown-linux-gnu/@turbo/repository.linux-x64-gnu.node packages/turbo-repository/npm/linux-x64-gnu/ mv native-packages/turbo-library-x86_64-unknown-linux-musl/@turbo/repository.linux-x64-musl.node packages/turbo-repository/npm/linux-x64-musl/ mv native-packages/turbo-library-aarch64-pc-windows-msvc/@turbo/repository.win32-arm64-msvc.node packages/turbo-repository/npm/win32-arm64-msvc/ mv native-packages/turbo-library-x86_64-pc-windows-msvc/@turbo/repository.win32-x64-msvc.node packages/turbo-repository/npm/win32-x64-msvc/ @@ -167,9 +141,7 @@ jobs: mkdir tarballs npm pack packages/turbo-repository/npm/darwin-arm64 npm pack packages/turbo-repository/npm/darwin-x64 - npm pack packages/turbo-repository/npm/linux-arm64-gnu npm pack packages/turbo-repository/npm/linux-arm64-musl - npm pack packages/turbo-repository/npm/linux-x64-gnu npm pack packages/turbo-repository/npm/linux-x64-musl npm pack packages/turbo-repository/npm/win32-arm64-msvc npm pack packages/turbo-repository/npm/win32-x64-msvc @@ -194,9 +166,7 @@ jobs: TAG="canary" npm publish -ddd --tag ${TAG} --access public turbo-repository-darwin-arm64-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-darwin-x64-${VERSION}.tgz - npm publish -ddd --tag ${TAG} --access public turbo-repository-linux-arm64-gnu-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-linux-arm64-musl-${VERSION}.tgz - npm publish -ddd --tag ${TAG} --access public turbo-repository-linux-x64-gnu-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-linux-x64-musl-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-win32-arm64-msvc-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-win32-x64-msvc-${VERSION}.tgz diff --git a/.github/workflows/turborepo-native-lib-test.yml b/.github/workflows/turborepo-native-lib-test.yml index 9ac95e12f6c42..53c1b1bc082c6 100644 --- a/.github/workflows/turborepo-native-lib-test.yml +++ b/.github/workflows/turborepo-native-lib-test.yml @@ -22,6 +22,7 @@ jobs: os: - name: ubuntu runner: ubuntu-latest + target: x86_64-unknown-linux-musl - name: macos runner: macos-latest node-version: @@ -50,12 +51,31 @@ jobs: github-token: "${{ secrets.GITHUB_TOKEN }}" node-version: ${{ matrix.node-version }} + # If we're on Linux we need to force musl as that's what napi uses to decide the target + - name: Setup compile target + if: ${{ matrix.os.target != '' }} + run: | + sudo apt-get update + sudo apt-get install -y build-essential clang lldb llvm libclang-dev curl musl-tools + rustup show active-toolchain + rustup target add ${{ matrix.os.target }} + - name: Install Global Turbo uses: ./.github/actions/install-global-turbo - name: Run tests + if: ${{ matrix.os.target == '' }} + # Manually set TURBO_API to an empty string to override Hetzner env + run: | + turbo run test --filter "turborepo-repository" --color + env: + NODE_VERSION: ${{ matrix.node-version }} + + - name: Run tests + if: ${{ matrix.os.target != '' }} # Manually set TURBO_API to an empty string to override Hetzner env run: | - TURBO_API= turbo run test --filter "turborepo-repository" --color --env-mode=strict + turbo run test --filter "turborepo-repository" --color env: NODE_VERSION: ${{ matrix.node-version }} + CARGO_BUILD_TARGET: ${{ matrix.os.target }} diff --git a/packages/turbo-repository/js/index.js b/packages/turbo-repository/js/index.js index 98ee1e2e64141..6d1f570e62a4e 100644 --- a/packages/turbo-repository/js/index.js +++ b/packages/turbo-repository/js/index.js @@ -10,44 +10,6 @@ const { join } = require("path"); const { platform, arch } = process; let nativeBinding = null; -let localFileExisted = false; -let loadError = null; - -function isMusl() { - // For Node 10 - if (!process.report || typeof process.report.getReport !== "function") { - try { - const lddPath = require("child_process") - .execSync("which ldd") - .toString() - .trim(); - return readFileSync(lddPath, "utf8").includes("musl"); - } catch (e) { - return true; - } - } else { - const orig = process.report.excludeNetwork; - process.report.excludeNetwork = true; - const { glibcVersionRuntime } = process.report.getReport().header; - process.report.excludeNetwork = orig; - if (typeof glibcVersionRuntime === "string") { - try { - // We support glibc v2.26+ - let [major, minor] = glibcVersionRuntime.split(".", 2); - if (parseInt(major, 10) !== 2) { - return true; - } - if (parseInt(minor, 10) < 26) { - return true; - } - return false; - } catch (e) { - return true; - } - } - return !glibcVersionRuntime; - } -} // TODO: find-up to turbo-repository? This currently only works from turbo-repository/js/dist const localPath = join(__dirname, "..", "..", "native", "@turbo"); @@ -88,28 +50,15 @@ switch (platform) { } break; case "linux": - if (isMusl()) { - switch (arch) { - case "x64": - suffix = "linux-x64-musl"; - break; - case "arm64": - suffix = "linux-arm64-musl"; - break; - default: - throw new Error(`Unsupported architecture on Linux: ${arch}`); - } - } else { - switch (arch) { - case "x64": - suffix = "linux-x64-gnu"; - break; - case "arm64": - suffix = "linux-arm64-gnu"; - break; - default: - throw new Error(`Unsupported architecture on Linux: ${arch}`); - } + switch (arch) { + case "x64": + suffix = "linux-x64-musl"; + break; + case "arm64": + suffix = "linux-arm64-musl"; + break; + default: + throw new Error(`Unsupported architecture on Linux: ${arch}`); } break; default: diff --git a/packages/turbo-repository/js/package.json b/packages/turbo-repository/js/package.json index ce0e9dd01b842..f1c03828eb4e5 100644 --- a/packages/turbo-repository/js/package.json +++ b/packages/turbo-repository/js/package.json @@ -17,8 +17,6 @@ "optionalDependencies": { "@turbo/repository-darwin-x64": "0.0.1-canary.11", "@turbo/repository-darwin-arm64": "0.0.1-canary.11", - "@turbo/repository-linux-x64-gnu": "0.0.1-canary.11", - "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.11", "@turbo/repository-linux-x64-musl": "0.0.1-canary.11", "@turbo/repository-linux-arm64-musl": "0.0.1-canary.11", "@turbo/repository-win32-x64-msvc": "0.0.1-canary.11", diff --git a/packages/turbo-repository/package.json b/packages/turbo-repository/package.json index b336d80b677f1..0bdb3b2807dd0 100644 --- a/packages/turbo-repository/package.json +++ b/packages/turbo-repository/package.json @@ -24,12 +24,12 @@ "napi": { "name": "@turbo/repository", "triples": { - "defaults": true, + "defaults": false, "additional": [ "x86_64-apple-darwin", "aarch64-apple-darwin", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc" ] diff --git a/packages/turbo-repository/turbo.json b/packages/turbo-repository/turbo.json index ba3111139fba9..04eb01b8f0324 100644 --- a/packages/turbo-repository/turbo.json +++ b/packages/turbo-repository/turbo.json @@ -4,7 +4,9 @@ "tasks": { "build": { "dependsOn": ["cli#rust-src"], - "env": ["NODE_VERSION"] + "env": ["NODE_VERSION", "CARGO_BUILD_TARGET"], + // napi swallow compile errors so we cannot cache as we might save a failure + "cache": false }, "test": { "dependsOn": ["build"] From 0895be9323cbde9686ac650d30bfa62149945a99 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 28 Feb 2025 17:05:13 -0500 Subject: [PATCH 07/17] bump version --- packages/turbo-repository/js/package.json | 16 +++++++++------- .../npm/darwin-arm64/package.json | 2 +- .../turbo-repository/npm/darwin-x64/package.json | 2 +- .../npm/linux-arm64-gnu/package.json | 2 +- .../npm/linux-arm64-musl/package.json | 2 +- .../npm/linux-x64-gnu/package.json | 2 +- .../npm/linux-x64-musl/package.json | 2 +- .../npm/win32-arm64-msvc/package.json | 2 +- .../npm/win32-x64-msvc/package.json | 2 +- 9 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/turbo-repository/js/package.json b/packages/turbo-repository/js/package.json index f1c03828eb4e5..c2e4d5bce0a25 100644 --- a/packages/turbo-repository/js/package.json +++ b/packages/turbo-repository/js/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "description": "", "bugs": "https://github.com/vercel/turborepo/issues", "homepage": "https://turbo.build/repo", @@ -15,11 +15,13 @@ ], "types": "dist/index.d.ts", "optionalDependencies": { - "@turbo/repository-darwin-x64": "0.0.1-canary.11", - "@turbo/repository-darwin-arm64": "0.0.1-canary.11", - "@turbo/repository-linux-x64-musl": "0.0.1-canary.11", - "@turbo/repository-linux-arm64-musl": "0.0.1-canary.11", - "@turbo/repository-win32-x64-msvc": "0.0.1-canary.11", - "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.11" + "@turbo/repository-darwin-x64": "0.0.1-canary.12", + "@turbo/repository-darwin-arm64": "0.0.1-canary.12", + "@turbo/repository-linux-x64-musl": "0.0.1-canary.12", + "@turbo/repository-linux-arm64-musl": "0.0.1-canary.12", + "@turbo/repository-win32-x64-msvc": "0.0.1-canary.12", + "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.12", + "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.12", + "@turbo/repository-linux-x64-gnu": "0.0.1-canary.12" } } diff --git a/packages/turbo-repository/npm/darwin-arm64/package.json b/packages/turbo-repository/npm/darwin-arm64/package.json index 29f8d0a45b68e..11246bcc74b1e 100644 --- a/packages/turbo-repository/npm/darwin-arm64/package.json +++ b/packages/turbo-repository/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-darwin-arm64", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/darwin-x64/package.json b/packages/turbo-repository/npm/darwin-x64/package.json index a997c82d6cac3..94209492d161a 100644 --- a/packages/turbo-repository/npm/darwin-x64/package.json +++ b/packages/turbo-repository/npm/darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-darwin-x64", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-arm64-gnu/package.json b/packages/turbo-repository/npm/linux-arm64-gnu/package.json index e3cd548846452..13d6388096b3e 100644 --- a/packages/turbo-repository/npm/linux-arm64-gnu/package.json +++ b/packages/turbo-repository/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-arm64-gnu", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-arm64-musl/package.json b/packages/turbo-repository/npm/linux-arm64-musl/package.json index fa2e43c3eb178..588e598825133 100644 --- a/packages/turbo-repository/npm/linux-arm64-musl/package.json +++ b/packages/turbo-repository/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-arm64-musl", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-x64-gnu/package.json b/packages/turbo-repository/npm/linux-x64-gnu/package.json index af1f5c2ab7876..259b2f27dda77 100644 --- a/packages/turbo-repository/npm/linux-x64-gnu/package.json +++ b/packages/turbo-repository/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-x64-gnu", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-x64-musl/package.json b/packages/turbo-repository/npm/linux-x64-musl/package.json index 337e4ac3242db..fb64a55db4a6e 100644 --- a/packages/turbo-repository/npm/linux-x64-musl/package.json +++ b/packages/turbo-repository/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-x64-musl", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/win32-arm64-msvc/package.json b/packages/turbo-repository/npm/win32-arm64-msvc/package.json index 0038398355e7b..7305a534f0548 100644 --- a/packages/turbo-repository/npm/win32-arm64-msvc/package.json +++ b/packages/turbo-repository/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-win32-arm64-msvc", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/win32-x64-msvc/package.json b/packages/turbo-repository/npm/win32-x64-msvc/package.json index b861ab970ac4d..55e8078e5a078 100644 --- a/packages/turbo-repository/npm/win32-x64-msvc/package.json +++ b/packages/turbo-repository/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-win32-x64-msvc", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.12", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", From 84f58aaf07f2ad4b11184c64ee92cc33a48b409f Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 28 Feb 2025 17:07:38 -0500 Subject: [PATCH 08/17] remove generated gnu packages --- packages/turbo-repository/js/package.json | 4 +-- .../npm/linux-arm64-gnu/package.json | 26 ------------------- .../npm/linux-x64-gnu/package.json | 26 ------------------- 3 files changed, 1 insertion(+), 55 deletions(-) delete mode 100644 packages/turbo-repository/npm/linux-arm64-gnu/package.json delete mode 100644 packages/turbo-repository/npm/linux-x64-gnu/package.json diff --git a/packages/turbo-repository/js/package.json b/packages/turbo-repository/js/package.json index c2e4d5bce0a25..29cf426bca310 100644 --- a/packages/turbo-repository/js/package.json +++ b/packages/turbo-repository/js/package.json @@ -20,8 +20,6 @@ "@turbo/repository-linux-x64-musl": "0.0.1-canary.12", "@turbo/repository-linux-arm64-musl": "0.0.1-canary.12", "@turbo/repository-win32-x64-msvc": "0.0.1-canary.12", - "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.12", - "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.12", - "@turbo/repository-linux-x64-gnu": "0.0.1-canary.12" + "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.12" } } diff --git a/packages/turbo-repository/npm/linux-arm64-gnu/package.json b/packages/turbo-repository/npm/linux-arm64-gnu/package.json deleted file mode 100644 index 13d6388096b3e..0000000000000 --- a/packages/turbo-repository/npm/linux-arm64-gnu/package.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "@turbo/repository-linux-arm64-gnu", - "version": "0.0.1-canary.12", - "repository": { - "type": "git", - "url": "https://github.com/vercel/turborepo", - "directory": "packages/turbo-repository/npm/linux-arm64-gnu" - }, - "os": [ - "linux" - ], - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "main": "repository.linux-arm64-gnu.node", - "files": [ - "repository.linux-arm64-gnu.node" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - } -} diff --git a/packages/turbo-repository/npm/linux-x64-gnu/package.json b/packages/turbo-repository/npm/linux-x64-gnu/package.json deleted file mode 100644 index 259b2f27dda77..0000000000000 --- a/packages/turbo-repository/npm/linux-x64-gnu/package.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "@turbo/repository-linux-x64-gnu", - "version": "0.0.1-canary.12", - "repository": { - "type": "git", - "url": "https://github.com/vercel/turborepo", - "directory": "packages/turbo-repository/npm/linux-x64-gnu" - }, - "os": [ - "linux" - ], - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "main": "repository.linux-x64-gnu.node", - "files": [ - "repository.linux-x64-gnu.node" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - } -} From 749a13f6f179f443f70819dea1eda8a7d06f3bd1 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Mon, 3 Mar 2025 21:58:57 -0500 Subject: [PATCH 09/17] chore(release): add back gnu builds (#10082) ### Description Revert #10062 as we need them and now we avoid cross compiling `git2` ### Testing Instructions [Test run](https://github.com/vercel/turborepo/actions/runs/13644965628) --- .../workflows/turborepo-library-release.yml | 30 ++++++++ .../workflows/turborepo-native-lib-test.yml | 22 +----- packages/turbo-repository/js/index.js | 69 ++++++++++++++++--- packages/turbo-repository/js/package.json | 2 + .../npm/linux-arm64-gnu/package.json | 26 +++++++ .../npm/linux-x64-gnu/package.json | 26 +++++++ packages/turbo-repository/package.json | 6 +- packages/turbo-repository/turbo.json | 4 +- 8 files changed, 149 insertions(+), 36 deletions(-) create mode 100644 packages/turbo-repository/npm/linux-arm64-gnu/package.json create mode 100644 packages/turbo-repository/npm/linux-x64-gnu/package.json diff --git a/.github/workflows/turborepo-library-release.yml b/.github/workflows/turborepo-library-release.yml index e501b964cafe3..c4d04c8e87fca 100644 --- a/.github/workflows/turborepo-library-release.yml +++ b/.github/workflows/turborepo-library-release.yml @@ -23,6 +23,30 @@ jobs: - host: macos-latest target: x86_64-apple-darwin + - host: ubuntu-latest + target: aarch64-unknown-linux-gnu + setup: | + sudo apt update + sudo apt install -y g++-aarch64-linux-gnu libc6-dev-arm64-cross xz-utils + mkdir zig + curl --show-error --location https://ziglang.org/builds/zig-linux-x86_64-0.14.0-dev.3028+cdc9d65b0.tar.xz | tar -J -xf - -C zig --strip-components 1 + export PATH=$PATH:$(pwd)/zig + echo "$(pwd)/zig" >> $GITHUB_PATH + + - host: ubuntu-latest + target: x86_64-unknown-linux-gnu + container: amazon/aws-lambda-nodejs:20 + install: | + microdnf install -y gcc gcc-c++ git tar xz + curl https://sh.rustup.rs -sSf | bash -s -- -y + npm i -g pnpm@8.9.0 + mkdir ../zig + curl --show-error --location https://ziglang.org/builds/zig-linux-x86_64-0.14.0-dev.3028+cdc9d65b0.tar.xz | tar -J -xf - -C ../zig --strip-components 1 + export PATH=$PATH:$(pwd)/../zig + echo "$(pwd)/../zig" >> $GITHUB_PATH + setup: | + pnpm install + - host: ubuntu-latest target: x86_64-unknown-linux-musl container: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2023-09-17-alpine @@ -126,7 +150,9 @@ jobs: run: | mv native-packages/turbo-library-aarch64-apple-darwin/@turbo/repository.darwin-arm64.node packages/turbo-repository/npm/darwin-arm64/ mv native-packages/turbo-library-x86_64-apple-darwin/@turbo/repository.darwin-x64.node packages/turbo-repository/npm/darwin-x64/ + mv native-packages/turbo-library-aarch64-unknown-linux-gnu/@turbo/repository.linux-arm64-gnu.node packages/turbo-repository/npm/linux-arm64-gnu/ mv native-packages/turbo-library-aarch64-unknown-linux-musl/@turbo/repository.linux-arm64-musl.node packages/turbo-repository/npm/linux-arm64-musl/ + mv native-packages/turbo-library-x86_64-unknown-linux-gnu/@turbo/repository.linux-x64-gnu.node packages/turbo-repository/npm/linux-x64-gnu/ mv native-packages/turbo-library-x86_64-unknown-linux-musl/@turbo/repository.linux-x64-musl.node packages/turbo-repository/npm/linux-x64-musl/ mv native-packages/turbo-library-aarch64-pc-windows-msvc/@turbo/repository.win32-arm64-msvc.node packages/turbo-repository/npm/win32-arm64-msvc/ mv native-packages/turbo-library-x86_64-pc-windows-msvc/@turbo/repository.win32-x64-msvc.node packages/turbo-repository/npm/win32-x64-msvc/ @@ -141,7 +167,9 @@ jobs: mkdir tarballs npm pack packages/turbo-repository/npm/darwin-arm64 npm pack packages/turbo-repository/npm/darwin-x64 + npm pack packages/turbo-repository/npm/linux-arm64-gnu npm pack packages/turbo-repository/npm/linux-arm64-musl + npm pack packages/turbo-repository/npm/linux-x64-gnu npm pack packages/turbo-repository/npm/linux-x64-musl npm pack packages/turbo-repository/npm/win32-arm64-msvc npm pack packages/turbo-repository/npm/win32-x64-msvc @@ -166,7 +194,9 @@ jobs: TAG="canary" npm publish -ddd --tag ${TAG} --access public turbo-repository-darwin-arm64-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-darwin-x64-${VERSION}.tgz + npm publish -ddd --tag ${TAG} --access public turbo-repository-linux-arm64-gnu-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-linux-arm64-musl-${VERSION}.tgz + npm publish -ddd --tag ${TAG} --access public turbo-repository-linux-x64-gnu-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-linux-x64-musl-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-win32-arm64-msvc-${VERSION}.tgz npm publish -ddd --tag ${TAG} --access public turbo-repository-win32-x64-msvc-${VERSION}.tgz diff --git a/.github/workflows/turborepo-native-lib-test.yml b/.github/workflows/turborepo-native-lib-test.yml index 53c1b1bc082c6..9ac95e12f6c42 100644 --- a/.github/workflows/turborepo-native-lib-test.yml +++ b/.github/workflows/turborepo-native-lib-test.yml @@ -22,7 +22,6 @@ jobs: os: - name: ubuntu runner: ubuntu-latest - target: x86_64-unknown-linux-musl - name: macos runner: macos-latest node-version: @@ -51,31 +50,12 @@ jobs: github-token: "${{ secrets.GITHUB_TOKEN }}" node-version: ${{ matrix.node-version }} - # If we're on Linux we need to force musl as that's what napi uses to decide the target - - name: Setup compile target - if: ${{ matrix.os.target != '' }} - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang lldb llvm libclang-dev curl musl-tools - rustup show active-toolchain - rustup target add ${{ matrix.os.target }} - - name: Install Global Turbo uses: ./.github/actions/install-global-turbo - name: Run tests - if: ${{ matrix.os.target == '' }} - # Manually set TURBO_API to an empty string to override Hetzner env - run: | - turbo run test --filter "turborepo-repository" --color - env: - NODE_VERSION: ${{ matrix.node-version }} - - - name: Run tests - if: ${{ matrix.os.target != '' }} # Manually set TURBO_API to an empty string to override Hetzner env run: | - turbo run test --filter "turborepo-repository" --color + TURBO_API= turbo run test --filter "turborepo-repository" --color --env-mode=strict env: NODE_VERSION: ${{ matrix.node-version }} - CARGO_BUILD_TARGET: ${{ matrix.os.target }} diff --git a/packages/turbo-repository/js/index.js b/packages/turbo-repository/js/index.js index 6d1f570e62a4e..98ee1e2e64141 100644 --- a/packages/turbo-repository/js/index.js +++ b/packages/turbo-repository/js/index.js @@ -10,6 +10,44 @@ const { join } = require("path"); const { platform, arch } = process; let nativeBinding = null; +let localFileExisted = false; +let loadError = null; + +function isMusl() { + // For Node 10 + if (!process.report || typeof process.report.getReport !== "function") { + try { + const lddPath = require("child_process") + .execSync("which ldd") + .toString() + .trim(); + return readFileSync(lddPath, "utf8").includes("musl"); + } catch (e) { + return true; + } + } else { + const orig = process.report.excludeNetwork; + process.report.excludeNetwork = true; + const { glibcVersionRuntime } = process.report.getReport().header; + process.report.excludeNetwork = orig; + if (typeof glibcVersionRuntime === "string") { + try { + // We support glibc v2.26+ + let [major, minor] = glibcVersionRuntime.split(".", 2); + if (parseInt(major, 10) !== 2) { + return true; + } + if (parseInt(minor, 10) < 26) { + return true; + } + return false; + } catch (e) { + return true; + } + } + return !glibcVersionRuntime; + } +} // TODO: find-up to turbo-repository? This currently only works from turbo-repository/js/dist const localPath = join(__dirname, "..", "..", "native", "@turbo"); @@ -50,15 +88,28 @@ switch (platform) { } break; case "linux": - switch (arch) { - case "x64": - suffix = "linux-x64-musl"; - break; - case "arm64": - suffix = "linux-arm64-musl"; - break; - default: - throw new Error(`Unsupported architecture on Linux: ${arch}`); + if (isMusl()) { + switch (arch) { + case "x64": + suffix = "linux-x64-musl"; + break; + case "arm64": + suffix = "linux-arm64-musl"; + break; + default: + throw new Error(`Unsupported architecture on Linux: ${arch}`); + } + } else { + switch (arch) { + case "x64": + suffix = "linux-x64-gnu"; + break; + case "arm64": + suffix = "linux-arm64-gnu"; + break; + default: + throw new Error(`Unsupported architecture on Linux: ${arch}`); + } } break; default: diff --git a/packages/turbo-repository/js/package.json b/packages/turbo-repository/js/package.json index 29cf426bca310..c22b717a22581 100644 --- a/packages/turbo-repository/js/package.json +++ b/packages/turbo-repository/js/package.json @@ -17,6 +17,8 @@ "optionalDependencies": { "@turbo/repository-darwin-x64": "0.0.1-canary.12", "@turbo/repository-darwin-arm64": "0.0.1-canary.12", + "@turbo/repository-linux-x64-gnu": "0.0.1-canary.12", + "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.12", "@turbo/repository-linux-x64-musl": "0.0.1-canary.12", "@turbo/repository-linux-arm64-musl": "0.0.1-canary.12", "@turbo/repository-win32-x64-msvc": "0.0.1-canary.12", diff --git a/packages/turbo-repository/npm/linux-arm64-gnu/package.json b/packages/turbo-repository/npm/linux-arm64-gnu/package.json new file mode 100644 index 0000000000000..13d6388096b3e --- /dev/null +++ b/packages/turbo-repository/npm/linux-arm64-gnu/package.json @@ -0,0 +1,26 @@ +{ + "name": "@turbo/repository-linux-arm64-gnu", + "version": "0.0.1-canary.12", + "repository": { + "type": "git", + "url": "https://github.com/vercel/turborepo", + "directory": "packages/turbo-repository/npm/linux-arm64-gnu" + }, + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "main": "repository.linux-arm64-gnu.node", + "files": [ + "repository.linux-arm64-gnu.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} diff --git a/packages/turbo-repository/npm/linux-x64-gnu/package.json b/packages/turbo-repository/npm/linux-x64-gnu/package.json new file mode 100644 index 0000000000000..259b2f27dda77 --- /dev/null +++ b/packages/turbo-repository/npm/linux-x64-gnu/package.json @@ -0,0 +1,26 @@ +{ + "name": "@turbo/repository-linux-x64-gnu", + "version": "0.0.1-canary.12", + "repository": { + "type": "git", + "url": "https://github.com/vercel/turborepo", + "directory": "packages/turbo-repository/npm/linux-x64-gnu" + }, + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "main": "repository.linux-x64-gnu.node", + "files": [ + "repository.linux-x64-gnu.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} diff --git a/packages/turbo-repository/package.json b/packages/turbo-repository/package.json index 0bdb3b2807dd0..b336d80b677f1 100644 --- a/packages/turbo-repository/package.json +++ b/packages/turbo-repository/package.json @@ -24,12 +24,12 @@ "napi": { "name": "@turbo/repository", "triples": { - "defaults": false, + "defaults": true, "additional": [ "x86_64-apple-darwin", "aarch64-apple-darwin", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc" ] diff --git a/packages/turbo-repository/turbo.json b/packages/turbo-repository/turbo.json index 04eb01b8f0324..ba3111139fba9 100644 --- a/packages/turbo-repository/turbo.json +++ b/packages/turbo-repository/turbo.json @@ -4,9 +4,7 @@ "tasks": { "build": { "dependsOn": ["cli#rust-src"], - "env": ["NODE_VERSION", "CARGO_BUILD_TARGET"], - // napi swallow compile errors so we cannot cache as we might save a failure - "cache": false + "env": ["NODE_VERSION"] }, "test": { "dependsOn": ["build"] From 90b46ea7e3f49de2ffdc7b833697713482390c7a Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Mon, 3 Mar 2025 22:00:43 -0500 Subject: [PATCH 10/17] bump version --- packages/turbo-repository/js/package.json | 18 +++++++++--------- .../npm/darwin-arm64/package.json | 2 +- .../npm/darwin-x64/package.json | 2 +- .../npm/linux-arm64-gnu/package.json | 2 +- .../npm/linux-arm64-musl/package.json | 2 +- .../npm/linux-x64-gnu/package.json | 2 +- .../npm/linux-x64-musl/package.json | 2 +- .../npm/win32-arm64-msvc/package.json | 2 +- .../npm/win32-x64-msvc/package.json | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/turbo-repository/js/package.json b/packages/turbo-repository/js/package.json index c22b717a22581..7a330220545da 100644 --- a/packages/turbo-repository/js/package.json +++ b/packages/turbo-repository/js/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "description": "", "bugs": "https://github.com/vercel/turborepo/issues", "homepage": "https://turbo.build/repo", @@ -15,13 +15,13 @@ ], "types": "dist/index.d.ts", "optionalDependencies": { - "@turbo/repository-darwin-x64": "0.0.1-canary.12", - "@turbo/repository-darwin-arm64": "0.0.1-canary.12", - "@turbo/repository-linux-x64-gnu": "0.0.1-canary.12", - "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.12", - "@turbo/repository-linux-x64-musl": "0.0.1-canary.12", - "@turbo/repository-linux-arm64-musl": "0.0.1-canary.12", - "@turbo/repository-win32-x64-msvc": "0.0.1-canary.12", - "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.12" + "@turbo/repository-darwin-x64": "0.0.1-canary.13", + "@turbo/repository-darwin-arm64": "0.0.1-canary.13", + "@turbo/repository-linux-x64-gnu": "0.0.1-canary.13", + "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.13", + "@turbo/repository-linux-x64-musl": "0.0.1-canary.13", + "@turbo/repository-linux-arm64-musl": "0.0.1-canary.13", + "@turbo/repository-win32-x64-msvc": "0.0.1-canary.13", + "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.13" } } diff --git a/packages/turbo-repository/npm/darwin-arm64/package.json b/packages/turbo-repository/npm/darwin-arm64/package.json index 11246bcc74b1e..f993201c0bade 100644 --- a/packages/turbo-repository/npm/darwin-arm64/package.json +++ b/packages/turbo-repository/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-darwin-arm64", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/darwin-x64/package.json b/packages/turbo-repository/npm/darwin-x64/package.json index 94209492d161a..be86290965fe7 100644 --- a/packages/turbo-repository/npm/darwin-x64/package.json +++ b/packages/turbo-repository/npm/darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-darwin-x64", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-arm64-gnu/package.json b/packages/turbo-repository/npm/linux-arm64-gnu/package.json index 13d6388096b3e..eb528ac64053c 100644 --- a/packages/turbo-repository/npm/linux-arm64-gnu/package.json +++ b/packages/turbo-repository/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-arm64-gnu", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-arm64-musl/package.json b/packages/turbo-repository/npm/linux-arm64-musl/package.json index 588e598825133..6e7edd5f468bb 100644 --- a/packages/turbo-repository/npm/linux-arm64-musl/package.json +++ b/packages/turbo-repository/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-arm64-musl", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-x64-gnu/package.json b/packages/turbo-repository/npm/linux-x64-gnu/package.json index 259b2f27dda77..c9f5709ac0176 100644 --- a/packages/turbo-repository/npm/linux-x64-gnu/package.json +++ b/packages/turbo-repository/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-x64-gnu", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-x64-musl/package.json b/packages/turbo-repository/npm/linux-x64-musl/package.json index fb64a55db4a6e..932d334716877 100644 --- a/packages/turbo-repository/npm/linux-x64-musl/package.json +++ b/packages/turbo-repository/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-x64-musl", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/win32-arm64-msvc/package.json b/packages/turbo-repository/npm/win32-arm64-msvc/package.json index 7305a534f0548..3fb1417a2a06d 100644 --- a/packages/turbo-repository/npm/win32-arm64-msvc/package.json +++ b/packages/turbo-repository/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-win32-arm64-msvc", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/win32-x64-msvc/package.json b/packages/turbo-repository/npm/win32-x64-msvc/package.json index 55e8078e5a078..5e5fc57ac3916 100644 --- a/packages/turbo-repository/npm/win32-x64-msvc/package.json +++ b/packages/turbo-repository/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-win32-x64-msvc", - "version": "0.0.1-canary.12", + "version": "0.0.1-canary.13", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", From eb4d6e0067a06afdc162209b3a2af7fe04df32fd Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Tue, 4 Mar 2025 11:08:41 -0500 Subject: [PATCH 11/17] comparison is base --- packages/turbo-repository/rust/src/lib.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/turbo-repository/rust/src/lib.rs b/packages/turbo-repository/rust/src/lib.rs index 71bd0e8728922..ae8d6f7f7b7ba 100644 --- a/packages/turbo-repository/rust/src/lib.rs +++ b/packages/turbo-repository/rust/src/lib.rs @@ -217,16 +217,14 @@ impl Workspace { pub async fn affected_packages( &self, files: Vec, - comparison: Option<&str>, // this is required when optimize_global_invalidations is true + base: Option<&str>, // this is required when optimize_global_invalidations is true optimize_global_invalidations: Option, ) -> Result, Error> { - let comparison = optimize_global_invalidations + let base = optimize_global_invalidations .unwrap_or(false) .then(|| { - comparison.ok_or_else(|| { - Error::from_reason( - "optimizeGlobalInvalidations true, but no comparison commit given", - ) + base.ok_or_else(|| { + Error::from_reason("optimizeGlobalInvalidations true, but no base commit given") }) }) .transpose()?; @@ -244,14 +242,14 @@ impl Workspace { .collect(); // Create a ChangeMapper with no ignore patterns - let change_detector = comparison + let change_detector = base .is_some() .then(|| Either::Left(DefaultPackageChangeMapperWithLockfile::new(&self.graph))) .unwrap_or_else(|| Either::Right(DefaultPackageChangeMapper::new(&self.graph))); let mapper = ChangeMapper::new(&self.graph, vec![], change_detector); - let lockfile_contents = if let Some(comparison) = comparison { - self.get_lockfile_contents(&changed_files, workspace_root, comparison) + let lockfile_contents = if let Some(base) = base { + self.get_lockfile_contents(&changed_files, workspace_root, base) } else if changed_files.contains( AnchoredSystemPath::new(self.graph.package_manager().lockfile_name()) .expect("the lockfile name will not be an absolute path"), From 09ce77c702f45234ad2d999b3e9f5df9165473ec Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Tue, 4 Mar 2025 11:12:09 -0500 Subject: [PATCH 12/17] rebuild --- packages/turbo-repository/js/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/turbo-repository/js/index.d.ts b/packages/turbo-repository/js/index.d.ts index 9459bffcfb71f..0ebf830aad724 100644 --- a/packages/turbo-repository/js/index.d.ts +++ b/packages/turbo-repository/js/index.d.ts @@ -57,7 +57,7 @@ export class Workspace { */ affectedPackages( files: Array, - comparison?: string | undefined | null, + base?: string | undefined | null, optimizeGlobalInvalidations?: boolean | undefined | null ): Promise>; } From 231980127fc87dbcbe491d42b4a897593bc15d7b Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Tue, 4 Mar 2025 11:12:16 -0500 Subject: [PATCH 13/17] bump version --- packages/turbo-repository/js/package.json | 18 +++++++++--------- .../npm/darwin-arm64/package.json | 2 +- .../npm/darwin-x64/package.json | 2 +- .../npm/linux-arm64-gnu/package.json | 2 +- .../npm/linux-arm64-musl/package.json | 2 +- .../npm/linux-x64-gnu/package.json | 2 +- .../npm/linux-x64-musl/package.json | 2 +- .../npm/win32-arm64-msvc/package.json | 2 +- .../npm/win32-x64-msvc/package.json | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/turbo-repository/js/package.json b/packages/turbo-repository/js/package.json index 7a330220545da..553afe42b6a0b 100644 --- a/packages/turbo-repository/js/package.json +++ b/packages/turbo-repository/js/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "description": "", "bugs": "https://github.com/vercel/turborepo/issues", "homepage": "https://turbo.build/repo", @@ -15,13 +15,13 @@ ], "types": "dist/index.d.ts", "optionalDependencies": { - "@turbo/repository-darwin-x64": "0.0.1-canary.13", - "@turbo/repository-darwin-arm64": "0.0.1-canary.13", - "@turbo/repository-linux-x64-gnu": "0.0.1-canary.13", - "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.13", - "@turbo/repository-linux-x64-musl": "0.0.1-canary.13", - "@turbo/repository-linux-arm64-musl": "0.0.1-canary.13", - "@turbo/repository-win32-x64-msvc": "0.0.1-canary.13", - "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.13" + "@turbo/repository-darwin-x64": "0.0.1-canary.14", + "@turbo/repository-darwin-arm64": "0.0.1-canary.14", + "@turbo/repository-linux-x64-gnu": "0.0.1-canary.14", + "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.14", + "@turbo/repository-linux-x64-musl": "0.0.1-canary.14", + "@turbo/repository-linux-arm64-musl": "0.0.1-canary.14", + "@turbo/repository-win32-x64-msvc": "0.0.1-canary.14", + "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.14" } } diff --git a/packages/turbo-repository/npm/darwin-arm64/package.json b/packages/turbo-repository/npm/darwin-arm64/package.json index f993201c0bade..34260b2f91014 100644 --- a/packages/turbo-repository/npm/darwin-arm64/package.json +++ b/packages/turbo-repository/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-darwin-arm64", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/darwin-x64/package.json b/packages/turbo-repository/npm/darwin-x64/package.json index be86290965fe7..1a3c2d7f17da8 100644 --- a/packages/turbo-repository/npm/darwin-x64/package.json +++ b/packages/turbo-repository/npm/darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-darwin-x64", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-arm64-gnu/package.json b/packages/turbo-repository/npm/linux-arm64-gnu/package.json index eb528ac64053c..098dc2a2f6e86 100644 --- a/packages/turbo-repository/npm/linux-arm64-gnu/package.json +++ b/packages/turbo-repository/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-arm64-gnu", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-arm64-musl/package.json b/packages/turbo-repository/npm/linux-arm64-musl/package.json index 6e7edd5f468bb..bab045e0a1c45 100644 --- a/packages/turbo-repository/npm/linux-arm64-musl/package.json +++ b/packages/turbo-repository/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-arm64-musl", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-x64-gnu/package.json b/packages/turbo-repository/npm/linux-x64-gnu/package.json index c9f5709ac0176..03b50b9f0a448 100644 --- a/packages/turbo-repository/npm/linux-x64-gnu/package.json +++ b/packages/turbo-repository/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-x64-gnu", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-x64-musl/package.json b/packages/turbo-repository/npm/linux-x64-musl/package.json index 932d334716877..da68acdb521cb 100644 --- a/packages/turbo-repository/npm/linux-x64-musl/package.json +++ b/packages/turbo-repository/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-x64-musl", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/win32-arm64-msvc/package.json b/packages/turbo-repository/npm/win32-arm64-msvc/package.json index 3fb1417a2a06d..24b64880eafb5 100644 --- a/packages/turbo-repository/npm/win32-arm64-msvc/package.json +++ b/packages/turbo-repository/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-win32-arm64-msvc", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/win32-x64-msvc/package.json b/packages/turbo-repository/npm/win32-x64-msvc/package.json index 5e5fc57ac3916..0f725e433f07f 100644 --- a/packages/turbo-repository/npm/win32-x64-msvc/package.json +++ b/packages/turbo-repository/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-win32-x64-msvc", - "version": "0.0.1-canary.13", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", From ba4a794915daaa33f80d0bafd9a2002ed83f8afc Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Wed, 5 Mar 2025 07:35:02 -0500 Subject: [PATCH 14/17] Update packages/turbo-repository/rust/src/lib.rs Co-authored-by: Chris Olszewski --- packages/turbo-repository/rust/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/turbo-repository/rust/src/lib.rs b/packages/turbo-repository/rust/src/lib.rs index ae8d6f7f7b7ba..773a9b0699cc8 100644 --- a/packages/turbo-repository/rust/src/lib.rs +++ b/packages/turbo-repository/rust/src/lib.rs @@ -256,7 +256,7 @@ impl Workspace { ) { LockfileContents::UnknownChange } else { - LockfileContents::Unknown + LockfileContents::Unchanged }; let package_changes = match mapper.changed_packages(changed_files, lockfile_contents) { From 373c4fe186dae878e43052bd1e81ff562426c38c Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Wed, 5 Mar 2025 07:35:11 -0500 Subject: [PATCH 15/17] Update packages/turbo-repository/rust/src/lib.rs Co-authored-by: Chris Olszewski --- packages/turbo-repository/rust/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/turbo-repository/rust/src/lib.rs b/packages/turbo-repository/rust/src/lib.rs index 773a9b0699cc8..85dd4aeb9a773 100644 --- a/packages/turbo-repository/rust/src/lib.rs +++ b/packages/turbo-repository/rust/src/lib.rs @@ -206,7 +206,7 @@ impl Workspace { .ok() .unwrap_or(LockfileContents::UnknownChange) }) - .unwrap_or(LockfileContents::Unknown) + .unwrap_or(LockfileContents::Unchanged) } /// Given a set of "changed" files, returns a set of packages that are From 5d0c5b6e4d7497b217aaadb8c156b02e8b0040ee Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Wed, 5 Mar 2025 07:38:06 -0500 Subject: [PATCH 16/17] keep status quo for package_changes_watcher --- crates/turborepo-lib/src/package_changes_watcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index 27452a219bfcf..dbd5298f77c7f 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -345,7 +345,7 @@ impl Subscriber { } let changed_packages = change_mapper - .changed_packages(changed_files.clone(), LockfileContents::Unknown); + .changed_packages(changed_files.clone(), LockfileContents::Unchanged); tracing::warn!("changed_files: {:?}", changed_files); tracing::warn!("changed_packages: {:?}", changed_packages); From bd24c7c623c058a67e1ba51e34182edecbd41d78 Mon Sep 17 00:00:00 2001 From: Dimitri Mitropoulos Date: Wed, 5 Mar 2025 07:40:24 -0500 Subject: [PATCH 17/17] removes LockfileContents::Unknown --- crates/turborepo-repository/src/change_mapper/mod.rs | 6 ------ crates/turborepo-repository/src/change_mapper/package.rs | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/turborepo-repository/src/change_mapper/mod.rs b/crates/turborepo-repository/src/change_mapper/mod.rs index 95e3ee0ab49d8..b5ef71c533c58 100644 --- a/crates/turborepo-repository/src/change_mapper/mod.rs +++ b/crates/turborepo-repository/src/change_mapper/mod.rs @@ -37,8 +37,6 @@ pub enum LockfileContents { /// previous lockfile (i.e. `git status`, or perhaps a lockfile that was /// deleted or otherwise inaccessible with the information we have) UnknownChange, - /// We are unsure whether the lockfile changed or not - Unknown, /// We know the lockfile changed and have the contents of the previous /// lockfile Changed(Vec), @@ -205,10 +203,6 @@ impl<'a, PD: PackageChangeMapper> ChangeMapper<'a, PD> { } // We don't know if the lockfile changed or not, so we can't assume anything - LockfileContents::Unknown => { - debug!("no previous lockfile available, assuming all packages changed"); - Ok(PackageChanges::Some(changed_pkgs)) - } LockfileContents::Unchanged => { debug!("the lockfile did not change"); Ok(PackageChanges::Some(changed_pkgs)) diff --git a/crates/turborepo-repository/src/change_mapper/package.rs b/crates/turborepo-repository/src/change_mapper/package.rs index 2fb79fc52044c..4d3e387211e68 100644 --- a/crates/turborepo-repository/src/change_mapper/package.rs +++ b/crates/turborepo-repository/src/change_mapper/package.rs @@ -242,7 +242,7 @@ mod tests { [AnchoredSystemPathBuf::from_raw("README.md")?] .into_iter() .collect(), - LockfileContents::Unknown, + LockfileContents::Unchanged, )?; // We should return All because we don't have global deps and @@ -262,7 +262,7 @@ mod tests { [AnchoredSystemPathBuf::from_raw("README.md")?] .into_iter() .collect(), - LockfileContents::Unknown, + LockfileContents::Unchanged, )?; // We only get a root workspace change since we have global deps specified and