diff --git a/Cargo.lock b/Cargo.lock index 38fdac8c4f000..d06ce10e339b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6699,14 +6699,17 @@ dependencies = [ name = "turborepo-napi" version = "0.1.0" dependencies = [ + "either", "napi", "napi-build", "napi-derive", "pretty_assertions", "thiserror", "tokio", + "tracing", "turbopath", "turborepo-repository", + "turborepo-scm", ] [[package]] @@ -6720,6 +6723,7 @@ dependencies = [ "biome_diagnostics", "biome_json_parser", "biome_json_syntax", + "either", "globwalk", "itertools 0.10.5", "lazy-regex", diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index cc4cf7bfb19e2..dbd5298f77c7f 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -15,7 +15,9 @@ use turborepo_filewatch::{ NotifyError, OptionalWatch, }; use turborepo_repository::{ - change_mapper::{ChangeMapper, GlobalDepsPackageChangeMapper, PackageChanges}, + change_mapper::{ + ChangeMapper, GlobalDepsPackageChangeMapper, LockfileContents, PackageChanges, + }, package_graph::{PackageGraph, PackageGraphBuilder, PackageName, WorkspacePackage}, package_json::PackageJson, }; @@ -342,7 +344,8 @@ impl Subscriber { continue; } - let changed_packages = change_mapper.changed_packages(changed_files.clone(), None); + let changed_packages = change_mapper + .changed_packages(changed_files.clone(), LockfileContents::Unchanged); tracing::warn!("changed_files: {:?}", changed_files); tracing::warn!("changed_packages: {:?}", changed_packages); diff --git a/crates/turborepo-lib/src/run/scope/change_detector.rs b/crates/turborepo-lib/src/run/scope/change_detector.rs index e0f5b289ac17e..670b17d491ae6 100644 --- a/crates/turborepo-lib/src/run/scope/change_detector.rs +++ b/crates/turborepo-lib/src/run/scope/change_detector.rs @@ -5,7 +5,7 @@ use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf}; use turborepo_repository::{ change_mapper::{ AllPackageChangeReason, ChangeMapper, DefaultPackageChangeMapper, Error, - GlobalDepsPackageChangeMapper, PackageChanges, PackageInclusionReason, + GlobalDepsPackageChangeMapper, LockfileContents, PackageChanges, PackageInclusionReason, }, package_graph::{PackageGraph, PackageName}, }; @@ -52,13 +52,12 @@ impl<'a> ScopeChangeDetector<'a> { } /// Gets the lockfile content from SCM if it has changed. - /// Does *not* error if cannot get content, instead just - /// returns a Some(None) - fn get_lockfile_contents( + /// Does *not* error if cannot get content. + pub fn get_lockfile_contents( &self, from_ref: Option<&str>, changed_files: &HashSet, - ) -> Option>> { + ) -> LockfileContents { let lockfile_path = self .pkg_graph .package_manager() @@ -70,23 +69,21 @@ impl<'a> ScopeChangeDetector<'a> { &lockfile_path, ) { debug!("lockfile did not change"); - return None; + return LockfileContents::Unchanged; } - let lockfile_path = self - .pkg_graph - .package_manager() - .lockfile_path(self.turbo_root); - let Ok(content) = self.scm.previous_content(from_ref, &lockfile_path) else { - return Some(None); + debug!("lockfile did change but could not get previous content"); + return LockfileContents::UnknownChange; }; - Some(Some(content)) + debug!("lockfile changed, have the previous content"); + LockfileContents::Changed(content) } } impl<'a> GitChangeDetector for ScopeChangeDetector<'a> { + /// get the actual changed packages between two git refs fn changed_packages( &self, from_ref: Option<&str>, diff --git a/crates/turborepo-repository/Cargo.toml b/crates/turborepo-repository/Cargo.toml index b562597e12cfe..9f95d9f39f0e6 100644 --- a/crates/turborepo-repository/Cargo.toml +++ b/crates/turborepo-repository/Cargo.toml @@ -17,6 +17,7 @@ biome_diagnostics = { workspace = true } biome_json_parser = { workspace = true } biome_json_syntax = { workspace = true } +either = { workspace = true } globwalk = { version = "0.1.0", path = "../turborepo-globwalk" } itertools = { workspace = true } lazy-regex = "2.5.0" diff --git a/crates/turborepo-repository/src/change_mapper/mod.rs b/crates/turborepo-repository/src/change_mapper/mod.rs index dce6b0325d323..b5ef71c533c58 100644 --- a/crates/turborepo-repository/src/change_mapper/mod.rs +++ b/crates/turborepo-repository/src/change_mapper/mod.rs @@ -7,8 +7,8 @@ use std::{ }; pub use package::{ - DefaultPackageChangeMapper, Error, GlobalDepsPackageChangeMapper, PackageChangeMapper, - PackageMapping, + DefaultPackageChangeMapper, DefaultPackageChangeMapperWithLockfile, Error, + GlobalDepsPackageChangeMapper, PackageChangeMapper, PackageMapping, }; use tracing::debug; use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf}; @@ -29,6 +29,19 @@ pub enum LockfileChange { ChangedPackages(HashSet), } +/// This describes the state of a change to a lockfile. +pub enum LockfileContents { + /// We know the lockfile did not change + Unchanged, + /// We know the lockfile changed but don't have the file contents of the + /// previous lockfile (i.e. `git status`, or perhaps a lockfile that was + /// deleted or otherwise inaccessible with the information we have) + UnknownChange, + /// We know the lockfile changed and have the contents of the previous + /// lockfile + Changed(Vec), +} + #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub enum PackageInclusionReason { /// All the packages are invalidated @@ -122,13 +135,7 @@ impl<'a, PD: PackageChangeMapper> ChangeMapper<'a, PD> { pub fn changed_packages( &self, changed_files: HashSet, - // None - we don't know if the lockfile changed - // - // Some(None) - we know the lockfile changed, but don't know exactly why (i.e. `git status` - // and the lockfile is there) - // - // Some(Some(content)) - we know the lockfile changed and have the contents - lockfile_change: Option>>, + lockfile_contents: LockfileContents, ) -> Result { if let Some(file) = Self::default_global_file_changed(&changed_files) { debug!("global file changed"); @@ -142,15 +149,16 @@ impl<'a, PD: PackageChangeMapper> ChangeMapper<'a, PD> { // get filtered files and add the packages that contain them let filtered_changed_files = self.filter_ignored_files(changed_files.iter())?; + // calculate lockfile_change here based on changed_files match self.get_changed_packages(filtered_changed_files.into_iter()) { PackageChanges::All(reason) => Ok(PackageChanges::All(reason)), PackageChanges::Some(mut changed_pkgs) => { - match lockfile_change { - Some(Some(content)) => { + match lockfile_contents { + LockfileContents::Changed(previous_lockfile_contents) => { // if we run into issues, don't error, just assume all packages have changed let Ok(lockfile_changes) = - self.get_changed_packages_from_lockfile(&content) + self.get_changed_packages_from_lockfile(&previous_lockfile_contents) else { debug!( "unable to determine lockfile changes, assuming all packages \ @@ -183,13 +191,22 @@ impl<'a, PD: PackageChangeMapper> ChangeMapper<'a, PD> { } // We don't have the actual contents, so just invalidate everything - Some(None) => { - debug!("no previous lockfile available, assuming all packages changed"); + LockfileContents::UnknownChange => { + // this can happen in a blobless checkout + debug!( + "we know the lockfile changed but we don't have the contents so we \ + have to assume all packages changed and rebuild everything" + ); Ok(PackageChanges::All( AllPackageChangeReason::LockfileChangedWithoutDetails, )) } - None => Ok(PackageChanges::Some(changed_pkgs)), + + // We don't know if the lockfile changed or not, so we can't assume anything + LockfileContents::Unchanged => { + debug!("the lockfile did not change"); + Ok(PackageChanges::Some(changed_pkgs)) + } } } } diff --git a/crates/turborepo-repository/src/change_mapper/package.rs b/crates/turborepo-repository/src/change_mapper/package.rs index 7c6d57f96265b..4d3e387211e68 100644 --- a/crates/turborepo-repository/src/change_mapper/package.rs +++ b/crates/turborepo-repository/src/change_mapper/package.rs @@ -24,6 +24,19 @@ pub trait PackageChangeMapper { fn detect_package(&self, file: &AnchoredSystemPath) -> PackageMapping; } +impl PackageChangeMapper for either::Either +where + L: PackageChangeMapper, + R: PackageChangeMapper, +{ + fn detect_package(&self, file: &AnchoredSystemPath) -> PackageMapping { + match self { + either::Either::Left(l) => l.detect_package(file), + either::Either::Right(r) => r.detect_package(file), + } + } +} + /// Detects package by checking if the file is inside the package. /// /// Does *not* use the `globalDependencies` in turbo.json. @@ -73,6 +86,43 @@ impl PackageChangeMapper for DefaultPackageChangeMapper<'_> { } } +pub struct DefaultPackageChangeMapperWithLockfile<'a> { + base: DefaultPackageChangeMapper<'a>, +} + +impl<'a> DefaultPackageChangeMapperWithLockfile<'a> { + pub fn new(pkg_dep_graph: &'a PackageGraph) -> Self { + Self { + base: DefaultPackageChangeMapper::new(pkg_dep_graph), + } + } +} + +impl PackageChangeMapper for DefaultPackageChangeMapperWithLockfile<'_> { + fn detect_package(&self, path: &AnchoredSystemPath) -> PackageMapping { + // If we have a lockfile change, we consider this as a root package change, + // since there's a chance that the root package uses a workspace package + // dependency (this is cursed behavior but sadly possible). There's a chance + // that we can make this more accurate by checking which package + // manager, since not all package managers may permit root pulling from + // workspace package dependencies + if PackageManager::supported_managers() + .iter() + .any(|pm| pm.lockfile_name() == path.as_str()) + { + PackageMapping::Package(( + WorkspacePackage { + name: PackageName::Root, + path: AnchoredSystemPathBuf::from_raw("").unwrap(), + }, + PackageInclusionReason::ConservativeRootLockfileChanged, + )) + } else { + self.base.detect_package(path) + } + } +} + #[derive(Error, Debug)] pub enum Error { #[error(transparent)] @@ -88,7 +138,7 @@ pub enum Error { /// changes all packages. Since we have a list of global deps, /// we can check against that and avoid invalidating in unnecessary cases. pub struct GlobalDepsPackageChangeMapper<'a> { - pkg_dep_graph: &'a PackageGraph, + base: DefaultPackageChangeMapperWithLockfile<'a>, global_deps_matcher: wax::Any<'a>, } @@ -97,10 +147,11 @@ impl<'a> GlobalDepsPackageChangeMapper<'a> { pkg_dep_graph: &'a PackageGraph, global_deps: I, ) -> Result { + let base = DefaultPackageChangeMapperWithLockfile::new(pkg_dep_graph); let global_deps_matcher = wax::any(global_deps)?; Ok(Self { - pkg_dep_graph, + base, global_deps_matcher, }) } @@ -108,25 +159,7 @@ impl<'a> GlobalDepsPackageChangeMapper<'a> { impl PackageChangeMapper for GlobalDepsPackageChangeMapper<'_> { fn detect_package(&self, path: &AnchoredSystemPath) -> PackageMapping { - // If we have a lockfile change, we consider this as a root package change, - // since there's a chance that the root package uses a workspace package - // dependency (this is cursed behavior but sadly possible). There's a chance - // that we can make this more accurate by checking which package - // manager, since not all package managers may permit root pulling from - // workspace package dependencies - if PackageManager::supported_managers() - .iter() - .any(|pm| pm.lockfile_name() == path.as_str()) - { - return PackageMapping::Package(( - WorkspacePackage { - name: PackageName::Root, - path: AnchoredSystemPathBuf::from_raw("").unwrap(), - }, - PackageInclusionReason::ConservativeRootLockfileChanged, - )); - } - match DefaultPackageChangeMapper::new(self.pkg_dep_graph).detect_package(path) { + match self.base.detect_package(path) { // Since `DefaultPackageChangeMapper` is overly conservative, we can check here if // the path is actually in globalDeps and if not, return it as // PackageDetection::Package(WorkspacePackage::root()). @@ -160,7 +193,8 @@ mod tests { use super::{DefaultPackageChangeMapper, GlobalDepsPackageChangeMapper}; use crate::{ change_mapper::{ - AllPackageChangeReason, ChangeMapper, PackageChanges, PackageInclusionReason, + AllPackageChangeReason, ChangeMapper, LockfileContents, PackageChanges, + PackageInclusionReason, }, discovery::{self, PackageDiscovery}, package_graph::{PackageGraphBuilder, WorkspacePackage}, @@ -208,7 +242,7 @@ mod tests { [AnchoredSystemPathBuf::from_raw("README.md")?] .into_iter() .collect(), - None, + LockfileContents::Unchanged, )?; // We should return All because we don't have global deps and @@ -228,7 +262,7 @@ mod tests { [AnchoredSystemPathBuf::from_raw("README.md")?] .into_iter() .collect(), - None, + LockfileContents::Unchanged, )?; // We only get a root workspace change since we have global deps specified and diff --git a/crates/turborepo-scm/src/git.rs b/crates/turborepo-scm/src/git.rs index c9f57e454f4d8..9e6d51ef8b234 100644 --- a/crates/turborepo-scm/src/git.rs +++ b/crates/turborepo-scm/src/git.rs @@ -37,6 +37,7 @@ impl SCM { } } + /// get the actual changed files between two git refs pub fn changed_files( &self, turbo_root: &AbsoluteSystemPath, diff --git a/packages/turbo-repository/__tests__/affected-packages.test.ts b/packages/turbo-repository/__tests__/affected-packages.test.ts index 4cb62e0b50f6b..7c4a1008fea10 100644 --- a/packages/turbo-repository/__tests__/affected-packages.test.ts +++ b/packages/turbo-repository/__tests__/affected-packages.test.ts @@ -1,4 +1,4 @@ -import { describe, it } from "node:test"; +import { beforeEach, describe, it } from "node:test"; import { strict as assert } from "node:assert"; import * as path from "node:path"; import { Workspace, Package, PackageManager } from "../js/dist/index.js"; @@ -8,7 +8,6 @@ type PackageReduced = Pick; interface AffectedPackagesTestParams { description: string; files: string[]; - changedLockfile?: string | undefined | null; expected: PackageReduced[]; } @@ -33,49 +32,22 @@ describe("affectedPackages", () => { ], }, { - description: - "a lockfile change will only affect packages impacted by the change", - files: [], - changedLockfile: `lockfileVersion: '6.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: {} - - apps/app: - dependencies: - microdiff: - specifier: ^1.4.0 - version: 1.5.0 - ui: - specifier: workspace:* - version: link:../../packages/ui - - packages/blank: {} - - packages/ui: {} - -packages: - - /microdiff@1.5.0: - resolution: {integrity: sha512-Drq+/THMvDdzRYrK0oxJmOKiC24ayUV8ahrt8l3oRK51PWt6gdtrIGrlIH3pT/lFh1z93FbAcidtsHcWbnRz8Q==} - dev: false -`, - expected: [{ name: "app-a", relativePath: "apps/app" }], + description: "a lockfile change will affect all packages", + files: ["pnpm-lock.yaml"], + expected: [ + { name: "app-a", relativePath: "apps/app" }, + { name: "ui", relativePath: "packages/ui" }, + ], }, ]; - for (const { description, files, expected, changedLockfile } of tests) { + for (const { description, files, expected } of tests) { it(description, async () => { const dir = path.resolve(__dirname, "./fixtures/monorepo"); const workspace = await Workspace.find(dir); const reduced: PackageReduced[] = ( - await workspace.affectedPackages(files, changedLockfile) + await workspace.affectedPackages(files) ).map((pkg) => { return { name: pkg.name, @@ -86,4 +58,38 @@ packages: assert.deepEqual(reduced, expected); }); } + + describe("optimizedLockfileUpdates", () => { + it("errors if not provided comparison ref", async () => { + const dir = path.resolve(__dirname, "./fixtures/monorepo"); + const workspace = await Workspace.find(dir); + + assert.rejects( + workspace.affectedPackages(["pnpm-lock.yaml"], null, true) + ); + }); + + it("still considers root file changes as global", async () => { + const dir = path.resolve(__dirname, "./fixtures/monorepo"); + const workspace = await Workspace.find(dir); + + const reduced: PackageReduced[] = ( + await workspace.affectedPackages( + ["file-we-do-not-understand.txt"], + "HEAD", + true + ) + ).map((pkg) => { + return { + name: pkg.name, + relativePath: pkg.relativePath, + }; + }); + + assert.deepEqual(reduced, [ + { name: "app-a", relativePath: "apps/app" }, + { name: "ui", relativePath: "packages/ui" }, + ]); + }); + }); }); diff --git a/packages/turbo-repository/js/index.d.ts b/packages/turbo-repository/js/index.d.ts index 26d252b6f8741..0ebf830aad724 100644 --- a/packages/turbo-repository/js/index.d.ts +++ b/packages/turbo-repository/js/index.d.ts @@ -57,6 +57,7 @@ export class Workspace { */ affectedPackages( files: Array, - changedLockfile?: string | undefined | null + base?: string | undefined | null, + optimizeGlobalInvalidations?: boolean | undefined | null ): Promise>; } diff --git a/packages/turbo-repository/js/package.json b/packages/turbo-repository/js/package.json index ce0e9dd01b842..553afe42b6a0b 100644 --- a/packages/turbo-repository/js/package.json +++ b/packages/turbo-repository/js/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "description": "", "bugs": "https://github.com/vercel/turborepo/issues", "homepage": "https://turbo.build/repo", @@ -15,13 +15,13 @@ ], "types": "dist/index.d.ts", "optionalDependencies": { - "@turbo/repository-darwin-x64": "0.0.1-canary.11", - "@turbo/repository-darwin-arm64": "0.0.1-canary.11", - "@turbo/repository-linux-x64-gnu": "0.0.1-canary.11", - "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.11", - "@turbo/repository-linux-x64-musl": "0.0.1-canary.11", - "@turbo/repository-linux-arm64-musl": "0.0.1-canary.11", - "@turbo/repository-win32-x64-msvc": "0.0.1-canary.11", - "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.11" + "@turbo/repository-darwin-x64": "0.0.1-canary.14", + "@turbo/repository-darwin-arm64": "0.0.1-canary.14", + "@turbo/repository-linux-x64-gnu": "0.0.1-canary.14", + "@turbo/repository-linux-arm64-gnu": "0.0.1-canary.14", + "@turbo/repository-linux-x64-musl": "0.0.1-canary.14", + "@turbo/repository-linux-arm64-musl": "0.0.1-canary.14", + "@turbo/repository-win32-x64-msvc": "0.0.1-canary.14", + "@turbo/repository-win32-arm64-msvc": "0.0.1-canary.14" } } diff --git a/packages/turbo-repository/npm/darwin-arm64/package.json b/packages/turbo-repository/npm/darwin-arm64/package.json index 29f8d0a45b68e..34260b2f91014 100644 --- a/packages/turbo-repository/npm/darwin-arm64/package.json +++ b/packages/turbo-repository/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-darwin-arm64", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/darwin-x64/package.json b/packages/turbo-repository/npm/darwin-x64/package.json index a997c82d6cac3..1a3c2d7f17da8 100644 --- a/packages/turbo-repository/npm/darwin-x64/package.json +++ b/packages/turbo-repository/npm/darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-darwin-x64", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-arm64-gnu/package.json b/packages/turbo-repository/npm/linux-arm64-gnu/package.json index e3cd548846452..098dc2a2f6e86 100644 --- a/packages/turbo-repository/npm/linux-arm64-gnu/package.json +++ b/packages/turbo-repository/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-arm64-gnu", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-arm64-musl/package.json b/packages/turbo-repository/npm/linux-arm64-musl/package.json index fa2e43c3eb178..bab045e0a1c45 100644 --- a/packages/turbo-repository/npm/linux-arm64-musl/package.json +++ b/packages/turbo-repository/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-arm64-musl", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-x64-gnu/package.json b/packages/turbo-repository/npm/linux-x64-gnu/package.json index af1f5c2ab7876..03b50b9f0a448 100644 --- a/packages/turbo-repository/npm/linux-x64-gnu/package.json +++ b/packages/turbo-repository/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-x64-gnu", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/linux-x64-musl/package.json b/packages/turbo-repository/npm/linux-x64-musl/package.json index 337e4ac3242db..da68acdb521cb 100644 --- a/packages/turbo-repository/npm/linux-x64-musl/package.json +++ b/packages/turbo-repository/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-linux-x64-musl", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/win32-arm64-msvc/package.json b/packages/turbo-repository/npm/win32-arm64-msvc/package.json index 0038398355e7b..24b64880eafb5 100644 --- a/packages/turbo-repository/npm/win32-arm64-msvc/package.json +++ b/packages/turbo-repository/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-win32-arm64-msvc", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/npm/win32-x64-msvc/package.json b/packages/turbo-repository/npm/win32-x64-msvc/package.json index b861ab970ac4d..0f725e433f07f 100644 --- a/packages/turbo-repository/npm/win32-x64-msvc/package.json +++ b/packages/turbo-repository/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/repository-win32-x64-msvc", - "version": "0.0.1-canary.11", + "version": "0.0.1-canary.14", "repository": { "type": "git", "url": "https://github.com/vercel/turborepo", diff --git a/packages/turbo-repository/rust/Cargo.toml b/packages/turbo-repository/rust/Cargo.toml index 6a97de05ced1e..3dd33dfd1d9ad 100644 --- a/packages/turbo-repository/rust/Cargo.toml +++ b/packages/turbo-repository/rust/Cargo.toml @@ -11,12 +11,15 @@ crate-type = ["cdylib"] workspace = true [dependencies] +either = { workspace = true } napi = { version = "2.14.0", features = ["tokio_rt"] } napi-derive = "2.14.0" thiserror = { workspace = true } tokio = { workspace = true } turbopath = { workspace = true } turborepo-repository = { workspace = true } +turborepo-scm = { workspace = true } +tracing = "0.1.37" [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/packages/turbo-repository/rust/src/lib.rs b/packages/turbo-repository/rust/src/lib.rs index 8460427b6c39f..85dd4aeb9a773 100644 --- a/packages/turbo-repository/rust/src/lib.rs +++ b/packages/turbo-repository/rust/src/lib.rs @@ -3,14 +3,20 @@ use std::{ hash::Hash, }; +use either::Either; use napi::Error; use napi_derive::napi; -use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf}; +use tracing::debug; +use turbopath::{AbsoluteSystemPath, AnchoredSystemPath, AnchoredSystemPathBuf}; use turborepo_repository::{ - change_mapper::{ChangeMapper, GlobalDepsPackageChangeMapper, PackageChanges}, + change_mapper::{ + ChangeMapper, DefaultPackageChangeMapper, DefaultPackageChangeMapperWithLockfile, + LockfileContents, PackageChanges, + }, inference::RepoState as WorkspaceState, package_graph::{PackageGraph, PackageName, PackageNode, WorkspacePackage, ROOT_PKG_NAME}, }; +use turborepo_scm::SCM; mod internal; #[napi] @@ -182,6 +188,27 @@ impl Workspace { Ok(map) } + pub fn get_lockfile_contents( + &self, + changed_files: &HashSet, + workspace_root: &AbsoluteSystemPath, + from_commit: &str, + ) -> LockfileContents { + let lockfile_name = self.graph.package_manager().lockfile_name(); + changed_files + .contains(AnchoredSystemPath::new(&lockfile_name).unwrap()) + .then(|| { + let git = SCM::new(workspace_root); + let anchored_path = workspace_root.join_component(lockfile_name); + git.previous_content(Some(from_commit), &anchored_path) + .map(LockfileContents::Changed) + .inspect_err(|e| debug!("{e}")) + .ok() + .unwrap_or(LockfileContents::UnknownChange) + }) + .unwrap_or(LockfileContents::Unchanged) + } + /// Given a set of "changed" files, returns a set of packages that are /// "affected" by the changes. The `files` argument is expected to be a list /// of strings relative to the monorepo root and use the current system's @@ -190,13 +217,22 @@ impl Workspace { pub async fn affected_packages( &self, files: Vec, - changed_lockfile: Option, + base: Option<&str>, // this is required when optimize_global_invalidations is true + optimize_global_invalidations: Option, ) -> Result, Error> { + let base = optimize_global_invalidations + .unwrap_or(false) + .then(|| { + base.ok_or_else(|| { + Error::from_reason("optimizeGlobalInvalidations true, but no base commit given") + }) + }) + .transpose()?; let workspace_root = match AbsoluteSystemPath::new(&self.absolute_path) { Ok(path) => path, Err(e) => return Err(Error::from_reason(e.to_string())), }; - let hash_set_of_paths: HashSet = files + let changed_files: HashSet = files .into_iter() .filter_map(|path| { let path_components = path.split(std::path::MAIN_SEPARATOR).collect::>(); @@ -206,11 +242,24 @@ impl Workspace { .collect(); // Create a ChangeMapper with no ignore patterns - let global_deps_package_detector = - GlobalDepsPackageChangeMapper::new(&self.graph, std::iter::empty::<&str>()).unwrap(); - let mapper = ChangeMapper::new(&self.graph, vec![], global_deps_package_detector); - let lockfile_change = changed_lockfile.map(|s| Some(s.into_bytes())); - let package_changes = match mapper.changed_packages(hash_set_of_paths, lockfile_change) { + let change_detector = base + .is_some() + .then(|| Either::Left(DefaultPackageChangeMapperWithLockfile::new(&self.graph))) + .unwrap_or_else(|| Either::Right(DefaultPackageChangeMapper::new(&self.graph))); + let mapper = ChangeMapper::new(&self.graph, vec![], change_detector); + + let lockfile_contents = if let Some(base) = base { + self.get_lockfile_contents(&changed_files, workspace_root, base) + } else if changed_files.contains( + AnchoredSystemPath::new(self.graph.package_manager().lockfile_name()) + .expect("the lockfile name will not be an absolute path"), + ) { + LockfileContents::UnknownChange + } else { + LockfileContents::Unchanged + }; + + let package_changes = match mapper.changed_packages(changed_files, lockfile_contents) { Ok(changes) => changes, Err(e) => return Err(Error::from_reason(e.to_string())), };