diff --git a/crates/turborepo-lib/src/cli/error.rs b/crates/turborepo-lib/src/cli/error.rs index ef1df55fedf9e..a60e8c9496363 100644 --- a/crates/turborepo-lib/src/cli/error.rs +++ b/crates/turborepo-lib/src/cli/error.rs @@ -26,6 +26,8 @@ pub enum Error { #[error(transparent)] Boundaries(#[from] crate::boundaries::Error), #[error(transparent)] + Clone(#[from] crate::commands::clone::Error), + #[error(transparent)] Path(#[from] turbopath::PathError), #[error(transparent)] #[diagnostic(transparent)] diff --git a/crates/turborepo-lib/src/cli/mod.rs b/crates/turborepo-lib/src/cli/mod.rs index 4b1ee799da6a5..21c4b305d8f75 100644 --- a/crates/turborepo-lib/src/cli/mod.rs +++ b/crates/turborepo-lib/src/cli/mod.rs @@ -28,8 +28,8 @@ use turborepo_ui::{ColorConfig, GREY}; use crate::{ cli::error::print_potential_tasks, commands::{ - bin, boundaries, config, daemon, generate, info, link, login, logout, ls, prune, query, - run, scan, telemetry, unlink, CommandBase, + bin, boundaries, clone, config, daemon, generate, info, link, login, logout, ls, prune, + query, run, scan, telemetry, unlink, CommandBase, }, get_version, run::watch::WatchClient, @@ -588,6 +588,17 @@ pub enum Command { #[clap(short = 'F', long, group = "scope-filter-group")] filter: Vec, }, + #[clap(hide = true)] + Clone { + url: String, + dir: Option, + #[clap(long, conflicts_with = "local")] + ci: bool, + #[clap(long, conflicts_with = "ci")] + local: bool, + #[clap(long)] + depth: Option, + }, /// Generate the autocompletion script for the specified shell Completion { shell: Shell }, /// Runs the Turborepo background daemon @@ -1360,7 +1371,6 @@ pub async fn run( }; cli_args.command = Some(command); - cli_args.cwd = Some(repo_root.as_path().to_owned()); let root_telemetry = GenericEventBuilder::new(); root_telemetry.track_start(); @@ -1389,6 +1399,18 @@ pub async fn run( Ok(boundaries::run(base, event).await?) } + Command::Clone { + url, + dir, + ci, + local, + depth, + } => { + let event = CommandEventBuilder::new("clone").with_parent(&root_telemetry); + event.track_call(); + + Ok(clone::run(cwd, url, dir.as_deref(), *ci, *local, *depth)?) + } #[allow(unused_variables)] Command::Daemon { command, idle_time } => { let event = CommandEventBuilder::new("daemon").with_parent(&root_telemetry); diff --git a/crates/turborepo-lib/src/commands/clone.rs b/crates/turborepo-lib/src/commands/clone.rs new file mode 100644 index 0000000000000..cb93aee95d76d --- /dev/null +++ b/crates/turborepo-lib/src/commands/clone.rs @@ -0,0 +1,53 @@ +use std::env::current_dir; + +use camino::Utf8Path; +use thiserror::Error; +use turbopath::AbsoluteSystemPathBuf; +use turborepo_ci::is_ci; +use turborepo_scm::clone::{CloneMode, Git}; + +#[derive(Debug, Error)] +pub enum Error { + #[error("path is not valid UTF-8")] + Path(#[from] camino::FromPathBufError), + + #[error(transparent)] + Turbopath(#[from] turbopath::PathError), + + #[error("failed to clone repository")] + Scm(#[from] turborepo_scm::Error), +} + +pub fn run( + cwd: Option<&Utf8Path>, + url: &str, + dir: Option<&str>, + ci: bool, + local: bool, + depth: Option, +) -> Result { + // We do *not* want to use the repo root but the actual, literal cwd for clone + let cwd = if let Some(cwd) = cwd { + cwd.to_owned() + } else { + current_dir() + .expect("could not get current directory") + .try_into()? + }; + let abs_cwd = AbsoluteSystemPathBuf::from_cwd(cwd)?; + + let clone_mode = if ci { + CloneMode::CI + } else if local { + CloneMode::Local + } else if is_ci() { + CloneMode::CI + } else { + CloneMode::Local + }; + + let git = Git::find()?; + git.clone(url, abs_cwd, dir, None, clone_mode, depth)?; + + Ok(0) +} diff --git a/crates/turborepo-lib/src/commands/mod.rs b/crates/turborepo-lib/src/commands/mod.rs index 4acee04c53f0b..efe48d781bf43 100644 --- a/crates/turborepo-lib/src/commands/mod.rs +++ b/crates/turborepo-lib/src/commands/mod.rs @@ -15,6 +15,7 @@ use crate::{ pub(crate) mod bin; pub(crate) mod boundaries; +pub(crate) mod clone; pub(crate) mod config; pub(crate) mod daemon; pub(crate) mod generate; diff --git a/crates/turborepo-lib/src/diagnostics.rs b/crates/turborepo-lib/src/diagnostics.rs index de836c04e17f7..a3a2330055339 100644 --- a/crates/turborepo-lib/src/diagnostics.rs +++ b/crates/turborepo-lib/src/diagnostics.rs @@ -11,7 +11,7 @@ use tokio::{ }; use turbo_updater::check_for_updates; use turbopath::AbsoluteSystemPathBuf; -use turborepo_scm::Git; +use turborepo_scm::GitRepo; use crate::{ commands::{ @@ -157,7 +157,7 @@ impl Diagnostic for GitDaemonDiagnostic { // get the current setting let stdout = Stdio::piped(); - let Ok(git_path) = Git::find_bin() else { + let Ok(git_path) = GitRepo::find_bin() else { return Err("git not found"); }; diff --git a/crates/turborepo-scm/src/clone.rs b/crates/turborepo-scm/src/clone.rs new file mode 100644 index 0000000000000..8d0c1dcb8981a --- /dev/null +++ b/crates/turborepo-scm/src/clone.rs @@ -0,0 +1,88 @@ +use std::{backtrace::Backtrace, process::Command}; + +use turbopath::AbsoluteSystemPathBuf; + +use crate::{Error, GitRepo}; + +pub enum CloneMode { + /// Cloning locally, do a blobless clone (good UX and reasonably fast) + Local, + /// Cloning on CI, do a treeless clone (worse UX but fastest) + CI, +} + +// Provide a sane maximum depth for cloning. If a user needs more history than +// this, they can override or fetch it themselves. +const MAX_CLONE_DEPTH: usize = 64; + +/// A wrapper around the git binary that is not tied to a specific repo. +pub struct Git { + bin: AbsoluteSystemPathBuf, +} + +impl Git { + pub fn find() -> Result { + Ok(Self { + bin: GitRepo::find_bin()?, + }) + } + + pub fn spawn_git_command( + &self, + cwd: &AbsoluteSystemPathBuf, + args: &[&str], + pathspec: &str, + ) -> Result<(), Error> { + let mut command = Command::new(self.bin.as_std_path()); + command + .args(args) + .current_dir(cwd) + .env("GIT_OPTIONAL_LOCKS", "0"); + + if !pathspec.is_empty() { + command.arg("--").arg(pathspec); + } + + let output = command.spawn()?.wait_with_output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + Err(Error::Git(stderr, Backtrace::capture())) + } else { + Ok(()) + } + } + + pub fn clone( + &self, + url: &str, + cwd: AbsoluteSystemPathBuf, + dir: Option<&str>, + branch: Option<&str>, + mode: CloneMode, + depth: Option, + ) -> Result<(), Error> { + let depth = depth.unwrap_or(MAX_CLONE_DEPTH).to_string(); + let mut args = vec!["clone", "--depth", &depth]; + if let Some(branch) = branch { + args.push("--branch"); + args.push(branch); + } + match mode { + CloneMode::Local => { + args.push("--filter=blob:none"); + } + CloneMode::CI => { + args.push("--filter=tree:0"); + } + } + args.push(url); + if let Some(dir) = dir { + args.push(dir); + } + + self.spawn_git_command(&cwd, &args, "")?; + + Ok(()) + } +} diff --git a/crates/turborepo-scm/src/git.rs b/crates/turborepo-scm/src/git.rs index c9f57e454f4d8..923d0c1f6e1e7 100644 --- a/crates/turborepo-scm/src/git.rs +++ b/crates/turborepo-scm/src/git.rs @@ -14,7 +14,7 @@ use turbopath::{ }; use turborepo_ci::Vendor; -use crate::{Error, Git, SCM}; +use crate::{Error, GitRepo, SCM}; #[derive(Debug, PartialEq, Eq)] pub struct InvalidRange { @@ -170,7 +170,7 @@ impl CIEnv { } } -impl Git { +impl GitRepo { fn get_current_branch(&self) -> Result { let output = self.execute_git_command(&["branch", "--show-current"], "")?; let output = String::from_utf8(output)?; @@ -323,7 +323,7 @@ impl Git { Ok(files) } - fn execute_git_command(&self, args: &[&str], pathspec: &str) -> Result, Error> { + pub fn execute_git_command(&self, args: &[&str], pathspec: &str) -> Result, Error> { let mut command = Command::new(self.bin.as_std_path()); command .args(args) @@ -431,7 +431,7 @@ mod tests { use super::{previous_content, CIEnv, InvalidRange}; use crate::{ git::{GitHubCommit, GitHubEvent}, - Error, Git, SCM, + Error, GitRepo, SCM, }; fn setup_repository( @@ -1044,7 +1044,7 @@ mod tests { repo.branch(branch, &commit, true).unwrap(); }); - let thing = Git::find(&root).unwrap(); + let thing = GitRepo::find(&root).unwrap(); let actual = thing.resolve_base(target_branch, CIEnv::none()).ok(); assert_eq!(actual.as_deref(), expected); @@ -1280,7 +1280,7 @@ mod tests { Err(VarError::NotPresent) }; - let actual = Git::get_github_base_ref(CIEnv { + let actual = GitRepo::get_github_base_ref(CIEnv { is_github_actions: test_case.env.is_github_actions, github_base_ref: test_case.env.github_base_ref, github_event_path: temp_file diff --git a/crates/turborepo-scm/src/lib.rs b/crates/turborepo-scm/src/lib.rs index 88399aaa4a402..c06aec27bbddc 100644 --- a/crates/turborepo-scm/src/lib.rs +++ b/crates/turborepo-scm/src/lib.rs @@ -19,6 +19,7 @@ use thiserror::Error; use tracing::debug; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError, RelativeUnixPathBuf}; +pub mod clone; pub mod git; mod hash_object; mod ls_tree; @@ -171,7 +172,7 @@ pub(crate) fn wait_for_success( } #[derive(Debug, Clone)] -pub struct Git { +pub struct GitRepo { root: AbsoluteSystemPathBuf, bin: AbsoluteSystemPathBuf, } @@ -184,7 +185,7 @@ enum GitError { Root(AbsoluteSystemPathBuf, Error), } -impl Git { +impl GitRepo { fn find(path_in_repo: &AbsoluteSystemPath) -> Result { // If which produces an invalid absolute path, it's not an execution error, it's // a programming error. We expect it to always give us an absolute path @@ -236,17 +237,19 @@ fn find_git_root(turbo_root: &AbsoluteSystemPath) -> Result SCM { - Git::find(path_in_repo).map(SCM::Git).unwrap_or_else(|e| { - debug!("{}, continuing with manual hashing", e); - SCM::Manual - }) + GitRepo::find(path_in_repo) + .map(SCM::Git) + .unwrap_or_else(|e| { + debug!("{}, continuing with manual hashing", e); + SCM::Manual + }) } pub fn is_manual(&self) -> bool { diff --git a/crates/turborepo-scm/src/ls_tree.rs b/crates/turborepo-scm/src/ls_tree.rs index 0c2415bf90c60..d6dfbef594cc8 100644 --- a/crates/turborepo-scm/src/ls_tree.rs +++ b/crates/turborepo-scm/src/ls_tree.rs @@ -6,9 +6,9 @@ use std::{ use nom::Finish; use turbopath::{AbsoluteSystemPathBuf, RelativeUnixPathBuf}; -use crate::{wait_for_success, Error, Git, GitHashes}; +use crate::{wait_for_success, Error, GitHashes, GitRepo}; -impl Git { +impl GitRepo { #[tracing::instrument(skip(self))] pub fn git_ls_tree(&self, root_path: &AbsoluteSystemPathBuf) -> Result { let mut hashes = GitHashes::new(); diff --git a/crates/turborepo-scm/src/package_deps.rs b/crates/turborepo-scm/src/package_deps.rs index 85fba9051d467..b1090a6b50852 100644 --- a/crates/turborepo-scm/src/package_deps.rs +++ b/crates/turborepo-scm/src/package_deps.rs @@ -8,7 +8,7 @@ use turborepo_telemetry::events::task::{FileHashMethod, PackageTaskEventBuilder} #[cfg(feature = "git2")] use crate::hash_object::hash_objects; -use crate::{Error, Git, GitHashes, SCM}; +use crate::{Error, GitHashes, GitRepo, SCM}; pub const INPUT_INCLUDE_DEFAULT_FILES: &str = "$TURBO_DEFAULT$"; @@ -112,7 +112,7 @@ impl SCM { } } -impl Git { +impl GitRepo { fn get_package_file_hashes>( &self, turbo_root: &AbsoluteSystemPath, diff --git a/crates/turborepo-scm/src/status.rs b/crates/turborepo-scm/src/status.rs index ee42ca96286f0..c8384060ce909 100644 --- a/crates/turborepo-scm/src/status.rs +++ b/crates/turborepo-scm/src/status.rs @@ -9,9 +9,9 @@ use std::{ use nom::Finish; use turbopath::{AbsoluteSystemPath, RelativeUnixPathBuf}; -use crate::{wait_for_success, Error, Git, GitHashes}; +use crate::{wait_for_success, Error, GitHashes, GitRepo}; -impl Git { +impl GitRepo { #[tracing::instrument(skip(self, root_path, hashes))] pub(crate) fn append_git_status( &self, diff --git a/turborepo-tests/integration/tests/clone/basic-monorepo.t b/turborepo-tests/integration/tests/clone/basic-monorepo.t new file mode 100644 index 0000000000000..93e11e4484072 --- /dev/null +++ b/turborepo-tests/integration/tests/clone/basic-monorepo.t @@ -0,0 +1,19 @@ +Setup + $ . ${TESTDIR}/../../../helpers/setup_integration_test.sh + + $ git config uploadpack.allowFilter true + $ cd .. +Make sure we allow partial clones + +Do a blobless clone + $ ${TURBO} clone file://$(pwd)/basic-monorepo.t basic-monorepo-blobless --local + Cloning into 'basic-monorepo-blobless'... + $ cd basic-monorepo-blobless + $ ${TURBO} build > /dev/null 2>&1 + $ cd .. + +Do a treeless clone + $ ${TURBO} clone file://$(pwd)/basic-monorepo.t basic-monorepo-treeless --local + Cloning into 'basic-monorepo-treeless'... + $ cd basic-monorepo-treeless + $ ${TURBO} build > /dev/null 2>&1 \ No newline at end of file diff --git a/turborepo-tests/integration/tests/clone/new-repo.t b/turborepo-tests/integration/tests/clone/new-repo.t new file mode 100644 index 0000000000000..203cf661addb8 --- /dev/null +++ b/turborepo-tests/integration/tests/clone/new-repo.t @@ -0,0 +1,33 @@ +Setup + $ . ${TESTDIR}/../../../helpers/setup.sh + +Create a repo + $ mkdir my-repo + $ cd my-repo + $ git init --quiet + $ echo "Hello World" > README.md + $ git add README.md + $ git config user.email "test@example.com" + $ git config user.name "Test" + $ git commit -m "Initial commit" --quiet + +Make sure we allow partial clones + $ git config uploadpack.allowFilter true + $ cd .. + +Clone repo with `--ci` + $ ${TURBO} clone file://$(pwd)/my-repo my-repo-treeless --ci + Cloning into 'my-repo-treeless'... + $ cd my-repo-treeless +Assert it's a treeless clone + $ git config remote.origin.partialclonefilter + tree:0 + $ cd .. + +Clone repo with `--local` + $ ${TURBO} clone file://$(pwd)/my-repo my-repo-blobless --local + Cloning into 'my-repo-blobless'... + $ cd my-repo-blobless +Assert it's a blobless clone + $ git config remote.origin.partialclonefilter + blob:none