From 437a57b665786a3d3b5142ce5f79d839decd486f Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 7 Jul 2025 11:24:00 -0400 Subject: [PATCH 1/7] sparse-checkout: remove use of the_repository The logic for the 'git sparse-checkout' builtin uses the_repository all over the place, despite some use of a repository struct in different method parameters. Complete this removal of the_repository by using 'repo' when possible. In one place, there was already a local variable 'r' that was set to the_repository, so move that to a method parameter. We cannot remove the USE_THE_REPOSITORY_VARIABLE declaration as we are still using global constants for the state of the sparse-checkout. Signed-off-by: Derrick Stolee --- builtin/sparse-checkout.c | 117 ++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 55 deletions(-) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 8c333b3e2e145b..06de61bd9d0d33 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -204,12 +204,12 @@ static void clean_tracked_sparse_directories(struct repository *r) ensure_full_index(r->index); } -static int update_working_directory(struct pattern_list *pl) +static int update_working_directory(struct repository *r, + struct pattern_list *pl) { enum update_sparsity_result result; struct unpack_trees_options o; struct lock_file lock_file = LOCK_INIT; - struct repository *r = the_repository; struct pattern_list *old_pl; /* If no branch has been checked out, there are no updates to make. */ @@ -327,7 +327,8 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl) string_list_clear(&sl, 0); } -static int write_patterns_and_update(struct pattern_list *pl) +static int write_patterns_and_update(struct repository *repo, + struct pattern_list *pl) { char *sparse_filename; FILE *fp; @@ -336,15 +337,15 @@ static int write_patterns_and_update(struct pattern_list *pl) sparse_filename = get_sparse_checkout_filename(); - if (safe_create_leading_directories(the_repository, sparse_filename)) + if (safe_create_leading_directories(repo, sparse_filename)) die(_("failed to create directory for sparse-checkout file")); hold_lock_file_for_update(&lk, sparse_filename, LOCK_DIE_ON_ERROR); - result = update_working_directory(pl); + result = update_working_directory(repo, pl); if (result) { rollback_lock_file(&lk); - update_working_directory(NULL); + update_working_directory(repo, NULL); goto out; } @@ -372,25 +373,26 @@ enum sparse_checkout_mode { MODE_CONE_PATTERNS = 2, }; -static int set_config(enum sparse_checkout_mode mode) +static int set_config(struct repository *repo, + enum sparse_checkout_mode mode) { /* Update to use worktree config, if not already. */ - if (init_worktree_config(the_repository)) { + if (init_worktree_config(repo)) { error(_("failed to initialize worktree config")); return 1; } - if (repo_config_set_worktree_gently(the_repository, + if (repo_config_set_worktree_gently(repo, "core.sparseCheckout", mode ? "true" : "false") || - repo_config_set_worktree_gently(the_repository, + repo_config_set_worktree_gently(repo, "core.sparseCheckoutCone", mode == MODE_CONE_PATTERNS ? "true" : "false")) return 1; if (mode == MODE_NO_PATTERNS) - return set_sparse_index_config(the_repository, 0); + return set_sparse_index_config(repo, 0); return 0; } @@ -410,7 +412,7 @@ static enum sparse_checkout_mode update_cone_mode(int *cone_mode) { return MODE_ALL_PATTERNS; } -static int update_modes(int *cone_mode, int *sparse_index) +static int update_modes(struct repository *repo, int *cone_mode, int *sparse_index) { int mode, record_mode; @@ -418,20 +420,20 @@ static int update_modes(int *cone_mode, int *sparse_index) record_mode = (*cone_mode != -1) || !core_apply_sparse_checkout; mode = update_cone_mode(cone_mode); - if (record_mode && set_config(mode)) + if (record_mode && set_config(repo, mode)) return 1; /* Set sparse-index/non-sparse-index mode if specified */ if (*sparse_index >= 0) { - if (set_sparse_index_config(the_repository, *sparse_index) < 0) + if (set_sparse_index_config(repo, *sparse_index) < 0) die(_("failed to modify sparse-index config")); /* force an index rewrite */ - repo_read_index(the_repository); - the_repository->index->updated_workdir = 1; + repo_read_index(repo); + repo->index->updated_workdir = 1; if (!*sparse_index) - ensure_full_index(the_repository->index); + ensure_full_index(repo->index); } return 0; @@ -448,7 +450,7 @@ static struct sparse_checkout_init_opts { } init_opts; static int sparse_checkout_init(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { struct pattern_list pl; char *sparse_filename; @@ -464,7 +466,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, }; setup_work_tree(); - repo_read_index(the_repository); + repo_read_index(repo); init_opts.cone_mode = -1; init_opts.sparse_index = -1; @@ -473,7 +475,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, builtin_sparse_checkout_init_options, builtin_sparse_checkout_init_usage, 0); - if (update_modes(&init_opts.cone_mode, &init_opts.sparse_index)) + if (update_modes(repo, &init_opts.cone_mode, &init_opts.sparse_index)) return 1; memset(&pl, 0, sizeof(pl)); @@ -485,14 +487,14 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, if (res >= 0) { free(sparse_filename); clear_pattern_list(&pl); - return update_working_directory(NULL); + return update_working_directory(repo, NULL); } - if (repo_get_oid(the_repository, "HEAD", &oid)) { + if (repo_get_oid(repo, "HEAD", &oid)) { FILE *fp; /* assume we are in a fresh repo, but update the sparse-checkout file */ - if (safe_create_leading_directories(the_repository, sparse_filename)) + if (safe_create_leading_directories(repo, sparse_filename)) die(_("unable to create leading directories of %s"), sparse_filename); fp = xfopen(sparse_filename, "w"); @@ -511,7 +513,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, add_pattern("!/*/", empty_base, 0, &pl, 0); pl.use_cone_patterns = init_opts.cone_mode; - return write_patterns_and_update(&pl); + return write_patterns_and_update(repo, &pl); } static void insert_recursive_pattern(struct pattern_list *pl, struct strbuf *path) @@ -674,7 +676,8 @@ static void add_patterns_literal(int argc, const char **argv, add_patterns_from_input(pl, argc, argv, use_stdin ? stdin : NULL); } -static int modify_pattern_list(struct strvec *args, int use_stdin, +static int modify_pattern_list(struct repository *repo, + struct strvec *args, int use_stdin, enum modify_type m) { int result; @@ -696,22 +699,23 @@ static int modify_pattern_list(struct strvec *args, int use_stdin, } if (!core_apply_sparse_checkout) { - set_config(MODE_ALL_PATTERNS); + set_config(repo, MODE_ALL_PATTERNS); core_apply_sparse_checkout = 1; changed_config = 1; } - result = write_patterns_and_update(pl); + result = write_patterns_and_update(repo, pl); if (result && changed_config) - set_config(MODE_NO_PATTERNS); + set_config(repo, MODE_NO_PATTERNS); clear_pattern_list(pl); free(pl); return result; } -static void sanitize_paths(struct strvec *args, +static void sanitize_paths(struct repository *repo, + struct strvec *args, const char *prefix, int skip_checks) { int i; @@ -752,7 +756,7 @@ static void sanitize_paths(struct strvec *args, for (i = 0; i < args->nr; i++) { struct cache_entry *ce; - struct index_state *index = the_repository->index; + struct index_state *index = repo->index; int pos = index_name_pos(index, args->v[i], strlen(args->v[i])); if (pos < 0) @@ -779,7 +783,7 @@ static struct sparse_checkout_add_opts { } add_opts; static int sparse_checkout_add(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_add_options[] = { OPT_BOOL_F(0, "skip-checks", &add_opts.skip_checks, @@ -796,7 +800,7 @@ static int sparse_checkout_add(int argc, const char **argv, const char *prefix, if (!core_apply_sparse_checkout) die(_("no sparse-checkout to add to")); - repo_read_index(the_repository); + repo_read_index(repo); argc = parse_options(argc, argv, prefix, builtin_sparse_checkout_add_options, @@ -804,9 +808,9 @@ static int sparse_checkout_add(int argc, const char **argv, const char *prefix, for (int i = 0; i < argc; i++) strvec_push(&patterns, argv[i]); - sanitize_paths(&patterns, prefix, add_opts.skip_checks); + sanitize_paths(repo, &patterns, prefix, add_opts.skip_checks); - ret = modify_pattern_list(&patterns, add_opts.use_stdin, ADD); + ret = modify_pattern_list(repo, &patterns, add_opts.use_stdin, ADD); strvec_clear(&patterns); return ret; @@ -825,7 +829,7 @@ static struct sparse_checkout_set_opts { } set_opts; static int sparse_checkout_set(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int default_patterns_nr = 2; const char *default_patterns[] = {"/*", "!/*/", NULL}; @@ -847,7 +851,7 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, int ret; setup_work_tree(); - repo_read_index(the_repository); + repo_read_index(repo); set_opts.cone_mode = -1; set_opts.sparse_index = -1; @@ -856,7 +860,7 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, builtin_sparse_checkout_set_options, builtin_sparse_checkout_set_usage, 0); - if (update_modes(&set_opts.cone_mode, &set_opts.sparse_index)) + if (update_modes(repo, &set_opts.cone_mode, &set_opts.sparse_index)) return 1; /* @@ -870,10 +874,10 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, } else { for (int i = 0; i < argc; i++) strvec_push(&patterns, argv[i]); - sanitize_paths(&patterns, prefix, set_opts.skip_checks); + sanitize_paths(repo, &patterns, prefix, set_opts.skip_checks); } - ret = modify_pattern_list(&patterns, set_opts.use_stdin, REPLACE); + ret = modify_pattern_list(repo, &patterns, set_opts.use_stdin, REPLACE); strvec_clear(&patterns); return ret; @@ -891,7 +895,7 @@ static struct sparse_checkout_reapply_opts { static int sparse_checkout_reapply(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_reapply_options[] = { OPT_BOOL(0, "cone", &reapply_opts.cone_mode, @@ -912,12 +916,12 @@ static int sparse_checkout_reapply(int argc, const char **argv, builtin_sparse_checkout_reapply_options, builtin_sparse_checkout_reapply_usage, 0); - repo_read_index(the_repository); + repo_read_index(repo); - if (update_modes(&reapply_opts.cone_mode, &reapply_opts.sparse_index)) + if (update_modes(repo, &reapply_opts.cone_mode, &reapply_opts.sparse_index)) return 1; - return update_working_directory(NULL); + return update_working_directory(repo, NULL); } static char const * const builtin_sparse_checkout_disable_usage[] = { @@ -927,7 +931,7 @@ static char const * const builtin_sparse_checkout_disable_usage[] = { static int sparse_checkout_disable(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_disable_options[] = { OPT_END(), @@ -955,7 +959,7 @@ static int sparse_checkout_disable(int argc, const char **argv, * are expecting to do that when disabling sparse-checkout. */ give_advice_on_expansion = 0; - repo_read_index(the_repository); + repo_read_index(repo); memset(&pl, 0, sizeof(pl)); hashmap_init(&pl.recursive_hashmap, pl_hashmap_cmp, NULL, 0); @@ -966,13 +970,13 @@ static int sparse_checkout_disable(int argc, const char **argv, add_pattern("/*", empty_base, 0, &pl, 0); prepare_repo_settings(the_repository); - the_repository->settings.sparse_index = 0; + repo->settings.sparse_index = 0; - if (update_working_directory(&pl)) + if (update_working_directory(repo, &pl)) die(_("error while refreshing working directory")); clear_pattern_list(&pl); - return set_config(MODE_NO_PATTERNS); + return set_config(repo, MODE_NO_PATTERNS); } static char const * const builtin_sparse_checkout_check_rules_usage[] = { @@ -987,14 +991,17 @@ static struct sparse_checkout_check_rules_opts { char *rules_file; } check_rules_opts; -static int check_rules(struct pattern_list *pl, int null_terminated) { +static int check_rules(struct repository *repo, + struct pattern_list *pl, + int null_terminated) +{ struct strbuf line = STRBUF_INIT; struct strbuf unquoted = STRBUF_INIT; char *path; int line_terminator = null_terminated ? 0 : '\n'; strbuf_getline_fn getline_fn = null_terminated ? strbuf_getline_nul : strbuf_getline; - the_repository->index->sparse_checkout_patterns = pl; + repo->index->sparse_checkout_patterns = pl; while (!getline_fn(&line, stdin)) { path = line.buf; if (!null_terminated && line.buf[0] == '"') { @@ -1006,7 +1013,7 @@ static int check_rules(struct pattern_list *pl, int null_terminated) { path = unquoted.buf; } - if (path_in_sparse_checkout(path, the_repository->index)) + if (path_in_sparse_checkout(path, repo->index)) write_name_quoted(path, stdout, line_terminator); } strbuf_release(&line); @@ -1016,7 +1023,7 @@ static int check_rules(struct pattern_list *pl, int null_terminated) { } static int sparse_checkout_check_rules(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_check_rules_options[] = { OPT_BOOL('z', NULL, &check_rules_opts.null_termination, @@ -1055,7 +1062,7 @@ static int sparse_checkout_check_rules(int argc, const char **argv, const char * free(sparse_filename); } - ret = check_rules(&pl, check_rules_opts.null_termination); + ret = check_rules(repo, &pl, check_rules_opts.null_termination); clear_pattern_list(&pl); free(check_rules_opts.rules_file); return ret; @@ -1084,8 +1091,8 @@ int cmd_sparse_checkout(int argc, repo_config(the_repository, git_default_config, NULL); - prepare_repo_settings(the_repository); - the_repository->settings.command_requires_full_index = 0; + prepare_repo_settings(repo); + repo->settings.command_requires_full_index = 0; return fn(argc, argv, prefix, repo); } From a1564f74cfa19eee528a1a0c54b87ac52d1e8975 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 7 Jul 2025 11:42:06 -0400 Subject: [PATCH 2/7] sparse-checkout: add basics of 'clean' command When users change their sparse-checkout definitions to add new directories and remove old ones, there may be a few reasons why directories no longer in scope remain (ignored or excluded files still exist, Windows handles are still open, etc.). When these files still exist, the sparse index feature notices that a tracked, but sparse, directory still exists on disk and thus the index expands. This causes a performance hit _and_ the advice printed isn't very helpful. Using 'git clean' isn't enough (generally '-dfx' may be needed) but also this may not be sufficient. Add a new subcommand to 'git sparse-checkout' that removes these tracked-but-sparse directories. The implementation details provide a clear definition of what is happening, but it is difficult to describe this without including the internal implementation details. The core operation converts the index to a sparse index (in memory if not already on disk) and then deletes any directories in the worktree that correspond with a sparse directory entry in that sparse index. In the most common case, this means that a file will be removed if it is contained within a directory that is both tracked and outside of the sparse-checkout definition. However, there can be exceptions depending on the current state of the index: * If the worktree has a modification to a tracked, sparse file, then that file's parent directories will be expanded instead of represented as sparse directories. Siblings of those parent directories may be considered sparse. * If the user staged a sparse file with "git add --sparse", then that file loses the SKIP_WORKTREE bit until the sparse-checkout is reapplied. Until then, that file's parent directories are not represented as sparse directory entries and thus will not be removed. Siblings of those parent directories may be considered sparse. (There may be other reasons why the SKIP_WORKTREE bit was removed for a file and this impact on the sparse directories will apply to those as well.) * If the user has a merge conflict outside of the sparse-checkout definition, then those conflict entries prevent the parent directories from being represented as sparse directory entries and thus are not removed. * The cases above present reasons why certain _file conditions_ will impact which _directories_ are considered sparse. The list of tracked directories that are outside of the sparse-checkout definition but not represented as a sparse directory further reduces the list of files that will be removed. For these complicated reasons, the documentation details a potential list of files that will be "considered for removal" instead of defining the list concretely. The special cases can be handled by resolving conflicts, committing staged changes, and running 'git sparse-checkout reapply' to update the SKIP_WORKTREE bits as expected by the sparse-checkout definition. It is important to make clear that this operation will remove ignored and excluded files which would normally be ignored even by 'git clean -f' unless the '-x' or '-X' option is provided. This is the most extreme method for doing this, but it works when the sparse-checkout is in cone mode and is expected to rescope based on directories, not files. The current implementation always deletes these sparse directories without warning. This is unacceptable for a released version, but those features will be added in changes coming immediately after this one. Note that this will not remove an untracked directory (or any of its contents) if its parent is a tracked directory within the sparse-checkout definition. This is required to prevent removing data created by tools that perform caching operations for editors or build tools. Thus, 'git sparse-checkout clean' is both more aggressive and more careful than 'git clean -fx': * It is more aggressive because it will remove _tracked_ files within the sparse directories. * It is less aggressive because it will leave _untracked_ files that are not contained in sparse directories. These special cases will be handled more explicitly in a future change that expands tests for the 'git sparse-checkout clean' command. We handle some of the modified, staged, and committed states including some impact on 'git status' after cleaning. Signed-off-by: Derrick Stolee --- Documentation/git-sparse-checkout.adoc | 19 ++++- builtin/sparse-checkout.c | 64 ++++++++++++++- t/t1091-sparse-checkout-builtin.sh | 103 +++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 2 deletions(-) diff --git a/Documentation/git-sparse-checkout.adoc b/Documentation/git-sparse-checkout.adoc index 529a8edd9c1ed8..baaebce746d2a1 100644 --- a/Documentation/git-sparse-checkout.adoc +++ b/Documentation/git-sparse-checkout.adoc @@ -9,7 +9,7 @@ git-sparse-checkout - Reduce your working tree to a subset of tracked files SYNOPSIS -------- [verse] -'git sparse-checkout' (init | list | set | add | reapply | disable | check-rules) [] +'git sparse-checkout' (init | list | set | add | reapply | disable | check-rules | clean) [] DESCRIPTION @@ -111,6 +111,23 @@ flags, with the same meaning as the flags from the `set` command, in order to change which sparsity mode you are using without needing to also respecify all sparsity paths. +'clean':: + Opportunistically remove files outside of the sparse-checkout + definition. This command requires cone mode to use recursive + directory matches to determine which files should be removed. A + file is considered for removal if it is contained within a tracked + directory that is outside of the sparse-checkout definition. ++ +Some special cases, such as merge conflicts or modified files outside of +the sparse-checkout definition could lead to keeping files that would +otherwise be removed. Resolve conflicts, stage modifications, and use +`git sparse-checkout reapply` in conjunction with `git sparse-checkout +clean` to resolve these cases. ++ +This command can be used to be sure the sparse index works efficiently, +though it does not require enabling the sparse index feature via the +`index.sparse=true` configuration. + 'disable':: Disable the `core.sparseCheckout` config setting, and restore the working directory to include all files. diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 06de61bd9d0d33..f7caa28f3f00aa 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -2,6 +2,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "abspath.h" #include "config.h" #include "dir.h" #include "environment.h" @@ -23,7 +24,7 @@ static const char *empty_base = ""; static char const * const builtin_sparse_checkout_usage[] = { - N_("git sparse-checkout (init | list | set | add | reapply | disable | check-rules) []"), + N_("git sparse-checkout (init | list | set | add | reapply | disable | check-rules | clean) []"), NULL }; @@ -924,6 +925,66 @@ static int sparse_checkout_reapply(int argc, const char **argv, return update_working_directory(repo, NULL); } +static char const * const builtin_sparse_checkout_clean_usage[] = { + "git sparse-checkout clean [-n|--dry-run]", + NULL +}; + +static const char *msg_remove = N_("Removing %s\n"); + +static int sparse_checkout_clean(int argc, const char **argv, + const char *prefix, + struct repository *repo) +{ + struct strbuf full_path = STRBUF_INIT; + const char *msg = msg_remove; + size_t worktree_len; + + struct option builtin_sparse_checkout_clean_options[] = { + OPT_END(), + }; + + setup_work_tree(); + if (!core_apply_sparse_checkout) + die(_("must be in a sparse-checkout to clean directories")); + if (!core_sparse_checkout_cone) + die(_("must be in a cone-mode sparse-checkout to clean directories")); + + argc = parse_options(argc, argv, prefix, + builtin_sparse_checkout_clean_options, + builtin_sparse_checkout_clean_usage, 0); + + if (repo_read_index(repo) < 0) + die(_("failed to read index")); + + if (convert_to_sparse(repo->index, SPARSE_INDEX_MEMORY_ONLY) || + repo->index->sparse_index == INDEX_EXPANDED) + die(_("failed to convert index to a sparse index; resolve merge conflicts and try again")); + + strbuf_addstr(&full_path, repo->worktree); + strbuf_addch(&full_path, '/'); + worktree_len = full_path.len; + + for (size_t i = 0; i < repo->index->cache_nr; i++) { + struct cache_entry *ce = repo->index->cache[i]; + if (!S_ISSPARSEDIR(ce->ce_mode)) + continue; + strbuf_setlen(&full_path, worktree_len); + strbuf_add(&full_path, ce->name, ce->ce_namelen); + + if (!is_directory(full_path.buf)) + continue; + + printf(msg, ce->name); + + if (remove_dir_recursively(&full_path, 0)) + warning_errno(_("failed to remove '%s'"), ce->name); + } + + strbuf_release(&full_path); + return 0; +} + static char const * const builtin_sparse_checkout_disable_usage[] = { "git sparse-checkout disable", NULL @@ -1080,6 +1141,7 @@ int cmd_sparse_checkout(int argc, OPT_SUBCOMMAND("set", &fn, sparse_checkout_set), OPT_SUBCOMMAND("add", &fn, sparse_checkout_add), OPT_SUBCOMMAND("reapply", &fn, sparse_checkout_reapply), + OPT_SUBCOMMAND("clean", &fn, sparse_checkout_clean), OPT_SUBCOMMAND("disable", &fn, sparse_checkout_disable), OPT_SUBCOMMAND("check-rules", &fn, sparse_checkout_check_rules), OPT_END(), diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index ab3a105ffff253..bdb7b21e327b2a 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -1050,5 +1050,108 @@ test_expect_success 'check-rules null termination' ' test_cmp expect actual ' +test_expect_success 'clean' ' + git -C repo sparse-checkout set --cone deep/deeper1 && + git -C repo sparse-checkout reapply && + mkdir repo/deep/deeper2 repo/folder1 && + + # Add untracked files + touch repo/deep/deeper2/file && + touch repo/folder1/file && + + cat >expect <<-\EOF && + Removing deep/deeper2/ + Removing folder1/ + EOF + + git -C repo sparse-checkout clean >out && + test_cmp expect out && + + test_path_is_missing repo/deep/deeper2 && + test_path_is_missing repo/folder1 +' + +test_expect_success 'clean with sparse file states' ' + test_when_finished git reset --hard && + git -C repo sparse-checkout set --cone deep/deeper1 && + mkdir repo/folder2 && + + # create an untracked file and a modified file + touch repo/folder2/file && + echo dirty >repo/folder2/a && + + # First clean/reapply pass will do nothing. + git -C repo sparse-checkout clean >out && + test_must_be_empty out && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + git -C repo sparse-checkout reapply 2>err && + test_grep folder2 err && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + # Now, stage the change to the tracked file. + git -C repo add --sparse folder2/a && + + # Clean will continue not doing anything. + git -C repo sparse-checkout clean >out && + test_line_count = 0 out && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + # But we can reapply to remove the staged change. + git -C repo sparse-checkout reapply 2>err && + test_grep folder2 err && + test_path_is_missing repo/folder2/a && + test_path_exists repo/folder2/file && + + # We can clean now. + cat >expect <<-\EOF && + Removing folder2/ + EOF + git -C repo sparse-checkout clean >out && + test_cmp expect out && + test_path_is_missing repo/folder2 && + + # At the moment, the file is staged. + cat >expect <<-\EOF && + M folder2/a + EOF + + git -C repo status -s >out && + test_cmp expect out && + + # Reapply persists the modified state. + git -C repo sparse-checkout reapply && + cat >expect <<-\EOF && + M folder2/a + EOF + git -C repo status -s >out && + test_cmp expect out && + + # Committing the change leads to resolved status. + git -C repo commit -m "modified" && + git -C repo status -s >out && + test_must_be_empty out && + + # Repeat, but this time commit before reapplying. + mkdir repo/folder2/ && + echo dirtier >repo/folder2/a && + git -C repo add --sparse folder2/a && + git -C repo sparse-checkout clean >out && + test_must_be_empty out && + test_path_exists repo/folder2/a && + + # Committing without reapplying makes it look like a deletion + # due to no skip-worktree bit. + git -C repo commit -m "dirtier" && + git -C repo status -s >out && + test_must_be_empty out && + + git -C repo sparse-checkout reapply && + git -C repo status -s >out && + test_must_be_empty out +' test_done From 71a498db65440ad40c6cbde92b74ea1105075737 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 15 Jul 2025 10:47:31 -0400 Subject: [PATCH 3/7] sparse-checkout: match some 'clean' behavior The 'git sparse-checkout clean' subcommand is somewhat similar to 'git clean' in that it will delete files that should not be in the worktree. The big difference is that it focuses on the directories that should not be in the worktree due to cone-mode sparse-checkout. It also does not discriminate in the kinds of files and focuses on deleting entire directories. However, there are some restrictions that would be good to bring over from 'git clean', specifically how it refuses to do anything without the '-f'/'--force' or '-n'/'--dry-run' arguments. The 'clean.requireForce' config can be set to 'false' to imply '--force'. Add this behavior to avoid accidental deletion of files that cannot be recovered from Git. Signed-off-by: Derrick Stolee --- Documentation/git-sparse-checkout.adoc | 9 +++++ builtin/sparse-checkout.c | 15 ++++++- t/t1091-sparse-checkout-builtin.sh | 54 +++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/Documentation/git-sparse-checkout.adoc b/Documentation/git-sparse-checkout.adoc index baaebce746d2a1..42050ff5b50cf2 100644 --- a/Documentation/git-sparse-checkout.adoc +++ b/Documentation/git-sparse-checkout.adoc @@ -127,6 +127,15 @@ clean` to resolve these cases. This command can be used to be sure the sparse index works efficiently, though it does not require enabling the sparse index feature via the `index.sparse=true` configuration. ++ +To prevent accidental deletion of worktree files, the `clean` subcommand +will not delete any files without the `-f` or `--force` option, unless +the `clean.requireForce` config option is set to `false`. ++ +The `--dry-run` option will list the directories that would be removed +without deleting them. Running in this mode can be helpful to predict the +behavior of the clean comand or to determine which kinds of files are left +in the sparse directories. 'disable':: Disable the `core.sparseCheckout` config setting, and restore the diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index f7caa28f3f00aa..d777b64960668d 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -931,6 +931,7 @@ static char const * const builtin_sparse_checkout_clean_usage[] = { }; static const char *msg_remove = N_("Removing %s\n"); +static const char *msg_would_remove = N_("Would remove %s\n"); static int sparse_checkout_clean(int argc, const char **argv, const char *prefix, @@ -939,8 +940,12 @@ static int sparse_checkout_clean(int argc, const char **argv, struct strbuf full_path = STRBUF_INIT; const char *msg = msg_remove; size_t worktree_len; + int force = 0, dry_run = 0; + int require_force = 1; struct option builtin_sparse_checkout_clean_options[] = { + OPT__DRY_RUN(&dry_run, N_("dry run")), + OPT__FORCE(&force, N_("force"), PARSE_OPT_NOCOMPLETE), OPT_END(), }; @@ -954,6 +959,13 @@ static int sparse_checkout_clean(int argc, const char **argv, builtin_sparse_checkout_clean_options, builtin_sparse_checkout_clean_usage, 0); + repo_config_get_bool(repo, "clean.requireforce", &require_force); + if (require_force && !force && !dry_run) + die(_("for safety, refusing to clean without one of --force or --dry-run")); + + if (dry_run) + msg = msg_would_remove; + if (repo_read_index(repo) < 0) die(_("failed to read index")); @@ -977,7 +989,8 @@ static int sparse_checkout_clean(int argc, const char **argv, printf(msg, ce->name); - if (remove_dir_recursively(&full_path, 0)) + if (dry_run <= 0 && + remove_dir_recursively(&full_path, 0)) warning_errno(_("failed to remove '%s'"), ce->name); } diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index bdb7b21e327b2a..e6b768a8da959a 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -1059,12 +1059,29 @@ test_expect_success 'clean' ' touch repo/deep/deeper2/file && touch repo/folder1/file && + test_must_fail git -C repo sparse-checkout clean 2>err && + grep "refusing to clean" err && + + git -C repo config clean.requireForce true && + test_must_fail git -C repo sparse-checkout clean 2>err && + grep "refusing to clean" err && + + cat >expect <<-\EOF && + Would remove deep/deeper2/ + Would remove folder1/ + EOF + + git -C repo sparse-checkout clean --dry-run >out && + test_cmp expect out && + test_path_exists repo/deep/deeper2 && + test_path_exists repo/folder1 && + cat >expect <<-\EOF && Removing deep/deeper2/ Removing folder1/ EOF - git -C repo sparse-checkout clean >out && + git -C repo sparse-checkout clean -f >out && test_cmp expect out && test_path_is_missing repo/deep/deeper2 && @@ -1076,6 +1093,10 @@ test_expect_success 'clean with sparse file states' ' git -C repo sparse-checkout set --cone deep/deeper1 && mkdir repo/folder2 && + # The previous test case checked the -f option, so + # test the config option in this one. + git -C repo config clean.requireForce false && + # create an untracked file and a modified file touch repo/folder2/file && echo dirty >repo/folder2/a && @@ -1154,4 +1175,35 @@ test_expect_success 'clean with sparse file states' ' test_must_be_empty out ' +test_expect_success 'clean with merge conflict status' ' + git clone repo clean-merge && + + echo dirty >clean-merge/deep/deeper2/a && + touch clean-merge/folder2/extra && + + cat >input <<-EOF && + 0 $ZERO_OID folder1/a + 100644 $(git -C clean-merge rev-parse HEAD:folder1/a) 1 folder1/a + EOF + git -C clean-merge update-index --index-info err && + grep "failed to convert index to a sparse index" err && + + mkdir -p clean-merge/folder1/ && + echo merged >clean-merge/folder1/a && + git -C clean-merge add --sparse folder1/a && + + # deletes folder2/ but leaves staged change in folder1 + # and dirty change in deep/deeper2/ + cat >expect <<-\EOF && + Removing folder2/ + EOF + + git -C clean-merge sparse-checkout clean -f >out && + test_cmp expect out +' + test_done From 6b29dda4b8961a999855ada42996de120513f218 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 15 Jul 2025 13:11:07 -0400 Subject: [PATCH 4/7] dir: add generic "walk all files" helper There is sometimes a need to visit every file within a directory, recursively. The main example is remove_dir_recursively(), though it has some extra flags that make it want to iterate over paths in a custom way. There is also the fill_directory() approach but that involves an index and a pathspec. This change adds a new for_each_file_in_dir() method that will be helpful in the next change. Signed-off-by: Derrick Stolee --- dir.c | 28 ++++++++++++++++++++++++++++ dir.h | 14 ++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/dir.c b/dir.c index 71108ac79b78b8..194b36a8c4040d 100644 --- a/dir.c +++ b/dir.c @@ -30,6 +30,7 @@ #include "read-cache-ll.h" #include "setup.h" #include "sparse-index.h" +#include "strbuf.h" #include "submodule-config.h" #include "symlinks.h" #include "trace2.h" @@ -87,6 +88,33 @@ struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp) return e; } +int for_each_file_in_dir(struct strbuf *path, file_iterator fn, const void *data) +{ + struct dirent *e; + int res = 0; + size_t baselen = path->len; + DIR *dir = opendir(path->buf); + + if (!dir) + return 0; + + while (!res && (e = readdir_skip_dot_and_dotdot(dir)) != NULL) { + unsigned char dtype = get_dtype(e, path, 0); + strbuf_setlen(path, baselen); + strbuf_addstr(path, e->d_name); + + if (dtype == DT_REG) { + res = fn(path->buf, data); + } else if (dtype == DT_DIR) { + strbuf_addch(path, '/'); + res = for_each_file_in_dir(path, fn, data); + } + } + + closedir(dir); + return res; +} + int count_slashes(const char *s) { int cnt = 0; diff --git a/dir.h b/dir.h index fc9be7b427a134..20d4a078d61ef8 100644 --- a/dir.h +++ b/dir.h @@ -536,6 +536,20 @@ int get_sparse_checkout_patterns(struct pattern_list *pl); */ int remove_dir_recursively(struct strbuf *path, int flag); +/* + * This function pointer type is called on each file discovered in + * for_each_file_in_dir. The iteration stops if this method returns + * non-zero. + */ +typedef int (*file_iterator)(const char *path, const void *data); + +struct strbuf; +/* + * Given a directory path, recursively visit each file within, including + * within subdirectories. + */ +int for_each_file_in_dir(struct strbuf *path, file_iterator fn, const void *data); + /* * Tries to remove the path, along with leading empty directories so long as * those empty directories are not startup_info->original_cwd. Ignores From 2cde464fd4c225144489c222537e5d7549f81849 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 15 Jul 2025 13:15:32 -0400 Subject: [PATCH 5/7] sparse-checkout: add --verbose option to 'clean' The 'git sparse-checkout clean' subcommand is focused on directories, deleting any tracked sparse directories to clean up the worktree and make the sparse index feature work optimally. However, this directory-focused approach can leave users wondering why those directories exist at all. In my experience, these files are left over due to ignore or exclude patterns, Windows file handles, or possibly merge conflict resolutions. Add a new '--verbose' option for users to see all the files that are being deleted (with '--force') or would be deleted (with '--dry-run'). Based on usage, users may request further context on this list of files for states such as tracked/untracked, unstaged/staged/conflicted, etc. Signed-off-by: Derrick Stolee --- Documentation/git-sparse-checkout.adoc | 5 +++++ builtin/sparse-checkout.c | 28 ++++++++++++++++++++++++-- t/t1091-sparse-checkout-builtin.sh | 14 ++++++++++--- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/Documentation/git-sparse-checkout.adoc b/Documentation/git-sparse-checkout.adoc index 42050ff5b50cf2..113728a0e7c01d 100644 --- a/Documentation/git-sparse-checkout.adoc +++ b/Documentation/git-sparse-checkout.adoc @@ -136,6 +136,11 @@ The `--dry-run` option will list the directories that would be removed without deleting them. Running in this mode can be helpful to predict the behavior of the clean comand or to determine which kinds of files are left in the sparse directories. ++ +The `--verbose` option will list every file within the directories that +are considered for removal. This option is helpful to determine if those +files are actually important or perhaps to explain why the directory is +still present despite the current sparse-checkout. 'disable':: Disable the `core.sparseCheckout` config setting, and restore the diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index d777b64960668d..8d3c3485f53bdf 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -930,6 +930,26 @@ static char const * const builtin_sparse_checkout_clean_usage[] = { NULL }; +static int list_file_iterator(const char *path, const void *data) +{ + const char *msg = data; + + printf(msg, path); + return 0; +} + +static void list_every_file_in_dir(const char *msg, + const char *directory) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, directory); + fprintf(stderr, "list every file in %s\n", directory); + + for_each_file_in_dir(&path, list_file_iterator, msg); + strbuf_release(&path); +} + static const char *msg_remove = N_("Removing %s\n"); static const char *msg_would_remove = N_("Would remove %s\n"); @@ -940,12 +960,13 @@ static int sparse_checkout_clean(int argc, const char **argv, struct strbuf full_path = STRBUF_INIT; const char *msg = msg_remove; size_t worktree_len; - int force = 0, dry_run = 0; + int force = 0, dry_run = 0, verbose = 0; int require_force = 1; struct option builtin_sparse_checkout_clean_options[] = { OPT__DRY_RUN(&dry_run, N_("dry run")), OPT__FORCE(&force, N_("force"), PARSE_OPT_NOCOMPLETE), + OPT__VERBOSE(&verbose, N_("report each affected file, not just directories")), OPT_END(), }; @@ -987,7 +1008,10 @@ static int sparse_checkout_clean(int argc, const char **argv, if (!is_directory(full_path.buf)) continue; - printf(msg, ce->name); + if (verbose) + list_every_file_in_dir(msg, ce->name); + else + printf(msg, ce->name); if (dry_run <= 0 && remove_dir_recursively(&full_path, 0)) diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index e6b768a8da959a..7b15fa669c4662 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -1053,11 +1053,11 @@ test_expect_success 'check-rules null termination' ' test_expect_success 'clean' ' git -C repo sparse-checkout set --cone deep/deeper1 && git -C repo sparse-checkout reapply && - mkdir repo/deep/deeper2 repo/folder1 && + mkdir -p repo/deep/deeper2 repo/folder1/extra/inside && # Add untracked files touch repo/deep/deeper2/file && - touch repo/folder1/file && + touch repo/folder1/extra/inside/file && test_must_fail git -C repo sparse-checkout clean 2>err && grep "refusing to clean" err && @@ -1074,7 +1074,15 @@ test_expect_success 'clean' ' git -C repo sparse-checkout clean --dry-run >out && test_cmp expect out && test_path_exists repo/deep/deeper2 && - test_path_exists repo/folder1 && + test_path_exists repo/folder1/extra/inside/file && + + cat >expect <<-\EOF && + Would remove deep/deeper2/file + Would remove folder1/extra/inside/file + EOF + + git -C repo sparse-checkout clean --dry-run --verbose >out && + test_cmp expect out && cat >expect <<-\EOF && Removing deep/deeper2/ From 460e5e8157fc87a4246c49f53e34495bc33d4432 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 7 Jul 2025 11:55:46 -0400 Subject: [PATCH 6/7] sparse-index: point users to new 'clean' action In my experience, the most-common reason that the sparse index must expand to a full one is because there is some leftover file in a tracked directory that is now outside of the sparse-checkout. The new 'git sparse-checkout clean' command will find and delete these directories, so point users to it when they hit the sparse index expansion advice. Signed-off-by: Derrick Stolee --- sparse-index.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparse-index.c b/sparse-index.c index 5634abafaa07ed..5d14795063b578 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -32,7 +32,8 @@ int give_advice_on_expansion = 1; "Your working directory likely has contents that are outside of\n" \ "your sparse-checkout patterns. Use 'git sparse-checkout list' to\n" \ "see your sparse-checkout definition and compare it to your working\n" \ - "directory contents. Running 'git clean' may assist in this cleanup." + "directory contents. Running 'git sparse-checkout clean' may assist\n" \ + "in this cleanup." struct modify_index_context { struct index_state *write; From 7f6f62bce607fd4d0438f05f9ee76f8547b96caf Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 16 Jul 2025 10:59:11 -0400 Subject: [PATCH 7/7] t: expand tests around sparse merges and clean With the current implementation of 'git sparse-checkout clean', we notice that a file that was in a conflicted state does not get cleaned up because of some internal details around the SKIP_WORKTREE bit. This test is documenting the current behavior before we update it in the following change. Signed-off-by: Derrick Stolee --- t/t1091-sparse-checkout-builtin.sh | 56 ++++++++++++++++++------------ 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 7b15fa669c4662..b2da4feaeff9ec 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -1183,35 +1183,47 @@ test_expect_success 'clean with sparse file states' ' test_must_be_empty out ' -test_expect_success 'clean with merge conflict status' ' - git clone repo clean-merge && +test_expect_success 'sparse-checkout operations with merge conflicts' ' + git clone repo merge && - echo dirty >clean-merge/deep/deeper2/a && - touch clean-merge/folder2/extra && + ( + cd merge && + mkdir -p folder1/even/more/dirs && + echo base >folder1/even/more/dirs/file && + git add folder1 && + git commit -m "base" && - cat >input <<-EOF && - 0 $ZERO_OID folder1/a - 100644 $(git -C clean-merge rev-parse HEAD:folder1/a) 1 folder1/a - EOF - git -C clean-merge update-index --index-info folder1/even/more/dirs/file && + git commit -a -m "right" && - git -C clean-merge sparse-checkout set deep/deeper1 && + git checkout -b left HEAD~1 && + echo left >folder1/even/more/dirs/file && + git commit -a -m "left" && - test_must_fail git -C clean-merge sparse-checkout clean -f 2>err && - grep "failed to convert index to a sparse index" err && + git checkout -b merge && + git sparse-checkout set deep/deeper1 && - mkdir -p clean-merge/folder1/ && - echo merged >clean-merge/folder1/a && - git -C clean-merge add --sparse folder1/a && + test_must_fail git merge -m "will-conflict" right && - # deletes folder2/ but leaves staged change in folder1 - # and dirty change in deep/deeper2/ - cat >expect <<-\EOF && - Removing folder2/ - EOF + test_must_fail git sparse-checkout clean -f 2>err && + grep "failed to convert index to a sparse index" err && - git -C clean-merge sparse-checkout clean -f >out && - test_cmp expect out + echo merged >folder1/even/more/dirs/file && + git add --sparse folder1 && + git merge --continue && + + test_path_exists folder1/even/more/dirs/file && + + # clean does not remove the file, because the + # SKIP_WORKTREE bit was not cleared by the merge command. + git sparse-checkout clean -f >out && + test_line_count = 0 out && + test_path_exists folder1/even/more/dirs/file && + + git sparse-checkout reapply && + test_path_is_missing folder1 + ) ' test_done