[DO NOT MERGE] sparse index prototype

Documentation/config/extensions.txt

@@ -6,3 +6,10 @@ extensions.objectFormat::
 Note that this setting should only be set by linkgit:git-init[1] or
 linkgit:git-clone[1].  Trying to change it after initialization will not
 work and will produce hard-to-diagnose issues.
+
+extensions.sparseIndex::
+	When combined with `core.sparseCheckout=true` and
+	`core.sparseCheckoutCone=true`, the index may contain entries
+	corresponding to directories outside of the sparse-checkout
+	definition. Versions of Git that do not understand this extension
+	do not expect directory entries in the index.

Documentation/git-sparse-checkout.txt

@@ -45,6 +45,20 @@ To avoid interfering with other worktrees, it first enables the
 When `--cone` is provided, the `core.sparseCheckoutCone` setting is
 also set, allowing for better performance with a limited set of
 patterns (see 'CONE PATTERN SET' below).
++
+Use the `--[no-]sparse-index` option to toggle the use of the sparse
+index format. This reduces the size of the index to be more closely
+aligned with your sparse-checkout definition. This can have significant
+performance advantages for commands such as `git status` or `git add`.
+This feature is still experimental. Some commands might be slower with
+a sparse index until they are properly integrated with the feature.
++
+**WARNING:** Using a sparse index requires modifying the index in a way
+that is not completely understood by other tools. Enabling sparse index
+enables the `extensions.spareseIndex` config value, which might cause
+other tools to stop working with your repository. If you have trouble with
+this compatibility, then run `git sparse-checkout sparse-index disable` to
+remove this config and rewrite your index to not be sparse.
 
 'set'::
 	Write a set of patterns to the sparse-checkout file, as given as

Documentation/technical/index-format.txt

@@ -26,7 +26,7 @@ Git index format
      Extensions are identified by signature. Optional extensions can
      be ignored if Git does not understand them.
 
-     Git currently supports cached tree and resolve undo extensions.
+     Git currently supports cache tree and resolve undo extensions.
 
      4-byte extension signature. If the first byte is 'A'..'Z' the
      extension is optional and can be ignored.
@@ -136,14 +136,35 @@ Git index format
 
 == Extensions
 
-=== Cached tree
-
-  Cached tree extension contains pre-computed hashes for trees that can
-  be derived from the index. It helps speed up tree object generation
-  from index for a new commit.
-
-  When a path is updated in index, the path must be invalidated and
-  removed from tree cache.
+=== Cache tree
+
+  Since the index does not record entries for directories, the cache
+  entries cannot describe tree objects that already exist in the object
+  database for regions of the index that are unchanged from an existing
+  commit. The cache tree extension stores a recursive tree structure that
+  describes the trees that already exist and completely match sections of
+  the cache entries. This speeds up tree object generation from the index
+  for a new commit by only computing the trees that are "new" to that
+  commit. It also assists when comparing the index to another tree, such
+  as `HEAD^{tree}`, since sections of the index can be skipped when a tree
+  comparison demonstrates equality.
+
+  The recursive tree structure uses nodes that store a number of cache
+  entries, a list of subnodes, and an object ID (OID). The OID references
+  the existing tree for that node, if it is known to exist. The subnodes
+  correspond to subdirectories that themselves have cache tree nodes. The
+  number of cache entries corresponds to the number of cache entries in
+  the index that describe paths within that tree's directory.
+
+  The extension tracks the full directory structure in the cache tree
+  extension, but this is generally smaller than the full cache entry list.
+
+  When a path is updated in index, Git invalidates all nodes of the
+  recursive cache tree corresponding to the parent directories of that
+  path. We store these tree nodes as being "invalid" by using "-1" as the
+  number of cache entries. Invalid nodes still store a span of index
+  entries, allowing Git to focus its efforts when reconstructing a full
+  cache tree.
 
   The signature for this extension is { 'T', 'R', 'E', 'E' }.
 
@@ -174,7 +195,8 @@ Git index format
   first entry represents the root level of the repository, followed by the
   first subtree--let's call this A--of the root level (with its name
   relative to the root level), followed by the first subtree of A (with
-  its name relative to A), ...
+  its name relative to A), and so on. The specified number of subtrees
+  indicates when the current level of the recursive stack is complete.
 
 === Resolve undo
 

Documentation/technical/racy-git.txt

@@ -26,7 +26,7 @@ information obtained from the filesystem via `lstat(2)` system
 call when they were last updated.  When checking if they differ,
 Git first runs `lstat(2)` on the files and compares the result
 with this information (this is what was originally done by the
-`ce_match_stat()` function, but the current code does it in
+`ie_match_stat()` function, but the current code does it in
 `ce_match_stat_basic()` function).  If some of these "cached
 stat information" fields do not match, Git can tell that the
 files are modified without even looking at their contents.
@@ -102,7 +102,7 @@ timestamp as the index file itself.
 
 The callers that want to check if an index entry matches the
 corresponding file in the working tree continue to call
-`ce_match_stat()`, but with this change, `ce_match_stat()` uses
+`ie_match_stat()`, but with this change, `ie_match_stat()` uses
 `ce_modified_check_fs()` to see if racily clean ones are
 actually clean after comparing the cached stat information using
 `ce_match_stat_basic()`.
@@ -128,7 +128,7 @@ Runtime penalty
 ---------------
 
 The runtime penalty of falling back to `ce_modified_check_fs()`
-from `ce_match_stat()` can be very expensive when there are many
+from `ie_match_stat()` can be very expensive when there are many
 racily clean entries.  An obvious way to artificially create
 this situation is to give the same timestamp to all the files in
 the working tree in a large project, run `git update-index` on

Makefile

@@ -603,9 +603,6 @@ unexport CDPATH
 SCRIPT_SH += git-bisect.sh
 SCRIPT_SH += git-difftool--helper.sh
 SCRIPT_SH += git-filter-branch.sh
-SCRIPT_SH += git-merge-octopus.sh
-SCRIPT_SH += git-merge-one-file.sh
-SCRIPT_SH += git-merge-resolve.sh
 SCRIPT_SH += git-mergetool.sh
 SCRIPT_SH += git-quiltimport.sh
 SCRIPT_SH += git-request-pull.sh
@@ -930,6 +927,7 @@ LIB_OBJS += merge-blobs.o
 LIB_OBJS += merge-ort.o
 LIB_OBJS += merge-ort-wrappers.o
 LIB_OBJS += merge-recursive.o
+LIB_OBJS += merge-strategies.o
 LIB_OBJS += merge.o
 LIB_OBJS += mergesort.o
 LIB_OBJS += midx.o
@@ -1003,6 +1001,7 @@ LIB_OBJS += sha1-name.o
 LIB_OBJS += shallow.o
 LIB_OBJS += sideband.o
 LIB_OBJS += sigchain.o
+LIB_OBJS += sparse-index.o
 LIB_OBJS += split-index.o
 LIB_OBJS += stable-qsort.o
 LIB_OBJS += strbuf.o
@@ -1120,8 +1119,11 @@ BUILTIN_OBJS += builtin/mailsplit.o
 BUILTIN_OBJS += builtin/merge-base.o
 BUILTIN_OBJS += builtin/merge-file.o
 BUILTIN_OBJS += builtin/merge-index.o
+BUILTIN_OBJS += builtin/merge-octopus.o
+BUILTIN_OBJS += builtin/merge-one-file.o
 BUILTIN_OBJS += builtin/merge-ours.o
 BUILTIN_OBJS += builtin/merge-recursive.o
+BUILTIN_OBJS += builtin/merge-resolve.o
 BUILTIN_OBJS += builtin/merge-tree.o
 BUILTIN_OBJS += builtin/merge.o
 BUILTIN_OBJS += builtin/mktag.o

apply.c

@@ -3541,6 +3541,8 @@ static int load_current(struct apply_state *state,
 	if (!patch->is_new)
 		BUG("patch to %s is not a creation", patch->old_name);
 
+	ensure_full_index(state->repo->index);
+
 	pos = index_name_pos(state->repo->index, name, strlen(name));
 	if (pos < 0)
 		return error(_("%s: does not exist in index"), name);
@@ -3710,7 +3712,11 @@ static int check_preimage(struct apply_state *state,
 	}
 
 	if (state->check_index && !previous) {
-		int pos = index_name_pos(state->repo->index, old_name,
+		int pos;
+
+		ensure_full_index(state->repo->index);
+
+		pos = index_name_pos(state->repo->index, old_name,
 					 strlen(old_name));
 		if (pos < 0) {
 			if (patch->is_new < 0)
@@ -3769,6 +3775,8 @@ static int check_to_create(struct apply_state *state,
 	if (state->check_index && (!ok_if_exists || !state->cached)) {
 		int pos;
 
+		ensure_full_index(state->repo->index);
+
 		pos = index_name_pos(state->repo->index, new_name, strlen(new_name));
 		if (pos >= 0) {
 			struct cache_entry *ce = state->repo->index->cache[pos];

archive.c

@@ -309,6 +309,7 @@ int write_archive_entries(struct archiver_args *args,
 		opts.head_idx = -1;
 		opts.src_index = args->repo->index;
 		opts.dst_index = args->repo->index;
+		ensure_full_index(opts.src_index);
 		opts.fn = oneway_merge;
 		init_tree_desc(&t, args->tree->buffer, args->tree->size);
 		if (unpack_trees(1, &t, &opts))

blame.c

@@ -108,6 +108,7 @@ static void verify_working_tree_path(struct repository *r,
 			return;
 	}
 
+	ensure_full_index(r->index);
 	pos = index_name_pos(r->index, path, strlen(path));
 	if (pos >= 0)
 		; /* path is in the index */
@@ -277,7 +278,11 @@ static struct commit *fake_working_tree_commit(struct repository *r,
 
 	len = strlen(path);
 	if (!mode) {
-		int pos = index_name_pos(r->index, path, len);
+		int pos;
+
+		ensure_full_index(r->index);
+
+		pos = index_name_pos(r->index, path, len);
 		if (0 <= pos)
 			mode = r->index->cache[pos]->ce_mode;
 		else

builtin.h

@@ -177,9 +177,12 @@ int cmd_maintenance(int argc, const char **argv, const char *prefix);
 int cmd_merge(int argc, const char **argv, const char *prefix);
 int cmd_merge_base(int argc, const char **argv, const char *prefix);
 int cmd_merge_index(int argc, const char **argv, const char *prefix);
+int cmd_merge_octopus(int argc, const char **argv, const char *prefix);
 int cmd_merge_ours(int argc, const char **argv, const char *prefix);
 int cmd_merge_file(int argc, const char **argv, const char *prefix);
+int cmd_merge_one_file(int argc, const char **argv, const char *prefix);
 int cmd_merge_recursive(int argc, const char **argv, const char *prefix);
+int cmd_merge_resolve(int argc, const char **argv, const char *prefix);
 int cmd_merge_tree(int argc, const char **argv, const char *prefix);
 int cmd_mktag(int argc, const char **argv, const char *prefix);
 int cmd_mktree(int argc, const char **argv, const char *prefix);

builtin/add.c

@@ -491,6 +491,9 @@ int cmd_add(int argc, const char **argv, const char *prefix)
 	add_new_files = !take_worktree_changes && !refresh_only && !add_renormalize;
 	require_pathspec = !(take_worktree_changes || (0 < addremove_explicit));
 
+	prepare_repo_settings(the_repository);
+	the_repository->settings.command_requires_full_index = 0;
+
 	hold_locked_index(&lock_file, LOCK_DIE_ON_ERROR);
 
 	/*

builtin/am.c

@@ -1874,6 +1874,7 @@ static int fast_forward_to(struct tree *head, struct tree *remote, int reset)
 	init_tree_desc(&t[0], head->buffer, head->size);
 	init_tree_desc(&t[1], remote->buffer, remote->size);
 
+	ensure_full_index(opts.src_index);
 	if (unpack_trees(2, t, &opts)) {
 		rollback_lock_file(&lock_file);
 		return -1;
@@ -1908,6 +1909,7 @@ static int merge_tree(struct tree *tree)
 	opts.fn = oneway_merge;
 	init_tree_desc(&t[0], tree->buffer, tree->size);
 
+	ensure_full_index(opts.src_index);
 	if (unpack_trees(1, t, &opts)) {
 		rollback_lock_file(&lock_file);
 		return -1;

builtin/checkout-index.c

@@ -48,11 +48,14 @@ static void write_tempfile_record(const char *name, const char *prefix)
 static int checkout_file(const char *name, const char *prefix)
 {
 	int namelen = strlen(name);
-	int pos = cache_name_pos(name, namelen);
+	int pos;
 	int has_same_name = 0;
 	int did_checkout = 0;
 	int errs = 0;
 
+	ensure_full_index(the_repository->index);
+	pos = index_name_pos(the_repository->index, name, namelen);
+
 	if (pos < 0)
 		pos = -pos - 1;
 

builtin/checkout.c

@@ -644,6 +644,7 @@ static int reset_tree(struct tree *tree, const struct checkout_opts *o,
 			       NULL);
 	parse_tree(tree);
 	init_tree_desc(&tree_desc, tree->buffer, tree->size);
+	ensure_full_index(opts.src_index);
 	switch (unpack_trees(1, &tree_desc, &opts)) {
 	case -2:
 		*writeout_error = 1;
@@ -824,9 +825,6 @@ static int merge_working_tree(const struct checkout_opts *opts,
 		}
 	}
 
-	if (!active_cache_tree)
-		active_cache_tree = cache_tree();
-
 	if (!cache_tree_fully_valid(active_cache_tree))
 		cache_tree_update(&the_index, WRITE_TREE_SILENT | WRITE_TREE_REPAIR);
 

builtin/clone.c

@@ -808,6 +808,7 @@ static int checkout(int submodule_progress)
 	tree = parse_tree_indirect(&oid);
 	parse_tree(tree);
 	init_tree_desc(&t, tree->buffer, tree->size);
+	ensure_full_index(opts.src_index);
 	if (unpack_trees(1, &t, &opts) < 0)
 		die(_("unable to checkout working tree"));
 

builtin/commit.c

@@ -428,6 +428,7 @@ static void create_base_index(const struct commit *current_head)
 		die(_("failed to unpack HEAD tree object"));
 	parse_tree(tree);
 	init_tree_desc(&t, tree->buffer, tree->size);
+	ensure_full_index(opts.src_index);
 	if (unpack_trees(1, &t, &opts))
 		exit(128); /* We've already reported the error, finish dying */
 }
@@ -1559,6 +1560,9 @@ int cmd_status(int argc, const char **argv, const char *prefix)
 	if (argc == 2 && !strcmp(argv[1], "-h"))
 		usage_with_options(builtin_status_usage, builtin_status_options);
 
+	prepare_repo_settings(the_repository);
+	the_repository->settings.command_requires_full_index = 0;
+
 	status_init_config(&s, git_status_config);
 	argc = parse_options(argc, argv, prefix,
 			     builtin_status_options,

builtin/grep.c

@@ -506,6 +506,8 @@ static int grep_cache(struct grep_opt *opt,
 	if (repo_read_index(repo) < 0)
 		die(_("index file corrupt"));
 
+	ensure_full_index(repo->index);
+
 	for (nr = 0; nr < repo->index->cache_nr; nr++) {
 		const struct cache_entry *ce = repo->index->cache[nr];
 		strbuf_setlen(&name, name_base_len);

builtin/ls-files.c

@@ -150,7 +150,7 @@ static void show_other_files(const struct index_state *istate,
 	}
 }
 
-static void show_killed_files(const struct index_state *istate,
+static void show_killed_files(struct index_state *istate,
 			      const struct dir_struct *dir)
 {
 	int i;
@@ -159,6 +159,8 @@ static void show_killed_files(const struct index_state *istate,
 		char *cp, *sp;
 		int pos, len, killed = 0;
 
+		ensure_full_index(istate);
+
 		for (cp = ent->name; cp - ent->name < ent->len; cp = sp + 1) {
 			sp = strchr(cp, '/');
 			if (!sp) {
@@ -313,6 +315,7 @@ static void show_files(struct repository *repo, struct dir_struct *dir)
 			show_killed_files(repo->index, dir);
 	}
 	if (show_cached || show_stage) {
+		ensure_full_index(repo->index);
 		for (i = 0; i < repo->index->cache_nr; i++) {
 			const struct cache_entry *ce = repo->index->cache[i];
 
@@ -332,6 +335,7 @@ static void show_files(struct repository *repo, struct dir_struct *dir)
 		}
 	}
 	if (show_deleted || show_modified) {
+		ensure_full_index(repo->index);
 		for (i = 0; i < repo->index->cache_nr; i++) {
 			const struct cache_entry *ce = repo->index->cache[i];
 			struct stat st;
@@ -368,6 +372,7 @@ static void prune_index(struct index_state *istate,
 
 	if (!prefix || !istate->cache_nr)
 		return;
+	ensure_full_index(istate);
 	pos = index_name_pos(istate, prefix, prefixlen);
 	if (pos < 0)
 		pos = -pos-1;
@@ -428,6 +433,8 @@ void overlay_tree_on_index(struct index_state *istate,
 	if (!tree)
 		die("bad tree-ish %s", tree_name);
 
+	ensure_full_index(istate);
+
 	/* Hoist the unmerged entries up to stage #3 to make room */
 	for (i = 0; i < istate->cache_nr; i++) {
 		struct cache_entry *ce = istate->cache[i];