bundle-uri: store fetch.bundleCreationToken

When a bundle list specifies the "creationToken" heuristic, the Git
client downloads the list and then starts downloading bundles in
descending creationToken order. This process stops as soon as all
downloaded bundles can be applied to the repository (because all
required commits are present in the repository or in the downloaded
bundles).

When checking the same bundle list twice, this strategy requires
downloading the bundle with the maximum creationToken again, which is
wasteful. The creationToken heuristic promises that the client will not
have a use for that bundle if its creationToken value is the at most the
previous creationToken value.

To prevent these wasteful downloads, create a fetch.bundleCreationToken
config setting that the Git client sets after downloading bundles. This
value allows skipping that maximum bundle download when this config
value is the same value (or larger).

To test that this works correctly, we can insert some "duplicate"
fetches into existing tests and demonstrate that only the bundle list is
downloaded.

The previous logic for downloading bundles by creationToken worked even
if the bundle list was empty, but now we have logic that depends on the
first entry of the list. Terminate early in the (non-sensical) case of
an empty bundle list.

Signed-off-by: Derrick Stolee <[email protected]>

Author: derrickstolee

Documentation/config/fetch.txt

@@ -104,3 +104,11 @@ fetch.bundleURI::
 	first running `git fetch --bundle-uri=<uri>` immediately before
 	`git fetch <args>`. See details of the `--bundle-uri` option in
 	linkgit:git-fetch[1].
+
+fetch.bundleCreationToken::
+	When using `fetch.bundleURI` to fetch incrementally from a bundle
+	list that uses the "creationToken" heuristic, this config value
+	stores the maximum `creationToken` value of the downloaded bundles.
+	This value is used to prevent downloading bundles in the future
+	if the advertised `creationToken` is not strictly larger than this
+	value.

bundle-uri.c

@@ -481,6 +481,8 @@ static int fetch_bundles_by_token(struct repository *r,
 {
 	int cur;
 	int pop_or_push = 0;
+	const char *creationTokenStr;
+	uint64_t maxCreationToken;
 	struct bundle_list_context ctx = {
 		.r = r,
 		.list = list,
@@ -494,8 +496,27 @@ static int fetch_bundles_by_token(struct repository *r,
 
 	for_all_bundles_in_list(list, append_bundle, &bundles);
 
+	if (!bundles.nr) {
+		free(bundles.items);
+		return 0;
+	}
+
 	QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
 
+	/*
+	 * If fetch.bundleCreationToken exists, parses to a uint64t, and
+	 * is not strictly smaller than the maximum creation token in the
+	 * bundle list, then do not download any bundles.
+	 */
+	if (!repo_config_get_value(r,
+				   "fetch.bundlecreationtoken",
+				   &creationTokenStr) &&
+	    sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) == 1 &&
+	    bundles.items[0]->creationToken <= maxCreationToken) {
+		free(bundles.items);
+		return 0;
+	}
+
 	/*
 	 * Use a stack-based approach to download the bundles and attempt
 	 * to unbundle them in decreasing order by creation token. If we
@@ -558,14 +579,24 @@ static int fetch_bundles_by_token(struct repository *r,
 		cur += pop_or_push;
 	}
 
-	free(bundles.items);
-
 	/*
 	 * We succeed if the loop terminates because 'cur' drops below
 	 * zero. The other case is that we terminate because 'cur'
 	 * reaches the end of the list, so we have a failure no matter
 	 * which bundles we apply from the list.
 	 */
+	if (cur < 0) {
+		struct strbuf value = STRBUF_INIT;
+		strbuf_addf(&value, "%"PRIu64"", bundles.items[0]->creationToken);
+		if (repo_config_set_multivar_gently(ctx.r,
+						    "fetch.bundleCreationToken",
+						    value.buf, NULL, 0))
+			warning(_("failed to store maximum creation token"));
+
+		strbuf_release(&value);
+	}
+
+	free(bundles.items);
 	return cur >= 0;
 }
 

t/t5558-clone-bundle-uri.sh

@@ -455,6 +455,7 @@ test_expect_success 'http clone with bundle.heuristic creates fetch.bundleURI' '
 		"$HTTPD_URL/smart/fetch.git" fetch-http-4 &&
 
 	test_cmp_config -C fetch-http-4 "$HTTPD_URL/bundle-list" fetch.bundleuri &&
+	test_cmp_config -C fetch-http-4 1 fetch.bundlecreationtoken &&
 
 	# The clone should copy two files: the list and bundle-1.
 	test_bundle_downloaded bundle-list trace-clone.txt &&
@@ -479,6 +480,8 @@ test_expect_success 'http clone with bundle.heuristic creates fetch.bundleURI' '
 		refs/heads/left:refs/heads/left \
 		refs/heads/right:refs/heads/right &&
 
+	test_cmp_config -C fetch-http-4 2 fetch.bundlecreationtoken &&
+
 	# This fetch should copy two files: the list and bundle-2.
 	test_bundle_downloaded bundle-list trace1.txt &&
 	test_bundle_downloaded bundle-2.bundle trace1.txt &&
@@ -492,6 +495,15 @@ test_expect_success 'http clone with bundle.heuristic creates fetch.bundleURI' '
 	EOF
 	test_cmp expect refs &&
 
+	# No-op fetch
+	GIT_TRACE2_EVENT="$(pwd)/trace1b.txt" \
+		git -C fetch-http-4 fetch origin --no-tags \
+		refs/heads/left:refs/heads/left \
+		refs/heads/right:refs/heads/right &&
+	test_bundle_downloaded bundle-list trace1b.txt &&
+	! test_bundle_downloaded bundle-1.bundle trace1b.txt &&
+	! test_bundle_downloaded bundle-2.bundle trace1b.txt &&
+
 	cat >>"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
 	[bundle "bundle-3"]
 		uri = bundle-3.bundle
@@ -508,6 +520,8 @@ test_expect_success 'http clone with bundle.heuristic creates fetch.bundleURI' '
 		git -C fetch-http-4 fetch origin --no-tags \
 		refs/heads/merge:refs/heads/merge &&
 
+	test_cmp_config -C fetch-http-4 4 fetch.bundlecreationtoken &&
+
 	# This fetch should copy three files: the list, bundle-3, and bundle-4.
 	test_bundle_downloaded bundle-list trace2.txt &&
 	test_bundle_downloaded bundle-4.bundle trace2.txt &&
@@ -524,7 +538,16 @@ test_expect_success 'http clone with bundle.heuristic creates fetch.bundleURI' '
 	refs/bundles/left
 	refs/bundles/merge
 	EOF
-	test_cmp expect refs
+	test_cmp expect refs &&
+
+	# No-op fetch
+	GIT_TRACE2_EVENT="$(pwd)/trace2b.txt" \
+		git -C fetch-http-4 fetch origin &&
+	test_bundle_downloaded bundle-list trace2b.txt &&
+	! test_bundle_downloaded bundle-1.bundle trace2b.txt &&
+	! test_bundle_downloaded bundle-2.bundle trace2b.txt &&
+	! test_bundle_downloaded bundle-3.bundle trace2b.txt &&
+	! test_bundle_downloaded bundle-4.bundle trace2b.txt
 '
 
 # Do not add tests here unless they use the HTTP server, as they will