git-add : Respect submodule ignore=all and only add changes with --force <path>

.devcontainer/Dockerfile

@@ -0,0 +1,70 @@
+FROM ubuntu:latest
+
+ARG USER_ID
+ARG GROUP_ID
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Copenhagen
+
+RUN apt-get update \
+    && \
+    apt-get install -y \
+        sudo \
+        build-essential \
+        libcurl4-gnutls-dev \
+        libexpat1-dev \
+        gettext \
+        libz-dev \
+        libssl-dev \
+        asciidoc \
+        xmlto \
+        docbook-xsl \
+        \
+        tzdata \
+        git \
+        coccinelle \
+    && \
+    ln -fs /usr/share/zoneinfo/$TZ /etc/localtime \
+    && \
+    dpkg-reconfigure --frontend noninteractive tzdata
+
+RUN  apt-get update && apt-get install -y autoconf
+
+RUN addgroup -gid 1001 gituser
+RUN adduser --disabled-password -u 1001 -gid 1001 gituser
+RUN usermod -aG sudo gituser
+RUN echo 'gituser ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/010-gituser
+RUN chmod 0440 /etc/sudoers.d/010-gituser
+
+USER 1001
+
+#WORKDIR /home/gituser/git-src/
+
+#RUN  make configure
+#RUN ./configure --prefix=${HOME}/.local/
+
+#RUN make -j$(nproc) gitweb || make gitweb 
+#RUN make install-gitweb
+
+#RUN make -j$(nproc) || make
+#RUN make -j$(nproc) gitweb || make gitweb 
+#RUN make install-gitweb
+#RUN make -j$(nproc) NO_PERL=YesPlease install || make NO_PERL=YesPlease install
+#RUN make install
+
+
+RUN mkdir -p  ${HOME}/.local/
+#RUN ls -la ${HOME}/.local/bin
+#ENV PATH="/home/gituser/.local/bin:${PATH}"
+#RUN ls -l $HOME/.local/bin
+#RUN echo $PATH $HOME 
+
+#RUN git --version
+#RUN which git 
+
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "Git User"
+
+#WORKDIR /home/gituser/git-test/
+
+#CMD [ "git" ]

.devcontainer/Dockerfile.standalone

@@ -0,0 +1,76 @@
+FROM ubuntu:latest
+
+ARG USER_ID
+ARG GROUP_ID
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Copenhagen
+
+RUN apt-get update \
+    && \
+    apt-get install -y \
+        build-essential \
+        libcurl4-gnutls-dev \
+        libexpat1-dev \
+        gettext \
+        libz-dev \
+        libssl-dev \
+        asciidoc \
+        xmlto \
+        docbook-xsl \
+        \
+        tzdata \
+        \
+        nano \
+        vim \
+    && \
+    ln -fs /usr/share/zoneinfo/$TZ /etc/localtime \
+    && \
+    dpkg-reconfigure --frontend noninteractive tzdata
+
+RUN  apt-get update && apt-get install -y autoconf
+
+RUN addgroup -gid ${GROUP_ID} gituser
+RUN adduser --disabled-password -u ${USER_ID} -gid ${GROUP_ID} gituser
+
+
+COPY --chown=${USER_ID}:${GROUP_ID} ./git /home/gituser/git-src/
+
+USER ${USER_ID}
+
+WORKDIR /home/gituser/git-src/
+
+RUN  make configure
+RUN ./configure --prefix=${HOME}/.local/
+
+#RUN make -j$(nproc) gitweb || make gitweb 
+#RUN make install-gitweb
+
+#RUN make -j$(nproc) || make
+#RUN make -j$(nproc) gitweb || make gitweb 
+#RUN make install-gitweb
+RUN make -j$(nproc) NO_PERL=YesPlease install || make NO_PERL=YesPlease install
+RUN make install
+
+WORKDIR /home/gituser/git-src/t
+RUN ./t2206-add-submodule-ignored.sh -v
+
+RUN ls -la ${HOME}/.local/
+RUN ls -la ${HOME}/.local/bin
+ENV PATH="/home/gituser/.local/bin:${PATH}"
+RUN ls -l $HOME/.local/bin
+RUN echo $PATH $HOME 
+
+RUN git --version
+RUN which git 
+
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "Git User"
+
+WORKDIR /home/gituser/git-src/t
+RUN pwd && ls -la 
+RUN ./t2206-add-submodule-ignored.sh -v
+
+WORKDIR /home/gituser/git-test/
+
+CMD [ "git" ]

.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+{
+  "name": "Git Dev Container",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": "",
+    "args": {
+      "user_id": "1001",
+      "group_id": "1001"
+    }
+  },
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "terminal.integrated.shell.linux": "/bin/bash"
+      }
+    }
+  },
+  "remoteUser": "gituser",
+  "features": {},
+  "mounts": [
+    "source=${localWorkspaceFolder}/,target=/home/gitusers/git-src,type=bind,consistency=cached",
+    "source=${localWorkspaceFolder}/,target=/home/gitusers/git-test,type=bind,consistency=cached"
+  ],
+  "postCreateCommand": "echo"
+}

Documentation/config/submodule.adoc

@@ -32,15 +32,16 @@ submodule.<name>.fetchRecurseSubmodules::
 
 submodule.<name>.ignore::
 	Defines under what circumstances "git status" and the diff family show
-	a submodule as modified. When set to "all", it will never be considered
-	modified (but it will nonetheless show up in the output of status and
-	commit when it has been staged), "dirty" will ignore all changes
-	to the submodule's work tree and
+	a submodule as modified.
+	Set to "all" will never considered the submodule modified. It can
+	nevertheless be staged using the option --include_ignored_submodules and
+	it will then show up in the output of status.
+	Set to "dirty" will ignore all changes to the submodule's work tree and
 	takes only differences between the HEAD of the submodule and the commit
 	recorded in the superproject into account. "untracked" will additionally
 	let submodules with modified tracked files in their work tree show up.
-	Using "none" (the default when this option is not set) also shows
-	submodules that have untracked files in their work tree as changed.
+	Set to "none"(default) It is also shows submodules that have untracked
+	files in their work tree as changed.
 	This setting overrides any setting made in .gitmodules for this submodule,
 	both settings can be overridden on the command line by using the
 	"--ignore-submodules" option. The 'git submodule' commands are not

Documentation/git-add.adoc

@@ -79,6 +79,11 @@ in linkgit:gitglossary[7].
 `--force`::
 	Allow adding otherwise ignored files.
 
+`--include_ignored_submodules`::
+	The option is also used when `submodule.<name>.ignore=all`
+	is set, but you want to stage an update of the submodule. The
+	`path` to the submodule must be explicitly specified.
+
 `--sparse`::
 	Allow updating index entries outside of the sparse-checkout cone.
 	Normally, `git add` refuses to update index entries whose paths do

Documentation/gitmodules.adoc

@@ -70,7 +70,10 @@ submodule.<name>.ignore::
 --
 	all;; The submodule will never be considered modified (but will
 	    nonetheless show up in the output of status and commit when it has
-	    been staged).
+	    been staged). Add `(new commits)` can be overruled using the
+	    `git add --include_ignored_submodules <submodule.path>` 
+		The setting affects `status`, `update-index`, `diff` and `log`(due 
+		to underlaying `diff`).
 
 	dirty;; All changes to the submodule's work tree will be ignored, only
 	    committed differences between the `HEAD` of the submodule and its

builtin/add.c

@@ -233,6 +233,7 @@ N_("The following paths are ignored by one of your .gitignore files:\n");
 static int verbose, show_only, ignored_too, refresh_only;
 static int ignore_add_errors, intent_to_add, ignore_missing;
 static int warn_on_embedded_repo = 1;
+static int include_ignored_submodules;
 
 #define ADDREMOVE_DEFAULT 1
 static int addremove = ADDREMOVE_DEFAULT;
@@ -271,6 +272,7 @@ static struct option builtin_add_options[] = {
 	OPT_BOOL( 0 , "ignore-errors", &ignore_add_errors, N_("just skip files which cannot be added because of errors")),
 	OPT_BOOL( 0 , "ignore-missing", &ignore_missing, N_("check if - even missing - files are ignored in dry run")),
 	OPT_BOOL(0, "sparse", &include_sparse, N_("allow updating entries outside of the sparse-checkout cone")),
+    OPT_BOOL(0, "include-ignored-submodules", &include_ignored_submodules, N_("add submodules even if they has configuration ignore=all")),
 	OPT_STRING(0, "chmod", &chmod_arg, "(+|-)x",
 		   N_("override the executable bit of the listed files")),
 	OPT_HIDDEN_BOOL(0, "warn-embedded-repo", &warn_on_embedded_repo,
@@ -582,7 +584,7 @@ int cmd_add(int argc,
 	else
 		exit_status |= add_files_to_cache(repo, prefix,
 						  &pathspec, ps_matched,
-						  include_sparse, flags);
+						  include_sparse, flags, include_ignored_submodules);
 
 	if (take_worktree_changes && !add_renormalize && !ignore_add_errors &&
 	    report_path_error(ps_matched, &pathspec))

builtin/checkout.c

@@ -899,7 +899,7 @@ static int merge_working_tree(const struct checkout_opts *opts,
 			 */
 
 			add_files_to_cache(the_repository, NULL, NULL, NULL, 0,
-					   0);
+					   0, 0 );
 			init_ui_merge_options(&o, the_repository);
 			o.verbosity = 0;
 			work = write_in_core_index_as_tree(the_repository);

builtin/commit.c

@@ -455,7 +455,7 @@ static const char *prepare_index(const char **argv, const char *prefix,
 		repo_hold_locked_index(the_repository, &index_lock,
 				       LOCK_DIE_ON_ERROR);
 		add_files_to_cache(the_repository, also ? prefix : NULL,
-				   &pathspec, ps_matched, 0, 0);
+				   &pathspec, ps_matched, 0, 0, 0 );
 		if (!all && report_path_error(ps_matched, &pathspec))
 			exit(128);
 

read-cache-ll.h

@@ -481,7 +481,7 @@ int cmp_cache_name_compare(const void *a_, const void *b_);
 
 int add_files_to_cache(struct repository *repo, const char *prefix,
 		       const struct pathspec *pathspec, char *ps_matched,
-		       int include_sparse, int flags);
+		       int include_sparse, int flags, int ignored_too );
 
 void overlay_tree_on_index(struct index_state *istate,
 			   const char *tree_name, const char *prefix);

read-cache.c

@@ -48,6 +48,8 @@
 #include "csum-file.h"
 #include "promisor-remote.h"
 #include "hook.h"
+#include "submodule.h"
+#include "submodule-config.h"
 
 /* Mask for the name length in ce_flags in the on-disk index */
 
@@ -3880,9 +3882,12 @@ void overlay_tree_on_index(struct index_state *istate,
 
 struct update_callback_data {
 	struct index_state *index;
+	struct repository *repo;
+	struct pathspec *pathspec;
 	int include_sparse;
 	int flags;
 	int add_errors;
+	int include_ignored_submodules;
 };
 
 static int fix_unmerged_status(struct diff_filepair *p,
@@ -3924,13 +3929,55 @@ static void update_callback(struct diff_queue_struct *q,
 		default:
 			die(_("unexpected diff status %c"), p->status);
 		case DIFF_STATUS_MODIFIED:
-		case DIFF_STATUS_TYPE_CHANGED:
+		case DIFF_STATUS_TYPE_CHANGED: {
+			struct stat st;
+			if (!lstat(path, &st) && S_ISDIR(st.st_mode)) { // only consider submodule if it is a directory
+				const struct submodule *sub = submodule_from_path(data->repo, null_oid(the_hash_algo), path);
+				if (sub && sub->name && sub->ignore && !strcmp(sub->ignore, "all")) {
+					int pathspec_matches = 0;
+					char *norm_pathspec = NULL;
+					int ps_i;
+					trace_printf("ignore=all %s\n", path);
+					trace_printf("pathspec %s\n",
+							(data->pathspec && data->pathspec->nr) ? "has pathspec" : "no pathspec");
+					/* Safely scan all pathspec items (q->nr may exceed pathspec->nr). */
+					if (data->pathspec) {
+						for (ps_i = 0; ps_i < data->pathspec->nr; ps_i++) {
+							const char *m = data->pathspec->items[ps_i].match;
+							if (!m)
+								continue;
+							norm_pathspec = xstrdup(m);
+							strip_dir_trailing_slashes(norm_pathspec);
+							if (!strcmp(path, norm_pathspec)) {
+								pathspec_matches = 1;
+								FREE_AND_NULL(norm_pathspec);
+								break;
+							}
+							FREE_AND_NULL(norm_pathspec);
+						}
+					}
+					if (pathspec_matches) {
+						if (data->include_ignored_submodules && data->include_ignored_submodules > 0) {
+							trace_printf("Add submodule due to --include_ignored_submodules: %s\n", path);
+						} else {
+							printf(_("Skipping submodule due to ignore=all: %s"), path);
+							printf(_("Use --include_ignored_submodules, if you really want to add them.") );
+							continue;
+						}
+					} else {
+						/* No explicit pathspec match -> skip silently (or with trace). */
+						trace_printf("Pathspec to submodule does not match explicitly: %s\n", path);
+						continue;
+					}
+				}
+			}
 			if (add_file_to_index(data->index, path, data->flags)) {
 				if (!(data->flags & ADD_CACHE_IGNORE_ERRORS))
 					die(_("updating files failed"));
 				data->add_errors++;
 			}
 			break;
+		}
 		case DIFF_STATUS_DELETED:
 			if (data->flags & ADD_CACHE_IGNORE_REMOVAL)
 				break;
@@ -3945,7 +3992,7 @@ static void update_callback(struct diff_queue_struct *q,
 
 int add_files_to_cache(struct repository *repo, const char *prefix,
 		       const struct pathspec *pathspec, char *ps_matched,
-		       int include_sparse, int flags)
+		       int include_sparse, int flags, int include_ignored_submodules )
 {
 	struct update_callback_data data;
 	struct rev_info rev;
@@ -3954,6 +4001,9 @@ int add_files_to_cache(struct repository *repo, const char *prefix,
 	data.index = repo->index;
 	data.include_sparse = include_sparse;
 	data.flags = flags;
+	data.repo = repo;
+	data.include_ignored_submodules = include_ignored_submodules;
+	data.pathspec = (struct pathspec *)pathspec;
 
 	repo_init_revisions(repo, &rev, prefix);
 	setup_revisions(0, NULL, &rev, NULL);

t/lib-submodule-update.sh

@@ -95,14 +95,14 @@ create_lib_submodule_repo () {
 			git commit -m "modified file2 and added file3" &&
 			git push origin modifications
 		) &&
-		git add sub1 &&
+		git add --include-ignored-submodules sub1 &&
 		git commit -m "Modify sub1" &&
 
 		git checkout -b add_nested_sub modify_sub1 &&
 		git -C sub1 checkout -b "add_nested_sub" &&
 		git -C sub1 submodule add --branch no_submodule ../submodule_update_sub2 sub2 &&
 		git -C sub1 commit -a -m "add a nested submodule" &&
-		git add sub1 &&
+		git add --include-ignored-submodules sub1 &&
 		git commit -a -m "update submodule, that updates a nested submodule" &&
 		git checkout -b modify_sub1_recursively &&
 		git -C sub1 checkout -b modify_sub1_recursively &&
@@ -112,7 +112,7 @@ create_lib_submodule_repo () {
 		git -C sub1/sub2 commit -m "make a change in nested sub" &&
 		git -C sub1 add sub2 &&
 		git -C sub1 commit -m "update nested sub" &&
-		git add sub1 &&
+		git add --include-ignored-submodules sub1 &&
 		git commit -m "update sub1, that updates nested sub" &&
 		git -C sub1 push origin modify_sub1_recursively &&
 		git -C sub1/sub2 push origin modify_sub1_recursively &&