Merge pull request #19424 from github/tausbn/python-extract-hidden-file-by-default

Python: Extract files in hidden dirs by default

Author: tausbn

python/extractor/cli-integration-test/hidden-files/config.yml

@@ -0,0 +1,3 @@
+name: Test Config
+paths-ignore:
+  - "**/.*/**"

python/extractor/cli-integration-test/hidden-files/query-default.expected

@@ -0,0 +1,6 @@
+|             name              |
++-------------------------------+
+| .hidden_file.py               |
+| another_non_hidden.py         |
+| foo.py                        |
+| visible_file_in_hidden_dir.py |

python/extractor/cli-integration-test/hidden-files/query-skipped.expected

@@ -0,0 +1,4 @@
+|      name       |
++-----------------+
+| .hidden_file.py |
+| foo.py          |

python/extractor/cli-integration-test/hidden-files/query.ql

@@ -0,0 +1,3 @@
+import python
+
+select any(File f).getShortName() as name order by name

python/extractor/cli-integration-test/hidden-files/repo_dir/.hidden_dir/internal_non_hidden/another_non_hidden.py

@@ -0,0 +1 @@
+print(42)

python/extractor/cli-integration-test/hidden-files/repo_dir/.hidden_dir/visible_file_in_hidden_dir.py

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+
+set -x
+
+CODEQL=${CODEQL:-codeql}
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$SCRIPTDIR"
+
+rm -rf db db-skipped
+
+# Test 1: Default behavior should be to extract files in hidden directories
+$CODEQL database create db --language python --source-root repo_dir/
+$CODEQL query run --database db query.ql > query-default.actual
+diff query-default.expected query-default.actual
+
+# Test 2: The default behavior can be overridden by setting `paths-ignore` in the config file
+$CODEQL database create db-skipped --language python --source-root repo_dir/ --codescanning-config=config.yml
+$CODEQL query run --database db-skipped query.ql > query-skipped.actual
+diff query-skipped.expected query-skipped.actual
+
+rm -rf db db-skipped

python/extractor/cli-integration-test/hidden-files/repo_dir/.hidden_file.py

@@ -41,19 +41,27 @@ def glob_part_to_regex(glob, add_sep):
 
 def glob_to_regex(glob, prefix=""):
     '''Convert entire glob to a compiled regex'''
+    # When the glob ends in `/`, we need to remember this so that we don't accidentally add an
+    # extra separator to the final regex.
+    end_sep = "" if glob.endswith("/") else SEP
     glob = glob.strip().strip("/")
     parts = glob.split("/")
     #Trailing '**' is redundant, so strip it off.
     if parts[-1] == "**":
         parts = parts[:-1]
         if not parts:
             return ".*"
+    # The `glob.strip("/")` call above will have removed all trailing slashes, but if there was at
+    # least one trailing slash, we want there to be an extra part, so we add it explicitly here in
+    # that case, using the emptyness of `end_sep` as a proxy.
+    if end_sep == "":
+        parts += [""]
     parts = [ glob_part_to_regex(escape(p), True) for p in parts[:-1] ] + [ glob_part_to_regex(escape(parts[-1]), False) ]
     # we need to escape the prefix, specifically because on windows the prefix will be
     # something like `C:\\folder\\subfolder\\` and without escaping the
     # backslash-path-separators will get interpreted as regex escapes (which might be
     # invalid sequences, causing the extractor to crash)
-    full_pattern = escape(prefix) + ''.join(parts) + "(?:" + SEP + ".*|$)"
+    full_pattern = escape(prefix) + ''.join(parts) + "(?:" + end_sep + ".*|$)"
     return re.compile(full_pattern)
 
 def filter_from_pattern(pattern, prev_filter, prefix):