fix: pip_compile to handle multiple generated requirements.in (#909)

* fix: pip_compile to handle multiple generated requirements.in

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: only use requirements.in absolute when generated

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: use hermetic toolchain to lock the requirements

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: tests on CI

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: windows path separator madness

Signed-off-by: Thulio Ferraz Assis <[email protected]>

Signed-off-by: Thulio Ferraz Assis <[email protected]>

Author: f0rmiga

.bazelci/presubmit.yml

@@ -167,6 +167,22 @@ tasks:
     working_directory: examples/pip_repository_annotations
     platform: windows
 
+  integration_test_compile_pip_requirements_linux:
+    <<: *reusable_build_test_all
+    name: compile_pip_requirements integration tests on Linux
+    working_directory: tests/compile_pip_requirements
+    platform: ubuntu2004
+  integration_test_compile_pip_requirements_macos:
+    <<: *reusable_build_test_all
+    name: compile_pip_requirements integration tests on macOS
+    working_directory: tests/compile_pip_requirements
+    platform: macos
+  integration_test_compile_pip_requirements_windows:
+    <<: *reusable_build_test_all
+    name: compile_pip_requirements integration tests on Windows
+    working_directory: tests/compile_pip_requirements
+    platform: windows
+
   integration_test_pip_repository_entry_points_linux:
     <<: *reusable_build_test_all
     name: pip_repository_entry_points integration tests on Linux

.bazelrc

@@ -3,8 +3,8 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
-build --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps
-query --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps
+build --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps
+query --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps
 
 test --test_output=errors
 

examples/pip_parse/BUILD

@@ -55,16 +55,6 @@ alias(
     actual = entry_point("yamllint"),
 )
 
-# The requirements.in file can be checked in to the source tree or it can be
-# generated. Pretend that we do some generating of the file. For this example,
-# the "template" is already the file we want.
-genrule(
-    name = "generate_requirements_in",
-    srcs = ["requirements.in.template"],
-    outs = ["requirements.in"],
-    cmd = "cp $(SRCS) $(OUTS)",
-)
-
 # This rule adds a convenient way to update the requirements file.
 compile_pip_requirements(
     name = "requirements",

examples/pip_parse/requirements.in.template renamed to examples/pip_parse/requirements.in

@@ -1,6 +1,7 @@
 "Set defaults for the pip-compile command to run it under Bazel"
 
 import os
+import re
 import sys
 from pathlib import Path
 from shutil import copyfile
@@ -26,19 +27,17 @@ def _select_golden_requirements_file(
         return requirements_txt
 
 
-def _fix_up_requirements_in_path(
-    resolved_requirements_in, requirements_in, output_file
-):
+def _fix_up_requirements_in_path(absolute_prefix, output_file):
     """Fix up references to the input file inside of the generated requirements file.
 
     We don't want fully resolved, absolute paths in the generated requirements file.
     The paths could differ for every invocation. Replace them with a predictable path.
     """
     output_file = Path(output_file)
-    fixed_requirements_text = output_file.read_text().replace(
-        resolved_requirements_in, requirements_in
-    )
-    output_file.write_text(fixed_requirements_text)
+    contents = output_file.read_text()
+    contents = contents.replace(absolute_prefix, "")
+    contents = re.sub(r"\\(?!(\n|\r\n))", "/", contents)
+    output_file.write_text(contents)
 
 
 if __name__ == "__main__":
@@ -58,9 +57,11 @@ def _fix_up_requirements_in_path(
     requirements_windows = parse_str_none(sys.argv.pop(1))
     update_target_label = sys.argv.pop(1)
 
-    # The requirements_in file could be generated. We need to get the path to it before we change
-    # directory into the workspace directory.
-    resolved_requirements_in = str(Path(requirements_in).resolve())
+    # The requirements_in file could be generated, so we will need to remove the
+    # absolute prefixes in the locked requirements output file.
+    requirements_in_path = Path(requirements_in)
+    resolved_requirements_in = str(requirements_in_path.resolve())
+    absolute_prefix = resolved_requirements_in[: -len(str(requirements_in_path))]
 
     # Before loading click, set the locale for its parser.
     # If it leaks through to the system setting, it may fail:
@@ -116,17 +117,17 @@ def _fix_up_requirements_in_path(
     sys.argv.append("--generate-hashes")
     sys.argv.append("--output-file")
     sys.argv.append(requirements_txt if UPDATE else requirements_out)
-    sys.argv.append(resolved_requirements_in)
+    sys.argv.append(
+        requirements_in if requirements_in_path.exists() else resolved_requirements_in
+    )
 
     if UPDATE:
         print("Updating " + requirements_txt)
         try:
             cli()
         except SystemExit as e:
             if e.code == 0:
-                _fix_up_requirements_in_path(
-                    resolved_requirements_in, requirements_in, requirements_txt
-                )
+                _fix_up_requirements_in_path(absolute_prefix, requirements_txt)
             raise
     else:
         # cli will exit(0) on success
@@ -145,9 +146,7 @@ def _fix_up_requirements_in_path(
                 )
                 sys.exit(1)
             elif e.code == 0:
-                _fix_up_requirements_in_path(
-                    resolved_requirements_in, requirements_in, requirements_out
-                )
+                _fix_up_requirements_in_path(absolute_prefix, requirements_out)
                 golden_filename = _select_golden_requirements_file(
                     requirements_txt,
                     requirements_linux,

python/pip_install/pip_compile.py

@@ -0,0 +1,5 @@
+test --test_output=errors
+
+# Windows requires these for multi-python support:
+build --enable_runfiles
+startup --windows_enable_symlinks

tests/compile_pip_requirements/.bazelrc

@@ -0,0 +1 @@
+bazel-*

tests/compile_pip_requirements/.gitignore

@@ -0,0 +1,33 @@
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+
+genrule(
+    name = "generate_requirements_extra_in",
+    srcs = [],
+    outs = ["requirements_extra.in"],
+    cmd = "echo 'setuptools~=65.6.3' > $@",
+)
+
+genrule(
+    name = "generate_requirements_in",
+    srcs = [],
+    outs = ["requirements.in"],
+    cmd = """
+cat > $@ <<EOF
+-r requirements_extra.in
+pip~=22.3.1
+EOF
+""",
+)
+
+compile_pip_requirements(
+    name = "pip_dependencies",
+    data = [
+        "requirements_extra.in",
+    ],
+    extra_args = [
+        "--allow-unsafe",
+        "--resolver=backtracking",
+    ],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock.txt",
+)

tests/compile_pip_requirements/BUILD.bazel

@@ -0,0 +1,3 @@
+# compile_pip_requirements
+
+This test ensures that generated requirements.in can be used by compile_pip_requirements.

tests/compile_pip_requirements/README.md

@@ -0,0 +1,15 @@
+local_repository(
+    name = "rules_python",
+    path = "../..",
+)
+
+load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
+
+pip_install_dependencies()
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python39",
+    python_version = "3.9",
+)

tests/compile_pip_requirements/WORKSPACE

@@ -0,0 +1,16 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+#    bazel run //:pip_dependencies.update
+#
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==22.3.1 \
+    --hash=sha256:65fd48317359f3af8e593943e6ae1506b66325085ea64b706a998c6e83eeaf38 \
+    --hash=sha256:908c78e6bc29b676ede1c4d57981d490cb892eb45cd8c214ab6298125119e077
+    # via -r ./requirements.in
+setuptools==65.6.3 \
+    --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \
+    --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75
+    # via -r ./requirements_extra.in