diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index b468970fbb..ac24113d03 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -435,6 +435,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_linux.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_debian: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Debian @@ -447,6 +453,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_linux.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_macos: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on macOS @@ -459,6 +471,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_darwin.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_windows: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Windows @@ -471,6 +489,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_windows.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_pip_repository_entry_points_ubuntu_min: <<: *minimum_supported_version diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 89e355806c..ceb20db7ef 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -95,25 +95,30 @@ def _locate(bazel_runfiles, file): requirements_windows = parse_str_none(sys.argv.pop(1)) update_target_label = sys.argv.pop(1) + requirements_file = _select_golden_requirements_file( + requirements_txt=requirements_txt, requirements_linux=requirements_linux, + requirements_darwin=requirements_darwin, requirements_windows=requirements_windows + ) + resolved_requirements_in = _locate(bazel_runfiles, requirements_in) - resolved_requirements_txt = _locate(bazel_runfiles, requirements_txt) + resolved_requirements_file = _locate(bazel_runfiles, requirements_file) # Files in the runfiles directory has the following naming schema: # Main repo: __main__/ # External repo: / # We want to strip both __main__ and from the absolute prefix # to keep the requirements lock file agnostic. - repository_prefix = requirements_txt[: requirements_txt.index("/") + 1] - absolute_path_prefix = resolved_requirements_txt[ - : -(len(requirements_txt) - len(repository_prefix)) + repository_prefix = requirements_file[: requirements_file.index("/") + 1] + absolute_path_prefix = resolved_requirements_file[ + : -(len(requirements_file) - len(repository_prefix)) ] # As requirements_in might contain references to generated files we want to # use the runfiles file first. Thus, we need to compute the relative path # from the execution root. # Note: Windows cannot reference generated files without runfiles support enabled. - requirements_in_relative = requirements_in[len(repository_prefix) :] - requirements_txt_relative = requirements_txt[len(repository_prefix) :] + requirements_in_relative = requirements_in[len(repository_prefix):] + requirements_file_relative = requirements_file[len(repository_prefix):] # Before loading click, set the locale for its parser. # If it leaks through to the system setting, it may fail: @@ -135,11 +140,11 @@ def _locate(bazel_runfiles, file): sys.argv.append(os.environ["TEST_TMPDIR"]) # Make a copy for pip-compile to read and mutate. requirements_out = os.path.join( - os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" + os.environ["TEST_TMPDIR"], os.path.basename(requirements_file) + ".out" ) # Those two files won't necessarily be on the same filesystem, so we can't use os.replace # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link. - shutil.copy(resolved_requirements_txt, requirements_out) + shutil.copy(resolved_requirements_file, requirements_out) update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( update_target_label, @@ -150,7 +155,7 @@ def _locate(bazel_runfiles, file): sys.argv.append("--generate-hashes") sys.argv.append("--output-file") - sys.argv.append(requirements_txt_relative if UPDATE else requirements_out) + sys.argv.append(requirements_file_relative if UPDATE else requirements_out) sys.argv.append( requirements_in_relative if Path(requirements_in_relative).exists() @@ -159,28 +164,28 @@ def _locate(bazel_runfiles, file): print(sys.argv) if UPDATE: - print("Updating " + requirements_txt_relative) + print("Updating " + requirements_file_relative) if "BUILD_WORKSPACE_DIRECTORY" in os.environ: workspace = os.environ["BUILD_WORKSPACE_DIRECTORY"] - requirements_txt_tree = os.path.join(workspace, requirements_txt_relative) - # In most cases, requirements_txt will be a symlink to the real file in the source tree. - # If symlinks are not enabled (e.g. on Windows), then requirements_txt will be a copy, + requirements_file_tree = os.path.join(workspace, requirements_file_relative) + # In most cases, requirements_file will be a symlink to the real file in the source tree. + # If symlinks are not enabled (e.g. on Windows), then requirements_file will be a copy, # and we should copy the updated requirements back to the source tree. - if not os.path.samefile(resolved_requirements_txt, requirements_txt_tree): + if not os.path.samefile(resolved_requirements_file, requirements_file_tree): atexit.register( lambda: shutil.copy( - resolved_requirements_txt, requirements_txt_tree + resolved_requirements_file, requirements_file_tree ) ) cli() - requirements_txt_relative_path = Path(requirements_txt_relative) - content = requirements_txt_relative_path.read_text() + requirements_file_relative_path = Path(requirements_file_relative) + content = requirements_file_relative_path.read_text() content = content.replace(absolute_path_prefix, "") - requirements_txt_relative_path.write_text(content) + requirements_file_relative_path.write_text(content) else: # cli will exit(0) on success try: - print("Checking " + requirements_txt) + print("Checking " + requirements_file) cli() print("cli() should exit", file=sys.stderr) sys.exit(1) @@ -194,13 +199,7 @@ def _locate(bazel_runfiles, file): ) sys.exit(1) elif e.code == 0: - golden_filename = _select_golden_requirements_file( - requirements_txt, - requirements_linux, - requirements_darwin, - requirements_windows, - ) - golden = open(_locate(bazel_runfiles, golden_filename)).readlines() + golden = open(_locate(bazel_runfiles, requirements_file)).readlines() out = open(requirements_out).readlines() out = [line.replace(absolute_path_prefix, "") for line in out] if golden != out: diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel index d6ac0086ab..87ffe706dd 100644 --- a/tests/compile_pip_requirements/BUILD.bazel +++ b/tests/compile_pip_requirements/BUILD.bazel @@ -32,3 +32,33 @@ compile_pip_requirements( requirements_in = "requirements.txt", requirements_txt = "requirements_lock.txt", ) + +genrule( + name = "generate_os_specific_requirements_in", + srcs = [], + outs = ["requirements_os_specific.in"], + cmd = """ +cat > $@ <