fix(bzlmod): keep the lockfile platform independent when resolving python (#2135)

Before this PR the lockfile would become platform dependent when the
`requirements` file would have env markers. This was not caught because
we do not have MODULE.bazel.lock checked into the `rules_python`
repository because the CI is running against many versions and the lock
file is different, therefore we would not be able to run with
`bazel build --lockfile_mode=error`.

With this change we use the label to `BUILD.bazel` which is living next
to the `python` symlink and since the `BUILD.bazel` is the same on all
platforms, the lockfile will remain the same.

Summary
* refactor(uv): create a reusable macro for using uv for locking reqs.
* test(bzlmod): enable testing the MODULE.bazel.lock breakage across
platforms.
* test(bzlmod): use a universal requirements file for 3.9.
This breaks the CI, because the python interpreter file hash is added to
the lock file.
* fix(bzlmod): keep the lockfile platform independent when resolving
python


Fixes #1105 and #1868 for real this time.
Implements an additional helper for #1975.

Author: aignas

.bazelci/presubmit.yml

@@ -223,6 +223,12 @@ tasks:
     name: "examples/bzlmod: Ubuntu, minimum Bazel"
     working_directory: examples/bzlmod
     platform: ubuntu2004
+    build_flags:
+      - "--lockfile_mode=update"
+    test_flags:
+      - "--lockfile_mode=update"
+    coverage_flags:
+      - "--lockfile_mode=update"
   integration_test_bzlmod_ubuntu:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod

.gitignore

@@ -52,3 +52,4 @@ user.bazelrc
 # MODULE.bazel.lock is ignored for now as per recommendation from upstream.
 # See https://github.com/bazelbuild/bazel/issues/20369
 MODULE.bazel.lock
+!/examples/bzlmod/MODULE.bazel.lock

CHANGELOG.md

@@ -28,6 +28,12 @@ A brief description of the categories of changes:
 * (gazelle): Update error messages when unable to resolve a dependency to be more human-friendly.
 
 ### Fixed
+* (bzlmod) get the path to the host python interpreter in a way that results in
+  platform non-dependent hashes in the lock file when the requirement markers need
+  to be evaluated.
+* (bzlmod) correctly watch sources used for evaluating requirement markers for
+  any changes so that the repository rule or module extensions can be
+  re-evaluated when the said files change.
 * (gazelle): Fix incorrect use of `t.Fatal`/`t.Fatalf` in tests.
 * (toolchain) Omit third-party python packages from coverage reports from
   stage2 bootstrap template.

docs/BUILD.bazel

@@ -13,11 +13,10 @@
 # limitations under the License.
 
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@dev_pip//:requirements.bzl", "requirement")
-load("//python:py_binary.bzl", "py_binary")
 load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: disable=bzl-visibility
 load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable=bzl-visibility
+load("//python/uv/private:lock.bzl", "lock")  # buildifier: disable=bzl-visibility
 load("//sphinxdocs:readthedocs.bzl", "readthedocs_install")
 load("//sphinxdocs:sphinx.bzl", "sphinx_build_binary", "sphinx_docs")
 load("//sphinxdocs:sphinx_stardoc.bzl", "sphinx_stardoc", "sphinx_stardocs")
@@ -140,71 +139,12 @@ sphinx_build_binary(
     ],
 )
 
-_REQUIREMENTS_TARGET_COMPATIBLE_WITH = select({
-    "@platforms//os:linux": [],
-    "@platforms//os:macos": [],
-    "@platforms//os:windows": [],
-    "//conditions:default": ["@platforms//:incompatible"],
-}) if BZLMOD_ENABLED else ["@platforms//:incompatible"]
-
 # Run bazel run //docs:requirements.update
-genrule(
+lock(
     name = "requirements",
     srcs = ["pyproject.toml"],
-    outs = ["_requirements.txt"],
-    cmd = "$(UV_BIN) pip compile " + " ".join([
-        "--custom-compile-command='bazel run //docs:requirements.update'",
-        "--generate-hashes",
-        "--universal",
-        "--emit-index-url",
-        "--no-strip-extras",
-        "--no-build",
-        "--python=$(PYTHON3)",
-        "$<",
-        "--output-file=$@",
-        # Always try upgrading
-        "--upgrade",
-    ]),
-    tags = [
-        "local",
-        "manual",
-        "no-cache",
-    ],
-    target_compatible_with = _REQUIREMENTS_TARGET_COMPATIBLE_WITH,
-    toolchains = [
-        "//python/uv:current_toolchain",
-        "//python:current_py_toolchain",
-    ],
-)
-
-# Write a script that can be used for updating the in-tree version of the
-# requirements file
-write_file(
-    name = "gen_update_requirements",
-    out = "requirements.update.py",
-    content = [
-        "from os import environ",
-        "from pathlib import Path",
-        "from sys import stderr",
-        "",
-        'src = Path(environ["REQUIREMENTS_FILE"])',
-        'dst = Path(environ["BUILD_WORKSPACE_DIRECTORY"]) / "docs" / "requirements.txt"',
-        'print(f"Writing requirements contents from {src} to {dst}", file=stderr)',
-        "dst.write_text(src.read_text())",
-        'print("Success!", file=stderr)',
-    ],
-    target_compatible_with = _REQUIREMENTS_TARGET_COMPATIBLE_WITH,
-)
-
-py_binary(
-    name = "requirements.update",
-    srcs = ["requirements.update.py"],
-    data = [":requirements"],
-    env = {
-        "REQUIREMENTS_FILE": "$(location :requirements)",
-    },
-    tags = ["manual"],
-    target_compatible_with = _REQUIREMENTS_TARGET_COMPATIBLE_WITH,
+    out = "requirements.txt",
+    upgrade = True,
 )
 
 licenses(["notice"])  # Apache 2.0

examples/BUILD.bazel

@@ -12,4 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# The following is experimental API and currently not intended for use outside this example.
+load("@rules_python//python/uv/private:lock.bzl", "lock")  # buildifier: disable=bzl-visibility
+
 licenses(["notice"])  # Apache 2.0
+
+lock(
+    name = "bzlmod_requirements_3_9",
+    srcs = ["bzlmod/requirements.in"],
+    out = "bzlmod/requirements_lock_3_9.txt",
+    python_version = "3.9.19",
+)

examples/bzlmod/.bazelrc

@@ -1,5 +1,9 @@
 common --enable_bzlmod
 
+# Update the lockfile by running:
+# bazel mod deps --lockfile_mode=update
+common --lockfile_mode=error
+
 coverage --java_runtime_version=remotejdk_11
 
 test --test_output=errors --enable_runfiles

examples/bzlmod/BUILD.bazel

@@ -9,20 +9,10 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@pip//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement")
 load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test")
 load("@python_versions//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements")
-load("@python_versions//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requirements")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
 
 # This stanza calls a rule that generates targets for managing pip dependencies
-# with pip-compile.
-compile_pip_requirements_3_9(
-    name = "requirements_3_9",
-    src = "requirements.in",
-    requirements_txt = "requirements_lock_3_9.txt",
-    requirements_windows = "requirements_windows_3_9.txt",
-)
-
-# This stanza calls a rule that generates targets for managing pip dependencies
-# with pip-compile.
+# with pip-compile for a particular python version.
 compile_pip_requirements_3_10(
     name = "requirements_3_10",
     timeout = "moderate",

examples/bzlmod/MODULE.bazel

@@ -135,17 +135,7 @@ pip.parse(
     ],
     hub_name = "pip",
     python_version = "3.9",
-    # The requirements files for each platform that we want to support.
-    requirements_by_platform = {
-        # Default requirements file for needs to explicitly provide the platforms
-        "//:requirements_lock_3_9.txt": "linux_*,osx_*",
-        # This API allows one to specify additional platforms that the users
-        # configure the toolchains for themselves. In this example we add
-        # `windows_aarch64` to illustrate that `rules_python` won't fail to
-        # process the value, but it does not mean that this example will work
-        # on Windows ARM.
-        "//:requirements_windows_3_9.txt": "windows_x86_64,windows_aarch64",
-    },
+    requirements_lock = "requirements_lock_3_9.txt",
     # These modifications were created above and we
     # are providing pip.parse with the label of the mod
     # and the name of the wheel.
@@ -179,8 +169,17 @@ pip.parse(
     ],
     hub_name = "pip",
     python_version = "3.10",
-    requirements_lock = "//:requirements_lock_3_10.txt",
-    requirements_windows = "//:requirements_windows_3_10.txt",
+    # The requirements files for each platform that we want to support.
+    requirements_by_platform = {
+        # Default requirements file for needs to explicitly provide the platforms
+        "//:requirements_lock_3_10.txt": "linux_*,osx_*",
+        # This API allows one to specify additional platforms that the users
+        # configure the toolchains for themselves. In this example we add
+        # `windows_aarch64` to illustrate that `rules_python` won't fail to
+        # process the value, but it does not mean that this example will work
+        # on Windows ARM.
+        "//:requirements_windows_3_10.txt": "windows_x86_64,windows_aarch64",
+    },
     # These modifications were created above and we
     # are providing pip.parse with the label of the mod
     # and the name of the wheel.