Merge branch 'venvs-single-files' into exp/distinfo-venv

Author: aignas

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single
 
 test --test_output=errors
 

python/config_settings/BUILD.bazel

@@ -217,6 +217,15 @@ string_flag(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "is_venvs_site_packages",
+    flag_values = {
+        ":venvs_site_packages": VenvsSitePackages.YES,
+    },
+    # NOTE: Only public because it is used in whl_library repos.
+    visibility = ["//visibility:public"],
+)
+
 define_pypi_internal_flags(
     name = "define_pypi_internal_flags",
 )

python/private/py_library.bzl

@@ -253,6 +253,7 @@ def _get_site_packages_symlinks(ctx):
 
     repo_runfiles_dirname = None
     dirs_with_init = {}  # dirname -> runfile path
+    site_packages_symlinks = []
     for src in ctx.files.srcs:
         if src.extension not in PYTHON_FILE_EXTENSIONS:
             continue
@@ -261,16 +262,19 @@ def _get_site_packages_symlinks(ctx):
             continue
         path = path.removeprefix(site_packages_root)
         dir_name, _, filename = path.rpartition("/")
-        if not dir_name:
-            # This would be e.g. `site-packages/__init__.py`, which isn't valid
-            # because it's not within a directory for an importable Python package.
-            # However, the pypi integration over-eagerly adds a pkgutil-style
-            # __init__.py file during the repo phase. Just ignore them for now.
-            continue
 
-        if filename.startswith("__init__."):
+        if dir_name and filename.startswith("__init__."):
             dirs_with_init[dir_name] = None
             repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0]
+        elif not dir_name:
+            repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0]
+
+            # This would be files that do not have directories and we just need to add
+            # direct symlinks to them as is:
+            site_packages_symlinks.append((
+                paths.join(repo_runfiles_dirname, site_packages_root, filename),
+                filename,
+            ))
 
     # Sort so that we encounter `foo` before `foo/bar`. This ensures we
     # see the top-most explicit package first.
@@ -286,7 +290,6 @@ def _get_site_packages_symlinks(ctx):
         if not is_sub_package:
             first_level_explicit_packages.append(d)
 
-    site_packages_symlinks = []
     for dirname in first_level_explicit_packages:
         site_packages_symlinks.append((
             paths.join(repo_runfiles_dirname, site_packages_root, dirname),

python/private/pypi/BUILD.bazel

@@ -18,6 +18,11 @@ package(default_visibility = ["//:__subpackages__"])
 
 licenses(["notice"])
 
+exports_files(
+    srcs = ["namespace_pkg_tmpl.py"],
+    visibility = ["//visibility:public"],
+)
+
 filegroup(
     name = "distribution",
     srcs = glob(

python/private/pypi/namespace_pkg_tmpl.py

@@ -0,0 +1,2 @@
+# __path__ manipulation added by bazel-contrib/rules_python to support namespace pkgs.
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

python/private/pypi/namespace_pkgs.bzl

@@ -0,0 +1,83 @@
+"""Utilities to get where we should write namespace pkg paths."""
+
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+
+_ext = struct(
+    py = ".py",
+    pyd = ".pyd",
+    so = ".so",
+    pyc = ".pyc",
+)
+
+_TEMPLATE = Label("//python/private/pypi:namespace_pkg_tmpl.py")
+
+def _add_all(dirname, dirs):
+    dir_path = "."
+    for dir_name in dirname.split("/"):
+        dir_path = "{}/{}".format(dir_path, dir_name)
+        dirs[dir_path[2:]] = None
+
+def get_files(*, srcs, ignored_dirnames = [], root = None):
+    """Get the list of filenames to write the namespace pkg files.
+
+    Args:
+        srcs: {type}`src` a list of files to be passed to {bzl:obj}`py_library`
+            as `srcs` and `data`. This is usually a result of a {obj}`glob`.
+        ignored_dirnames: {type}`str` a list of patterns to ignore.
+        root: {type}`str` the prefix to use as the root.
+
+    Returns:
+        {type}`src` a list of paths to write the namespace pkg `__init__.py` file.
+    """
+    dirs = {}
+    ignored = {i: None for i in ignored_dirnames}
+
+    if root:
+        _add_all(root, ignored)
+
+    for file in srcs:
+        dirname, _, filename = file.rpartition("/")
+
+        if filename == "__init__.py":
+            ignored[dirname] = None
+            dirname, _, _ = dirname.rpartition("/")
+        elif filename.endswith(_ext.py):
+            pass
+        elif filename.endswith(_ext.pyc):
+            pass
+        elif filename.endswith(_ext.pyd):
+            pass
+        elif filename.endswith(_ext.so):
+            pass
+        else:
+            continue
+
+        if dirname in dirs or not dirname:
+            continue
+
+        _add_all(dirname, dirs)
+
+    return sorted([d for d in dirs if d not in ignored])
+
+def create_inits(**kwargs):
+    """Create init files and return the list to be included `py_library` srcs.
+
+    Args:
+        **kwargs: passed to {obj}`get_files`.
+
+    Returns:
+        {type}`list[str]` to be included as part of `py_library`.
+    """
+    srcs = []
+    for out in get_files(**kwargs):
+        src = "{}/__init__.py".format(out)
+        srcs.append(srcs)
+
+        copy_file(
+            name = "_cp_{}_namespace".format(out),
+            src = _TEMPLATE,
+            out = src,
+            **kwargs
+        )
+
+    return srcs

python/private/pypi/whl_installer/arguments.py

@@ -57,11 +57,6 @@ def parser(**kwargs: Any) -> argparse.ArgumentParser:
         action="store",
         help="Additional data exclusion parameters to add to the pip packages BUILD file.",
     )
-    parser.add_argument(
-        "--enable_implicit_namespace_pkgs",
-        action="store_true",
-        help="Disables conversion of implicit namespace packages into pkg-util style packages.",
-    )
     parser.add_argument(
         "--environment",
         action="store",

python/private/pypi/whl_installer/wheel_installer.py

@@ -27,7 +27,7 @@
 
 from pip._vendor.packaging.utils import canonicalize_name
 
-from python.private.pypi.whl_installer import arguments, namespace_pkgs, wheel
+from python.private.pypi.whl_installer import arguments, wheel
 
 
 def _configure_reproducible_wheels() -> None:
@@ -77,35 +77,10 @@ def _parse_requirement_for_extra(
     return None, None
 
 
-def _setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
-    """Converts native namespace packages to pkgutil-style packages
-
-    Namespace packages can be created in one of three ways. They are detailed here:
-    https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
-
-    'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but
-    'native namespace packages' (1) do not.
-
-    We ensure compatibility with Bazel of method 1 by converting them into method 2.
-
-    Args:
-        wheel_dir: the directory of the wheel to convert
-    """
-
-    namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
-        wheel_dir,
-        ignored_dirnames=["%s/bin" % wheel_dir],
-    )
-
-    for ns_pkg_dir in namespace_pkg_dirs:
-        namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
-
-
 def _extract_wheel(
     wheel_file: str,
     extras: Dict[str, Set[str]],
     enable_pipstar: bool,
-    enable_implicit_namespace_pkgs: bool,
     platforms: List[wheel.Platform],
     installation_dir: Path = Path("."),
 ) -> None:
@@ -116,15 +91,11 @@ def _extract_wheel(
         installation_dir: the destination directory for installation of the wheel.
         extras: a list of extras to add as dependencies for the installed wheel
         enable_pipstar: if true, turns off certain operations.
-        enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is
     """
 
     whl = wheel.Wheel(wheel_file)
     whl.unzip(installation_dir)
 
-    if not enable_implicit_namespace_pkgs:
-        _setup_namespace_pkg_compatibility(installation_dir)
-
     metadata = {
         "entry_points": [
             {
@@ -168,7 +139,6 @@ def main() -> None:
             wheel_file=whl,
             extras=extras,
             enable_pipstar=args.enable_pipstar,
-            enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
             platforms=arguments.get_platforms(args),
         )
         return

python/private/pypi/whl_library.bzl

@@ -173,9 +173,6 @@ def _parse_optional_attrs(rctx, args, extra_pip_args = None):
             json.encode(struct(arg = rctx.attr.pip_data_exclude)),
         ]
 
-    if rctx.attr.enable_implicit_namespace_pkgs:
-        args.append("--enable_implicit_namespace_pkgs")
-
     env = {}
     if rctx.attr.environment != None:
         for key, value in rctx.attr.environment.items():
@@ -389,6 +386,8 @@ def _whl_library_impl(rctx):
             metadata_name = metadata.name,
             metadata_version = metadata.version,
             requires_dist = metadata.requires_dist,
+            # TODO @aignas 2025-05-17: maybe have a build flag for this instead
+            enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs,
             # TODO @aignas 2025-04-14: load through the hub:
             annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
             data_exclude = rctx.attr.pip_data_exclude,
@@ -457,6 +456,8 @@ def _whl_library_impl(rctx):
             name = whl_path.basename,
             dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
             entry_points = entry_points,
+            # TODO @aignas 2025-05-17: maybe have a build flag for this instead
+            enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs,
             # TODO @aignas 2025-04-14: load through the hub:
             dependencies = metadata["deps"],
             dependencies_by_platform = metadata["deps_by_platform"],
@@ -580,7 +581,6 @@ attr makes `extra_pip_args` and `download_only` ignored.""",
             Label("//python/private/pypi/whl_installer:wheel.py"),
             Label("//python/private/pypi/whl_installer:wheel_installer.py"),
             Label("//python/private/pypi/whl_installer:arguments.py"),
-            Label("//python/private/pypi/whl_installer:namespace_pkgs.py"),
         ] + record_files.values(),
     ),
     "_rule_name": attr.string(default = "whl_library"),