diff --git a/.python-version b/.python-version index 2c20ac9bea..e4fba21835 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.13.3 +3.12 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9655b90487..bc9cb586b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,10 @@ END_UNRELEASED_TEMPLATE Set the `RULES_PYTHON_ENABLE_PIPSTAR=1` environment variable to enable it. * (utils) Add a way to run a REPL for any `rules_python` target that returns a `PyInfo` provider. +* (toolchains) Arbitrary python-build-standalone runtimes can be registered + and activated with custom flags. See the [Registering custom runtimes] + docs and {obj}`single_version_platform_override()` API docs for more + information. {#v0-0-0-removed} ### Removed diff --git a/MODULE.bazel b/MODULE.bazel index d0f7cc4afa..b10423baf6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -125,6 +125,23 @@ dev_python = use_extension( dev_python.override( register_all_versions = True, ) +dev_python.toolchain(python_version = "3.13") + +# For testing an arbitrary runtime triggered by a custom flag. +# See //tests/toolchains:custom_platform_toolchain_test +dev_python.single_version_platform_override( + platform = "linux-x86-install-only-stripped", + python_version = "3.13.3", + sha256 = "01d08b9bc8a96698b9d64c2fc26da4ecc4fa9e708ce0a34fb88f11ab7e552cbd", + #target_compatible_with = [ + # "@platforms//os:linux", + # "@platforms//cpu:x86_64", + #], + #target_settings = [ + # "@@//tests/support:is_custom_runtime_linux-x86-install-only-stripped", + #], + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3+20250409-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"], +) dev_pip = use_extension( "//python/extensions:pip.bzl", diff --git a/docs/toolchains.md b/docs/toolchains.md index ada887c945..3f256476f5 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -243,6 +243,76 @@ existing attributes: * Adding additional Python versions via {bzl:obj}`python.single_version_override` or {bzl:obj}`python.single_version_platform_override`. +### Registering custom runtimes + +Because the python-build-standalone project has _thousands_ of prebuilt runtimes +available, rules_python only includes popular runtimes in its built in +configurations. If you want to use a runtime that isn't already known to +rules_python then {obj}`single_version_platform_override()` can be used to do +so. In short, it allows specifying an arbitrary URL and using custom flags +to control when a runtime is used. + +In the example below, we register a particular python-build-standalone runtime +that is activated for Linux x86 builds when the custom flag +`--//:runtime=my-custom-runtime` is set. + +``` +# File: MODULE.bazel +bazel_dep(name = "bazel_skylib", version = "1.7.1.") +bazel_dep(name = "rules_python", version = "1.5.0") +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version="3.13.3") +python.single_version_platform_override( + platform = "my-platform", + python_version = "3.13.3", + sha256 = "01d08b9bc8a96698b9d64c2fc26da4ecc4fa9e708ce0a34fb88f11ab7e552cbd", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + target_settings = [ + "@@//:runtime=my-custom-runtime", + ], + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3+20250409-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"], +) +# File: //:BUILD.bazel +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +string_flag( + name = "custom_runtime", + build_setting_default = "", +) +config_setting( + name = "is_custom_runtime_linux-x86-install-only-stripped", + flag_values = { + ":custom_runtime": "linux-x86-install-only-stripped", + }, +) +``` + +Notes: +- While any URL and archive can be used, it's assumed their content looks how + a python-build-standalone archive looks. +- `python.toolchain()` is required if the version is unknown; if the version + is already known, it can be omitted. +- A "version aware" toolchain is registered, which means the Python version flag + must also match (e.g. `--@rules_python//python/config_settings:python_version=3.13.3` + must be set -- see `minor_mapping` and `is_default` for controls and docs + about version matching and selection). +- The labels in `target_settings` must be absolute; `@@` refers to the main repo. +- The `target_settings` are `config_setting` targets, which means you can + customize how matching occurs. + +:::{seealso} +See {obj}`//python/config_settings` for flags rules_python already defines +that can be used with `target_settings`. Some particular ones of note are: +{flag}`--py_linux_libc` and {flag}`--py_freethreaded`, among others. +::: + +:::{versionadded} VERSION_NEXT_FEATURE +Added support for custom platform names, `target_compatible_with`, and +`target_settings` with `single_version_platform_override`. +::: + ### Using defined toolchains from WORKSPACE It is possible to use toolchains defined in `MODULE.bazel` in `WORKSPACE`. For example diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index ce22421300..b319919305 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -241,11 +241,17 @@ bzl_library( ], ) +bzl_library( + name = "platform_info_bzl", + srcs = ["platform_info.bzl"], +) + bzl_library( name = "python_bzl", srcs = ["python.bzl"], deps = [ ":full_version_bzl", + ":platform_info_bzl", ":python_register_toolchains_bzl", ":pythons_hub_bzl", ":repo_utils_bzl", diff --git a/python/private/platform_info.bzl b/python/private/platform_info.bzl new file mode 100644 index 0000000000..3f7dc00165 --- /dev/null +++ b/python/private/platform_info.bzl @@ -0,0 +1,34 @@ +"""Helper to define a struct used to define platform metadata.""" + +def platform_info( + *, + compatible_with = [], + flag_values = {}, + target_settings = [], + os_name, + arch): + """Creates a struct of platform metadata. + + This is just a helper to ensure structs are created the same and + the meaning/values are documented. + + Args: + compatible_with: list[str], where the values are string labels. These + are the target_compatible_with values to use with the toolchain + flag_values: dict[str|Label, Any] of config_setting.flag_values + compatible values. DEPRECATED -- use target_settings instead + target_settings: list[str], where the values are string labels. These + are the target_settings values to use with the toolchain. + os_name: str, the os name; must match the name used in `@platfroms//os` + arch: str, the cpu name; must match the name used in `@platforms//cpu` + + Returns: + A struct with attributes and values matching the args. + """ + return struct( + compatible_with = compatible_with, + flag_values = flag_values, + target_settings = target_settings, + os_name = os_name, + arch = arch, + ) diff --git a/python/private/python.bzl b/python/private/python.bzl index a7e257601f..f452b98d2e 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -15,13 +15,19 @@ "Python toolchain module extensions for use with bzlmod." load("@bazel_features//:features.bzl", "bazel_features") -load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") +load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS", "platform_info") load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") load(":python_register_toolchains.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") -load(":toolchains_repo.bzl", "host_compatible_python_repo", "multi_toolchain_aliases", "sorted_host_platforms") +load( + ":toolchains_repo.bzl", + "host_compatible_python_repo", + "multi_toolchain_aliases", + "sorted_host_platform_names", + "sorted_host_platforms", +) load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") load(":version.bzl", "version") @@ -267,6 +273,24 @@ def parse_modules(*, module_ctx, _fail = fail): def _python_impl(module_ctx): py = parse_modules(module_ctx = module_ctx) + # Host compatible runtime repos + # dict[str version, struct] where struct has: + # * full_python_version: str + # * platform: platform_info struct + # * platform_name: str platform name + # * impl_repo_name: str repo name of the runtime's python_repository() repo + all_host_compatible_impls = {} + + # Host compatible repos that still need to be created because, when + # creating the actual runtime repo, there wasn't a host-compatible + # variant defined for it. + # dict[str reponame, struct] where struct has: + # * compatible_version: str, e.g. 3.10 or 3.10.1. The version the host + # repo should be compatible with + # * full_python_version: str, e.g. 3.10.1, the full python version of + # the toolchain that still needs a host repo created. + needed_host_repos = {} + # list of structs; see inline struct call within the loop below. toolchain_impls = [] @@ -293,11 +317,24 @@ def _python_impl(module_ctx): kwargs.update(py.config.kwargs.get(toolchain_info.python_version, {})) kwargs.update(py.config.kwargs.get(full_python_version, {})) kwargs.update(py.config.default) + if "3.13" in full_python_version: + print("bzlmod register:", toolchain_info.name, full_python_version) + print("kwargs.platforms:", kwargs["platforms"].keys()) + ##print("kwargs.tool_versions:", kwargs["tool_versions"]) + + # todo: this part is failing. python_register_toolchains doesn't have + # the new platform in its platform map, so never tries register_result = python_register_toolchains( name = toolchain_info.name, _internal_bzlmod_toolchain_call = True, **kwargs ) + if not register_result.impl_repos: + # If nothing was registered, something has gone wrong. This probably + # means the `platforms` map and `tool_versions[version]["shas"]` + # aren't in sync. + # todo: ignore instead of fail? + fail("No impls registered for", toolchain_info) host_platforms = {} for repo_name, (platform_name, platform_info) in register_result.impl_repos.items(): @@ -318,23 +355,111 @@ def _python_impl(module_ctx): set_python_version_constraint = is_last, )) if _is_compatible_with_host(module_ctx, platform_info): - host_platforms[platform_name] = platform_info - - host_platforms = sorted_host_platforms(host_platforms) - host_compatible_python_repo( - name = toolchain_info.name + "_host", - # NOTE: Order matters. The first found to be compatible is (usually) used. - platforms = host_platforms.keys(), - os_names = { - str(i): platform_info.os_name - for i, platform_info in enumerate(host_platforms.values()) - }, - arch_names = { - str(i): platform_info.arch - for i, platform_info in enumerate(host_platforms.values()) - }, - python_version = full_python_version, - ) + host_compat_entry = struct( + full_python_version = full_python_version, + platform = platform_info, + platform_name = platform_name, + impl_repo_name = repo_name, + ) + host_platforms[platform_name] = host_compat_entry + all_host_compatible_impls.setdefault(full_python_version, []).append( + host_compat_entry, + ) + all_host_compatible_impls.setdefault( + full_python_version.rpartition(".")[0], + [], + ).append(host_compat_entry) + + host_repo_name = toolchain_info.name + "_host" + if not host_platforms: + print("need:", host_repo_name) + needed_host_repos[host_repo_name] = struct( + compatible_version = toolchain_info.python_version, + full_python_version = full_python_version, + ) + else: + print("create:", host_repo_name) + host_platforms = sorted_host_platforms(host_platforms) + entries = host_platforms.values() + host_compatible_python_repo( + name = host_repo_name, + base_name = host_repo_name, + # NOTE: Order matters. The first found to be compatible is (usually) used. + platforms = host_platforms.keys(), + os_names = { + str(i): entry.platform.os_name + for i, entry in enumerate(entries) + }, + arch_names = { + str(i): entry.platform.arch + for i, entry in enumerate(entries) + }, + python_versions = { + str(i): entry.full_python_version + for i, entry in enumerate(entries) + }, + impl_repo_names = { + str(i): entry.impl_repo_name + for i, entry in enumerate(entries) + }, + ) + + def vt(s): + return tuple([int(x) for x in s.split(".")]) + + if needed_host_repos: + print("host repos still needed:", needed_host_repos) + for key, entries in all_host_compatible_impls.items(): + all_host_compatible_impls[key] = sorted( + entries, + reverse = True, + key = lambda e: vt(e.full_python_version), + ) + + for host_repo_name, info in needed_host_repos.items(): + choices = [] + if info.compatible_version not in all_host_compatible_impls: + print( + "No host compatible for:", + info.compatible_version, + "available:", + all_host_compatible_impls.keys(), + ) + continue + ##fail(" version missing", info.compatible_version) + + for entry in all_host_compatible_impls[info.compatible_version]: + # todo: numeric version comparison + # todo: should we restrict at all? Maybe just take the highest? + if vt(entry.full_python_version) <= vt(info.full_python_version): + choices.append(entry) + if choices: + platform_keys = [ + # We have to prepend the offset because the same platform + # name might occur across different versions + "{}_{}".format(i, entry.platform_name) + for i, entry in enumerate(choices) + ] + platform_keys = sorted_host_platform_names(platform_keys) + + print("create alt: {} for {}".format(host_repo_name, info.compatible_version)) + print("platforms=", platform_keys) + + host_compatible_python_repo( + name = host_repo_name, + base_name = host_repo_name, + platforms = platform_keys, + impl_repo_names = { + str(i): entry.impl_repo_name + for i, entry in enumerate(choices) + }, + os_names = {str(i): entry.platform.os_name for i, entry in enumerate(choices)}, + arch_names = {str(i): entry.platform.arch for i, entry in enumerate(choices)}, + python_versions = {str(i): entry.full_python_version for i, entry in enumerate(choices)}, + ) + else: + # todo: figure out what to do. Define nothing, if we can. + fail("No host-compatible found") # List of the base names ("python_3_10") for the toolchain repos base_toolchain_repo_names = [] @@ -586,6 +711,49 @@ def _process_single_version_platform_overrides(*, tag, _fail = fail, default): if tag.urls: available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls + if tag.platform not in default["platforms"]: + os_name = tag.os_name + arch = tag.arch + if not os_name or not arch: + for v in tag.target_compatible_with: + if os_name and arch: + break + if not os_name: + if v.startswith("@platforms//os:"): + if v.endswith(":linux"): + os_name = "linux" + + if not arch: + if v.startswith("@platforms//cpu:"): + if v.endswith(":x86_64"): + arch = "x86_64" + + if not os_name: + os_name = "UNKNOWN_CUSTOM" + if not arch: + arch = "UNKNOWN_CUSTOM" + + # todo: figure out why these can't be none + os_name = struct() + arch = struct() + + default["platforms"][tag.platform] = platform_info( + compatible_with = tag.target_compatible_with, + target_settings = tag.target_settings, + os_name = os_name, + arch = arch, + ) + elif ( + tag.target_compatible_with or tag.target_settings or + tag.os_name or tag.arch + ): + # todo: fail, or ignore? + fail(( + "Cannot override platform {} with custom platform settings" + ).format( + tag.platform, + )) + def _process_global_overrides(*, tag, default, _fail = fail): if tag.available_python_versions: available_versions = default["tool_versions"] @@ -1084,12 +1252,14 @@ configuration, please use {obj}`single_version_override`. ::: """, attrs = { + "arch": attr.string(), "coverage_tool": attr.label( doc = """\ The coverage tool to be used for a particular Python interpreter. This can override `rules_python` defaults. """, ), + "os_name": attr.string(), "patch_strip": attr.int( mandatory = False, doc = "Same as the --strip argument of Unix patch.", @@ -1101,7 +1271,6 @@ The coverage tool to be used for a particular Python interpreter. This can overr ), "platform": attr.string( mandatory = True, - values = PLATFORMS.keys(), doc = "The platform to override the values for, must be one of:\n{}.".format("\n".join(sorted(["* `{}`".format(p) for p in PLATFORMS]))), ), "python_version": attr.string( @@ -1117,6 +1286,8 @@ The coverage tool to be used for a particular Python interpreter. This can overr doc = "The 'strip_prefix' for the archive, defaults to 'python'.", default = "python", ), + "target_compatible_with": attr.string_list(), + "target_settings": attr.string_list(), "urls": attr.string_list( mandatory = False, doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of `{python_version}`, `{platform}` and `{build}` will be interpolated based on the contents in the override and the known {attr}`platform` values.", diff --git a/python/private/python_register_toolchains.bzl b/python/private/python_register_toolchains.bzl index 2e0748deb0..b3454d4823 100644 --- a/python/private/python_register_toolchains.bzl +++ b/python/private/python_register_toolchains.bzl @@ -117,6 +117,8 @@ def python_register_toolchains( # dict[str repo name, tuple[str, platform_info]] impl_repos = {} + if "3.13" in python_version: + print("plat map:", platforms.keys()) for platform, platform_info in platforms.items(): sha256 = tool_versions[python_version]["sha256"].get(platform, None) if not sha256: @@ -144,6 +146,12 @@ def python_register_toolchains( impl_repo_name = "{}_{}".format(name, platform) impl_repos[impl_repo_name] = (platform, platform_info) + if "3.13" in python_version: + print("py repo:", impl_repo_name, python_version) + print(sha256, patches, patch_strip, platform) + print(release_filename, urls, strip_prefix) + print(coverage_tool) + print(kwargs) python_repository( name = impl_repo_name, sha256 = sha256, diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index fd86b415cc..74936c3ac5 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -15,7 +15,7 @@ """This file contains repository rules and macros to support toolchain registration. """ -load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY", "PLATFORMS") +load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY") load(":auth.bzl", "get_auth") load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":text_util.bzl", "render") @@ -327,7 +327,7 @@ function defaults (e.g. `single_version_override` for `MODULE.bazel` files. "platform": attr.string( doc = "The platform name for the Python interpreter tarball.", mandatory = True, - values = PLATFORMS.keys(), + ##values = PLATFORMS.keys(), ), "python_version": attr.string( doc = "The Python version.", diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 2476889583..10678d25b9 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -309,11 +309,11 @@ actions.""", environ = [REPO_DEBUG_ENV_VAR], ) -def _host_compatible_python_repo(rctx): +def _host_compatible_python_repo_impl(rctx): rctx.file("BUILD.bazel", _HOST_TOOLCHAIN_BUILD_CONTENT) os_name = repo_utils.get_platforms_os_name(rctx) - host_platform = _get_host_platform( + impl_repo_name = _get_host_impl_repo_name( rctx = rctx, logger = repo_utils.logger(rctx), python_version = rctx.attr.python_version, @@ -321,10 +321,16 @@ def _host_compatible_python_repo(rctx): cpu_name = repo_utils.get_platforms_cpu_name(rctx), platforms = rctx.attr.platforms, ) - repo = "@@{py_repository}_{host_platform}".format( - py_repository = rctx.attr.name[:-len("_host")], - host_platform = host_platform, - ) + + # Bzlmod quirk: A repository rule can't, in its **implemention function**, + # resolve an apparent repo name referring to a repo created by the same + # bzlmod extension. To work around this, we use a canonical label. + repo = "@@{}".format(impl_repo_name) + + ##repo = "@@{py_repository}_{host_platform}".format( + ## py_repository = rctx.attr.name[:-len("_host")], + ## host_platform = host_platform, + ##) rctx.report_progress("Symlinking interpreter files to the target platform") host_python_repo = rctx.path(Label("{repo}//:BUILD.bazel".format(repo = repo))) @@ -380,26 +386,75 @@ def _host_compatible_python_repo(rctx): # NOTE: The term "toolchain" is a misnomer for this rule. This doesn't define # a repo with toolchains or toolchain implementations. host_compatible_python_repo = repository_rule( - _host_compatible_python_repo, + implementation = _host_compatible_python_repo_impl, doc = """\ Creates a repository with a shorter name meant to be used in the repository_ctx, which needs to have `symlinks` for the interpreter. This is separate from the toolchain_aliases repo because referencing the `python` interpreter target from this repo causes an eager fetch of the toolchain for the host platform. - """, + +This repo has two ways in which is it called: + +1. Workspace. The `platforms` attribute is set, which are keys into the + PLATFORMS global. It assumes `name` + is a + valid repo name which it can use as the backing repo. + +2. Bzlmod. All platform and backing repo information is passed in. +""", attrs = { "arch_names": attr.string_dict( doc = """ -If set, overrides the platform metadata. Keyed by index in `platforms` +Arch (cpu) names. Only set in bzlmod. Keyed by index in `platforms` +""", + ), + "base_name": attr.string( + doc = """ +The name arg, but without bzlmod canonicalization applied. Only set in bzlmod. +""", + ), + "impl_repo_names": attr.string_dict( + doc = """ +The names of backing runtime repos. Only set in bzlmod. The names must be repos +in the same extension as creates the host repo. Keyed by index in `platforms`. """, ), "os_names": attr.string_dict( doc = """ -If set, overrides the platform metadata. Keyed by index in `platforms` +If set, overrides the platform metadata. Only set in bzlmod. Keyed by +index in `platforms` +""", + ), + "platforms": attr.string_list( + mandatory = True, + doc = """ +Platform names (workspace) or platform name-like keys (bzlmod) + +NOTE: The order of this list matters. The first platform that is compatible +with the host will be selected; this can be customized by using the +`RULES_PYTHON_REPO_TOOLCHAIN_*` env vars. + +The values passed vary depending on workspace vs bzlmod. + +Workspace: the values are keys into the `PLATFORMS` dict and are the suffix +to append to `name` to point to the backing repo name. + +Bzlmod: The values are arbitrary keys to create the platform map from the +other attributes (os_name, arch_names, et al). +""", + ), + "python_version": attr.string( + doc = """ +Full python version, Major.Minor.Micro. + +Only set in workspace calls. +""", + ), + "python_versions": attr.string_dict( + doc = """ +If set, the Python version for the corresponding selected platform. Values in +Major.Minor.Patch format. Keyed by index in `platforms`. """, ), - "platforms": attr.string_list(mandatory = True), - "python_version": attr.string(mandatory = True), "_rule_name": attr.string(default = "host_compatible_python_repo"), "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), }, @@ -435,8 +490,8 @@ multi_toolchain_aliases = repository_rule( }, ) -def sorted_host_platforms(platform_map): - """Sort the keys in the platform map to give correct precedence. +def sorted_host_platform_names(platform_names): + """Sort platform names to give correct precedence. The order of keys in the platform mapping matters for the host toolchain selection. When multiple runtimes are compatible with the host, we take the @@ -453,11 +508,10 @@ def sorted_host_platforms(platform_map): is an innocous looking formatter disable directive. Args: - platform_map: a mapping of platforms and their metadata. + platform_names: a list of platform names Returns: - dict; the same values, but with the keys inserted in the desired - order so that iteration happens in the desired order. + list[str] the same values, but in the desired order. """ def platform_keyer(name): @@ -467,13 +521,26 @@ def sorted_host_platforms(platform_map): 1 if FREETHREADED in name else 0, ) - sorted_platform_keys = sorted(platform_map.keys(), key = platform_keyer) + return sorted(platform_names, key = platform_keyer) + +def sorted_host_platforms(platform_map): + """Sort the keys in the platform map to give correct precedence. + + See sorted_host_platform_names for explanation. + + Args: + platform_map: a mapping of platforms and their metadata. + + Returns: + dict; the same values, but with the keys inserted in the desired + order so that iteration happens in the desired order. + """ return { key: platform_map[key] - for key in sorted_platform_keys + for key in sorted_host_platform_names(platform_map.keys()) } -def _get_host_platform(*, rctx, logger, python_version, os_name, cpu_name, platforms): +def _get_host_impl_repo_name(*, rctx, logger, python_version, os_name, cpu_name, platforms): """Gets the host platform. Args: @@ -488,25 +555,42 @@ def _get_host_platform(*, rctx, logger, python_version, os_name, cpu_name, platf """ if rctx.attr.os_names: platform_map = {} + base_name = rctx.attr.base_name + if not base_name: + fail("base name missing") for i, platform_name in enumerate(platforms): key = str(i) + impl_repo_name = rctx.attr.impl_repo_names[key] + impl_repo_name = rctx.name.replace(base_name, impl_repo_name) platform_map[platform_name] = struct( os_name = rctx.attr.os_names[key], arch = rctx.attr.arch_names[key], + python_version = rctx.attr.python_versions[key], + impl_repo_name = impl_repo_name, ) else: - platform_map = sorted_host_platforms(PLATFORMS) + base_name = rctx.name.removesuffix("_host") + platform_map = {} + for platform_name, info in sorted_host_platforms(PLATFORMS).items(): + platform_map[platform_name] = struct( + os_name = info.os_name, + arch = info.arch, + python_version = python_version, + impl_repo_name = "{}_{}".format(base_name, platform_name), + ) candidates = [] for platform in platforms: meta = platform_map[platform] if meta.os_name == os_name and meta.arch == cpu_name: - candidates.append(platform) + candidates.append((platform, meta)) if len(candidates) == 1: - return candidates[0] + platform_name, meta = candidates[0] + return getattr(meta, "impl_repo_name", platform_name) + # todo: have this handle multiple python versions if candidates: env_var = "RULES_PYTHON_REPO_TOOLCHAIN_{}_{}_{}".format( python_version.replace(".", "_"), @@ -520,12 +604,26 @@ def _get_host_platform(*, rctx, logger, python_version, os_name, cpu_name, platf candidates, )) elif preference not in candidates: + # todo: need to map names like 3_13_0_linux_x86_64 back to + # the input values. Ah, er, wait + # Is this working? + # The return value is appended to this repo's name. + # This repo's name is e.g. python_3_13. + # the net result would be + # python_3_10_3_13_0_linux_x86_64 + # which isn't a valid name return logger.fail("Please choose a preferred interpreter out of the following platforms: {}".format(candidates)) else: candidates = [preference] if candidates: - return candidates[0] + platform_name, meta = candidates[0] + print("multiple candidates:", candidates) + suffix = getattr(meta, "impl_repo_name", None) + if not suffix: + suffix = platform_name + return suffix + ##return getattr(meta, "impl_repo_name", platform_name) return logger.fail("Could not find a compatible 'host' python for '{os_name}', '{cpu_name}' from the loaded platforms: {platforms}".format( os_name = os_name, diff --git a/python/versions.bzl b/python/versions.bzl index 166cc98851..e712a2e126 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -15,6 +15,8 @@ """The Python versions we use for the toolchains. """ +load("//python/private:platform_info.bzl", "platform_info") + # Values present in the @platforms//os package MACOS_NAME = "osx" LINUX_NAME = "linux" @@ -684,42 +686,12 @@ MINOR_MAPPING = { "3.13": "3.13.2", } -def _platform_info( - *, - compatible_with = [], - flag_values = {}, - target_settings = [], - os_name, - arch): - """Creates a struct of platform metadata. - - Args: - compatible_with: list[str], where the values are string labels. These - are the target_compatible_with values to use with the toolchain - flag_values: dict[str|Label, Any] of config_setting.flag_values - compatible values. DEPRECATED -- use target_settings instead - target_settings: list[str], where the values are string labels. These - are the target_settings values to use with the toolchain. - os_name: str, the os name; must match the name used in `@platfroms//os` - arch: str, the cpu name; must match the name used in `@platforms//cpu` - - Returns: - A struct with attributes and values matching the args. - """ - return struct( - compatible_with = compatible_with, - flag_values = flag_values, - target_settings = target_settings, - os_name = os_name, - arch = arch, - ) - def _generate_platforms(): is_libc_glibc = str(Label("//python/config_settings:_is_py_linux_libc_glibc")) is_libc_musl = str(Label("//python/config_settings:_is_py_linux_libc_musl")) platforms = { - "aarch64-apple-darwin": _platform_info( + "aarch64-apple-darwin": platform_info( compatible_with = [ "@platforms//os:macos", "@platforms//cpu:aarch64", @@ -727,7 +699,7 @@ def _generate_platforms(): os_name = MACOS_NAME, arch = "aarch64", ), - "aarch64-unknown-linux-gnu": _platform_info( + "aarch64-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:aarch64", @@ -738,7 +710,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "aarch64", ), - "armv7-unknown-linux-gnu": _platform_info( + "armv7-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:armv7", @@ -749,7 +721,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "arm", ), - "i386-unknown-linux-gnu": _platform_info( + "i386-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:i386", @@ -760,7 +732,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "x86_32", ), - "ppc64le-unknown-linux-gnu": _platform_info( + "ppc64le-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:ppc", @@ -771,7 +743,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "ppc", ), - "riscv64-unknown-linux-gnu": _platform_info( + "riscv64-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:riscv64", @@ -782,7 +754,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "riscv64", ), - "s390x-unknown-linux-gnu": _platform_info( + "s390x-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:s390x", @@ -793,7 +765,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "s390x", ), - "x86_64-apple-darwin": _platform_info( + "x86_64-apple-darwin": platform_info( compatible_with = [ "@platforms//os:macos", "@platforms//cpu:x86_64", @@ -801,7 +773,7 @@ def _generate_platforms(): os_name = MACOS_NAME, arch = "x86_64", ), - "x86_64-pc-windows-msvc": _platform_info( + "x86_64-pc-windows-msvc": platform_info( compatible_with = [ "@platforms//os:windows", "@platforms//cpu:x86_64", @@ -809,7 +781,7 @@ def _generate_platforms(): os_name = WINDOWS_NAME, arch = "x86_64", ), - "x86_64-unknown-linux-gnu": _platform_info( + "x86_64-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", @@ -820,7 +792,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "x86_64", ), - "x86_64-unknown-linux-musl": _platform_info( + "x86_64-unknown-linux-musl": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", @@ -836,7 +808,7 @@ def _generate_platforms(): is_freethreaded_yes = str(Label("//python/config_settings:_is_py_freethreaded_yes")) is_freethreaded_no = str(Label("//python/config_settings:_is_py_freethreaded_no")) return { - p + suffix: _platform_info( + p + suffix: platform_info( compatible_with = v.compatible_with, target_settings = [ freethreadedness, diff --git a/tests/bootstrap_impls/bin.py b/tests/bootstrap_impls/bin.py index 1176107384..3d467dcf29 100644 --- a/tests/bootstrap_impls/bin.py +++ b/tests/bootstrap_impls/bin.py @@ -23,3 +23,4 @@ print("sys.flags.safe_path:", sys.flags.safe_path) print("file:", __file__) print("sys.executable:", sys.executable) +print("sys._base_executable:", sys._base_executable) diff --git a/tests/support/BUILD.bazel b/tests/support/BUILD.bazel index 9fb5cd0760..303dbafbdf 100644 --- a/tests/support/BUILD.bazel +++ b/tests/support/BUILD.bazel @@ -18,6 +18,7 @@ # to force them to resolve in the proper context. # ==================== +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load(":sh_py_run_test.bzl", "current_build_settings") package( @@ -90,3 +91,15 @@ platform( current_build_settings( name = "current_build_settings", ) + +string_flag( + name = "custom_runtime", + build_setting_default = "", +) + +config_setting( + name = "is_custom_runtime_linux-x86-install-only-stripped", + flag_values = { + ":custom_runtime": "linux-x86-install-only-stripped", + }, +) diff --git a/tests/toolchains/BUILD.bazel b/tests/toolchains/BUILD.bazel index c55dc92a7d..c3021765b4 100644 --- a/tests/toolchains/BUILD.bazel +++ b/tests/toolchains/BUILD.bazel @@ -12,8 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") load(":defs.bzl", "define_toolchain_tests") define_toolchain_tests( name = "toolchain_tests", ) + +py_reconfig_test( + name = "custom_platform_toolchain_test", + srcs = ["custom_platform_toolchain_test.py"], + custom_runtime = "linux-x86-install-only-stripped", + python_version = "3.13.3", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) diff --git a/tests/toolchains/custom_platform_toolchain_test.py b/tests/toolchains/custom_platform_toolchain_test.py new file mode 100644 index 0000000000..d6c083a6a2 --- /dev/null +++ b/tests/toolchains/custom_platform_toolchain_test.py @@ -0,0 +1,15 @@ +import sys +import unittest + + +class VerifyCustomPlatformToolchainTest(unittest.TestCase): + + def test_custom_platform_interpreter_used(self): + # We expect the repo name, and thus path, to have the + # platform name in it. + self.assertIn("linux-x86-install-only-stripped", sys._base_executable) + print(sys._base_executable) + + +if __name__ == "__main__": + unittest.main()