feat: use rules_python implemented py_runtime, py_runtime_pair, PyRuntimeInfo (#1574)

This switches over to using the rules_python implementation of
`py_runtime`,
`py_runtime_pair`, and `PyRuntimeInfo` for Bazel 6 and higher. Bazel 5
lacks features
(specifically provider ctors) to allow enabling it on that version.

This is possible because the rules don't directly use the PyRuntimeInfo
provider
(mostly, see below), they only care about the structure of it as exposed
from the
ToolchainInfo provider.

Switching the toolchain providers and rules over early allows some
development of
the toolchain prior to Bazel 7 and the rest of the rules_python Starlark
implementation
being enabled.

The builtin PyRuntimeInfo is still returned and accepted for two
reasons:
* Better compatibility with the builtin rules to make transitioning
easier
* `py_binary` has an old, possibly defunct (not sure) code path that
will look up the
  the PyRuntimeInfo from a flag/implicit attribute.

Author: rickeylev

CHANGELOG.md

@@ -19,6 +19,12 @@ A brief description of the categories of changes:
 
 ## Unreleased
 
+### Changed
+
+* (toolchains) `py_runtime`, `py_runtime_pair`, and `PyRuntimeInfo` now use the
+  rules_python Starlark implementation, not the one built into Bazel. NOTE: This
+  only applies to Bazel 6+; Bazel 5 still uses the builtin implementation.
+
 [0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
 
 ## [0.27.0] - 2023-11-16

python/BUILD.bazel

@@ -157,7 +157,6 @@ bzl_library(
     deps = [
         "//python/private:util_bzl",
         "//python/private/common:py_runtime_macro_bzl",
-        "@rules_python_internal//:rules_python_config_bzl",
     ],
 )
 
@@ -167,7 +166,7 @@ bzl_library(
     deps = [
         "//python/private:bazel_tools_bzl",
         "//python/private:py_runtime_pair_macro_bzl",
-        "@rules_python_internal//:rules_python_config_bzl",
+        "//python/private:util_bzl",
     ],
 )
 
@@ -176,8 +175,8 @@ bzl_library(
     srcs = ["py_runtime_info.bzl"],
     deps = [
         "//python/private:reexports_bzl",
+        "//python/private:util_bzl",
         "//python/private/common:providers_bzl",
-        "@rules_python_internal//:rules_python_config_bzl",
     ],
 )
 

python/private/common/BUILD.bazel

@@ -74,7 +74,7 @@ bzl_library(
     srcs = ["providers.bzl"],
     deps = [
         ":semantics_bzl",
-        "@rules_python_internal//:rules_python_config_bzl",
+        "//python/private:util_bzl",
     ],
 )
 
@@ -171,9 +171,10 @@ bzl_library(
     srcs = ["py_runtime_rule.bzl"],
     deps = [
         ":attributes_bzl",
-        ":common_bzl",
         ":providers_bzl",
         ":py_internal_bzl",
+        "//python/private:reexports_bzl",
+        "//python/private:util_bzl",
         "@bazel_skylib//lib:dicts",
         "@bazel_skylib//lib:paths",
     ],

python/private/common/providers.bzl

@@ -13,14 +13,15 @@
 # limitations under the License.
 """Providers for Python rules."""
 
-load("@rules_python_internal//:rules_python_config.bzl", "config")
+load("//python/private:util.bzl", "IS_BAZEL_6_OR_HIGHER")
 
 # TODO: load CcInfo from rules_cc
 _CcInfo = CcInfo
 
 DEFAULT_STUB_SHEBANG = "#!/usr/bin/env python3"
 
-DEFAULT_BOOTSTRAP_TEMPLATE = "@bazel_tools//tools/python:python_bootstrap_template.txt"
+DEFAULT_BOOTSTRAP_TEMPLATE = Label("//python/private:python_bootstrap_template.txt")
+
 _PYTHON_VERSION_VALUES = ["PY2", "PY3"]
 
 # Helper to make the provider definitions not crash under Bazel 5.4:
@@ -31,7 +32,7 @@ _PYTHON_VERSION_VALUES = ["PY2", "PY3"]
 # This isn't actually used under Bazel 5.4, so just stub out the values
 # to get past the loading phase.
 def _define_provider(doc, fields, **kwargs):
-    if not config.enable_pystar:
+    if not IS_BAZEL_6_OR_HIGHER:
         return provider("Stub, not used", fields = []), None
     return provider(doc = doc, fields = fields, **kwargs)
 

python/private/common/py_runtime_rule.bzl

@@ -15,15 +15,15 @@
 
 load("@bazel_skylib//lib:dicts.bzl", "dicts")
 load("@bazel_skylib//lib:paths.bzl", "paths")
+load("//python/private:reexports.bzl", "BuiltinPyRuntimeInfo")
+load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")
 load(":attributes.bzl", "NATIVE_RULES_ALLOWLIST_ATTRS")
-load(":common.bzl", "check_native_allowed")
-load(":providers.bzl", "DEFAULT_BOOTSTRAP_TEMPLATE", "DEFAULT_STUB_SHEBANG", _PyRuntimeInfo = "PyRuntimeInfo")
+load(":providers.bzl", "DEFAULT_BOOTSTRAP_TEMPLATE", "DEFAULT_STUB_SHEBANG", "PyRuntimeInfo")
 load(":py_internal.bzl", "py_internal")
 
 _py_builtins = py_internal
 
 def _py_runtime_impl(ctx):
-    check_native_allowed(ctx)
     interpreter_path = ctx.attr.interpreter_path or None  # Convert empty string to None
     interpreter = ctx.file.interpreter
     if (interpreter_path and interpreter) or (not interpreter_path and not interpreter):
@@ -44,7 +44,7 @@ def _py_runtime_impl(ctx):
     if ctx.attr.coverage_tool:
         coverage_di = ctx.attr.coverage_tool[DefaultInfo]
 
-        if _py_builtins.is_singleton_depset(coverage_di.files):
+        if _is_singleton_depset(coverage_di.files):
             coverage_tool = coverage_di.files.to_list()[0]
         elif coverage_di.files_to_run and coverage_di.files_to_run.executable:
             coverage_tool = coverage_di.files_to_run.executable
@@ -60,39 +60,45 @@ def _py_runtime_impl(ctx):
         coverage_files = None
 
     python_version = ctx.attr.python_version
-    if python_version == "_INTERNAL_SENTINEL":
-        if ctx.fragments.py.use_toolchains:
-            fail(
-                "When using Python toolchains, this attribute must be set explicitly to either 'PY2' " +
-                "or 'PY3'. See https://github.com/bazelbuild/bazel/issues/7899 for more " +
-                "information. You can temporarily avoid this error by reverting to the legacy " +
-                "Python runtime mechanism (`--incompatible_use_python_toolchains=false`).",
-            )
-        else:
-            python_version = ctx.fragments.py.default_python_version
 
     # TODO: Uncomment this after --incompatible_python_disable_py2 defaults to true
     # if ctx.fragments.py.disable_py2 and python_version == "PY2":
     #     fail("Using Python 2 is not supported and disabled; see " +
     #          "https://github.com/bazelbuild/bazel/issues/15684")
 
+    py_runtime_info_kwargs = dict(
+        interpreter_path = interpreter_path or None,
+        interpreter = interpreter,
+        files = runtime_files if hermetic else None,
+        coverage_tool = coverage_tool,
+        coverage_files = coverage_files,
+        python_version = python_version,
+        stub_shebang = ctx.attr.stub_shebang,
+        bootstrap_template = ctx.file.bootstrap_template,
+    )
+    builtin_py_runtime_info_kwargs = dict(py_runtime_info_kwargs)
+    if not IS_BAZEL_7_OR_HIGHER:
+        builtin_py_runtime_info_kwargs.pop("bootstrap_template")
     return [
-        _PyRuntimeInfo(
-            interpreter_path = interpreter_path or None,
-            interpreter = interpreter,
-            files = runtime_files if hermetic else None,
-            coverage_tool = coverage_tool,
-            coverage_files = coverage_files,
-            python_version = python_version,
-            stub_shebang = ctx.attr.stub_shebang,
-            bootstrap_template = ctx.file.bootstrap_template,
-        ),
+        PyRuntimeInfo(**py_runtime_info_kwargs),
+        # Return the builtin provider for better compatibility.
+        # 1. There is a legacy code path in py_binary that
+        #    checks for the provider when toolchains aren't used
+        # 2. It makes it easier to transition from builtins to rules_python
+        BuiltinPyRuntimeInfo(**builtin_py_runtime_info_kwargs),
         DefaultInfo(
             files = runtime_files,
             runfiles = ctx.runfiles(),
         ),
     ]
 
+def _is_singleton_depset(files):
+    # Bazel 6 doesn't have this helper to optimize detecting singleton depsets.
+    if _py_builtins:
+        return _py_builtins.is_singleton_depset(files)
+    else:
+        return len(files.to_list()) == 1
+
 # Bind to the name "py_runtime" to preserve the kind/rule_class it shows up
 # as elsewhere.
 py_runtime = rule(
@@ -189,8 +195,8 @@ For a platform runtime, this is the absolute path of a Python interpreter on
 the target platform. For an in-build runtime this attribute must not be set.
 """),
         "python_version": attr.string(
-            default = "_INTERNAL_SENTINEL",
-            values = ["PY2", "PY3", "_INTERNAL_SENTINEL"],
+            default = "PY3",
+            values = ["PY2", "PY3"],
             doc = """
 Whether this runtime is for Python major version 2 or 3. Valid values are `"PY2"`
 and `"PY3"`.

python/private/py_runtime_pair_rule.bzl

@@ -15,18 +15,19 @@
 """Implementation of py_runtime_pair."""
 
 load("//python:py_runtime_info.bzl", "PyRuntimeInfo")
+load("//python/private:reexports.bzl", "BuiltinPyRuntimeInfo")
 
 def _py_runtime_pair_impl(ctx):
     if ctx.attr.py2_runtime != None:
-        py2_runtime = ctx.attr.py2_runtime[PyRuntimeInfo]
+        py2_runtime = _get_py_runtime_info(ctx.attr.py2_runtime)
         if py2_runtime.python_version != "PY2":
             fail("The Python runtime in the 'py2_runtime' attribute did not have " +
                  "version 'PY2'")
     else:
         py2_runtime = None
 
     if ctx.attr.py3_runtime != None:
-        py3_runtime = ctx.attr.py3_runtime[PyRuntimeInfo]
+        py3_runtime = _get_py_runtime_info(ctx.attr.py3_runtime)
         if py3_runtime.python_version != "PY3":
             fail("The Python runtime in the 'py3_runtime' attribute did not have " +
                  "version 'PY3'")
@@ -43,6 +44,12 @@ def _py_runtime_pair_impl(ctx):
         py3_runtime = py3_runtime,
     )]
 
+def _get_py_runtime_info(target):
+    if PyRuntimeInfo in target:
+        return target[PyRuntimeInfo]
+    else:
+        return target[BuiltinPyRuntimeInfo]
+
 # buildifier: disable=unused-variable
 def _is_py2_disabled(ctx):
     # Because this file isn't bundled with Bazel, so we have to conditionally
@@ -58,15 +65,15 @@ py_runtime_pair = rule(
         # The two runtimes are used by the py_binary at runtime, and so need to
         # be built for the target platform.
         "py2_runtime": attr.label(
-            providers = [PyRuntimeInfo],
+            providers = [[PyRuntimeInfo], [BuiltinPyRuntimeInfo]],
             cfg = "target",
             doc = """\
 The runtime to use for Python 2 targets. Must have `python_version` set to
 `PY2`.
 """,
         ),
         "py3_runtime": attr.label(
-            providers = [PyRuntimeInfo],
+            providers = [[PyRuntimeInfo], [BuiltinPyRuntimeInfo]],
             cfg = "target",
             doc = """\
 The runtime to use for Python 3 targets. Must have `python_version` set to

python/private/reexports.bzl

@@ -37,4 +37,4 @@ different name. Then we can load it from elsewhere.
 BuiltinPyInfo = PyInfo
 
 # buildifier: disable=name-conventions
-internal_PyRuntimeInfo = PyRuntimeInfo
+BuiltinPyRuntimeInfo = PyRuntimeInfo

python/private/util.bzl

@@ -83,3 +83,9 @@ def add_tag(attrs, tag):
             attrs["tags"] = tags + [tag]
     else:
         attrs["tags"] = [tag]
+
+IS_BAZEL_7_OR_HIGHER = hasattr(native, "starlark_doc_extract")
+
+# Bazel 5.4 has a bug where every access of testing.ExecutionInfo is a
+# different object that isn't equal to any other. This is fixed in bazel 6+.
+IS_BAZEL_6_OR_HIGHER = testing.ExecutionInfo == testing.ExecutionInfo

python/py_runtime.bzl

@@ -14,12 +14,11 @@
 
 """Public entry point for py_runtime."""
 
-load("@rules_python_internal//:rules_python_config.bzl", "config")
-load("//python/private:util.bzl", "add_migration_tag")
+load("//python/private:util.bzl", "IS_BAZEL_6_OR_HIGHER", "add_migration_tag")
 load("//python/private/common:py_runtime_macro.bzl", _starlark_py_runtime = "py_runtime")
 
 # buildifier: disable=native-python
-_py_runtime_impl = _starlark_py_runtime if config.enable_pystar else native.py_runtime
+_py_runtime_impl = _starlark_py_runtime if IS_BAZEL_6_OR_HIGHER else native.py_runtime
 
 def py_runtime(**attrs):
     """See the Bazel core [py_runtime](https://docs.bazel.build/versions/master/be/python.html#py_runtime) documentation.

python/py_runtime_info.bzl

@@ -14,8 +14,8 @@
 
 """Public entry point for PyRuntimeInfo."""
 
-load("@rules_python_internal//:rules_python_config.bzl", "config")
-load("//python/private:reexports.bzl", "internal_PyRuntimeInfo")
+load("//python/private:reexports.bzl", "BuiltinPyRuntimeInfo")
+load("//python/private:util.bzl", "IS_BAZEL_6_OR_HIGHER")
 load("//python/private/common:providers.bzl", _starlark_PyRuntimeInfo = "PyRuntimeInfo")
 
-PyRuntimeInfo = _starlark_PyRuntimeInfo if config.enable_pystar else internal_PyRuntimeInfo
+PyRuntimeInfo = _starlark_PyRuntimeInfo if IS_BAZEL_6_OR_HIGHER else BuiltinPyRuntimeInfo