feat(rules): add PyExecutableInfo (bazel-contrib#2166)

The PyExecutableInfo provider exposes executable-specific information
that isn't easily accessible outside the target. The main purpose of
this provider is to facilitate packaging a binary or deriving a new
binary based upon the original.

Within rules_python, this will be used to pull the zip-building logic
out of executables and into separate rules. Within Google, this will be
used for a similar "package a binary" tool.

Along the way:
* Add runfiles references to sphinx inventory

Author: rickeylev

CHANGELOG.md

@@ -46,6 +46,9 @@ A brief description of the categories of changes:
 * (gazelle) Correctly resolve deps that have top-level module overlap with a gazelle_python.yaml dep module
 
 ### Added
+* (rules) Executables provide {obj}`PyExecutableInfo`, which contains
+  executable-specific information useful for packaging an executable or
+  or deriving a new one from the original.
 * (py_wheel) Removed use of bash to avoid failures on Windows machines which do not
   have it installed.
 

docs/BUILD.bazel

@@ -84,6 +84,7 @@ sphinx_stardocs(
         "//python:pip_bzl",
         "//python:py_binary_bzl",
         "//python:py_cc_link_params_info_bzl",
+        "//python:py_executable_info_bzl",
         "//python:py_library_bzl",
         "//python:py_runtime_bzl",
         "//python:py_runtime_info_bzl",

python/BUILD.bazel

@@ -139,6 +139,12 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "py_executable_info_bzl",
+    srcs = ["py_executable_info.bzl"],
+    deps = ["//python/private:py_executable_info_bzl"],
+)
+
 bzl_library(
     name = "py_import_bzl",
     srcs = ["py_import.bzl"],

python/private/BUILD.bazel

@@ -220,6 +220,11 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "py_executable_info_bzl",
+    srcs = ["py_executable_info.bzl"],
+)
+
 bzl_library(
     name = "py_interpreter_program_bzl",
     srcs = ["py_interpreter_program.bzl"],

python/private/common/BUILD.bazel

@@ -132,9 +132,11 @@ bzl_library(
         ":providers_bzl",
         ":py_internal_bzl",
         "//python/private:flags_bzl",
+        "//python/private:py_executable_info_bzl",
         "//python/private:rules_cc_srcs_bzl",
         "//python/private:toolchain_types_bzl",
         "@bazel_skylib//lib:dicts",
+        "@bazel_skylib//lib:structs",
         "@bazel_skylib//rules:common_settings",
     ],
 )

python/private/common/py_executable.bzl

@@ -14,9 +14,11 @@
 """Common functionality between test/binary executables."""
 
 load("@bazel_skylib//lib:dicts.bzl", "dicts")
+load("@bazel_skylib//lib:structs.bzl", "structs")
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("@rules_cc//cc:defs.bzl", "cc_common")
 load("//python/private:flags.bzl", "PrecompileAddToRunfilesFlag")
+load("//python/private:py_executable_info.bzl", "PyExecutableInfo")
 load("//python/private:reexports.bzl", "BuiltinPyRuntimeInfo")
 load(
     "//python/private:toolchain_types.bzl",
@@ -221,10 +223,14 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment =
     extra_exec_runfiles = exec_result.extra_runfiles.merge(
         ctx.runfiles(transitive_files = exec_result.extra_files_to_build),
     )
-    runfiles_details = struct(
-        default_runfiles = runfiles_details.default_runfiles.merge(extra_exec_runfiles),
-        data_runfiles = runfiles_details.data_runfiles.merge(extra_exec_runfiles),
-    )
+
+    # Copy any existing fields in case of company patches.
+    runfiles_details = struct(**(
+        structs.to_dict(runfiles_details) | dict(
+            default_runfiles = runfiles_details.default_runfiles.merge(extra_exec_runfiles),
+            data_runfiles = runfiles_details.data_runfiles.merge(extra_exec_runfiles),
+        )
+    ))
 
     return _create_providers(
         ctx = ctx,
@@ -400,8 +406,8 @@ def _get_base_runfiles_for_binary(
         semantics):
     """Returns the set of runfiles necessary prior to executable creation.
 
-    NOTE: The term "common runfiles" refers to the runfiles that both the
-    default and data runfiles have in common.
+    NOTE: The term "common runfiles" refers to the runfiles that are common to
+        runfiles_without_exe, default_runfiles, and data_runfiles.
 
     Args:
         ctx: The rule ctx.
@@ -418,6 +424,8 @@ def _get_base_runfiles_for_binary(
         struct with attributes:
         * default_runfiles: The default runfiles
         * data_runfiles: The data runfiles
+        * runfiles_without_exe: The default runfiles, but without the executable
+          or files specific to the original program/executable.
     """
     common_runfiles_depsets = [main_py_files]
 
@@ -431,7 +439,6 @@ def _get_base_runfiles_for_binary(
             common_runfiles_depsets.append(dep[PyInfo].transitive_pyc_files)
 
     common_runfiles = collect_runfiles(ctx, depset(
-        direct = [executable],
         transitive = common_runfiles_depsets,
     ))
     if extra_deps:
@@ -447,22 +454,27 @@ def _get_base_runfiles_for_binary(
             runfiles = common_runfiles,
         )
 
+    # Don't include build_data.txt in the non-exe runfiles. The build data
+    # may contain program-specific content (e.g. target name).
+    runfiles_with_exe = common_runfiles.merge(ctx.runfiles([executable]))
+
     # Don't include build_data.txt in data runfiles. This allows binaries to
     # contain other binaries while still using the same fixed location symlink
     # for the build_data.txt file. Really, the fixed location symlink should be
     # removed and another way found to locate the underlying build data file.
-    data_runfiles = common_runfiles
+    data_runfiles = runfiles_with_exe
 
     if is_stamping_enabled(ctx, semantics) and semantics.should_include_build_data(ctx):
-        default_runfiles = common_runfiles.merge(_create_runfiles_with_build_data(
+        default_runfiles = runfiles_with_exe.merge(_create_runfiles_with_build_data(
             ctx,
             semantics.get_central_uncachable_version_file(ctx),
             semantics.get_extra_write_build_data_env(ctx),
         ))
     else:
-        default_runfiles = common_runfiles
+        default_runfiles = runfiles_with_exe
 
     return struct(
+        runfiles_without_exe = common_runfiles,
         default_runfiles = default_runfiles,
         data_runfiles = data_runfiles,
     )
@@ -814,6 +826,11 @@ def _create_providers(
         ),
         create_instrumented_files_info(ctx),
         _create_run_environment_info(ctx, inherited_environment),
+        PyExecutableInfo(
+            main = main_py,
+            runfiles_without_exe = runfiles_details.runfiles_without_exe,
+            interpreter_path = runtime_details.executable_interpreter_path,
+        ),
     ]
 
     # TODO(b/265840007): Make this non-conditional once Google enables
@@ -904,6 +921,7 @@ def create_base_executable_rule(*, attrs, fragments = [], **kwargs):
     if "py" not in fragments:
         # The list might be frozen, so use concatentation
         fragments = fragments + ["py"]
+    kwargs.setdefault("provides", []).append(PyExecutableInfo)
     return rule(
         # TODO: add ability to remove attrs, i.e. for imports attr
         attrs = dicts.add(EXECUTABLE_ATTRS, attrs),

python/private/py_executable_info.bzl

@@ -0,0 +1,35 @@
+"""Implementation of PyExecutableInfo provider."""
+
+PyExecutableInfo = provider(
+    doc = """
+Information about an executable.
+
+This provider is for executable-specific information (e.g. tests and binaries).
+
+:::{versionadded} 0.36.0
+:::
+""",
+    fields = {
+        "interpreter_path": """
+:type: None | str
+
+Path to the Python interpreter to use for running the executable itself (not the
+bootstrap script). Either an absolute path (which means it is
+platform-specific), or a runfiles-relative path (which means the interpreter
+should be within `runtime_files`)
+""",
+        "main": """
+:type: File
+
+The user-level entry point file. Usually a `.py` file, but may also be `.pyc`
+file if precompiling is enabled.
+""",
+        "runfiles_without_exe": """
+:type: runfiles
+
+The runfiles the program needs, but without the original executable,
+files only added to support the original executable, or files specific to the
+original program.
+""",
+    },
+)

python/py_executable_info.bzl

@@ -0,0 +1,12 @@
+"""Provider for executable-specific information.
+
+The `PyExecutableInfo` provider contains information about an executable that
+isn't otherwise available from its public attributes or other providers.
+
+It exposes information primarily useful for consumers to package the executable,
+or derive a new executable from the base binary.
+"""
+
+load("//python/private:py_executable_info.bzl", _PyExecutableInfo = "PyExecutableInfo")
+
+PyExecutableInfo = _PyExecutableInfo

sphinxdocs/inventories/bazel_inventory.txt

@@ -65,6 +65,13 @@ native.package_name bzl:function 1 rules/lib/toplevel/native#package_name -
 native.package_relative_label bzl:function 1 rules/lib/toplevel/native#package_relative_label -
 native.repo_name bzl:function 1 rules/lib/toplevel/native#repo_name -
 native.repository_name bzl:function 1 rules/lib/toplevel/native#repository_name -
+runfiles bzl:type 1 rules/lib/builtins/runfiles -
+runfiles.empty_filenames bzl:type 1 rules/lib/builtins/runfiles#empty_filenames -
+runfiles.files bzl:type 1 rules/lib/builtins/runfiles#files -
+runfiles.merge bzl:type 1 rules/lib/builtins/runfiles#merge -
+runfiles.merge_all bzl:type 1 rules/lib/builtins/runfiles#merge_all -
+runfiles.root_symlinks bzl:type 1 rules/lib/builtins/runfiles#root_symlinks -
+runfiles.symlinks bzl:type 1 rules/lib/builtins/runfiles#symlinks -
 str bzl:type 1 rules/lib/string -
 struct bzl:type 1 rules/lib/builtins/struct -
 toolchain_type bzl:type 1 ules/lib/builtins/toolchain_type.html -

tests/base_rules/py_executable_base_tests.bzl

@@ -18,9 +18,11 @@ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config")
 load("@rules_testing//lib:analysis_test.bzl", "analysis_test")
 load("@rules_testing//lib:truth.bzl", "matching")
 load("@rules_testing//lib:util.bzl", rt_util = "util")
+load("//python:py_executable_info.bzl", "PyExecutableInfo")
 load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable=bzl-visibility
 load("//tests/base_rules:base_tests.bzl", "create_base_tests")
 load("//tests/base_rules:util.bzl", "WINDOWS_ATTR", pt_util = "util")
+load("//tests/support:py_executable_info_subject.bzl", "PyExecutableInfoSubject")
 load("//tests/support:support.bzl", "LINUX_X86_64", "WINDOWS_X86_64")
 
 _BuiltinPyRuntimeInfo = PyRuntimeInfo
@@ -132,11 +134,19 @@ def _test_executable_in_runfiles_impl(env, target):
         exe = ".exe"
     else:
         exe = ""
-
     env.expect.that_target(target).runfiles().contains_at_least([
         "{workspace}/{package}/{test_name}_subject" + exe,
     ])
 
+    if rp_config.enable_pystar:
+        py_exec_info = env.expect.that_target(target).provider(PyExecutableInfo, factory = PyExecutableInfoSubject.new)
+        py_exec_info.main().path().contains("_subject.py")
+        py_exec_info.interpreter_path().contains("python")
+        py_exec_info.runfiles_without_exe().contains_none_of([
+            "{workspace}/{package}/{test_name}_subject" + exe,
+            "{workspace}/{package}/{test_name}_subject",
+        ])
+
 def _test_default_main_can_be_generated(name, config):
     rt_util.helper_target(
         config.rule,

tests/support/py_executable_info_subject.bzl

@@ -0,0 +1,70 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyExecutableInfo testing subject."""
+
+load("@rules_testing//lib:truth.bzl", "subjects")
+
+def _py_executable_info_subject_new(info, *, meta):
+    """Creates a new `PyExecutableInfoSubject` for a PyExecutableInfo provider instance.
+
+    Method: PyExecutableInfoSubject.new
+
+    Args:
+        info: The PyExecutableInfo object
+        meta: ExpectMeta object.
+
+    Returns:
+        A `PyExecutableInfoSubject` struct
+    """
+
+    # buildifier: disable=uninitialized
+    public = struct(
+        # go/keep-sorted start
+        actual = info,
+        interpreter_path = lambda *a, **k: _py_executable_info_subject_interpreter_path(self, *a, **k),
+        main = lambda *a, **k: _py_executable_info_subject_main(self, *a, **k),
+        runfiles_without_exe = lambda *a, **k: _py_executable_info_subject_runfiles_without_exe(self, *a, **k),
+        # go/keep-sorted end
+    )
+    self = struct(
+        actual = info,
+        meta = meta,
+    )
+    return public
+
+def _py_executable_info_subject_interpreter_path(self):
+    """Returns a subject for `PyExecutableInfo.interpreter_path`."""
+    return subjects.str(
+        self.actual.interpreter_path,
+        meta = self.meta.derive("interpreter_path()"),
+    )
+
+def _py_executable_info_subject_main(self):
+    """Returns a subject for `PyExecutableInfo.main`."""
+    return subjects.file(
+        self.actual.main,
+        meta = self.meta.derive("main()"),
+    )
+
+def _py_executable_info_subject_runfiles_without_exe(self):
+    """Returns a subject for `PyExecutableInfo.runfiles_without_exe`."""
+    return subjects.runfiles(
+        self.actual.runfiles_without_exe,
+        meta = self.meta.derive("runfiles_without_exe()"),
+    )
+
+# buildifier: disable=name-conventions
+PyExecutableInfoSubject = struct(
+    new = _py_executable_info_subject_new,
+)