[NO TESTS] WIP

Author: arrdem

examples/py_venv/BUILD.bazel

@@ -20,13 +20,29 @@ py_venv(
 )
 
 py_venv_binary(
-    name = "py_venv",
+    name = "external_venv",
     srcs = [
         ":stamped"
     ],
+    # FIXME: Allow use of venv from a separate target?
     venv = ":venv",
     main = ":stamped",
     imports = [
         "."
     ],
 )
+
+py_venv_binary(
+    name = "internal_venv",
+    srcs = [
+        ":stamped"
+    ],
+    main = ":stamped",
+    deps = [
+        "@pypi_cowsay//:pkg",
+    ],
+    imports = [
+        "."
+    ],
+
+)

py/private/BUILD.bazel

@@ -9,6 +9,7 @@ exports_files(
     [
         "run.tmpl.sh",
         "venv.tmpl.sh",
+        "venv_binary.tmpl.sh",
         "pytest.py.tmpl",
     ],
     visibility = ["//visibility:public"],

py/private/py_venv.bzl

@@ -9,6 +9,10 @@ load("@bazel_skylib//lib:paths.bzl", "paths")
 load("//py/private:py_semantics.bzl", _py_semantics = "semantics")
 load("//py/private/toolchain:types.bzl", "PY_TOOLCHAIN", "VENV_TOOLCHAIN")
 
+VirtualenvInfo = provider(fields = {
+    "home": "Path of the virtualenv",
+})
+
 def _dict_to_exports(env):
     return [
         "export %s=\"%s\"" % (k, v)
@@ -19,7 +23,13 @@ def _dict_to_exports(env):
 # be a layer on top of it if we can figure out flowing the data around. This is
 # PoC quality.
 
-def _py_venv_rule_impl(ctx):
+def _py_venv_base_impl(ctx):
+
+    """
+    Common base implementation of taking a PyInfo transitive depset and shoving all that into a "virtualenv" tree.
+    Depended on by the implementation of venv building and venv-based binary building.
+    """
+
     venv_toolchain = ctx.toolchains[VENV_TOOLCHAIN]
     py_toolchain = _py_semantics.resolve_toolchain(ctx)
 
@@ -32,12 +42,14 @@ def _py_venv_rule_impl(ctx):
     pth_lines.set_param_file_format("multiline")
     pth_lines.add_all(imports_depset)
 
-    site_packages_pth_file = ctx.actions.declare_file("{}.venv.pth".format(ctx.attr.name))
+    site_packages_pth_file = ctx.actions.declare_file("{}.pth".format(ctx.attr.name))
     ctx.actions.write(
         output = site_packages_pth_file,
         content = pth_lines,
     )
 
+    env_file = ctx.actions.declare_file("{}.env".format(ctx.attr.name))
+
     default_env = {
         "BAZEL_TARGET": str(ctx.label).lstrip("@"),
         "BAZEL_WORKSPACE": ctx.workspace_name,
@@ -52,6 +64,11 @@ def _py_venv_rule_impl(ctx):
             attribute_name = "env",
         )
 
+    ctx.actions.write(
+        output = env_file,
+        content = "\n".join(_dict_to_exports(default_env)).strip(),
+    )
+
     srcs_depset = _py_library.make_srcs_depset(ctx)
 
     # Use the runfiles computation logic to figure out the files we need to
@@ -68,26 +85,29 @@ def _py_venv_rule_impl(ctx):
         ],
     )
 
-    venv_name = ".{}.venv".format(ctx.attr.name)
+    venv_name = ".{}".format(ctx.attr.name)
     venv_dir = ctx.actions.declare_directory(venv_name)
 
-    # FIXME: Copy in a Python interpreter and symlink it as `./bin/python`
-    # FIXME: Note that the UV internals we're invoking here do interpreter path canonicalization we don't want
     ctx.actions.run(
         executable = venv_toolchain.bin,
         arguments = [
             "--location=" + venv_dir.path,
             "--python=" + ctx.file._interpreter_shim.path,
             "--pth-file=" + site_packages_pth_file.path,
+            "--env-file=" + env_file.path,
             "--bin-dir=" + ctx.bin_dir.path,
             "--collision-strategy=" + ctx.attr.package_collisions,
             "--venv-name=" + venv_name,
             "--mode=static-symlink",
-            "--version={}.{}.{}".format(py_toolchain.interpreter_version_info.major, py_toolchain.interpreter_version_info.minor, py_toolchain.interpreter_version_info.micro,),  # FIXME: Micro not actually used.
+            "--version={}.{}".format(
+                py_toolchain.interpreter_version_info.major,
+                py_toolchain.interpreter_version_info.minor,
+            ),
         ],
         inputs = rfs.merge_all([
             ctx.runfiles(files=[
                 site_packages_pth_file,
+                env_file,
                 ctx.file._interpreter_shim,
             ]),
             venv_toolchain.default_info.default_runfiles,
@@ -97,48 +117,108 @@ def _py_venv_rule_impl(ctx):
         ],
     )
 
+    return venv_dir, rfs.merge_all([
+        ctx.runfiles(files=[
+            venv_dir,
+        ])
+    ])
+
+def _py_venv_rule_impl(ctx):
+    """
+    A virtualenv implementation the binary of which is a proxy to the Python interpreter of the venv.
+    """
+
+    py_toolchain = _py_semantics.resolve_toolchain(ctx)
+    venv_dir, rfs = _py_venv_base_impl(ctx)
+
     # Now we can generate an entrypoint script wrapping $VENV/bin/python
-    executable_launcher = ctx.outputs.executable
     ctx.actions.expand_template(
         template = ctx.file._run_tmpl, # FIXME: Should always be single file
-        output = executable_launcher,
+        output = ctx.outputs.executable,
         substitutions = {
             "{{BASH_RLOCATION_FN}}": BASH_RLOCATION_FUNCTION.strip(),
             "{{INTERPRETER_FLAGS}}": " ".join(py_toolchain.flags + ctx.attr.interpreter_options),
-            "{{ENTRYPOINT}}": to_rlocation_path(ctx, ctx.file.main),
-            "{{PYTHON_ENV}}": "\n".join(_dict_to_exports(default_env)).strip(),
+            "{{ENTRYPOINT}}": "${VIRTUAL_ENV}/bin/python",
             "{{ARG_VENV}}": to_rlocation_path(ctx, venv_dir),
         },
         is_executable = True,
     )
 
-    instrumented_files_info = _py_library.make_instrumented_files_info(
-        ctx,
-        extra_source_attributes = ["main"],
-    )
+    # TODO: Zip output group to allow for bypassing filtering et. all
 
     return [
         DefaultInfo(
             files = depset([
-                executable_launcher,
+                ctx.outputs.executable,
                 venv_dir,
             ]),
-            executable = executable_launcher,
+            executable = ctx.outputs.executable,
             runfiles = rfs.merge(ctx.runfiles(files=[
                 venv_dir,
             ])),
         ),
-        PyInfo(
-            imports = imports_depset,
-            transitive_sources = srcs_depset,
-            has_py2_only_sources = False,
-            has_py3_only_sources = True,
-            uses_shared_libraries = False,
+        # FIXME: Does not provide PyInfo because venvs are supposed to be terminal artifacts.
+        VirtualenvInfo(
+            home = venv_dir,
         ),
-        instrumented_files_info,
-        RunEnvironmentInfo(
-            environment = passed_env,
-            inherited_environment = getattr(ctx.attr, "env_inherit", []),
+    ]
+
+def _py_venv_binary_impl(ctx):
+    """
+    A virtualenv implementation the binary of which is a proxy to the Python interpreter of the venv.
+    """
+
+    py_toolchain = _py_semantics.resolve_toolchain(ctx)
+
+    # Make runfiles to handle direct srcs and deps which we need to bolt on top
+    # of the venv
+    srcs_depset = _py_library.make_srcs_depset(ctx)
+    virtual_resolution = _py_library.resolve_virtuals(ctx)
+
+    # Use the runfiles computation logic to figure out the files we need to
+    # _build_ the venv. The final venv is these runfiles _plus_ the venv's
+    # structures.
+    rfs = _py_library.make_merged_runfiles(
+        ctx,
+        extra_depsets = [
+            py_toolchain.files,
+            srcs_depset,
+        ] + virtual_resolution.srcs + virtual_resolution.runfiles,
+        extra_runfiles_depsets = [
+            ctx.attr._runfiles_lib[DefaultInfo].default_runfiles,
+        ],
+    )
+
+    if not ctx.attr.venv:
+        venv_dir, venv_rfs = _py_venv_base_impl(ctx)
+
+    else:
+        venv_dir = ctx.attr.venv[VirtualenvInfo].home
+        venv_rfs = ctx.attr.venv[DefaultInfo].default_runfiles
+
+    rfs = rfs.merge(venv_rfs)
+
+    # Now we can generate an entrypoint script wrapping $VENV/bin/python
+    ctx.actions.expand_template(
+        template = ctx.file._bin_tmpl, # FIXME: Should always be single file
+        output = ctx.outputs.executable,
+        substitutions = {
+            "{{BASH_RLOCATION_FN}}": BASH_RLOCATION_FUNCTION.strip(),
+            "{{INTERPRETER_FLAGS}}": " ".join(py_toolchain.flags + ctx.attr.interpreter_options),
+            "{{ENTRYPOINT}}": to_rlocation_path(ctx, ctx.file.main),
+            "{{ARG_VENV}}": to_rlocation_path(ctx, venv_dir),
+            "{{RUNFILES_INTERPRETER}}": str(py_toolchain.runfiles_interpreter).lower(),
+        },
+        is_executable = True,
+    )
+
+    return [
+        DefaultInfo(
+            files = depset([
+                ctx.outputs.executable,
+            ]),
+            executable = ctx.outputs.executable,
+            runfiles = rfs,
         ),
     ]
 
@@ -147,11 +227,6 @@ _attrs = dict({
         doc = "Environment variables to set when running the binary.",
         default = {},
     ),
-    "main": attr.label(
-        doc = "Script to execute with the Python interpreter.",
-        allow_single_file = True,
-        mandatory = True,
-    ),
     "python_version": attr.string(
         doc = """Whether to build this target and its transitive deps for a specific python version.""",
     ),
@@ -194,10 +269,24 @@ A collision can occur when multiple packages providing the same file are install
 
 _attrs.update(**_py_library.attrs)
 
-_venv_attrs = dict({
+_binary_attrs = dict({
+    "main": attr.label(
+        doc = "Script to execute with the Python interpreter.",
+        allow_single_file = True,
+        mandatory = True,
+    ),
+    "venv": attr.label(
+        doc = "A virtualenv; if provided all 3rdparty deps are assumed to come via the venv.",
+        providers = [[VirtualenvInfo]],
+    ),
+    "_bin_tmpl": attr.label(
+        allow_single_file = True,
+        default = "//py/private:venv_binary.tmpl.sh",
+    ),
 })
 
 _test_attrs = dict({
+    # FIXME: Where does this come from, do we need to keep it?
     "env_inherit": attr.string_list(
         doc = "Specifies additional environment variables to inherit from the external environment when the test is executed by bazel test.",
         default = [],
@@ -226,10 +315,10 @@ _python_version_transition = transition(
 )
 
 py_venv_base = struct(
-    implementation = _py_venv_rule_impl,
+    # implementation = _py_venv_rule_impl,
     attrs = _attrs,
+    binary_attrs = _binary_attrs,
     test_attrs = _test_attrs,
-    venv_attrs = _venv_attrs,
     toolchains = [
         PY_TOOLCHAIN,
         VENV_TOOLCHAIN,
@@ -239,8 +328,8 @@ py_venv_base = struct(
 
 py_venv = rule(
     doc = "Build a Python pseudo-virtual environment under Bazel which will execute a shell or console.",
-    implementation = py_venv_base.implementation,
-    attrs = py_venv_base.attrs | py_venv_base.venv_attrs,
+    implementation = _py_venv_rule_impl,
+    attrs = py_venv_base.attrs,
     toolchains = py_venv_base.toolchains,
     executable = True,
     cfg = py_venv_base.cfg,
@@ -249,17 +338,17 @@ py_venv = rule(
 
 py_venv_binary = rule(
     doc = "Run a Python program under Bazel using a pseudo-virtualenv. Most users should use the [py_binary macro](#py_binary) instead of loading this directly.",
-    implementation = py_venv_base.implementation,
-    attrs = py_venv_base.attrs,
+    implementation = _py_venv_binary_impl,
+    attrs = py_venv_base.attrs | py_venv_base.binary_attrs,
     toolchains = py_venv_base.toolchains,
     executable = True,
     cfg = py_venv_base.cfg,
 )
 
 py_venv_test = rule(
     doc = "Run a Python program under Bazel using a pseudo-virtualenv. Most users should use the [py_test macro](#py_test) instead of loading this directly.",
-    implementation = py_venv_base.implementation,
-    attrs = py_venv_base.attrs | py_venv_base.test_attrs,
+    implementation = _py_venv_binary_impl,
+    attrs = py_venv_base.attrs | py_venv_base.binary_attrs | py_venv_base.test_attrs,
     toolchains = py_venv_base.toolchains,
     test = True,
     cfg = py_venv_base.cfg,

py/private/venv.tmpl.sh

@@ -6,14 +6,6 @@ runfiles_export_envvars
 
 set -o errexit -o nounset -o pipefail
 
-# Psuedo-activate; would be better if we could just source "the" activate scripts
-VIRTUAL_ENV="$(rlocation "{{ARG_VENV}}")"
-export VIRTUAL_ENV
+source "$(rlocation "{{ARG_VENV}}")"/bin/activate
 
-PATH="${VIRTUAL_ENV}/bin:${PATH}"
-export PATH
-
-{{PYTHON_ENV}}
-
-# And punt to the virtualenv's "interpreter" which will be a link to the Python toolchain
-exec "${VIRTUAL_ENV}/bin/python" {{INTERPRETER_FLAGS}} "$(rlocation {{ENTRYPOINT}})" "$@"
+exec "$(rlocation "{{ARG_VENV}}")"/bin/python {{INTERPRETER_FLAGS}} "$@"

py/private/venv_binary.tmpl.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+{{BASH_RLOCATION_FN}}
+
+runfiles_export_envvars
+
+set -o errexit -o nounset -o pipefail
+
+# Psuedo-activate; would be better if we could just source "the" activate scripts
+source "$(rlocation "{{ARG_VENV}}")"/bin/activate
+
+# And punt to the virtualenv's "interpreter" which will be a link to the Python toolchain
+# FIXME: Assumes that the entrypoint isn't relocated into the site-packages tree
+exec "$(rlocation "{{ARG_VENV}}")"/bin/python {{INTERPRETER_FLAGS}} "$(rlocation {{ENTRYPOINT}})" "$@"

py/tests/py-external-venv/BUILD.bazel

@@ -0,0 +1,20 @@
+load("//py/private:py_venv.bzl", "py_venv", "py_venv_test")
+
+py_venv(
+    name = "venv",
+    deps = [
+        "@pypi_cowsay//:pkg",
+    ],
+)
+
+py_venv_test(
+    name = "py-external-venv",
+    srcs = [
+        "test.py"
+    ],
+    venv = ":venv",
+    main = "test.py",
+    imports = [
+        "."
+    ],
+)