Fixes to allow downstream project to set a different Python toolchain.

Previously, in the typical use case where `python_interpreter_target`
is passed, the @bazel_rules_hdl//:init.bzl init macro would always
call the vendored `install_deps`. However, this would only work if
the passed in toolchain name matched the one used to generate the
vendored requirements.bzl (since @python39 is hardcoded as a default
inside).

Now, we pass along any interpreter args to the vendored `install_deps`
to override the defaults as documented at:
https://github.com/bazelbuild/rules_python/blob/main/examples/pip_parse_vendored/README.md.
This also eliminates the documented requirement to use the generated
pip deps instead of the vendored version if an interpreter arg is
set.

We also update rules_python to the latest version, allowing the use
of the "_host" toolchain label to avoid loading the interpreter per
bazel-contrib/rules_python#1644.

Finally, we clean up other references to the pip_parse repo (which
was only intended for the vendoring) and move them to use the
vendored deps directly.

Author: mikex-oss

WORKSPACE

@@ -47,12 +47,20 @@ bazel_skylib_workspace()
 maybe(
     http_archive,
     name = "rules_python",
-    sha256 = "5868e73107a8e85d8f323806e60cad7283f34b32163ea6ff1020cf27abef6036",
-    strip_prefix = "rules_python-0.25.0",
-    url = "https://github.com/bazelbuild/rules_python/releases/download/0.25.0/rules_python-0.25.0.tar.gz",
+    sha256 = "e3f1cc7a04d9b09635afb3130731ed82b5f58eadc8233d4efb59944d92ffc06f",
+    strip_prefix = "rules_python-0.33.2",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.33.2/rules_python-0.33.2.tar.gz",
 )
 
-load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+load(
+    "@rules_python//python:repositories.bzl",
+    "py_repositories",
+    "python_register_toolchains",
+)
+
+# Must be called before using anything from rules_python.
+# https://github.com/bazelbuild/rules_python/issues/1560#issuecomment-1815118394
+py_repositories()
 
 python_register_toolchains(
     name = "python39",
@@ -157,7 +165,16 @@ load("//dependency_support:dependency_support.bzl", "dependency_support")
 dependency_support()
 # Third Party
 
-load("@python39//:defs.bzl", "interpreter")
 load("//:init.bzl", "init")
 
-init(python_interpreter_target = interpreter)
+init(python_interpreter_target = "@python39_host//:python")
+
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+# Used only by the rules that vendor requirements.bzl
+# Not needed by users of rules_hdl.
+pip_parse(
+    name = "rules_hdl_pip_deps_to_vendor",
+    python_interpreter_target = "@python39_host//:python",
+    requirements_lock = "//dependency_support:pip_requirements.txt",
+)

cocotb/tests/BUILD.bazel

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@rules_hdl_pip_deps//:requirements.bzl", "requirement")
 load("//cocotb:cocotb.bzl", "cocotb_test")
+load("//dependency_support:requirements.bzl", "requirement")
 load("//verilog:defs.bzl", "verilog_library")
 
 package(

dependency_support/BUILD.bazel

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# See https://github.com/bazelbuild/rules_python/blob/main/examples/pip_parse_vendored/BUILD.bazel
+# for canonical vendoring setup.
+
 load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
@@ -21,14 +24,25 @@ package(
     default_visibility = ["//visibility:private"],
 )
 
-compile_pip_requirements(name = "pip_requirements")
+compile_pip_requirements(
+    name = "pip_requirements",
+    src = "pip_requirements.in",
+)
 
 genrule(
     name = "clean_requirements",
-    srcs = ["@rules_hdl_pip_deps//:requirements.bzl"],
+    srcs = ["@rules_hdl_pip_deps_to_vendor//:requirements.bzl"],
     outs = ["requirements.clean.bzl"],
     cmd = " | ".join([
         "cat $<",
+        # Substitute for vendored dependencies directly.
+        # We want to remove any references to the pip_parse repo used to generate
+        # the dependencies,  otherwise the vendoring is not useful.
+        "sed -e 's/rules_hdl_pip_deps_to_vendor/rules_hdl_pip_deps_vendored/g'",
+        "sed -e 's|//\\([^:]*\\):pkg|_\\1//:pkg|g'",
+        "sed -e 's|//\\([^:]*\\):whl|_\\1//:whl|g'",
+        "sed -e 's|//\\([^:]*\\):data|_\\1//:data|g'",
+        "sed -e 's|//{}:{}|_{}//:{}|g' >$@",
         # Replace the bazel 6.0.0 specific comment with something that bazel 5.4.0 would produce.
         # This enables this example to be run as a test under bazel 5.4.0.
         """sed -e 's#@rules_hdl//#@//#'""",

dependency_support/com_google_skywater_pdk/bundled.BUILD.bazel

@@ -14,9 +14,9 @@
 
 """ BUILD file for Skywater PDK """
 
-load("@rules_python//python:defs.bzl", "py_binary", "py_library")
+load("@rules_hdl//dependency_support:requirements.bzl", "requirement")
 load("@rules_hdl//dependency_support/com_google_skywater_pdk:cell_libraries.bzl", "CELL_LIBRARIES")
-load("@rules_hdl_pip_deps//:requirements.bzl", "requirement")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 
 licenses(["notice"])
 

dependency_support/requirements.bzl

@@ -14,8 +14,8 @@
 
 # GDS write tool package.
 
-load("@rules_hdl_pip_deps//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary")
+load("//dependency_support:requirements.bzl", "requirement")
 
 package(
     default_applicable_licenses = ["//:package_license"],

gds_write/BUILD

@@ -16,9 +16,11 @@
 
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
-load("@rules_python//python:pip.bzl", "pip_parse")
-load("@rules_python//python:repositories.bzl", "py_repositories")
+load(
+    "@rules_proto//proto:repositories.bzl",
+    "rules_proto_dependencies",
+    "rules_proto_toolchains",
+)
 load("//dependency_support:requirements.bzl", install_pip_deps = "install_deps")
 load("//dependency_support/boost:init_boost.bzl", "init_boost")
 load("//dependency_support/pybind11:init_pybind11.bzl", "init_pybind11")
@@ -30,41 +32,31 @@ def init(python_interpreter = None, python_interpreter_target = None):
     must call `init` to allow @bazel_rules_hdl to set itself up.
 
     `python_interpreter` and `python_interpreter_target` are passed to
-    @bazel_rules_hdl's instance of `pip_parse`. They can normally be set to
-    the default None value, but if the outside workspace has a custom Python
-    toolchain configured, these must be set, otherwise @bazel_rules_hdl will
-    not use the right Python toolchain when installing pip dependencies.
-
-    If either is set, the outside workspace must also include:
-
-        load(
-            "@rules_hdl_pip_deps//:requirements.bzl",
-            rules_hdl_install_pip_deps = "install_deps",
-        )
-        rules_hdl_install_pip_deps()
+    @bazel_rules_hdl's vendored requirements.bzl `install_deps`. If unspecified,
+    this project's default toolchain will be used. If the outside workspace has
+    a custom Python toolchain configured, these should be set, otherwise
+    @bazel_rules_hdl will not use the right Python toolchain when installing
+    pip dependencies.
 
     Args:
       python_interpreter: Path to external Python interpreter to use with
-      `pip_parse`. This can be an absolute path or relative to the host's
-      `PATH` environment variable.
+        `install_deps` for PyPI dependencies. This can be an absolute path or
+        relative to the host's `PATH` environment variable.
       python_interpreter_target: Bazel target of a Python interpreter to build
-      to use with `pip_parse`. Using `python_interpreter_target` makes it
-      possible to have a hermetic Python toolchain. `python_interpreter_target`
-      takes precedence over `python_interpreter` if both are set.
+        to use with `install_deps` for PyPI dependencies. Using
+        `python_interpreter_target` makes it possible to have a hermetic
+        Python toolchain. `python_interpreter_target` takes precedence over
+        `python_interpreter` if both are set.
     """
     rules_proto_dependencies()
     rules_proto_toolchains()
-    py_repositories()
 
-    # Used only by the rules that vendor requirements.bzl
-    pip_parse(
-        name = "rules_hdl_pip_deps",
-        requirements_lock = "@rules_hdl//dependency_support:pip_requirements.txt",
-        python_interpreter = python_interpreter,
-        python_interpreter_target = python_interpreter_target,
-    )
-    if (not python_interpreter) or (python_interpreter_target != "@python39//:install/bin/python3"):
-        install_pip_deps()
+    install_deps_kwargs = {}
+    if python_interpreter:
+        install_deps_kwargs["python_interpreter"] = python_interpreter
+    if python_interpreter_target:
+        install_deps_kwargs["python_interpreter_target"] = python_interpreter_target
+    install_pip_deps(**install_deps_kwargs)
 
     init_boost()
     init_pybind11(python_interpreter_target = python_interpreter_target)

init.bzl

@@ -14,8 +14,8 @@
 
 """ASAP7 PDK Package"""
 
-load("@rules_hdl_pip_deps//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+load("//dependency_support:requirements.bzl", "requirement")
 
 package(
     default_applicable_licenses = ["//:package_license"],

pdk/liberty/BUILD

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@rules_hdl_pip_deps//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary")
+load("//dependency_support:requirements.bzl", "requirement")
 
 package(
     default_applicable_licenses = ["//:package_license"],