exp(pypi): introduce 'all_whls', 'all_pkgs' and 'all_data' targets to replace requirements

Work towards bazel-contrib#2559

Author: aignas

examples/build_file_generation/BUILD.bazel

@@ -3,7 +3,6 @@
 # The `load` statement imports the symbol for the rule, in the defined
 # ruleset. When the symbol is loaded you can use the rule.
 load("@bazel_gazelle//:def.bzl", "gazelle")
-load("@pip//:requirements.bzl", "all_whl_requirements")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 load("@rules_python//python:py_binary.bzl", "py_binary")
 load("@rules_python//python:py_library.bzl", "py_library")
@@ -28,7 +27,7 @@ modules_mapping(
         "^_|(\\._)+",  # This is the default.
         "(\\.tests)+",  # Add a custom one to get rid of the psutil tests.
     ],
-    wheels = all_whl_requirements,
+    wheels = ["@pip//:all_whls"],
 )
 
 # Gazelle python extension needs a manifest file mapping from

examples/bzlmod/BUILD.bazel

@@ -6,7 +6,7 @@
 # The names @pip and @python_39 are values that are repository
 # names. Those names are defined in the MODULES.bazel file.
 load("@bazel_skylib//rules:build_test.bzl", "build_test")
-load("@pip//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement")
+load("@pip//:requirements.bzl", "requirement")
 load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test")
 load("@python_versions//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements")
 load("@rules_python//python:py_binary.bzl", "py_binary")
@@ -72,17 +72,17 @@ py_test_with_transition(
 # See: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/build_test_doc.md
 build_test(
     name = "all_wheels_build_test",
-    targets = all_whl_requirements,
+    targets = ["@pip//:all_whls"],
 )
 
 build_test(
     name = "all_data_requirements_build_test",
-    targets = all_data_requirements,
+    targets = ["@pip//:all_data"],
 )
 
 build_test(
     name = "all_requirements_build_test",
-    targets = all_requirements,
+    targets = ["@pip//:all_pkgs"],
 )
 
 # Check the annotations API

examples/bzlmod_build_file_generation/BUILD.bazel

@@ -6,7 +6,6 @@
 # The following code loads the base python requirements and gazelle
 # requirements.
 load("@bazel_gazelle//:def.bzl", "gazelle")
-load("@pip//:requirements.bzl", "all_whl_requirements")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 load("@rules_python//python:py_binary.bzl", "py_binary")
 load("@rules_python//python:py_library.bzl", "py_library")
@@ -35,7 +34,7 @@ modules_mapping(
         "^tzdata",  # Get rid of tzdata on Windows.
         "^lazy_object_proxy\\.cext$",  # Get rid of this on Linux because it isn't included on Windows.
     ],
-    wheels = all_whl_requirements,
+    wheels = ["@pip//:all_whls"],
 )
 
 modules_mapping(
@@ -49,7 +48,7 @@ modules_mapping(
     ],
     include_stub_packages = True,
     modules_mapping_name = "modules_mapping_with_types.json",
-    wheels = all_whl_requirements,
+    wheels = ["@pip//:all_whls"],
 )
 
 # Gazelle python extension needs a manifest file mapping from

gazelle/README.md

@@ -88,15 +88,14 @@ Create an empty file with this name. It might be next to your `requirements.txt`
 To keep the metadata updated, put this in your `BUILD.bazel` file next to `gazelle_python.yaml`:
 
 ```starlark
-load("@pip//:requirements.bzl", "all_whl_requirements")
 load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest")
 load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
 
 # This rule fetches the metadata for python packages we depend on. That data is
 # required for the gazelle_python_manifest rule to update our manifest file.
 modules_mapping(
     name = "modules_map",
-    wheels = all_whl_requirements,
+    wheels = ["@pip//all_whls"],
 )
 
 # Gazelle python extension needs a manifest file mapping from

gazelle/modules_mapping/def.bzl

@@ -64,7 +64,7 @@ modules_mapping = rule(
         ),
         "wheels": attr.label_list(
             allow_files = True,
-            doc = "The list of wheels, usually the 'all_whl_requirements' from @<pip_repository>//:requirements.bzl",
+            doc = """The list of wheels, usually the '["@<pip_repository>//:all_whls"]'""",
             mandatory = True,
         ),
         "_generator": attr.label(

python/private/pypi/hub_repository.bzl

@@ -19,14 +19,32 @@ load(":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases")
 load(":whl_config_setting.bzl", "whl_config_setting")
 
 _BUILD_FILE_CONTENTS = """\
+load("@rules_python//python:py_library.bzl", "py_library")
+
 package(default_visibility = ["//visibility:public"])
 
 # Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
 exports_files(["requirements.bzl"])
+
+filegroup(
+    name = "all_whls",
+    srcs = {all_whls},
+)
+
+filegroup(
+    name = "all_data",
+    srcs = {all_data},
+)
+
+py_library(
+    name = "all_pkgs",
+    deps = {all_pkgs},
+)
 """
 
 def _impl(rctx):
-    bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys()
+    all_bzl_packages = sorted(rctx.attr.whl_map.keys())
+    bzl_packages = rctx.attr.packages or all_bzl_packages
     aliases = render_multiplatform_pkg_aliases(
         aliases = {
             key: _whl_config_settings_from_json(values)
@@ -44,7 +62,20 @@ def _impl(rctx):
     # `requirement`, et al. macros.
     macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name)
 
-    rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
+    rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS.format(
+        all_pkgs = render.indent(render.list([
+            "//" + pkg
+            for pkg in all_bzl_packages
+        ])).lstrip(),
+        all_data = render.indent(render.list([
+            "//{}:data".format(pkg)
+            for pkg in all_bzl_packages
+        ])).lstrip(),
+        all_whls = render.indent(render.list([
+            "//{}:whl".format(pkg)
+            for pkg in all_bzl_packages
+        ])).lstrip(),
+    ))
     rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
         "%%ALL_DATA_REQUIREMENTS%%": render.list([
             macro_tmpl.format(p, "data")

python/private/pypi/pip_repository.bzl

@@ -64,10 +64,27 @@ def use_isolated(ctx, attr):
     return use_isolated
 
 _BUILD_FILE_CONTENTS = """\
+load("@rules_python//python:py_library.bzl", "py_library")
+
 package(default_visibility = ["//visibility:public"])
 
 # Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
 exports_files(["requirements.bzl"])
+
+filegroup(
+    name = "all_whls",
+    srcs = {all_whls},
+)
+
+filegroup(
+    name = "all_data",
+    srcs = {all_data},
+)
+
+py_library(
+    name = "all_pkgs",
+    deps = {all_pkgs},
+)
 """
 
 def _pip_repository_impl(rctx):
@@ -183,7 +200,20 @@ def _pip_repository_impl(rctx):
     for path, contents in aliases.items():
         rctx.file(path, contents)
 
-    rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
+    rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS.format(
+        all_pkgs = render.indent(render.list([
+            "//" + pkg
+            for pkg in bzl_packages
+        ])).lstrip(),
+        all_data = render.indent(render.list([
+            "//{}:data".format(pkg)
+            for pkg in bzl_packages
+        ])).lstrip(),
+        all_whls = render.indent(render.list([
+            "//{}:whl".format(pkg)
+            for pkg in bzl_packages
+        ])).lstrip(),
+    ))
     rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
         "    # %%GROUP_LIBRARY%%": """\
     group_repo = "{name}__groups"