Skip to content

Commit d39d35c

Browse files
mblandmbland
committed
Begin toolchainized protoc implementation
I solved the rest of the build failures from bazel-contrib#1618. Now multiple jobs are failing due to hanging scalac worker jobs as described in the PR description. - Proto Toolchainisation Design Doc https://docs.google.com/document/d/1CE6wJHNfKbUPBr7-mmk_0Yo3a4TaqcTPE0OWNuQkhPs/edit - bazelbuild/bazel: Protobuf repo recompilation sensitivity bazelbuild/bazel#7095 - bazelbuild/rules_proto: Implement proto toolchainisation bazelbuild/rules_proto#179 - rules_proto 6.0.0 release notes mentioning Protobuf Toolchainisation https://github.com/bazelbuild/rules_proto/releases/tag/6.0.0 It occurred to me to try adopting Proto Toolchainisation to see if that might resolve the issue. I've got it successfully generating the `@io_bazel_rules_scala_protoc` repo and registering toolchains for `@rules_proto//proto:toolchain_type`. However, it's currently dying because there's no toolchain registered for '@rules_java//java/proto:toolchain_type'. ```txt bazel build //src/... ERROR: .../src/protobuf/io/bazel/rules_scala/BUILD:4:14: in @@_builtins//:common/java/proto/java_proto_library.bzl%bazel_java_proto_aspect aspect on proto_library rule //src/protobuf/io/bazel/rules_scala:diagnostics_proto: Traceback (most recent call last): File "/virtual_builtins_bzl/common/java/proto/java_proto_library.bzl", line 53, column 53, in _bazel_java_proto_aspect_impl File "/virtual_builtins_bzl/common/proto/proto_common.bzl", line 364, column 17, in _find_toolchain Error in fail: No toolchains registered for '@rules_java//java/proto:toolchain_type'. ERROR: Analysis of target '//src/protobuf/io/bazel/rules_scala:diagnostics_proto' failed ERROR: Analysis of target '//src/java/io/bazel/rulesscala/scalac:scalac' failed; build aborted: Analysis failed ```
1 parent 8d85252 commit d39d35c

File tree

4 files changed

+179
-0
lines changed

4 files changed

+179
-0
lines changed

‎.bazelrc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,8 @@ build:windows --worker_quit_after_build --enable_runfiles
77
# Remove these upon completing Bzlmod compatibility work.
88
# - https://github.com/bazelbuild/rules_scala/issues/1482
99
build --noenable_bzlmod --enable_workspace
10+
11+
# Remove once proto toolchainization becomes the default
12+
# - https://bazel.build/reference/command-line-reference#flag--incompatible_enable_proto_toolchain_resolution
13+
# - https://docs.google.com/document/d/1CE6wJHNfKbUPBr7-mmk_0Yo3a4TaqcTPE0OWNuQkhPs/edit
14+
build --incompatible_enable_proto_toolchain_resolution

‎WORKSPACE‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,15 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
4646

4747
rules_proto_toolchains()
4848

49+
load("//scala:protoc_deps.bzl", "scala_protoc_setup")
50+
51+
scala_protoc_setup()
52+
53+
load("//scala:protoc_toolchain.bzl", "protoc_toolchains")
54+
55+
protoc_toolchains(name = "io_bazel_rules_scala_protoc")
56+
register_toolchains("@io_bazel_rules_scala_protoc//:all")
57+
4958
load("//scala:scala_cross_version.bzl", "default_maven_server_urls")
5059
load("//twitter_scrooge:twitter_scrooge.bzl", "twitter_scrooge")
5160

‎scala/protoc_deps.bzl‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
"""com_google_protobuf dependencies for the protoc_toolchain"""
2+
3+
load (
4+
":protoc_toolchain.bzl",
5+
"PROTOBUF_SHA256",
6+
"PROTOBUF_URL",
7+
"PROTOBUF_VERSION",
8+
)
9+
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
10+
11+
def scala_protoc_setup():
12+
http_archive(
13+
name = "com_google_protobuf",
14+
sha256 = PROTOBUF_SHA256,
15+
strip_prefix = "protobuf-%s" % PROTOBUF_VERSION,
16+
url = PROTOBUF_URL,
17+
)

‎scala/protoc_toolchain.bzl‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
"""Prepares the precompiled protoc toolchain"""
2+
3+
PROTOBUF_VERSION = "28.2"
4+
PROTOBUF_SHA256 = (
5+
"b2340aa47faf7ef10a0328190319d3f3bee1b24f426d4ce8f4253b6f27ce16db"
6+
)
7+
PROTOBUF_URL = (
8+
"https://github.com/protocolbuffers/protobuf/releases/download/" +
9+
"v{version}/protobuf-{version}.tar.gz".format(version=PROTOBUF_VERSION)
10+
)
11+
12+
_PROTOC_RELEASES_URL = "https://github.com/protocolbuffers/protobuf/releases"
13+
_PROTOC_DOWNLOAD_URL = (
14+
_PROTOC_RELEASES_URL + "/download/" +
15+
"v{version}/protoc-{version}-".format(version=PROTOBUF_VERSION) +
16+
"{platform}.zip"
17+
)
18+
19+
_PROTOC_BUILDS = {
20+
"linux-aarch_64": struct(
21+
integrity = "sha256-kdglPNwPDw/FHCtpyAZ3mWYy9SWthFBL+ltO44rT5Jw=",
22+
exec_compat = [
23+
"@platforms//os:linux",
24+
"@platforms//cpu:aarch64",
25+
],
26+
),
27+
"linux-ppcle_64": struct(
28+
integrity = "sha256-xcFrR2f/iGYJDuEOhwkDABBSAbXbHs7zgO3XPCM03Ng=",
29+
exec_compat = [
30+
"@platforms//os:linux",
31+
"@platforms//cpu:ppc64le",
32+
],
33+
),
34+
"linux-s390_64": struct(
35+
integrity = "sha256-ESIsQ4+G6Hsv2vaKKmzB2ytiKBP9btiRUJ4vOw4F7hs=",
36+
exec_compat = [
37+
"@platforms//os:linux",
38+
"@platforms//cpu:s390x",
39+
],
40+
),
41+
"linux-x86_32": struct(
42+
integrity = "sha256-ucjToo5Lq5WcwQ5smjjxfFlGc3Npv+AT9RqG19Ns//E=",
43+
exec_compat = [
44+
"@platforms//os:linux",
45+
"@platforms//cpu:x86_32"
46+
],
47+
),
48+
"linux-x86_64": struct(
49+
integrity = "sha256-L+v9QrWc6Too63iQGaRwo90ESWGbwE+E2tEzPaJh3sE=",
50+
exec_compat = [
51+
"@platforms//os:linux",
52+
"@platforms//cpu:x86_64"
53+
],
54+
),
55+
"osx-aarch_64": struct(
56+
integrity = "sha256-e7BI9ShBeJ2exhmDvgzkyeT7O9mhQ0YoILqaO+CgN5c=",
57+
exec_compat = [
58+
"@platforms//os:osx",
59+
"@platforms//cpu:aarch64",
60+
],
61+
),
62+
"osx-x86_64": struct(
63+
integrity = "sha256-Iy8H0Sv0gGIHp57CxzeDAcUuby9+/dIcDdQW8L2hA+w=",
64+
exec_compat = [
65+
"@platforms//os:osx",
66+
"@platforms//cpu:x86_64"
67+
],
68+
),
69+
"win32": struct(
70+
integrity = "sha256-V6hpbqvtUgl19PpGkUGwuC+iEIYj0YtyOqi+yae0u1g=",
71+
exec_compat = [
72+
"@platforms//os:windows",
73+
"@platforms//cpu:x86_32"
74+
],
75+
),
76+
"win64": struct(
77+
integrity = "sha256-S94ZJx7XyrkANXDyjG5MTXGWPq8SEahr87sl2biVF3o=",
78+
exec_compat = [
79+
"@platforms//os:windows",
80+
"@platforms//cpu:x86_64"
81+
],
82+
),
83+
}
84+
85+
_PROTOC_BUILD_TEMPLATE="""load(
86+
"@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl",
87+
"proto_toolchain",
88+
)
89+
90+
proto_toolchains = {{
91+
{proto_toolchains}
92+
}}
93+
94+
[
95+
proto_toolchain(
96+
name = build,
97+
proto_compiler = ":%s/bin/protoc%s" % (
98+
build, ".exe" if build.startswith("win") else ""
99+
),
100+
exec_compatible_with = specs,
101+
)
102+
for build, specs in proto_toolchains.items()
103+
]
104+
"""
105+
106+
_PROTOC_TOOLCHAIN_TEMPLATE= """ {build}: [\n{specs}\n ],"""
107+
108+
def _protoc_toolchains_impl(repository_ctx):
109+
proto_toolchains = []
110+
111+
for build in repository_ctx.attr.protoc_builds:
112+
protoc_build = _PROTOC_BUILDS[build]
113+
repository_ctx.download_and_extract(
114+
url = _PROTOC_DOWNLOAD_URL.format(platform = build),
115+
output = build,
116+
integrity = protoc_build.integrity,
117+
)
118+
proto_toolchains.append(_PROTOC_TOOLCHAIN_TEMPLATE.format(
119+
build = '"%s"' % build,
120+
specs = "\n".join([
121+
' "%s",' % s for s in protoc_build.exec_compat
122+
])
123+
))
124+
125+
build_content = _PROTOC_BUILD_TEMPLATE.format(
126+
name = repository_ctx.attr.name,
127+
proto_toolchains = "\n".join(proto_toolchains)
128+
)
129+
repository_ctx.file("BUILD", content=build_content, executable=False)
130+
131+
protoc_toolchains = repository_rule(
132+
implementation = _protoc_toolchains_impl,
133+
attrs = {
134+
"protoc_builds": attr.string_list(
135+
doc = (
136+
"os and arch identifiers for precompiled protoc release " +
137+
"download filenames from " + _PROTOC_RELEASES_URL
138+
),
139+
default = [
140+
"linux-aarch_64",
141+
"linux-x86_64",
142+
"osx-aarch_64",
143+
"osx-x86_64",
144+
"win64",
145+
],
146+
),
147+
}
148+
)

0 commit comments

Comments
 (0)