And now for something completely different...

Experimenting with a different approach

Author: chadnorvell

refresh.template.py

@@ -1418,10 +1418,11 @@ def main():
     # the compile commands for the targets defined for those collections.
     compile_command_sets = collections.defaultdict(list)
 
-    for (target, flags) in target_flag_pairs:
+    for target_key, target_data in target_flag_pairs:
+        target, flags = target_data
         commands = list(_get_commands(target, flags))
         # If the target is assigned to any collections, put the compile commands in the compile command sets for those collections.
-        collections_for_target = [collection_name for target_name, collection_name in target_collections.items() if target == target_name]
+        collections_for_target = [collection_name for collection_name, target_keys in target_collections.items() if target_key in target_keys]
         for collection_name in collections_for_target:
             compile_command_sets[collection_name].extend(commands)
         # Also put them into the main file.
@@ -1437,24 +1438,21 @@ def main():
         root_dir.mkdir(parents=True)
 
     # Chain output into compile_commands.json
-    for target_name in compile_command_sets:
+    for collection in compile_command_sets:
         # If the target doesn't have a specified file name, put it into the "catch all"
         # compilation database.
-        if target_name == '__all__':
+        if collection == '__all__':
             file_path = root_dir / "compile_commands.json"
         # Otherwise, write the database to the specific target file.
         else:
-            target_dir = root_dir / target_name
+            target_dir = root_dir / collection
             target_dir.mkdir(exist_ok=True)
             file_path = target_dir / "compile_commands.json"
 
-        # This condition is only relevant to __all__. If each specified target has a specified
-        # file name, there won't be any compile commands in __all__, and we shouldn't write
-        # a file with an empty array.
-        if (len(compile_command_sets[target_name]) > 0):
+        if (len(compile_command_sets[collection]) > 0):
             with open(file_path, 'w') as output_file:
                 json.dump(
-                    compile_command_sets[target_name],
+                    compile_command_sets[collection],
                     output_file,
                     indent=2, # Yay, human readability!
                     check_circular=False # For speed.

refresh_compile_commands.bzl

@@ -37,12 +37,27 @@ refresh_compile_commands(
         # However, if you use clangd and are looking for speed, we strongly recommend you follow the instructions below instead, since clangd is going to regularly infer the wrong commands for headers and give you lots of annoyingly unnecessary red squigglies.
 
     # Need to create separate files for specific targets? Give those targets a name and their compile commands file will be written into a subdirectory with that name.
-        # target_file_names = {
-        #     "//:host_build": "host",
-        #     "//:target_build": "target",
+        # target_collections = {
+        #     "host": "//:host_build",
+        #     "target": "//:target_build",
         # }
 
-    # Need to write compile commands to some directory other than the workspace root? Provide a path relative to the workspace root.
+    # You can define target collections, sort of like "meta-targets" that contain combination of compile commands from the specified targets.
+    # This is useful for multi-architecture projects, where you might be building the same files in multiple different ways depending on what architecture or device the code will run on.
+    # It only makes sense to get code intelligence for a consistent build, so you can produce consistent compile commands with target collections.
+    # Targets only need to be specified in either `targets` or `target_collections`; there's no need to add them to both.
+        # target_collections = {
+        #     "host": [
+        #         "//:host_build",
+        #         "//:host_tests",
+        #     ],
+        #     "target": [
+        #         ["//:target_build", "--platform=//:target_device --important_flag"],
+        #     ],
+        # }
+
+
+    # Need to write compile commands to some directory other than the workspace root? Provide a path relative to the workspace root. This is useful if you use target collections.
         # out_dir = ".compile_commands"
 
     # Need things to run faster? [Either for compile_commands.json generation or clangd indexing.]
@@ -76,62 +91,122 @@ def refresh_compile_commands(
         exclude_headers = None,
         exclude_external_sources = False,
         **kwargs):  # For the other common attributes. Tags, compatible_with, etc. https://docs.bazel.build/versions/main/be/common-definitions.html#common-attributes.
-
-    if not target_collections:
-        target_collections = {}
 
-    target_collections = {
-        _make_label_absolute(target): collection_name for target, collection_name in target_collections.items()
-    }
-
-    # We want a list of targets specified in `target_collections`, and we want a list of all of the target collections.
-    # There's a many-to-many relationship between targets and target collections, there can be duplicates in both of those lists.
-    # We want to de-duplicate those lists, but Starlark doesn't have a set class like Python does, so we do it manually.
-    target_collection_names = []
-    target_collection_targets = []
-
-    for target, collection_name in target_collections.items():
-        if not target in target_collection_targets:
-            target_collection_targets.append(target)
-
-        if not collection_name in target_collection_names:
-            target_collection_names.append(collection_name)
+    # Given `targets` that may be absent, or be a single string target:
+    #    targets = "//:some_target"
+    #
+    # ... or be a list of string targets:
+    #    targets = [
+    #        "//:some_target",
+    #        "//:another_target"
+    #    ]
+    #
+    # ... or be a dict of string targets and build flags:
+    #    targets = {
+    #        "//:some_target": "--flag ...",
+    #        "//:another_target": "--arg ..."
+    #    }
+    # 
+    # And given `target_collections` that may be absent, or have a collection name associated with a list of string targets:
+    #    target_collections = {
+    #        "host": [
+    #           "//:host_target",
+    #           ...
+    #        ],
+    #        "device": [
+    #           "//:device_target",
+    #           ...
+    #        ]
+    #    }
+    #
+    # ... or have collection names associated with lists of string targets with build flags:
+    #    target_collections = {
+    #        "host": [
+    #           ("//:host_target", "--flag ..."),
+    #           ("//:test_target", "--platform=//:host_platform"),
+    #           ...
+    #        ],
+    #        "device": [
+    #           ("//:device_target", "--arg ..."),
+    #           ("//:test_target", "--platform=//:device_platform"),
+    #           ...
+    #        ]
+    #    }
+    #
+    # ... we want to produce a `string_dict_list` that we can pass into the Python script template to assemble the `target_flag_pairs`.
+    # A simple dict with target name keys isn't adequate, because we can specify a target more than once with different build flags (see the last example above).
+    # So we assemble a dict where the keys are unique to each target+flag combination, and the values are a list of strings in this format: [<target>, <flags>]
+    #
+    # That takes care of the `target_flag_pairs`, which determines which targets to generate compile commands for.
+    #
+    # Associating compile commands with target collections is easier; we pass `target_collections`, but with the target names and flags concatenated in the same way
+    # as described above. This lets us associate each target+flag combination we generate compile commands for with its membership(s) in target collections.
+
+    target_collection_targets_list = []
+    serialized_target_collections = {}
 
+    if target_collections:
+        # Convert the targets specified in `target_collections` into the format we'll use with `targets`, so we can combine them into one list of targets to generate compile commands for.
+        for targets_for_collection in target_collections.values():
+            # You can specify a bare string target if the collection only has one target.
+            if type(targets_for_collection) != "list":
+                targets_for_collection = [targets_for_collection]
+
+            for target in targets_for_collection:
+                # The target can either be a plain string target name, or a tuple of a target name and build flags, similar to the format of `targets`.
+                if type(target) == "list":
+                    target_name, flags = target
+                else:
+                    target_name = target
+                    flags = ""
 
+                target_data = (_make_label_absolute(target_name), flags)
+                
+                # Targets may appear in multiple collections. We don't want duplicates in the final list, but Starlark doesn't have Python's set class. So we de-duplicate manually.
+                if target_data not in target_collection_targets_list:
+                    target_collection_targets_list.append(target_data)
+
+        # Assemble the association between target collections and their targets.
+        for collection, targets_for_collection in target_collections.items():
+            serialized_targets = []
+
+            for target in targets_for_collection:
+                if type(target) == "list":
+                    # If the target has flags, concat them with the target name. That's how we'll associate compile commands with collections on the Python side.
+                    serialized_targets.append("".join(target))
+                else:
+                    serialized_targets.append(target)
+            
+            serialized_target_collections[collection] = serialized_targets
+    
+    target_collection_targets = {"{}{}".format(target, flags): [target, flags] for target, flags in target_collection_targets_list}
 
     # Convert the various, acceptable target shorthands into the dictionary format
     # In Python, `type(x) == y` is an antipattern, but [Starlark doesn't support inheritance](https://bazel.build/rules/language), so `isinstance` doesn't exist, and this is the correct way to switch on type.
-    if not targets and len(target_collection_targets) == 0:  # Default to all targets in main workspace
-        targets = {"@//...": ""}
-    if not targets:  # In this case, targets were defined only in `target_collections`
-        targets = {target: "" for target in target_collection_targets}
+    if not targets and not target_collections:  # Default to all targets in main workspace
+        targets = {"@//...": ["@//...", ""]}
+    elif not targets:  # In this case, targets were defined only in `target_collections`
+        targets = target_collection_targets
     elif type(targets) == "select":  # Allow select: https://bazel.build/reference/be/functions#select
         # Pass select() to _expand_template to make it work
         # see https://bazel.build/docs/configurable-attributes#faq-select-macro
         pass
     elif type(targets) == "list":  # Allow specifying a list of targets w/o arguments
-        # The reason we want the list of target collection targets is that if you specify a target in `target_collections`, you don't have to redundantly specify it in `targets`, *unless* you want to specify build flags too.
-        # So we want to cull the list of target collection targets down to only those that *aren't* specified in `targets`.
-        unique_target_collection_targets = [target for target in target_collection_targets if target not in targets]
-        targets = {target: "" for target in (targets + unique_target_collection_targets)}
+        absolute_targets = [_make_label_absolute(target) for target in targets]
+        targets = {target: [target, ""] for target in absolute_targets} | target_collection_targets
     elif type(targets) != "dict":  # Assume they've supplied a single string/label and wrap it
-        unique_target_collection_targets = [target for target in target_collection_targets if target not in targets]
-        targets = {target: "" for target in ([targets] + unique_target_collection_targets)}
+        targets = {targets: [targets, ""]} | target_collection_targets
     else:  # Assume that they've provided a dict of targets with flags
-        unique_target_collection_targets = [target for target in target_collection_targets if target not in targets]
-        targets = targets | {target: "" for target in target_collection_targets}
-
-    targets = {
-        _make_label_absolute(target): flags for target, flags in targets.items()
-    }
+        absolute_targets = {_make_label_absolute(target): flags for target, flags in targets.items()}
+        targets = {"{}{}".format(target, flags): [target, flags] for target, flags in absolute_targets.items()} | target_collection_targets
 
     # Create a wrapper script that prints a helpful error message if the python version is too old, generated from check_python_version.template.py
     version_checker_script_name = name + ".check_python_version.py"
     _check_python_version(name = version_checker_script_name, to_run = name)
 
     # Generate the core, runnable python script from refresh.template.py
     script_name = name + ".py"
-    _expand_template(name = script_name, labels_to_flags = targets, labels_to_collections = target_collections, out_dir = out_dir, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
+    _expand_template(name = script_name, labels_to_flags = targets, labels_to_collections = serialized_target_collections, out_dir = out_dir, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
 
     # Combine them so the wrapper calls the main script
     native.py_binary(
@@ -147,6 +222,9 @@ def _make_label_absolute(label):
     # Make any package-relative labels absolute
     return label if label.startswith("/") or label.startswith("@") else "{}//{}:{}".format(native.repository_name(), native.package_name(), label.removeprefix(":"))
 
+def _expand_target_collection_targets(targets):
+    return "[\n" + "\n".join(["            '{}',".format(target) for target in targets]) + "\n        ]"
+
 def _expand_template_impl(ctx):
     """Inject targets of interest--and other settings--into refresh.template.py, and set it up to be run."""
     script = ctx.actions.declare_file(ctx.attr.name)
@@ -157,7 +235,7 @@ def _expand_template_impl(ctx):
         substitutions = {
             # Note, don't delete whitespace. Correctly doing multiline indenting.
             "        {target_flag_pairs}": "\n".join(["        {},".format(pair) for pair in ctx.attr.labels_to_flags.items()]),
-            "        {target_collections}": "\n".join(["        '{}': '{}',".format(target, collection_name) for (target, collection_name) in ctx.attr.labels_to_collections.items()]),
+            "        {target_collections}": "\n".join(["        '{}': {},".format(collection_name, _expand_target_collection_targets(targets)) for collection_name, targets in ctx.attr.labels_to_collections.items()]),
             "        {windows_default_include_paths}": "\n".join(["        %r," % path for path in find_cpp_toolchain(ctx).built_in_include_directories]),  # find_cpp_toolchain is from https://docs.bazel.build/versions/main/integrating-with-rules-cc.html
             "{out_dir}": repr(ctx.attr.out_dir),
             "{exclude_headers}": repr(ctx.attr.exclude_headers),
@@ -169,8 +247,8 @@ def _expand_template_impl(ctx):
 
 _expand_template = rule(
     attrs = {
-        "labels_to_flags": attr.string_dict(mandatory = True),  # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
-        "labels_to_collections": attr.string_dict(),
+        "labels_to_flags": attr.string_list_dict(mandatory = True),  # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
+        "labels_to_collections": attr.string_list_dict(),
         "out_dir": attr.string(default = "."),
         "exclude_external_sources": attr.bool(default = False),
         "exclude_headers": attr.string(values = ["all", "external", ""]),  # "" needed only for compatibility with Bazel < 3.6.0