Skip to content

Commit 1a7cde6

Browse files
chadnorvellchadnorvell
committed
Support target collections
This change adds two main features: 1. You can configure compile commands for certain targets or groups of targets to go into separate files For embedded systems development in particular, `clangd` doesn't work well with `compile_commands.json` generated for multiple targets. For example, if you have a build target that runs on your host machine in one configuration and another that runs on device in another configuration, `compile_commands.json` will contain multiple conflicting compile commands for the same source files. `clangd` will just use the first one it finds, which may not be the one you want to use for code intelligence. It's convenient to have separate compile commands files for each target, and switch the file `clangd` uses depending on how you want to navigate your code. By providing the `target_collections` argument, you can associate targets with named collections. Separate compile commands files will be generated for each of the specified collections, and will be placed in subdirectories with the specified names. This is most useful when you associate multiple targets with a collection, for example, to configure code intelligence to use the compile commands for all of the targets that build for one architecture or device. This means that you can now specify a target more than once, generating compile commands for builds of the same target but with different flags (e.g. `--platform`). Before, you implicitly could only specify each target once since the targets were dict keys. 2. You can specify a different output path If you are generating multiple compile commands files, its preferable not to output them into the workspace root. So you can specify a separate output path, relative to the workspace root. This patch doesn't change any existing behavior; if you don't add either of the new arguments to your invocation of `refresh_compile_commands`, everything will work exactly as it did before.
1 parent a14ad3a commit 1a7cde6

File tree

2 files changed

+186
-23
lines changed

2 files changed

+186
-23
lines changed

‎refresh.template.py

+47-12
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
# Similarly, when upgrading, please search for that MIN_PY= tag.
1919

2020

21+
import collections
2122
import concurrent.futures
2223
import enum
2324
import functools # MIN_PY=3.9: Replace `functools.lru_cache(maxsize=None)` with `functools.cache`.
@@ -1405,20 +1406,54 @@ def main():
14051406
# End: template filled by Bazel
14061407
]
14071408

1408-
compile_command_entries = []
1409-
for (target, flags) in target_flag_pairs:
1410-
compile_command_entries.extend(_get_commands(target, flags))
1411-
1412-
if not compile_command_entries:
1409+
target_collections = {
1410+
# Begin: template filled by Bazel
1411+
{target_collections}
1412+
# End: template filled by Bazel
1413+
}
1414+
1415+
# Associates lists of compile commands with compile command "collections".
1416+
# __all__ is a special case: It contains all generated compile commands.
1417+
# Any other collections defined in `target_collections` will contain only
1418+
# the compile commands for the targets defined for those collections.
1419+
compile_command_sets = collections.defaultdict(list)
1420+
1421+
for target_key, target_data in target_flag_pairs:
1422+
target, flags = target_data
1423+
commands = list(_get_commands(target, flags))
1424+
# If the target is assigned to any collections, put the compile commands in the compile command sets for those collections.
1425+
collections_for_target = [collection_name for collection_name, target_keys in target_collections.items() if target_key in target_keys]
1426+
for collection_name in collections_for_target:
1427+
compile_command_sets[collection_name].extend(commands)
1428+
# Also put them into the main file.
1429+
compile_command_sets['__all__'].extend(commands)
1430+
1431+
if len(compile_command_sets) <= 1 and len(compile_command_sets['__all__']) == 0:
14131432
log_error(""">>> Not (over)writing compile_commands.json, since no commands were extracted and an empty file is of no use.
14141433
There should be actionable warnings, above, that led to this.""")
14151434
sys.exit(1)
14161435

1436+
1437+
if not (root_dir := pathlib.Path({out_dir})).exists():
1438+
root_dir.mkdir(parents=True)
1439+
14171440
# Chain output into compile_commands.json
1418-
with open('compile_commands.json', 'w') as output_file:
1419-
json.dump(
1420-
compile_command_entries,
1421-
output_file,
1422-
indent=2, # Yay, human readability!
1423-
check_circular=False # For speed.
1424-
)
1441+
for collection in compile_command_sets:
1442+
# If the target doesn't have a specified file name, put it into the "catch all"
1443+
# compilation database.
1444+
if collection == '__all__':
1445+
file_path = root_dir / "compile_commands.json"
1446+
# Otherwise, write the database to the specific target file.
1447+
else:
1448+
target_dir = root_dir / collection
1449+
target_dir.mkdir(exist_ok=True)
1450+
file_path = target_dir / "compile_commands.json"
1451+
1452+
if (len(compile_command_sets[collection]) > 0):
1453+
with open(file_path, 'w') as output_file:
1454+
json.dump(
1455+
compile_command_sets[collection],
1456+
output_file,
1457+
indent=2, # Yay, human readability!
1458+
check_circular=False # For speed.
1459+
)

‎refresh_compile_commands.bzl

+139-11
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,30 @@ refresh_compile_commands(
3636
# ^ excluding headers will speed up compile_commands.json generation *considerably* because we won't need to preprocess your code to figure out which headers you use.
3737
# However, if you use clangd and are looking for speed, we strongly recommend you follow the instructions below instead, since clangd is going to regularly infer the wrong commands for headers and give you lots of annoyingly unnecessary red squigglies.
3838
39+
# Need to create separate files for specific targets? Give those targets a name and their compile commands file will be written into a subdirectory with that name.
40+
# target_collections = {
41+
# "host": "//:host_build",
42+
# "target": "//:target_build",
43+
# }
44+
45+
# You can define target collections, sort of like "meta-targets" that contain combination of compile commands from the specified targets.
46+
# This is useful for multi-architecture projects, where you might be building the same files in multiple different ways depending on what architecture or device the code will run on.
47+
# It only makes sense to get code intelligence for a consistent build, so you can produce consistent compile commands with target collections.
48+
# Targets only need to be specified in either `targets` or `target_collections`; there's no need to add them to both.
49+
# target_collections = {
50+
# "host": [
51+
# "//:host_build",
52+
# "//:host_tests",
53+
# ],
54+
# "target": [
55+
# ["//:target_build", "--platform=//:target_device --important_flag"],
56+
# ],
57+
# }
58+
59+
60+
# Need to write compile commands to some directory other than the workspace root? Provide a path relative to the workspace root. This is useful if you use target collections.
61+
# out_dir = ".compile_commands"
62+
3963
# Need things to run faster? [Either for compile_commands.json generation or clangd indexing.]
4064
# First: You might be able to refresh compile_commands.json slightly less often, making the current runtime okay.
4165
# If you're adding files, clangd should make pretty decent guesses at completions, using commands from nearby files. And if you're deleting files, there's not a problem. So you may not need to rerun refresh.py on every change to BUILD files. Instead, maybe refresh becomes something you run every so often when you can spare the time, making the current runtime okay.
@@ -62,34 +86,127 @@ load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
6286
def refresh_compile_commands(
6387
name,
6488
targets = None,
89+
target_collections = None,
90+
out_dir = None,
6591
exclude_headers = None,
6692
exclude_external_sources = False,
6793
**kwargs): # For the other common attributes. Tags, compatible_with, etc. https://docs.bazel.build/versions/main/be/common-definitions.html#common-attributes.
94+
95+
# Given `targets` that may be absent, or be a single string target:
96+
# targets = "//:some_target"
97+
#
98+
# ... or be a list of string targets:
99+
# targets = [
100+
# "//:some_target",
101+
# "//:another_target"
102+
# ]
103+
#
104+
# ... or be a dict of string targets and build flags:
105+
# targets = {
106+
# "//:some_target": "--flag ...",
107+
# "//:another_target": "--arg ..."
108+
# }
109+
#
110+
# And given `target_collections` that may be absent, or have a collection name associated with a list of string targets:
111+
# target_collections = {
112+
# "host": [
113+
# "//:host_target",
114+
# ...
115+
# ],
116+
# "device": [
117+
# "//:device_target",
118+
# ...
119+
# ]
120+
# }
121+
#
122+
# ... or have collection names associated with lists of string targets with build flags:
123+
# target_collections = {
124+
# "host": [
125+
# ("//:host_target", "--flag ..."),
126+
# ("//:test_target", "--platform=//:host_platform"),
127+
# ...
128+
# ],
129+
# "device": [
130+
# ("//:device_target", "--arg ..."),
131+
# ("//:test_target", "--platform=//:device_platform"),
132+
# ...
133+
# ]
134+
# }
135+
#
136+
# ... we want to produce a `string_dict_list` that we can pass into the Python script template to assemble the `target_flag_pairs`.
137+
# A simple dict with target name keys isn't adequate, because we can specify a target more than once with different build flags (see the last example above).
138+
# So we assemble a dict where the keys are unique to each target+flag combination, and the values are a list of strings in this format: [<target>, <flags>]
139+
#
140+
# That takes care of the `target_flag_pairs`, which determines which targets to generate compile commands for.
141+
#
142+
# Associating compile commands with target collections is easier; we pass `target_collections`, but with the target names and flags concatenated in the same way
143+
# as described above. This lets us associate each target+flag combination we generate compile commands for with its membership(s) in target collections.
144+
145+
target_collection_targets_list = []
146+
serialized_target_collections = {}
147+
148+
if target_collections:
149+
# Convert the targets specified in `target_collections` into the format we'll use with `targets`, so we can combine them into one list of targets to generate compile commands for.
150+
for targets_for_collection in target_collections.values():
151+
# You can specify a bare string target if the collection only has one target.
152+
if type(targets_for_collection) != "list":
153+
targets_for_collection = [targets_for_collection]
154+
155+
for target in targets_for_collection:
156+
# The target can either be a plain string target name, or a tuple of a target name and build flags, similar to the format of `targets`.
157+
if type(target) == "list":
158+
target_name, flags = target
159+
else:
160+
target_name = target
161+
flags = ""
162+
163+
target_data = (_make_label_absolute(target_name), flags)
164+
165+
# Targets may appear in multiple collections. We don't want duplicates in the final list, but Starlark doesn't have Python's set class. So we de-duplicate manually.
166+
if target_data not in target_collection_targets_list:
167+
target_collection_targets_list.append(target_data)
168+
169+
# Assemble the association between target collections and their targets.
170+
for collection, targets_for_collection in target_collections.items():
171+
serialized_targets = []
172+
173+
for target in targets_for_collection:
174+
if type(target) == "list":
175+
# If the target has flags, concat them with the target name. That's how we'll associate compile commands with collections on the Python side.
176+
serialized_targets.append("".join(target))
177+
else:
178+
serialized_targets.append(target)
179+
180+
serialized_target_collections[collection] = serialized_targets
181+
182+
target_collection_targets = {"{}{}".format(target, flags): [target, flags] for target, flags in target_collection_targets_list}
183+
68184
# Convert the various, acceptable target shorthands into the dictionary format
69185
# In Python, `type(x) == y` is an antipattern, but [Starlark doesn't support inheritance](https://bazel.build/rules/language), so `isinstance` doesn't exist, and this is the correct way to switch on type.
70-
if not targets: # Default to all targets in main workspace
71-
targets = {"@//...": ""}
186+
if not targets and not target_collections: # Default to all targets in main workspace
187+
targets = {"@//...": ["@//...", ""]}
188+
elif not targets: # In this case, targets were defined only in `target_collections`
189+
targets = target_collection_targets
72190
elif type(targets) == "select": # Allow select: https://bazel.build/reference/be/functions#select
73191
# Pass select() to _expand_template to make it work
74192
# see https://bazel.build/docs/configurable-attributes#faq-select-macro
75193
pass
76194
elif type(targets) == "list": # Allow specifying a list of targets w/o arguments
77-
targets = {target: "" for target in targets}
195+
absolute_targets = [_make_label_absolute(target) for target in targets]
196+
targets = {target: [target, ""] for target in absolute_targets} | target_collection_targets
78197
elif type(targets) != "dict": # Assume they've supplied a single string/label and wrap it
79-
targets = {targets: ""}
80-
81-
# Make any package-relative labels absolute
82-
targets = {
83-
target if target.startswith("/") or target.startswith("@") else "{}//{}:{}".format(native.repository_name(), native.package_name(), target.removeprefix(":")): flags for target, flags in targets.items()
84-
}
198+
targets = {targets: [targets, ""]} | target_collection_targets
199+
else: # Assume that they've provided a dict of targets with flags
200+
absolute_targets = {_make_label_absolute(target): flags for target, flags in targets.items()}
201+
targets = {"{}{}".format(target, flags): [target, flags] for target, flags in absolute_targets.items()} | target_collection_targets
85202

86203
# Create a wrapper script that prints a helpful error message if the python version is too old, generated from check_python_version.template.py
87204
version_checker_script_name = name + ".check_python_version.py"
88205
_check_python_version(name = version_checker_script_name, to_run = name)
89206

90207
# Generate the core, runnable python script from refresh.template.py
91208
script_name = name + ".py"
92-
_expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
209+
_expand_template(name = script_name, labels_to_flags = targets, labels_to_collections = serialized_target_collections, out_dir = out_dir, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
93210

94211
# Combine them so the wrapper calls the main script
95212
native.py_binary(
@@ -101,6 +218,13 @@ def refresh_compile_commands(
101218
**kwargs
102219
)
103220

221+
def _make_label_absolute(label):
222+
# Make any package-relative labels absolute
223+
return label if label.startswith("/") or label.startswith("@") else "{}//{}:{}".format(native.repository_name(), native.package_name(), label.removeprefix(":"))
224+
225+
def _expand_target_collection_targets(targets):
226+
return "[\n" + "\n".join([" '{}',".format(target) for target in targets]) + "\n ]"
227+
104228
def _expand_template_impl(ctx):
105229
"""Inject targets of interest--and other settings--into refresh.template.py, and set it up to be run."""
106230
script = ctx.actions.declare_file(ctx.attr.name)
@@ -111,7 +235,9 @@ def _expand_template_impl(ctx):
111235
substitutions = {
112236
# Note, don't delete whitespace. Correctly doing multiline indenting.
113237
" {target_flag_pairs}": "\n".join([" {},".format(pair) for pair in ctx.attr.labels_to_flags.items()]),
238+
" {target_collections}": "\n".join([" '{}': {},".format(collection_name, _expand_target_collection_targets(targets)) for collection_name, targets in ctx.attr.labels_to_collections.items()]),
114239
" {windows_default_include_paths}": "\n".join([" %r," % path for path in find_cpp_toolchain(ctx).built_in_include_directories]), # find_cpp_toolchain is from https://docs.bazel.build/versions/main/integrating-with-rules-cc.html
240+
"{out_dir}": repr(ctx.attr.out_dir),
115241
"{exclude_headers}": repr(ctx.attr.exclude_headers),
116242
"{exclude_external_sources}": repr(ctx.attr.exclude_external_sources),
117243
"{print_args_executable}": repr(ctx.executable._print_args_executable.path),
@@ -121,7 +247,9 @@ def _expand_template_impl(ctx):
121247

122248
_expand_template = rule(
123249
attrs = {
124-
"labels_to_flags": attr.string_dict(mandatory = True), # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
250+
"labels_to_flags": attr.string_list_dict(mandatory = True), # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
251+
"labels_to_collections": attr.string_list_dict(),
252+
"out_dir": attr.string(default = "."),
125253
"exclude_external_sources": attr.bool(default = False),
126254
"exclude_headers": attr.string(values = ["all", "external", ""]), # "" needed only for compatibility with Bazel < 3.6.0
127255
"_script_template": attr.label(allow_single_file = True, default = "refresh.template.py"),

0 commit comments

Comments
 (0)