Skip to content

Adds the ability to use absolute paths for bazel managed artifacts #152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Adds the ability to use absolute paths for bazel managed artifacts #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 36 additions & 4 deletions refresh.template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -765,7 +765,21 @@ def _all_platform_patch(compile_args: typing.List[str]):
return compile_args


def _get_cpp_command_for_files(compile_action):
def _apply_path_replacements(compile_args: typing.List[str], replacements: typing.Mapping[str, str]):
def apply_path_replacement(arg):
for (prefix, replacement) in replacements.items():
if arg.startswith(prefix):
return replacement + arg[len(prefix):]
elif arg.startswith('-I' + prefix):
return replacement + arg[2 + len(prefix):]
elif '={}'.format(prefix) in arg:
return arg.replace('={}'.format(prefix), '={}'.format(replacement))
return arg

return [apply_path_replacement(arg) for arg in compile_args]


def _get_cpp_command_for_files(compile_action, replacements):
"""Reformat compile_action into a compile command clangd can understand.

Undo Bazel-isms and figures out which files clangd should apply the command to.
Expand All @@ -774,13 +788,14 @@ def _get_cpp_command_for_files(compile_action):
compile_action.arguments = _all_platform_patch(compile_action.arguments)
compile_action.arguments = _apple_platform_patch(compile_action.arguments)
# Android and Linux and grailbio LLVM toolchains: Fine as is; no special patching needed.
compile_action.arguments = _apply_path_replacements(compile_action.arguments, replacements)

source_files, header_files = _get_files(compile_action)

return source_files, header_files, compile_action.arguments


def _convert_compile_commands(aquery_output):
def _convert_compile_commands(aquery_output, replacements):
"""Converts from Bazel's aquery format to de-Bazeled compile_commands.json entries.

Input: jsonproto output from aquery, pre-filtered to (Objective-)C(++) compile actions for a given build.
Expand All @@ -799,12 +814,15 @@ def _convert_compile_commands(aquery_output):
assert not target.startswith('//external'), f"Expecting external targets will start with @. Found //external for action {action}, target {target}"
action.is_external = target.startswith('@') and not target.startswith('@//')

def worker(compile_action):
return _get_cpp_command_for_files(compile_action, replacements)

# Process each action from Bazelisms -> file paths and their clang commands
# Threads instead of processes because most of the execution time is farmed out to subprocesses. No need to sidestep the GIL. Might change after https://github.com/clangd/clangd/issues/123 resolved
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(32, (os.cpu_count() or 1) + 4) # Backport. Default in MIN_PY=3.8. See "using very large resources implicitly on many-core machines" in https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
) as threadpool:
outputs = threadpool.map(_get_cpp_command_for_files, aquery_output.actions)
outputs = threadpool.map(worker, aquery_output.actions)

# Yield as compile_commands.json entries
header_files_already_written = set()
Expand Down Expand Up @@ -924,7 +942,21 @@ def _get_commands(target: str, flags: str):
Continuing gracefully...""")
return

yield from _convert_compile_commands(parsed_aquery_output)
replacements = {}
if {rewrite_bazel_paths}:
info_process = subprocess.run(
['bazel', 'info', 'output_path'],
capture_output=True,
encoding=locale.getpreferredencoding(),
)

output_path = pathlib.Path(info_process.stdout.strip())
replacements = {
"bazel-out/": os.fspath(output_path) + "/",
"external/": os.fspath(output_path.parent / "external") + "/",
}

yield from _convert_compile_commands(parsed_aquery_output, replacements)


# Log clear completion messages
Expand Down
15 changes: 14 additions & 1 deletion refresh_compile_commands.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ refresh_compile_commands(
# exclude_headers = "external",
# Still not fast enough?
# Make sure you're specifying just the targets you care about by setting `targets`, above.

# refresh_compile_commands does not work with --experimental_convenience_symlinks=ignore.
# For these workspaces, you can use absolute paths to the bazel build artifacts by setting rewrite_bazel_paths = True.
```
"""

Expand All @@ -63,6 +66,7 @@ def refresh_compile_commands(
targets = None,
exclude_headers = None,
exclude_external_sources = False,
rewrite_bazel_paths = False,
**kwargs): # For the other common attributes. Tags, compatible_with, etc. https://docs.bazel.build/versions/main/be/common-definitions.html#common-attributes.
# Convert the various, acceptable target shorthands into the dictionary format
# In Python, `type(x) == y` is an antipattern, but [Starlark doesn't support inheritance](https://bazel.build/rules/language), so `isinstance` doesn't exist, and this is the correct way to switch on type.
Expand All @@ -88,7 +92,14 @@ def refresh_compile_commands(

# Generate the core, runnable python script from refresh.template.py
script_name = name + ".py"
_expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
_expand_template(
name = script_name,
labels_to_flags = targets,
exclude_headers = exclude_headers,
exclude_external_sources = exclude_external_sources,
rewrite_bazel_paths = rewrite_bazel_paths,
**kwargs
)

# Combine them so the wrapper calls the main script
native.py_binary(
Expand All @@ -115,6 +126,7 @@ def _expand_template_impl(ctx):
" {windows_default_include_paths}": "\n".join([" %r," % path for path in find_cpp_toolchain(ctx).built_in_include_directories]), # find_cpp_toolchain is from https://docs.bazel.build/versions/main/integrating-with-rules-cc.html
"{exclude_headers}": repr(ctx.attr.exclude_headers),
"{exclude_external_sources}": repr(ctx.attr.exclude_external_sources),
"{rewrite_bazel_paths}": repr(ctx.attr.rewrite_bazel_paths),
},
)
return DefaultInfo(files = depset([script]))
Expand All @@ -124,6 +136,7 @@ _expand_template = rule(
"labels_to_flags": attr.string_dict(mandatory = True), # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
"exclude_external_sources": attr.bool(default = False),
"exclude_headers": attr.string(values = ["all", "external", ""]), # "" needed only for compatibility with Bazel < 3.6.0
"rewrite_bazel_paths": attr.bool(default = False),
"_script_template": attr.label(allow_single_file = True, default = "refresh.template.py"),
# For Windows INCLUDE. If this were eliminated, for example by the resolution of https://github.com/clangd/clangd/issues/123, we'd be able to just use a macro and skylib's expand_template rule: https://github.com/bazelbuild/bazel-skylib/pull/330
# Once https://github.com/bazelbuild/bazel/pull/17108 is widely released, we should be able to eliminate this and get INCLUDE directly. Perhaps for 7.0? Should be released in the sucessor to 6.0
Expand Down