-
Notifications
You must be signed in to change notification settings - Fork 108
/
refresh.template.py
1424 lines (1217 loc) · 84.8 KB
/
refresh.template.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
As a template, this file helps implement the refresh_compile_commands rule and is not part of the user interface. See ImplementationReadme.md for top-level context -- or refresh_compile_commands.bzl for narrower context.
Interface (after template expansion):
- `bazel run` to regenerate compile_commands.json, so autocomplete (and any other clang tooling!) reflect the latest Bazel build files.
- No arguments are needed; info from the rule baked into the template expansion.
- Any arguments passed are interpreted as arguments needed for the builds being analyzed.
- Requires being run under Bazel so we can access the workspace root environment variable.
- Output: a compile_commands.json in the workspace root that clang tooling (or you!) can look at to figure out how files are being compiled by Bazel
- Crucially, this output is de-Bazeled; The result is a command that could be run from the workspace root directly, with no Bazel-specific requirements, environment variables, etc.
"""
# This file requires python 3.6, which is enforced by check_python_version.template.py
# 3.6 backwards compatibility required by @zhanyong-wan in https://github.com/hedronvision/bazel-compile-commands-extractor/issues/111.
# 3.7 backwards compatibility required by @lummax in https://github.com/hedronvision/bazel-compile-commands-extractor/pull/27.
# ^ Try to contact before upgrading.
# When adding things could be cleaner if we had a higher minimum version, please add a comment with MIN_PY=3.<v>.
# Similarly, when upgrading, please search for that MIN_PY= tag.
import concurrent.futures
import enum
import functools # MIN_PY=3.9: Replace `functools.lru_cache(maxsize=None)` with `functools.cache`.
import itertools
import json
import locale
import os
import pathlib
import re
import shlex
import shutil
import subprocess
import sys
import tempfile
import time
import types
import typing # MIN_PY=3.9: Switch e.g. typing.List[str] -> List[str]
@enum.unique
class SGR(enum.Enum):
"""Enumerate (some of the) available SGR (Select Graphic Rendition) control sequences."""
# For details on SGR control sequences (and ANSI escape codes in general), see: https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_(Select_Graphic_Rendition)_parameters
RESET = '\033[0m'
FG_RED = '\033[0;31m'
FG_GREEN = '\033[0;32m'
FG_YELLOW = '\033[0;33m'
FG_BLUE = '\033[0;34m'
def _log_with_sgr(sgr, colored_message, uncolored_message=''):
"""Log a message to stderr wrapped in an SGR context."""
print(sgr.value, colored_message, SGR.RESET.value, uncolored_message, sep='', file=sys.stderr, flush=True)
def log_error(colored_message, uncolored_message=''):
"""Log an error message (in red) to stderr."""
_log_with_sgr(SGR.FG_RED, colored_message, uncolored_message)
def log_warning(colored_message, uncolored_message=''):
"""Log a warning message (in yellow) to stderr."""
_log_with_sgr(SGR.FG_YELLOW, colored_message, uncolored_message)
def log_info(colored_message, uncolored_message=''):
"""Log an informative message (in blue) to stderr."""
_log_with_sgr(SGR.FG_BLUE, colored_message, uncolored_message)
def log_success(colored_message, uncolored_message=''):
"""Log a success message (in green) to stderr."""
_log_with_sgr(SGR.FG_GREEN, colored_message, uncolored_message)
def _print_header_finding_warning_once():
"""Gives users context about "compiler errors" while header finding. Namely that we're recovering."""
# Shared between platforms
# Just log once; subsequent messages wouldn't add anything.
if _print_header_finding_warning_once.has_logged: return
_print_header_finding_warning_once.has_logged = True
log_warning(""">>> While locating the headers you use, we encountered a compiler warning or error.
No need to worry; your code doesn't have to compile for this tool to work.
However, we'll still print the errors and warnings in case they're helpful for you in fixing them.
If the errors are about missing files that Bazel should generate:
You might want to run a build of your code with --keep_going.
That way, everything possible is generated, browsable and indexed for autocomplete.
But, if you have *already* built your code successfully:
Please make sure you're supplying this tool with the same flags you use to build.
You can either use a refresh_compile_commands rule or the special -- syntax. Please see the README.
[Supplying flags normally won't work. That just causes this tool to be built with those flags.]
Continuing gracefully...""")
_print_header_finding_warning_once.has_logged = False
@functools.lru_cache(maxsize=None)
def _get_bazel_version():
"""Gets the Bazel version as a tuple of (major, minor, patch).
The rolling release and the release candidate are treated as the LTS release.
E.g. both 7.0.0-pre.XXXXXXXX.X and 7.0.0rc1 are treated as 7.0.0.
If the version can't be determined, returns (0, 0, 0).
"""
bazel_version_process = subprocess.run(
['bazel', 'version'],
# MIN_PY=3.7: Replace PIPEs with capture_output.
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding=locale.getpreferredencoding(),
check=True, # Should always succeed.
)
version = ''
for line in bazel_version_process.stdout.splitlines():
line = line.strip()
if line.startswith('Build label: '):
version = line[len('Build label: '):]
match = re.search(r'^(\d+)\.(\d+)\.(\d+)', version)
if not match:
log_warning(f">>> Failed to get Bazel version.\nPlease file an issue with the following log:\n", bazel_version_process.stdout)
return (0, 0, 0)
return tuple(int(match.group(i)) for i in range(1, 4))
@functools.lru_cache(maxsize=None)
def _get_bazel_cached_action_keys():
"""Gets the set of actionKeys cached in bazel-out."""
action_cache_process = subprocess.run(
['bazel', 'dump', '--action_cache'],
# MIN_PY=3.7: Replace PIPEs with capture_output.
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding=locale.getpreferredencoding(),
check=True, # Should always succeed.
)
action_keys = set()
marked_as_empty = False # Sometimes the action cache is empty...despite having built this file, so we have to handle that case. See https://github.com/hedronvision/bazel-compile-commands-extractor/issues/64
for line in action_cache_process.stdout.splitlines():
line = line.strip()
if line.startswith('actionKey = '):
action_keys.add(line[12:]) # Remainder after actionKey =
elif line.startswith('Action cache (0 records):'):
marked_as_empty = True
# Make sure we get notified of changes to the format, since bazel dump --action_cache isn't public API.
# We continue gracefully, rather than asserting, because we can (conservatively) continue without hitting cache.
if not marked_as_empty and not action_keys:
log_warning(">>> Failed to get action keys from Bazel.\nPlease file an issue with the following log:\n", action_cache_process.stdout)
return action_keys
def _parse_headers_from_makefile_deps(d_file_content: str, source_path_for_sanity_check: typing.Optional[str] = None):
"""Parse a set of headers from the contents of a `*.d` dependency file generated by clang (or gcc).
A dependency file can be generated with the `-M`/`--dependencies` flag or its friends.
See https://clang.llvm.org/docs/ClangCommandLineReference.html#dependency-file-generation for more details.
"""
if not d_file_content: # There was an error generating.
return set()
# When reading file content as text with universal newlines mode enabled (the default), Python converts OS-specific line endings to '\n' (see https://docs.python.org/3/library/functions.html#open-newline-parameter for the thrilling details).
# This function takes an arbitrary string, so we also ensure no `\r` characters have snuck through, because that's almost certainly an upstream error.
assert '\r' not in d_file_content, "Something went wrong in makefile parsing to get headers. Dependency file content should not contain literal '\r' characters. Output:\n" + repr(d_file_content)
# We assume that this Makefile-like dependency file (`*.d`) contains exactly one `target: dependencies` rule.
# There can be an optional space after the target, and long lists of dependencies (often) carry over with a backslash and newline.
# For example, `d_file_content` might be: `"foo.o : foo.cc bar.h \\\n baz.hpp"`.
target, dependencies = d_file_content.split(': ', 1) # Needs to handle absolute Windows paths, like C:\
target = target.strip() # Remove the optional trailing space.
assert target.endswith(('.o', '.obj')), "Something went wrong in makefile parsing to get headers. The target should be an object file. Output:\n" + d_file_content
# Undo shell-like line wrapping because the newlines aren't eaten by shlex.join. Note also that it's the line wrapping is inconsistently generated across compilers and depends on the lengths of the filenames, so you can't just split on the escaped newlines.
dependencies = dependencies.replace('\\\n', '')
# On Windows, swap out (single) backslash path directory separators for forward slash. Shlex otherwise eats the separators...and Windows gcc intermixes backslash separators with backslash escaped spaces. For a real example of gcc run from Windows, see https://github.com/hedronvision/bazel-compile-commands-extractor/issues/81
if os.name == 'nt':
dependencies = re.sub(r'\\(?=[^ \\])', '/', dependencies)
# We'll use shlex.split as a good proxy for escaping, but note that Makefiles themselves [don't seem to really support escaping spaces](https://stackoverflow.com/questions/30687828/how-to-escape-spaces-inside-a-makefile).
dependencies = shlex.split(dependencies)
source, *headers = dependencies # The first dependency is a source entry, only used to (optionally) sanity-check the dependencies if a source path is provided.
assert source_path_for_sanity_check is None or source.endswith(source_path_for_sanity_check), "Something went wrong in makefile parsing to get headers. The first dependency should be the source file. Output:\n" + d_file_content
# Make the headers unique, because GCC [sometimes emits duplicate entries](https://github.com/hedronvision/bazel-compile-commands-extractor/issues/7#issuecomment-975109458).
return set(headers)
@functools.lru_cache(maxsize=None)
def _get_cached_modified_time(path: str):
"""Returns 0 if the file doesn't exist.
Without the cache, most of our runtime in the cached case is `stat`'ing the same headers repeatedly.
"""
try:
return os.path.getmtime(path)
except OSError: # The file doesn't exist or is inaccessible.
# For our purposes, this means we don't have a newer version, so we'll return a very old time that'll always qualify the cache as fresh in a comparison. There are two cases here:
# (1) Somehow it wasn't generated in the build that created the depfile. We therefore won't get any fresher by building, so we'll treat that as good enough; or
# (2) It has been deleted since we last cached, in which case we'd rather use the cached version if it's otherwise fresh.
return 0
def _get_cached_file_exists(path: str):
return _get_cached_modified_time(path) != 0
def _get_cached_adjusted_modified_time(path: str):
"""Get the modified time of a file, slightly adjusted for easy comparison.
This is primarily intended to check whether header include caches are fresh.
If the file doesn't exist or is inaccessible (either because it was deleted or wasn't generated), return 0.
For bazel's internal sources, which have timestamps 10 years in the future, also return 0.
Without the cache, most of our runtime in the cached case is `stat`'ing the same headers repeatedly.
Cache shared with _get_cached_modified_time
"""
mtime = _get_cached_modified_time(path)
# Bazel internal sources have timestamps 10 years in the future as part of a mechanism to detect and prevent modification, so we'll similarly ignore those, since they shouldn't be changing.
if mtime > BAZEL_INTERNAL_SOURCE_CUTOFF:
return 0
return mtime
# Roughly 1 year into the future. This is safely below bazel's 10 year margin, but large enough that no sane normal file should be past this.
BAZEL_INTERNAL_SOURCE_CUTOFF = time.time() + 60*60*24*365
def _is_nvcc(path: str):
return os.path.basename(path).startswith('nvcc')
def _get_headers_gcc(compile_action, source_path: str, action_key: str):
"""Gets the headers used by a particular compile command that uses gcc arguments formatting (including clang.)
Relatively slow. Requires running the C preprocessor if we can't hit Bazel's cache.
"""
# Flags reference here: https://clang.llvm.org/docs/ClangCommandLineReference.html
# Check to see if Bazel has an (approximately) fresh cache of the included headers, and if so, use them to avoid a slow preprocessing step.
if action_key in _get_bazel_cached_action_keys(): # Safe because Bazel only holds one cached action key per path, and the key contains the path.
for i, arg in enumerate(compile_action.arguments):
if arg.startswith('-MF'):
if len(arg) > 3: # Either appended, like -MF<file>
dep_file_path = arg[:3]
else: # Or after as a separate arg, like -MF <file>
dep_file_path = compile_action.arguments[i+1]
if os.path.isfile(dep_file_path):
dep_file_last_modified = os.path.getmtime(dep_file_path) # Do before opening just as a basic hedge against concurrent write, even though we won't handle the concurrent delete case perfectly.
with open(dep_file_path) as dep_file:
dep_file_contents = dep_file.read()
headers = _parse_headers_from_makefile_deps(dep_file_contents)
# Check freshness of dep file by making sure none of the files in it have been modified since its creation.
if (_get_cached_adjusted_modified_time(source_path) <= dep_file_last_modified
and all(_get_cached_adjusted_modified_time(header_path) <= dep_file_last_modified for header_path in headers)):
return headers, True # Fresh cache! exit early. Still put in the Hedron outer cache bc we're willing to hit stale if we're unable to get new headers.
break
# Strip out existing dependency file generation that could interfere with ours.
# Clang on Apple doesn't let later flags override earlier ones, unfortunately.
# These flags are prefixed with M for "make", because that's their output format.
# *-dependencies is the long form. And the output file is traditionally *.d
header_cmd = (arg for arg in compile_action.arguments
if not arg.startswith('-M') and not arg.endswith(('-dependencies', '.d')))
# Strip output flags. Apple clang tries to do a full compile if you don't.
header_cmd = (arg for arg in header_cmd
if arg != '-o' and not arg.endswith('.o'))
# Strip sanitizer ignore lists...so they don't show up in the dependency list.
# See https://clang.llvm.org/docs/SanitizerSpecialCaseList.html and https://github.com/hedronvision/bazel-compile-commands-extractor/issues/34 for more context.
header_cmd = (arg for arg in header_cmd
if not arg.startswith('-fsanitize'))
# Dump system and user headers to stdout...in makefile format.
# Relies on our having made the workspace directory simulate a complete version of the execroot with //external symlink
header_cmd = list(header_cmd)
is_nvcc = _is_nvcc(header_cmd[0])
# https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#nvcc-command-options
if is_nvcc:
header_cmd += ['--generate-dependencies']
else:
# -M rather than --dependencies allows us to support the zig compiler. See https://github.com/hedronvision/bazel-compile-commands-extractor/pull/130
header_cmd += ['-M', '--print-missing-file-dependencies'] # Allows us to continue on past missing (generated) files--whose paths may be wrong (listed as written in the include)!
header_search_process = _subprocess_run_spilling_over_to_param_file_if_needed( # Note: gcc/clang can be run from Windows, too.
header_cmd,
# MIN_PY=3.7: Replace PIPEs with capture_output.
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=compile_action.environmentVariables,
encoding=locale.getpreferredencoding(),
check=False, # We explicitly ignore errors and carry on.
)
# Tolerate failure gracefully--during editing the code may not compile!
if header_search_process.stderr:
_print_header_finding_warning_once()
print(header_search_process.stderr, file=sys.stderr, end='') # Stderr captured and dumped atomically to avoid interlaced output.
headers = _parse_headers_from_makefile_deps(header_search_process.stdout)
# Can't cache when headers are missing.
# Why? We'd wrongly get a subset of the headers and we might wrongly think the cache is still fresh because we wouldn't know that the formerly missing header had been generated.
if is_nvcc:
should_cache = bool(header_search_process.stdout) # Without --print-missing-file-dependencies, nothing is written if a file isn't found. (Something is still written if no headers are included.) This avoids hardcoding the error message. Note that some errors, like that for #bad_directive are okay to ignore! We'll know the cache isn't fresh when the user changes the file.
else: # Handle '--print-missing-file-dependencies'
num_headers_output = len(headers)
headers = {header for header in headers if _get_cached_file_exists(header)} # We can't filter on headers starting with bazel-out because missing headers don't have complete paths; they just listed as #included
should_cache = len(headers) == num_headers_output
return headers, should_cache
def windows_list2cmdline(seq):
"""
Copied from list2cmdline in https://github.com/python/cpython/blob/main/Lib/subprocess.py because we need it but it's not exported as part of the public API.
Translate a sequence of arguments into a command line
string, using the same rules as the MS C runtime:
1) Arguments are delimited by white space, which is either a
space or a tab.
2) A string surrounded by double quotation marks is
interpreted as a single argument, regardless of white space
contained within. A quoted string can be embedded in an
argument.
3) A double quotation mark preceded by a backslash is
interpreted as a literal double quotation mark.
4) Backslashes are interpreted literally, unless they
immediately precede a double quotation mark.
5) If backslashes immediately precede a double quotation mark,
every pair of backslashes is interpreted as a literal
backslash. If the number of backslashes is odd, the last
backslash escapes the next double quotation mark as
described in rule 3.
"""
# See
# http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
# or search http://msdn.microsoft.com for
# "Parsing C++ Command-Line Arguments"
result = []
needquote = False
for arg in map(os.fsdecode, seq):
bs_buf = []
# Add a space to separate this argument from the others
if result:
result.append(' ')
needquote = (" " in arg) or ("\t" in arg) or not arg
if needquote:
result.append('"')
for c in arg:
if c == '\\':
# Don't know if we need to double yet.
bs_buf.append(c)
elif c == '"':
# Double backslashes.
result.append('\\' * len(bs_buf)*2)
bs_buf = []
result.append('\\"')
else:
# Normal char
if bs_buf:
result.extend(bs_buf)
bs_buf = []
result.append(c)
# Add remaining backslashes, if any.
if bs_buf:
result.extend(bs_buf)
if needquote:
result.extend(bs_buf)
result.append('"')
return ''.join(result)
def _subprocess_run_spilling_over_to_param_file_if_needed(command: typing.List[str], **kwargs):
"""Same as subprocess.run, but it handles the case where the command line length is exceeded on Windows and we need a param file."""
# On non-Windows, we have to run directly via a special case.
# Otherwise, any exceptions will also trigger a NameError, since WindowsError is not defined outside of Windows.
if os.name != 'nt':
return subprocess.run(command, **kwargs)
# See https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation
try:
return subprocess.run(command, **kwargs)
except WindowsError as e:
# We handle the error instead of calculating the command length because the length includes escaping internal to the subprocess.run call
if e.winerror == 206: # Thrown when command is too long, despite the error message being "The filename or extension is too long". For a few more details see also https://stackoverflow.com/questions/2381241/what-is-the-subprocess-popen-max-length-of-the-args-parameter
# Write command to a temporary file, so we can use it as a parameter file to the compiler.
# E.g. cl.exe @params_file.txt
# tempfile.NamedTemporaryFile doesn't work because cl.exe can't open it--as the Python docs would indicate--so we have to do cleanup ourselves.
fd, path = tempfile.mkstemp(text=True)
try:
os.write(fd, windows_list2cmdline(command[1:]).encode()) # should skip cl.exe the 1st line.
os.close(fd)
return subprocess.run([command[0], f'@{path}'], **kwargs)
finally: # Safe cleanup even in the event of an error
os.remove(path)
else: # Some other WindowsError we didn't mean to catch.
raise
def _get_headers_msvc(compile_action, source_path: str):
"""Gets the headers used by a particular compile command that uses msvc argument formatting (including clang-cl.)
Relatively slow. Requires running the C preprocessor.
"""
# Flags reference here: https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options
# Relies on our having made the workspace directory simulate a complete version of the execroot with //external junction
header_cmd = list(compile_action.arguments) + [
'/showIncludes', # Print included headers to stderr. https://docs.microsoft.com/en-us/cpp/build/reference/showincludes-list-include-files
'/EP', # Preprocess (only, no compilation for speed), writing to stdout where we can easily ignore it instead of a file. https://docs.microsoft.com/en-us/cpp/build/reference/ep-preprocess-to-stdout-without-hash-line-directives
]
# cl.exe needs the `INCLUDE` environment variable to find the system headers, since they aren't specified in the action command
# Bazel neglects to include INCLUDE per action, so we'll do the best we can and infer them from the default (host) cc toolchain.
# These are set in https://github.com/bazelbuild/bazel/bloc/master/tools/cpp/windows_cc_configure.bzl. Search INCLUDE.
# Bazel should have supplied the environment variables in aquery output but doesn't https://github.com/bazelbuild/bazel/issues/12852
# Non-Bazel Windows users would normally configure these by calling vcvars
# For more, see https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line
environment = compile_action.environmentVariables.copy()
if 'INCLUDE' not in environment:
environment['INCLUDE'] = os.pathsep.join((
# Begin: template filled by Bazel
{windows_default_include_paths}
# End: template filled by Bazel
))
header_search_process = _subprocess_run_spilling_over_to_param_file_if_needed(
header_cmd,
stderr=subprocess.PIPE,
stdout=subprocess.DEVNULL,
env=environment,
encoding=locale.getpreferredencoding(),
check=False, # We explicitly ignore errors and carry on.
)
# Based on the locale, `cl.exe` will emit different marker strings. See also https://github.com/ninja-build/ninja/issues/613#issuecomment-885185024 and https://github.com/bazelbuild/bazel/pull/7966.
# We can't just set environment['VSLANG'] = "1033" (English) and be done with it, because we can't assume the user has the English language pack installed.
# Note that, if we're ever having problems with MSVC changing these strings too often, we can instead infer them by compiling some test files and passing /nologo. See https://github.com/ninja-build/ninja/issues/613#issuecomment-1465084387
include_marker = (
'Note: including file:', # English - United States
'注意: 包含文件: ', # Chinese - People's Republic of China
'注意: 包含檔案:', # Chinese - Taiwan
'Poznámka: Včetně souboru:', # Czech
'Hinweis: Einlesen der Datei:', # German - Germany
'Remarque : inclusion du fichier : ', # French - France
'Nota: file incluso ', # Italian - Italy
'メモ: インクルード ファイル: ', # Japanese
'참고: 포함 파일:', # Korean
'Uwaga: w tym pliku: ', # Polish
'Observação: incluindo arquivo:', # Portuguese - Brazil
'Примечание: включение файла: ', # Russian
'Not: eklenen dosya: ', # Turkish
'Nota: inclusión del archivo:', # Spanish - Spain (Modern Sort)
)
headers = set() # Make unique. MSVC emits duplicate entries.
error_lines = []
for line in header_search_process.stderr.splitlines():
# Gobble up the header inclusion information...
if source_path.endswith('/' + line) or source_path == line: # Munching the source filename echoed the first part of the include output
continue
for marker in include_marker:
if line.startswith(marker):
headers.add(line[len(marker):].strip())
break
else:
error_lines.append(line)
if error_lines: # Output all errors at the end so they aren't interlaced due to concurrency
_print_header_finding_warning_once()
print('\n'.join(error_lines), file=sys.stderr)
should_cache = all('fatal error C1083:' not in error_line for error_line in error_lines) # Error code for file not found (when trying to include). Without this, we'd wrongly get a subset of the headers and we might wrongly think the cache is still fresh because we wouldn't know that the formerly missing header had been generated.
return headers, should_cache
def _is_relative_to(sub: pathlib.PurePath, parent: pathlib.PurePath):
"""Determine if one path is relative to another."""
# MIN_PY=3.9: Eliminate helper in favor of `PurePath.is_relative_to()`.
try:
sub.relative_to(parent)
except ValueError:
return False
return True
def _file_is_in_main_workspace_and_not_external(file_str: str):
file_path = pathlib.PurePath(file_str)
if file_path.is_absolute():
workspace_absolute = pathlib.PurePath(os.environ["BUILD_WORKSPACE_DIRECTORY"])
if not _is_relative_to(file_path, workspace_absolute):
return False
file_path = _is_relative_to(file_path, workspace_absolute)
# You can now assume that the path is relative to the workspace.
# [Already assuming that relative paths are relative to the main workspace.]
# some/file.h, but not external/some/file.h
# also allows for things like bazel-out/generated/file.h
if _is_relative_to(file_path, pathlib.PurePath("external")):
return False
# ... but, ignore files in e.g. bazel-out/<configuration>/bin/external/
if file_path.parts[0] == 'bazel-out' and file_path.parts[3] == 'external':
return False
return True
def _get_headers(compile_action, source_path: str):
"""Gets the headers used by a particular compile command.
Relatively slow. Requires running the C preprocessor.
"""
# Hacky, but hopefully this is a temporary workaround for the clangd issue mentioned in the caller (https://github.com/clangd/clangd/issues/123)
# Runs a modified version of the compile command to piggyback on the compiler's preprocessing and header searching.
# As an alternative approach, you might consider trying to get the headers by inspecting the Middlemen actions in the aquery output, but I don't see a way to get just the ones actually #included--or an easy way to get the system headers--without invoking the preprocessor's header search logic.
# For more on this, see https://github.com/hedronvision/bazel-compile-commands-extractor/issues/5#issuecomment-1031148373
if {exclude_headers} == "all":
return set()
elif {exclude_headers} == "external" and not {exclude_external_sources} and compile_action.is_external:
# Shortcut - an external action can't include headers in the workspace (or, non-external headers)
# The `not {exclude_external_sources}`` clause makes sure is_external was precomputed; there are no external actions if they've already been filtered in the process of excluding external sources.
return set()
output_file = None
for i, arg in enumerate(compile_action.arguments):
# As a reference, clang docs: https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang1-o-file
if arg == '-o' or arg == '--output': # clang/gcc. Docs https://clang.llvm.org/docs/ClangCommandLineReference.html
output_file = compile_action.arguments[i+1]
break
elif arg.startswith('/Fo') or arg.startswith('-Fo'): # MSVC *and clang*. MSVC docs https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically
output_file = arg[3:]
break
elif arg.startswith('--output='):
output_file = arg[9:]
break
# Since our output file parsing isn't complete, fall back on a warning message to solicit help.
# A more full (if more involved) solution would be to get the primaryOutput for the action from the aquery output, but this should handle the cases Bazel emits.
if not output_file and not _get_headers.has_logged:
_get_headers.has_logged = True
log_warning(f""">>> Please file an issue containing the following: Output file not detected in arguments {compile_action.arguments}.
Not a big deal; things will work but will be a little slower.
Thanks for your help!
Continuing gracefully...""")
# Check for a fresh cache of headers
cached_headers = None
if output_file:
cache_file_path = output_file + ".hedron.compile-commands.headers" # Embed our cache in bazel's
if os.path.isfile(cache_file_path):
cache_last_modified = os.path.getmtime(cache_file_path) # Do before opening just as a basic hedge against concurrent write, even though we won't handle the concurrent delete case perfectly.
try:
with open(cache_file_path) as cache_file:
action_key, cached_headers = json.load(cache_file)
except json.JSONDecodeError:
# Corrupted cache, which can happen if, for example, the user kills the program, since writes aren't atomic.
# But if it is the result of a bug, we want to print it before it's overwritten, so it can be reported
# For a real instance, see https://github.com/hedronvision/bazel-compile-commands-extractor/issues/60
with open(cache_file_path) as cache_file:
log_warning(f""">>> Ignoring corrupted header cache {cache_file_path}
This is okay if you manually killed this tool earlier.
But if this message is appearing spontaneously or frequently, please file an issue containing the contents of the corrupted cache, below.
{cache_file.read()}
Thanks for your help!
Continuing gracefully...""")
else:
# Check cache freshness.
# Action key validates that it corresponds to the same action arguments
# And we also need to check that there aren't newer versions of the files
if (action_key == compile_action.actionKey
and _get_cached_adjusted_modified_time(source_path) <= cache_last_modified
and all(_get_cached_adjusted_modified_time(header_path) <= cache_last_modified for header_path in cached_headers)):
return set(cached_headers)
if compile_action.arguments[0].endswith('cl.exe'): # cl.exe and also clang-cl.exe
headers, should_cache = _get_headers_msvc(compile_action, source_path)
else:
headers, should_cache = _get_headers_gcc(compile_action, source_path, compile_action.actionKey)
# Cache for future use
if output_file and should_cache:
os.makedirs(os.path.dirname(cache_file_path), exist_ok=True)
with open(cache_file_path, 'w') as cache_file:
json.dump(
(compile_action.actionKey, list(headers)),
cache_file,
indent=2,
check_circular=False,
)
elif not headers and cached_headers: # If we failed to get headers, we'll fall back on a stale cache.
headers = set(cached_headers)
if {exclude_headers} == "external":
headers = {header for header in headers if _file_is_in_main_workspace_and_not_external(header)}
return headers
_get_headers.has_logged = False
def _get_files(compile_action):
"""Gets the ({source files}, {header files}) clangd should be told the command applies to."""
# Getting the source file is a little trickier than it might seem.
# First, we do the obvious thing: Filter args to those that look like source files.
source_file_candidates = [arg for arg in compile_action.arguments if not arg.startswith('-') and arg.endswith(_get_files.source_extensions)]
assert source_file_candidates, f"No source files found in compile args: {compile_action.arguments}.\nPlease file an issue with this information!"
source_file = source_file_candidates[0]
# If we've got multiple candidates for source files, apply heuristics based on how Bazel tends to format commands.
# Note: Bazel, with its incremental building strategy, should only be compiling one source file per action.
if len(source_file_candidates) > 1:
# How does this case arise? Sometimes header search directories have source-file extensions. Horrible, but unfortunately true. See https://github.com/hedronvision/bazel-compile-commands-extractor/pull/37 for context and history.
# You can't simply further filter the args to those that aren't existing directories...because they can be generated directories that don't yet exist. Indeed the example in the PR (linked above) is this case.
# Bazel seems to consistently put the source file being compiled either:
# before the -o flag, for GCC-formatted commands, or
# after the /c flag, for MSVC-formatted commands
# [See https://github.com/hedronvision/bazel-compile-commands-extractor/pull/72 for -c counterexample for GCC]
# This is a strong assumption about Bazel internals, so we're taking some care to check that this condition holds with asserts. That way things are less likely to fail silently if it changes some day.
# You can definitely have a proper invocation to clang/gcc/msvc where these assumptions don't hold.
# However, parsing the command line this way is our best simple option. The other alternatives seem worse:
# Parsing the clang invocation properly to get the positional file arguments is hard and not future-proof if new flags are added. Consider a new flag -foo. Does it also capture the next argument after it?
# You might be tempted to crawl the inputs depset in the aquery output structure, but it's a fair amount of recursive code and there are other erroneous source files there, at least when building for Android in Bazel 5.1. You could fix this by intersecting the set of source files in the inputs with those listed as arguments on the command line, but I can imagine perverse, problematic cases here. It's a lot more code to still have those caveats.
# You might be tempted to get the source files out of the action message listed (just) in aquery --output=text output, but the message differs for external workspaces and tools. Plus paths with spaces are going to be hard because it's space delimited. You'd have to make even stronger assumptions than the -c.
# Concretely, the message usually has the form "action 'Compiling foo.cpp'"" -> foo.cpp. But it also has "action 'Compiling src/tools/launcher/dummy.cc [for tool]'" -> external/bazel_tools/src/tools/launcher/dummy.cc
# If we did ever go this route, you can join the output from aquery --output=text and --output=jsonproto by actionKey.
if '-o' in compile_action.arguments: # GCC, pre -o case
source_index = compile_action.arguments.index('-o') - 1
else: # MSVC, post /C case
assert '/c' in compile_action.arguments, f"-o or /c, required for parsing sources in GCC or MSVC-formatted commands, respectively, not found in compile args: {compile_action.arguments}.\nPlease file an issue with this information!"
source_index = compile_action.arguments.index('/c') + 1
source_file = compile_action.arguments[source_index]
assert source_file.endswith(_get_files.source_extensions), f"Source file candidate, {source_file}, seems to be wrong.\nSelected from {compile_action.arguments}.\nPlease file an issue with this information!"
# Warn gently about missing files
if not os.path.isfile(source_file):
if not _get_files.has_logged_missing_file_error: # Just log once; subsequent messages wouldn't add anything.
_get_files.has_logged_missing_file_error = True
log_warning(f""">>> A source file you compile doesn't (yet) exist: {source_file}
It's probably a generated file, and you haven't yet run a build to generate it.
That's OK; your code doesn't even have to compile for this tool to work.
If you can, though, you might want to run a build of your code with --keep_going.
That way everything possible is generated, browsable and indexed for autocomplete.
However, if you have *already* built your code, and generated the missing file...
Please make sure you're supplying this tool with the same flags you use to build.
You can either use a refresh_compile_commands rule or the special -- syntax. Please see the README.
[Supplying flags normally won't work. That just causes this tool to be built with those flags.]
Continuing gracefully...""")
return {source_file}, set()
# Note: We need to apply commands to headers and sources.
# Why? clangd currently tries to infer commands for headers using files with similar paths. This often works really poorly for header-only libraries. The commands should instead have been inferred from the source files using those libraries... See https://github.com/clangd/clangd/issues/123 for more.
# When that issue is resolved, we can stop looking for headers and just return the single source file.
# Assembly sources that are not preprocessed can't include headers
if os.path.splitext(source_file)[1] in _get_files.assembly_source_extensions:
return {source_file}, set()
header_files = _get_headers(compile_action, source_file)
# Ambiguous .h headers need a language specified if they aren't C, or clangd sometimes makes mistakes
# Delete this and unused extension variables when clangd >= 16 is released, since their underlying issues are resolved at HEAD
# Reference issues:
# https://github.com/clangd/clangd/issues/1173
# https://github.com/clangd/clangd/issues/1263
if (any(header_file.endswith('.h') for header_file in header_files)
and not source_file.endswith(_get_files.c_source_extensions)
and not any(arg.startswith('-x') or arg.startswith('--language') or arg.lower() in ('-objc', '-objc++', '/tc', '/tp') for arg in compile_action.arguments)):
if compile_action.arguments[0].endswith('cl.exe'): # cl.exe and also clang-cl.exe
lang_flag = '/TP' # https://docs.microsoft.com/en-us/cpp/build/reference/tc-tp-tc-tp-specify-source-file-type
else:
lang_flag = _get_files.extensions_to_language_args[os.path.splitext(source_file)[1]]
# Insert at front of (non executable) args, because --language is only supposed to take effect on files listed thereafter
compile_action.arguments.insert(1, lang_flag)
return {source_file}, header_files
_get_files.has_logged_missing_file_error = False
# Setup extensions and flags for the whole C-language family.
# Clang has a list: https://github.com/llvm/llvm-project/blob/b9f3b7f89a4cb4cf541b7116d9389c73690f78fa/clang/lib/Driver/Types.cpp#L293
_get_files.c_source_extensions = ('.c', '.i')
_get_files.cpp_source_extensions = ('.cc', '.cpp', '.cxx', '.c++', '.C', '.CC', '.cp', '.CPP', '.C++', '.CXX', '.ii')
_get_files.objc_source_extensions = ('.m',)
_get_files.objcpp_source_extensions = ('.mm', '.M')
_get_files.cuda_source_extensions = ('.cu', '.cui')
_get_files.opencl_source_extensions = ('.cl',)
_get_files.openclxx_source_extensions = ('.clcpp',)
_get_files.assembly_source_extensions = ('.s', '.asm')
_get_files.assembly_needing_c_preprocessor_source_extensions = ('.S',)
_get_files.source_extensions = _get_files.c_source_extensions + _get_files.cpp_source_extensions + _get_files.objc_source_extensions + _get_files.objcpp_source_extensions + _get_files.cuda_source_extensions + _get_files.opencl_source_extensions + _get_files.openclxx_source_extensions + _get_files.assembly_source_extensions + _get_files.assembly_needing_c_preprocessor_source_extensions
_get_files.extensions_to_language_args = { # Note that clangd fails on the --language or -ObjC or -ObjC++ forms. See https://github.com/clangd/clangd/issues/1173#issuecomment-1226847416
_get_files.c_source_extensions: '-xc',
_get_files.cpp_source_extensions: '-xc++',
_get_files.objc_source_extensions: '-xobjective-c',
_get_files.objcpp_source_extensions: '-xobjective-c++',
_get_files.cuda_source_extensions: '-xcuda',
_get_files.opencl_source_extensions: '-xcl',
_get_files.openclxx_source_extensions: '-xclcpp',
_get_files.assembly_source_extensions: '-xassembler',
_get_files.assembly_needing_c_preprocessor_source_extensions: '-xassembler-with-cpp',
}
_get_files.extensions_to_language_args = {ext : flag for exts, flag in _get_files.extensions_to_language_args.items() for ext in exts} # Flatten map for easier use
@functools.lru_cache(maxsize=None)
def _get_apple_SDKROOT(SDK_name: str):
"""Get path to xcode-select'd root for the given OS."""
SDKROOT_maybe_versioned = subprocess.check_output(
('xcrun', '--show-sdk-path', '-sdk', SDK_name.lower()),
stderr=subprocess.DEVNULL,
encoding=locale.getpreferredencoding()
).rstrip()
# Unless xcode-select has been invoked (like for a beta) we'd expect, e.g., '/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS<version>.sdk' or '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk'.
version = subprocess.check_output(
('xcrun', '--show-sdk-version', '-sdk', SDK_name.lower()),
stderr=subprocess.DEVNULL,
encoding=locale.getpreferredencoding()
).rstrip()
return SDKROOT_maybe_versioned.replace(version, '') # Strip version and use unversioned SDK symlink so the compile commands are still valid after an SDK update.
# Traditionally stored in SDKROOT environment variable, but not provided by Bazel. See https://github.com/bazelbuild/bazel/issues/12852
def _get_apple_platform(compile_args: typing.List[str]):
"""Figure out which Apple platform a command is for.
Is the name used by Xcode in the SDK files, not the marketing name.
e.g. iPhoneOS, not iOS.
"""
# A bit gross, but Bazel specifies the platform name in one of the include paths, so we mine it from there.
for arg in compile_args:
match = re.search('/Platforms/([a-zA-Z]+).platform/Developer/', arg)
if match:
return match.group(1)
return None
@functools.lru_cache(maxsize=None)
def _get_apple_DEVELOPER_DIR():
"""Get path to xcode-select'd developer directory."""
return subprocess.check_output(('xcode-select', '--print-path'), encoding=locale.getpreferredencoding()).rstrip()
# Unless xcode-select has been invoked (like for a beta) we'd expect, e.g., '/Applications/Xcode.app/Contents/Developer' or '/Library/Developer/CommandLineTools'.
# Traditionally stored in DEVELOPER_DIR environment variable, but not provided by Bazel. See https://github.com/bazelbuild/bazel/issues/12852
def _apple_platform_patch(compile_args: typing.List[str]):
"""De-Bazel the command into something clangd can parse.
This function has fixes specific to Apple platforms, but you should call it on all platforms. It'll determine whether the fixes should be applied or not.
"""
# Bazel internal environment variable fragment that distinguishes Apple platforms that need unwrapping.
# Note that this occurs in the Xcode-installed wrapper, but not the CommandLineTools wrapper, which works fine as is.
if any('__BAZEL_XCODE_' in arg for arg in compile_args):
# Undo Bazel's Apple platform compiler wrapping.
# Bazel wraps the compiler as `external/local_config_cc/wrapped_clang` and exports that wrapped compiler in the proto. However, we need a clang call that clangd can introspect. (See notes in "how clangd uses compile_commands.json" in ImplementationReadme.md for more.)
# Removing the wrapper is also important because Bazel's Xcode (but not CommandLineTools) wrapper crashes if you don't specify particular environment variables (replaced below). We'd need the wrapper to be invokable by clangd's --query-driver if we didn't remove the wrapper.
compile_args[0] = 'clang'
# We have to manually substitute out Bazel's macros so clang can parse the command
# Code this mirrors is in https://github.com/bazelbuild/bazel/blob/master/tools/osx/crosstool/wrapped_clang.cc
# Not complete--we're just swapping out the essentials, because there seems to be considerable turnover in the hacks they have in the wrapper.
compile_args = [arg for arg in compile_args if not arg.startswith('DEBUG_PREFIX_MAP_PWD') or arg == 'OSO_PREFIX_MAP_PWD'] # No need for debug prefix maps if compiling in place, not that we're compiling anyway.
# We also have to manually figure out the values of SDKROOT and DEVELOPER_DIR, since they're missing from the environment variables Bazel provides.
# Filed Bazel issue about the missing environment variables: https://github.com/bazelbuild/bazel/issues/12852
compile_args = [arg.replace('__BAZEL_XCODE_DEVELOPER_DIR__', _get_apple_DEVELOPER_DIR()) for arg in compile_args]
apple_platform = _get_apple_platform(compile_args)
assert apple_platform, f"Apple platform not detected in CMD: {compile_args}"
compile_args = [arg.replace('__BAZEL_XCODE_SDKROOT__', _get_apple_SDKROOT(apple_platform)) for arg in compile_args]
return compile_args
def _emscripten_platform_patch(compile_action):
"""De-Bazel the command into something clangd can parse.
This function has fixes specific to Emscripten platforms, but you should call it on all platforms. It'll determine whether the fixes should be applied or not
"""
emcc_driver = pathlib.Path(compile_action.arguments[0])
if not emcc_driver.name.startswith('emcc'):
return compile_action.arguments
workspace_absolute = pathlib.PurePath(os.environ["BUILD_WORKSPACE_DIRECTORY"])
def _get_sysroot(args: typing.List[str]):
"""Get path to sysroot from command line arguments."""
for idx, arg in enumerate(args):
if arg == '--sysroot' or arg == '-isysroot':
if idx + 1 < len(args):
return pathlib.PurePath(args[idx + 1])
elif arg.startswith('--sysroot='):
return pathlib.PurePath(arg[len('--sysroot='):])
elif arg.startswith('-isysroot'):
return pathlib.PurePath(arg[len('-isysroot'):])
return None
def get_workspace_root(path_from_execroot: pathlib.PurePath):
if path_from_execroot.parts[0] != 'external':
return pathlib.PurePath('.')
return pathlib.PurePath('external') / path_from_execroot.parts[1]
environment = compile_action.environmentVariables.copy()
environment['EXT_BUILD_ROOT'] = str(workspace_absolute)
environment['EMCC_SKIP_SANITY_CHECK'] = '1'
environment['EM_COMPILER_WRAPPER'] = str(pathlib.PurePath({print_args_executable}))
if 'EM_BIN_PATH' not in environment:
sysroot = _get_sysroot(compile_action.arguments)
assert sysroot, f'Emscripten sysroot not detected in CMD: {compile_action.arguments}'
environment['EM_BIN_PATH'] = str(get_workspace_root(sysroot))
if 'EM_CONFIG_PATH' not in environment:
environment['EM_CONFIG_PATH'] = str(get_workspace_root(emcc_driver) / 'emscripten_toolchain' / 'emscripten_config')
# We run the emcc process with the environment variable EM_COMPILER_WRAPPER to intercept the command line arguments passed to `clang`.
emcc_process = subprocess.run(
# On Windows, it fails to spawn the subprocess when the path uses forward slashes as a separator.
# Here, we convert emcc driver path to use the native path separator.
[str(emcc_driver)] + compile_action.arguments[1:],
# MIN_PY=3.7: Replace PIPEs with capture_output.
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=environment,
encoding=locale.getpreferredencoding(),
check=False, # We explicitly ignore errors and carry on.
)
lines = emcc_process.stdout.splitlines()
# Parse the arguments from the output of the emcc process.
if BEGIN_ARGS_MARKER in lines:
begin_args_idx = lines.index(BEGIN_ARGS_MARKER)
end_args_idx = lines.index(END_ARGS_MARKER, begin_args_idx + 1)
args = lines[begin_args_idx + 1:end_args_idx]
clang_driver = pathlib.PurePath(args[0])
if _is_relative_to(clang_driver, workspace_absolute):
args[0] = clang_driver.relative_to(workspace_absolute).as_posix()
return args
assert False, f'Failed to parse emcc output: {emcc_process.stderr}'
BEGIN_ARGS_MARKER = '===HEDRON_COMPILE_COMMANDS_BEGIN_ARGS==='
END_ARGS_MARKER = '===HEDRON_COMPILE_COMMANDS_END_ARGS==='
def _all_platform_patch(compile_args: typing.List[str]):
"""Apply de-Bazeling fixes to the compile command that are shared across target platforms."""
# clangd writes module cache files to the wrong place
# Without this fix, you get tons of module caches dumped into the VSCode root folder.
# Filed clangd issue at: https://github.com/clangd/clangd/issues/655
# Seems to have disappeared when we switched to aquery from action_listeners, but we'll leave it in until the bug is patched in case we start using C++ modules
compile_args = (arg for arg in compile_args if not arg.startswith('-fmodules-cache-path=bazel-out/'))
# We're transfering the commands as though they were compiled in place in the workspace; no need for prefix maps, so we'll remove them. This eliminates some postentially confusing Bazel variables, though I think clangd just ignores them anyway.
# Some example:
# -fdebug-prefix-map=__BAZEL_EXECUTION_ROOT__=.
# -fdebug-prefix-map=__BAZEL_XCODE_DEVELOPER_DIR__=/PLACEHOLDER_DEVELOPER_DIR
compile_args = (arg for arg in compile_args if not arg.startswith('-fdebug-prefix-map'))
# When Bazel builds with gcc it adds -fno-canonical-system-headers to the command line, which clang tooling chokes on, since it does not understand this flag.
# We'll remove this flag, until such time as clangd & clang-tidy gracefully ignore it. Tracking issues: https://github.com/clangd/clangd/issues/1004 and https://github.com/llvm/llvm-project/issues/61699.
# For more context see: https://github.com/hedronvision/bazel-compile-commands-extractor/issues/21
compile_args = (arg for arg in compile_args if not arg == '-fno-canonical-system-headers')
# Strip out -gcc-toolchain to work around https://github.com/clangd/clangd/issues/1248
skip_next = False
new_compile_args = []
for arg in compile_args:
if arg.startswith('-gcc-toolchain'):
if len(arg) == len('-gcc-toolchain'):
skip_next = True
elif skip_next:
skip_next = False
else:
new_compile_args.append(arg)
compile_args = new_compile_args
# Discover compilers that are actually symlinks to ccache--and replace them with the underlying compiler
if os.path.islink(compile_args[0]):
compiler_path = os.readlink(compile_args[0]) # MIN_PY=3.9 Switch to pathlib path.readlink()
if os.path.basename(compiler_path) == "ccache":
compiler = os.path.basename(compile_args[0])
real_compiler_path = shutil.which(compiler)
if real_compiler_path:
compile_args[0] = real_compiler_path
# Any other general fixes would go here...
return compile_args
def _nvcc_patch(compile_args: typing.List[str]) -> typing.List[str]:
"""Apply fixes to args to nvcc.
Basically remove everything that's an nvcc arg that is not also a clang arg, converting what we can.
"""
# Reference: https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#nvcc-command-options
if not _is_nvcc(compile_args[0]):
return compile_args
new_compile_args = [compile_args[0],
# Make clangd's behavior closer to nvcc's.
# Will become the default in clangd 17: https://github.com/llvm/llvm-project/commit/0ad5d40fa19f27db0e5f999d0e17b0c18b811019
'-Xclang', '-fcuda-allow-variadic-functions']
skip_next = True # skip the compile_args[0] which we added above
for arg in compile_args:
if skip_next:
skip_next = False
continue
if arg in _nvcc_flags_to_skip_no_arg:
continue
skip = False
for flag_with_arg in _nvcc_flags_to_skip_with_arg:
if arg == flag_with_arg:
skip = True
skip_next = True
break
if len(flag_with_arg) > 2: # NVCC allows all one-character flags to have no = or space separator before their arguments
flag_with_arg += '='
if arg.startswith(flag_with_arg):
skip = True
break
if skip:
continue
option, did_partition, remainder = arg.partition('=')
rewrite_to = _nvcc_rewrite_flags.get(option) # Handles =value. Note: Note assumes no rewritten args are one character with a value that might take the -I<value> short form. <TAG_REWRITE>
if rewrite_to:
if did_partition:
arg = rewrite_to + '=' + remainder
else:
arg = rewrite_to
if ',' in arg: # Unpack NVCC's (fairly unique) comma-separated list format
if arg.startswith('-'):
option, did_partition, remainder = arg.partition('=')
if did_partition and not arg.startswith('-D'): # -D has a special -D=name=value case.
arg = remainder
else: # Must have been a one-character flag to have no = separator
option = arg[:2]
arg = arg[2:]
else:
option = new_compile_args.pop()
for list_item in arg.split(','):
new_compile_args.append(option)
new_compile_args.append(list_item)
else:
new_compile_args.append(arg)
return new_compile_args
# Generated via nvcc_clang_diff.py. Consider making use of it if you need to update this!
_nvcc_flags_to_skip_no_arg = {
# long name, short name
'--Wdefault-stream-launch', '-Wdefault-stream-launch',
'--Wext-lambda-captures-this', '-Wext-lambda-captures-this',
'--Wmissing-launch-bounds', '-Wmissing-launch-bounds',
'--Wno-deprecated-gpu-targets', '-Wno-deprecated-gpu-targets',
'--allow-unsupported-compiler', '-allow-unsupported-compiler',
'--augment-host-linker-script', '-aug-hls',
'--clean-targets', '-clean',
'--compile-as-tools-patch', '-astoolspatch',
'--cubin', '-cubin',
'--cuda', '-cuda',
'--device-c', '-dc',
'--device-link', '-dlink',
'--device-w', '-dw',
'--display-error-number', '-err-no',
'--dlink-time-opt', '-dlto',
'--dont-use-profile', '-noprof',
'--dryrun', '-dryrun',
'--expt-extended-lambda', '-expt-extended-lambda',
'--expt-relaxed-constexpr', '-expt-relaxed-constexpr',
'--extended-lambda', '-extended-lambda',
'--extensible-whole-program', '-ewp',
'--extra-device-vectorization', '-extra-device-vectorization',
'--fatbin', '-fatbin',
'--forward-unknown-opts', '-forward-unknown-opts',
'--forward-unknown-to-host-compiler', '-forward-unknown-to-host-compiler',
'--forward-unknown-to-host-linker', '-forward-unknown-to-host-linker', # Not handled quite right--but very hard to handle fully. If patching, consider adding a special case prepending '-Wl,', making sure to not trigger the list expander.
'--gen-opt-lto', '-gen-opt-lto',
'--generate-line-info', '-lineinfo',
'--host-relocatable-link', '-r',
'--keep', '-keep',
'--keep-device-functions', '-keep-device-functions',
'--lib', '-lib',
'--link', '-link',
'--list-gpu-arch', '-arch-ls',
'--list-gpu-code', '-code-ls',
'--lto', '-lto',
'--no-align-double', '--no-align-double',
'--no-compress', '-no-compress',
'--no-device-link', '-nodlink',