diff --git a/.github/problem-matcher.json b/.github/problem-matcher.json new file mode 100644 index 0000000000..4454b6cbea --- /dev/null +++ b/.github/problem-matcher.json @@ -0,0 +1,16 @@ +{ + "problemMatcher": [ + { + "owner": "iwyu-matcher", + "pattern": [ + { + "regexp": "^(.*):(\\d+):(\\d+):\\s*(.*)$", + "file": 1, + "line": 2, + "column": 3, + "message": 4 + } + ] + } + ] + } \ No newline at end of file diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index e674f58d03..bde78fb07f 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -28,7 +28,9 @@ jobs: ctest --test-dir cmake-build-unit-tests -j4 - name: Install lcov - run: sudo apt install lcov + run: | + sudo apt-get update + sudo apt-get install -y lcov # no-external command excludes the system libraries in the root directory - name: Capture code coverage @@ -36,6 +38,7 @@ jobs: echo "Capturing code coverage..." lcov --no-external --capture --directory . \ --output-file cmake-build-unit-tests/coverage_unfiltered.info + geninfo --ignore-errors mismatch - name: Filter out 3rd party and mock files run: | diff --git a/.github/workflows/iwyu.yml b/.github/workflows/iwyu.yml new file mode 100644 index 0000000000..4235018ed4 --- /dev/null +++ b/.github/workflows/iwyu.yml @@ -0,0 +1,61 @@ +name: IWYU Analysis + +on: [push, pull_request, workflow_dispatch] + + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + + - name: Setup cmake + uses: jwlawson/actions-setup-cmake@v2 + with: + cmake-version: '3.22.x' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + + - name: Install iwyu + run: | + sudo apt-get update + sudo apt-get install -y iwyu + + - name: Register problem matcher + run: echo "::add-matcher::./.github/problem-matcher.json" + + - name: Export CXXFLAGS + + #Since Clang 17 is installed in usr/bin, it will search for built-ins in /usr/lib/clang/17/include. + #The purpose of this step is to help the clang toolchain used by IWYU to locate the correct system + #header directories. + + run: | + cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON executables/referenceApp + export CXXFLAGS="-isystem /usr/lib/llvm-17/lib/clang/17/include/" + echo "done exporting" + python3 iwyu_tool.py -p compile_commands.json 2>&1 > iwyu_output.txt | tee iwyu_output.txt + echo "Output in iwyu_output.txt" + + - name: Print IWYU output + run: cat iwyu_output.txt + + - name: Unregister problem matcher + if: always() + run: echo "::remove-matcher owner=iwyu-matcher::" + + - name: Upload IWYU output + uses: actions/upload-artifact@v4 + with: + name: iwyu_output + path: iwyu_output.txt diff --git a/iwyu_tool.py b/iwyu_tool.py new file mode 100755 index 0000000000..18cf5ff527 --- /dev/null +++ b/iwyu_tool.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 + +##===--- iwyu_tool.py -----------------------------------------------------===## +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +##===----------------------------------------------------------------------===## + +""" Driver to consume a Clang compilation database and invoke IWYU. + +Example usage with CMake: + + # Unix systems + $ mkdir build && cd build + $ CC="clang" CXX="clang++" cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ... + $ iwyu_tool.py -p . + + # Windows systems + $ mkdir build && cd build + $ cmake -DCMAKE_CXX_COMPILER="%VCINSTALLDIR%/bin/cl.exe" \ + -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -G Ninja ... + $ python3 iwyu_tool.py -p . + +See iwyu_tool.py -h for more details on command-line arguments. +""" +from __future__ import print_function +import os +import re +import sys +import json +import time +import shlex +import shutil +import argparse +import tempfile +import subprocess + + +CORRECT_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$') +SHOULD_ADD_RE = re.compile(r'^(.*?) should add these lines:$') +ADD_RE = re.compile('^(.*?) +// (.*)$') +SHOULD_REMOVE_RE = re.compile(r'^(.*?) should remove these lines:$') +FULL_LIST_RE = re.compile(r'The full include-list for (.*?):$') +END_RE = re.compile(r'^---$') +LINES_RE = re.compile(r'^- (.*?) // lines ([0-9]+)-[0-9]+$') + + +GENERAL, ADD, REMOVE, LIST = range(4) + + +def clang_formatter(output, style): + """ Process iwyu's output into something clang-like. """ + formatted = [] + + state = (GENERAL, None) + for line in output.splitlines(): + match = CORRECT_RE.match(line) + if match: + formatted.append('%s:1:1: note: #includes/fwd-decls are correct' % + match.groups(1)) + continue + match = SHOULD_ADD_RE.match(line) + if match: + state = (ADD, match.group(1)) + continue + match = SHOULD_REMOVE_RE.match(line) + if match: + state = (REMOVE, match.group(1)) + continue + match = FULL_LIST_RE.match(line) + if match: + state = (LIST, match.group(1)) + elif END_RE.match(line): + state = (GENERAL, None) + elif not line.strip(): + continue + elif state[0] == GENERAL: + formatted.append(line) + elif state[0] == ADD: + match = ADD_RE.match(line) + if match: + formatted.append("%s:1:1: %s: add '%s' (%s)" % + (state[1], + style, + match.group(1), + match.group(2))) + else: + formatted.append("%s:1:1: %s: add '%s'" % + (state[1], style, line)) + elif state[0] == REMOVE: + match = LINES_RE.match(line) + line_no = match.group(2) if match else '1' + formatted.append("%s:%s:1: %s: superfluous '%s'" % + (state[1], line_no, style, match.group(1))) + + return os.linesep.join(formatted) + + +DEFAULT_FORMAT = 'iwyu' +FORMATTERS = { + 'iwyu': lambda output: output, + 'clang': lambda output: clang_formatter(output, style="error"), + 'clang-warning': lambda output: clang_formatter(output, style="warning"), +} + + +if sys.platform.startswith('win'): + # Case-insensitive match on Windows + def normcase(s): + return s.lower() +else: + def normcase(s): + return s + + +def is_subpath_of(path, parent): + """ Return True if path is equal to or fully contained within parent. + + Assumes both paths are canonicalized with os.path.realpath. + """ + parent = normcase(parent) + path = normcase(path) + + if path == parent: + return True + + if not path.startswith(parent): + return False + + # Now we know parent is a prefix of path, but they only share lineage if the + # difference between them starts with a path separator, e.g. /a/b/c/file + # is not a parent of /a/b/c/file.cpp, but /a/b/c and /a/b/c/ are. + parent = parent.rstrip(os.path.sep) + suffix = path[len(parent):] + return suffix.startswith(os.path.sep) + + +def is_msvc_driver(compile_command): + """ Return True if compile_command matches an MSVC CL-style driver. """ + compile_command = normcase(compile_command) + + if compile_command.endswith('cl.exe'): + # Native MSVC compiler or clang-cl.exe + return True + + if compile_command.endswith('clang-cl'): + # Cross clang-cl on non-Windows + return True + + return False + + +def win_split(cmdline): + """ Minimal implementation of shlex.split for Windows following + https://msdn.microsoft.com/en-us/library/windows/desktop/17w5ykft.aspx. + """ + def split_iter(cmdline): + in_quotes = False + backslashes = 0 + arg = '' + for c in cmdline: + if c == '\\': + # MSDN: Backslashes are interpreted literally, unless they + # immediately precede a double quotation mark. + # Buffer them until we know what comes next. + backslashes += 1 + elif c == '"': + # Quotes can either be an escaped quote or the start of a quoted + # string. Paraphrasing MSDN: + # Before quotes, place one backslash in the arg for every pair + # of leading backslashes. If the number of backslashes is odd, + # retain the double quotation mark, otherwise interpret it as a + # string delimiter and switch state. + arg += '\\' * (backslashes // 2) + if backslashes % 2 == 1: + arg += c + else: + in_quotes = not in_quotes + backslashes = 0 + elif c in (' ', '\t') and not in_quotes: + # MSDN: Arguments are delimited by white space, which is either + # a space or a tab [but only outside of a string]. + # Flush any buffered backslashes and yield arg, unless empty. + arg += '\\' * backslashes + if arg: + yield arg + arg = '' + backslashes = 0 + else: + # Flush buffered backslashes and append. + arg += '\\' * backslashes + arg += c + backslashes = 0 + + if arg: + arg += '\\' * backslashes + yield arg + + return list(split_iter(cmdline)) + + +def split_command(cmdstr): + """ Split a command string into a list, respecting shell quoting. """ + if sys.platform.startswith('win'): + # shlex.split does not work for Windows command-lines, so special-case + # to our own implementation. + cmd = win_split(cmdstr) + else: + cmd = shlex.split(cmdstr) + + return cmd + + +def find_include_what_you_use(): + """ Find IWYU executable and return its full pathname. """ + env_iwyu_path = os.environ.get('IWYU_BINARY') + if env_iwyu_path: + return os.path.realpath(env_iwyu_path) + + # Search in same dir as this script. + iwyu_path = shutil.which('include-what-you-use', + path=os.path.dirname(__file__)) + if iwyu_path: + return os.path.realpath(iwyu_path) + + # Search the system PATH. + iwyu_path = shutil.which('include-what-you-use') + if iwyu_path: + return os.path.realpath(iwyu_path) + + return None + + +IWYU_EXECUTABLE = find_include_what_you_use() + + +class Process(object): + """ Manages an IWYU process in flight """ + def __init__(self, proc, outfile): + self.proc = proc + self.outfile = outfile + self.output = None + + def poll(self): + """ Return the exit code if the process has completed, None otherwise. + """ + return self.proc.poll() + + @property + def returncode(self): + return self.proc.returncode + + def get_output(self): + """ Return stdout+stderr output of the process. + + This call blocks until the process is complete, then returns the output. + """ + if not self.output: + self.proc.wait() + self.outfile.seek(0) + self.output = self.outfile.read().decode("utf-8") + self.outfile.close() + + return self.output + + @classmethod + def start(cls, invocation): + """ Start a Process for the invocation and capture stdout+stderr. """ + outfile = tempfile.TemporaryFile(prefix='iwyu') + process = subprocess.Popen( + invocation.command, + cwd=invocation.cwd, + stdout=outfile, + stderr=subprocess.STDOUT) + return cls(process, outfile) + + +KNOWN_COMPILER_WRAPPERS=frozenset([ + "ccache" +]) + + +class Invocation(object): + """ Holds arguments of an IWYU invocation. """ + def __init__(self, command, cwd): + self.command = command + self.cwd = cwd + + def __str__(self): + return ' '.join(self.command) + + @classmethod + def from_compile_command(cls, entry, extra_args): + """ Parse a JSON compilation database entry into new Invocation. """ + if 'arguments' in entry: + # arguments is a command-line in list form. + command = entry['arguments'] + elif 'command' in entry: + # command is a command-line in string form, split to list. + command = split_command(entry['command']) + else: + raise ValueError('Invalid compilation database entry: %s' % entry) + + if command[0] in KNOWN_COMPILER_WRAPPERS: + # Remove the compiler wrapper from the command. + command = command[1:] + + # Rewrite the compile command for IWYU + compile_command, compile_args = command[0], command[1:] + if is_msvc_driver(compile_command): + # If the compiler is cl-compatible, let IWYU be cl-compatible. + extra_args = ['--driver-mode=cl'] + extra_args + + command = [IWYU_EXECUTABLE] + extra_args + compile_args + return cls(command, entry['directory']) + + def start(self, verbose): + """ Run invocation and collect output. """ + if verbose: + print('# %s' % self, file=sys.stderr) + + return Process.start(self) + + +def fixup_compilation_db(compilation_db): + """ Canonicalize paths in JSON compilation database. """ + for entry in compilation_db: + # Convert relative paths to absolute ones if possible, based on the entry's directory. + if 'directory' in entry and not os.path.isabs(entry['file']): + entry['file'] = os.path.join(entry['directory'], entry['file']) + + # Expand relative paths and symlinks + entry['file'] = os.path.realpath(entry['file']) + + return compilation_db + + +def slice_compilation_db(compilation_db, selection): + """ Return a new compilation database reduced to the paths in selection. """ + if not selection: + return compilation_db + + # Canonicalize selection paths to match compilation database. + selection = [os.path.realpath(p) for p in selection] + + new_db = [] + for path in selection: + if not os.path.exists(path): + print('warning: \'%s\' not found on disk.' % path, file=sys.stderr) + continue + + found = [e for e in compilation_db if is_subpath_of(e['file'], path)] + if not found: + print('warning: \'%s\' not found in compilation database.' % path, + file=sys.stderr) + continue + + new_db.extend(found) + + return new_db + + +def worst_exit_code(worst, cur): + """Return the most extreme exit code of two. + + Negative exit codes occur if the program exits due to a signal (Unix) or + structured exception (Windows). If we've seen a negative one before, keep + it, as it usually indicates a critical error. + + Otherwise return the biggest positive exit code. + """ + if cur < 0: + # Negative results take precedence, return the minimum + return min(worst, cur) + elif worst < 0: + # We know cur is non-negative, negative worst must be minimum + return worst + else: + # We know neither are negative, return the maximum + return max(worst, cur) + + +def execute(invocations, verbose, formatter, jobs, max_load_average=0): + """ Launch processes described by invocations. """ + exit_code = 0 + if jobs == 1: + for invocation in invocations: + proc = invocation.start(verbose) + print(formatter(proc.get_output())) + exit_code = worst_exit_code(exit_code, proc.returncode) + return exit_code + + pending = [] + while invocations or pending: + # Collect completed IWYU processes and print results. + complete = [proc for proc in pending if proc.poll() is not None] + for proc in complete: + pending.remove(proc) + print(formatter(proc.get_output())) + exit_code = worst_exit_code(exit_code, proc.returncode) + + # Schedule new processes if there's room. + capacity = jobs - len(pending) + + if max_load_average > 0: + one_min_load_average, _, _ = os.getloadavg() + load_capacity = max_load_average - one_min_load_average + if load_capacity < 0: + load_capacity = 0 + if load_capacity < capacity: + capacity = int(load_capacity) + if not capacity and not pending: + # Ensure there is at least one job running. + capacity = 1 + + pending.extend(i.start(verbose) for i in invocations[:capacity]) + invocations = invocations[capacity:] + + # Yield CPU. + time.sleep(0.0001) + return exit_code + + +def main(compilation_db_path, source_files, verbose, formatter, jobs, + max_load_average, extra_args): + """ Entry point. """ + + if not IWYU_EXECUTABLE: + print('error: include-what-you-use executable not found', + file=sys.stderr) + return 1 + + try: + if os.path.isdir(compilation_db_path): + compilation_db_path = os.path.join(compilation_db_path, + 'compile_commands.json') + + # Read compilation db from disk. + compilation_db_path = os.path.realpath(compilation_db_path) + with open(compilation_db_path, 'r') as fileobj: + compilation_db = json.load(fileobj) + except IOError as why: + print('error: failed to parse compilation database: %s' % why, + file=sys.stderr) + return 1 + + compilation_db = fixup_compilation_db(compilation_db) + compilation_db = slice_compilation_db(compilation_db, source_files) + + # Transform compilation db entries into a list of IWYU invocations. + invocations = [ + Invocation.from_compile_command(e, extra_args) for e in compilation_db + ] + + return execute(invocations, verbose, formatter, jobs, max_load_average) + + +def _bootstrap(sys_argv): + """ Parse arguments and dispatch to main(). """ + + # This hackery is necessary to add the forwarded IWYU args to the + # usage and help strings. + def customize_usage(parser): + """ Rewrite the parser's format_usage. """ + original_format_usage = parser.format_usage + parser.format_usage = lambda: original_format_usage().rstrip() + \ + ' -- []' + os.linesep + + def customize_help(parser): + """ Rewrite the parser's format_help. """ + original_format_help = parser.format_help + + def custom_help(): + """ Customized help string, calls the adjusted format_usage. """ + helpmsg = original_format_help() + helplines = helpmsg.splitlines() + helplines[0] = parser.format_usage().rstrip() + return os.linesep.join(helplines) + os.linesep + + parser.format_help = custom_help + + # Parse arguments. + parser = argparse.ArgumentParser( + description='Include-what-you-use compilation database driver.', + epilog='Assumes include-what-you-use is available on the PATH.') + customize_usage(parser) + customize_help(parser) + + parser.add_argument('-v', '--verbose', action='store_true', + help='Print IWYU commands') + parser.add_argument('-o', '--output-format', type=str, + choices=FORMATTERS.keys(), default=DEFAULT_FORMAT, + help='Output format (default: %s)' % DEFAULT_FORMAT) + parser.add_argument('-j', '--jobs', type=int, default=1, + nargs='?', const=0, + help=('Number of concurrent subprocesses. If zero, ' + 'will try to match the logical cores of the ' + 'system.')) + parser.add_argument('-l', '--load', type=float, default=0, + help=('Do not start new jobs if the 1min load average ' + 'is greater than the provided value')) + parser.add_argument('-p', metavar='', required=True, + help='Compilation database path', dest='dbpath') + parser.add_argument('source', nargs='*', + help=('Zero or more source files (or directories) to ' + 'run IWYU on. Defaults to all in compilation ' + 'database.')) + + def partition_args(argv): + """ Split around '--' into driver args and IWYU args. """ + try: + double_dash = argv.index('--') + return argv[:double_dash], argv[double_dash+1:] + except ValueError: + return argv, [] + argv, extra_args = partition_args(sys_argv[1:]) + args = parser.parse_args(argv) + + jobs = args.jobs + if jobs == 0: + jobs = os.cpu_count() or 1 + + return main(args.dbpath, args.source, args.verbose, + FORMATTERS[args.output_format], jobs, args.load, extra_args) + + +if __name__ == '__main__': + sys.exit(_bootstrap(sys.argv))