Repository: kudu Updated Branches: refs/heads/master c25b7cbd8 -> a9271b05d
iwyu: run IWYU on multiple cores This changes iwyu_tool to use a number of parallel IWYU processes equal to the number of cores, resulting in a huge speedup for common dev workflows. Change-Id: I97206318943b408c6fdd5637c6ccb79ee4d87d3f Reviewed-on: http://gerrit.cloudera.org:8080/10104 Reviewed-by: Dan Burkert <danburk...@apache.org> Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/63216152 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/63216152 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/63216152 Branch: refs/heads/master Commit: 6321615289c4dd35eeb0ca85b74bf6be778251aa Parents: c25b7cb Author: Todd Lipcon <t...@apache.org> Authored: Wed Apr 18 12:30:42 2018 -0700 Committer: Todd Lipcon <t...@apache.org> Committed: Wed Apr 18 23:50:58 2018 +0000 ---------------------------------------------------------------------- build-support/iwyu/iwyu_tool.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/63216152/build-support/iwyu/iwyu_tool.py ---------------------------------------------------------------------- diff --git a/build-support/iwyu/iwyu_tool.py b/build-support/iwyu/iwyu_tool.py index 8943226..80bea51 100755 --- a/build-support/iwyu/iwyu_tool.py +++ b/build-support/iwyu/iwyu_tool.py @@ -70,12 +70,14 @@ Example usage with CMake: See iwyu_tool.py -h for more details on command-line arguments. """ -import os -import sys -import json import argparse -import subprocess +import json +import multiprocessing +from multiprocessing.pool import ThreadPool +import os import re +import subprocess +import sys def iwyu_formatter(output): @@ -144,7 +146,7 @@ def get_output(cwd, args): return process.communicate()[0].decode("utf-8").splitlines() -def run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter): +def run_iwyu(cwd, compile_command, iwyu_args, verbose): """ Rewrite compile_command to an IWYU command, and run it. """ compiler, _, args = compile_command.partition(' ') if compiler.endswith('cl.exe'): @@ -162,7 +164,10 @@ def run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter): if verbose: print('{}'.format(cmd_args)) - formatter(get_output(cwd, cmd_args)) + return get_output(cwd, cmd_args) + + + def main(compilation_db_path, source_files, verbose, formatter, iwyu_args): @@ -203,14 +208,21 @@ def main(compilation_db_path, source_files, verbose, formatter, iwyu_args): source) # Run analysis + def run_iwyu_task(entry): + cwd, compile_command = entry['directory'], entry['command'] + return run_iwyu(cwd, compile_command, iwyu_args, verbose) + pool = ThreadPool(multiprocessing.cpu_count()) try: - for entry in entries: - cwd, compile_command = entry['directory'], entry['command'] - run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter) + for iwyu_output in pool.imap_unordered(run_iwyu_task, entries): + formatter(iwyu_output) + except KeyboardInterrupt as ki: + sys.exit(1) except OSError as why: print('ERROR: Failed to launch include-what-you-use: %s' % why) return 1 - + finally: + pool.terminate() + pool.join() return 0
