Author: Alexandre Ganea Date: 2020-01-13T10:40:18-05:00 New Revision: b4a99a061f517e60985667e39519f60186cbb469
URL: https://github.com/llvm/llvm-project/commit/b4a99a061f517e60985667e39519f60186cbb469 DIFF: https://github.com/llvm/llvm-project/commit/b4a99a061f517e60985667e39519f60186cbb469.diff LOG: [Clang][Driver] Re-use the calling process instead of creating a new process for the cc1 invocation With this patch, the clang tool will now call the -cc1 invocation directly inside the same process. Previously, the -cc1 invocation was creating, and waiting for, a new process. This patch therefore reduces the number of created processes during a build, thus it reduces build times on platforms where process creation can be costly (Windows) and/or impacted by a antivirus. It also makes debugging a bit easier, as there's no need to attach to the secondary -cc1 process anymore, breakpoints will be hit inside the same process. Crashes or signaling inside the -cc1 invocation will have the same side-effect as before, and will be reported through the same means. This behavior can be controlled at compile-time through the CLANG_SPAWN_CC1 cmake flag, which defaults to OFF. Setting it to ON will revert to the previous behavior, where any -cc1 invocation will create/fork a secondary process. At run-time, it is also possible to tweak the CLANG_SPAWN_CC1 environment variable. Setting it and will override the compile-time setting. A value of 0 calls -cc1 inside the calling process; a value of 1 will create a secondary process, as before. Differential Revision: https://reviews.llvm.org/D69825 Added: clang/test/Driver/cc1-spawnprocess.c Modified: clang/CMakeLists.txt clang/include/clang/Config/config.h.cmake clang/include/clang/Driver/Driver.h clang/include/clang/Driver/Job.h clang/lib/Driver/Job.cpp clang/lib/Driver/ToolChains/Clang.cpp clang/test/CMakeLists.txt clang/test/Driver/clang_f_opts.c clang/test/Driver/fsanitize-blacklist.c clang/test/Driver/unknown-arg.c clang/test/Driver/warning-options_pedantic.cpp clang/tools/driver/driver.cpp Removed: ################################################################################ diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index bc172b468512..781c3eb7f2f2 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -237,6 +237,9 @@ set(ENABLE_X86_RELAX_RELOCATIONS OFF CACHE BOOL set(ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER FALSE CACHE BOOL "Enable the experimental new pass manager by default.") +set(CLANG_SPAWN_CC1 OFF CACHE BOOL + "Whether clang should use a new process for the CC1 invocation") + # TODO: verify the values against LangStandards.def? set(CLANG_DEFAULT_STD_C "" CACHE STRING "Default standard to use for C/ObjC code (IDENT from LangStandards.def, empty for platform default)") diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake index 68125dbc6d4b..261b3841b86f 100644 --- a/clang/include/clang/Config/config.h.cmake +++ b/clang/include/clang/Config/config.h.cmake @@ -80,4 +80,7 @@ #cmakedefine01 CLANG_ENABLE_OBJC_REWRITER #cmakedefine01 CLANG_ENABLE_STATIC_ANALYZER +/* Spawn a new process clang.exe for the CC1 tool invocation, when necessary */ +#cmakedefine01 CLANG_SPAWN_CC1 + #endif diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h index d667fcb1efef..fd25663bd358 100644 --- a/clang/include/clang/Driver/Driver.h +++ b/clang/include/clang/Driver/Driver.h @@ -204,6 +204,13 @@ class Driver { /// Whether the driver is generating diagnostics for debugging purposes. unsigned CCGenDiagnostics : 1; + /// Pointer to the ExecuteCC1Tool function, if available. + /// When the clangDriver lib is used through clang.exe, this provides a + /// shortcut for executing the -cc1 command-line directly, in the same + /// process. + typedef int (*CC1ToolFunc)(ArrayRef<const char *> argv); + CC1ToolFunc CC1Main = nullptr; + private: /// Raw target triple. std::string TargetTriple; diff --git a/clang/include/clang/Driver/Job.h b/clang/include/clang/Driver/Job.h index 41d972280852..0765b3c67d4e 100644 --- a/clang/include/clang/Driver/Job.h +++ b/clang/include/clang/Driver/Job.h @@ -119,7 +119,7 @@ class Command { /// \param NewEnvironment An array of environment variables. /// \remark If the environment remains unset, then the environment /// from the parent process will be used. - void setEnvironment(llvm::ArrayRef<const char *> NewEnvironment); + virtual void setEnvironment(llvm::ArrayRef<const char *> NewEnvironment); const char *getExecutable() const { return Executable; } @@ -130,6 +130,24 @@ class Command { /// Set whether to print the input filenames when executing. void setPrintInputFilenames(bool P) { PrintInputFilenames = P; } + +protected: + /// Optionally print the filenames to be compiled + void PrintFileNames() const; +}; + +/// Use the CC1 tool callback when available, to avoid creating a new process +class CC1Command : public Command { +public: + using Command::Command; + + void Print(llvm::raw_ostream &OS, const char *Terminator, bool Quote, + CrashReportInfo *CrashInfo = nullptr) const override; + + int Execute(ArrayRef<Optional<StringRef>> Redirects, std::string *ErrMsg, + bool *ExecutionFailed) const override; + + void setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) override; }; /// Like Command, but with a fallback which is executed in case diff --git a/clang/lib/Driver/Job.cpp b/clang/lib/Driver/Job.cpp index 0a95e49694f9..d57c3a1cdbb8 100644 --- a/clang/lib/Driver/Job.cpp +++ b/clang/lib/Driver/Job.cpp @@ -19,8 +19,10 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/Program.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> @@ -313,15 +315,46 @@ void Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { Environment.push_back(nullptr); } -int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, - std::string *ErrMsg, bool *ExecutionFailed) const { +void Command::PrintFileNames() const { if (PrintInputFilenames) { for (const char *Arg : InputFilenames) llvm::outs() << llvm::sys::path::filename(Arg) << "\n"; llvm::outs().flush(); } +} - SmallVector<const char*, 128> Argv; +int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, + std::string *ErrMsg, bool *ExecutionFailed) const { + PrintFileNames(); + + SmallVector<const char *, 128> Argv; + if (ResponseFile == nullptr) { + Argv.push_back(Executable); + Argv.append(Arguments.begin(), Arguments.end()); + Argv.push_back(nullptr); + } else { + // If the command is too large, we need to put arguments in a response file. + std::string RespContents; + llvm::raw_string_ostream SS(RespContents); + + // Write file contents and build the Argv vector + writeResponseFile(SS); + buildArgvForResponseFile(Argv); + Argv.push_back(nullptr); + SS.flush(); + + // Save the response file in the appropriate encoding + if (std::error_code EC = writeFileWithEncoding( + ResponseFile, RespContents, Creator.getResponseFileEncoding())) { + if (ErrMsg) + *ErrMsg = EC.message(); + if (ExecutionFailed) + *ExecutionFailed = true; + // Return -1 by convention (see llvm/include/llvm/Support/Program.h) to + // indicate the requested executable cannot be started. + return -1; + } + } Optional<ArrayRef<StringRef>> Env; std::vector<StringRef> ArgvVectorStorage; @@ -332,42 +365,51 @@ int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, Env = makeArrayRef(ArgvVectorStorage); } - if (ResponseFile == nullptr) { - Argv.push_back(Executable); - Argv.append(Arguments.begin(), Arguments.end()); - Argv.push_back(nullptr); + auto Args = llvm::toStringRefArray(Argv.data()); + return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects, + /*secondsToWait*/ 0, + /*memoryLimit*/ 0, ErrMsg, ExecutionFailed); +} - auto Args = llvm::toStringRefArray(Argv.data()); - return llvm::sys::ExecuteAndWait( - Executable, Args, Env, Redirects, /*secondsToWait*/ 0, - /*memoryLimit*/ 0, ErrMsg, ExecutionFailed); - } +void CC1Command::Print(raw_ostream &OS, const char *Terminator, bool Quote, + CrashReportInfo *CrashInfo) const { + OS << " (in-process)"; + Command::Print(OS, Terminator, Quote, CrashInfo); +} - // We need to put arguments in a response file (command is too large) - // Open stream to store the response file contents - std::string RespContents; - llvm::raw_string_ostream SS(RespContents); +int CC1Command::Execute(ArrayRef<llvm::Optional<StringRef>> /*Redirects*/, + std::string *ErrMsg, bool *ExecutionFailed) const { + PrintFileNames(); - // Write file contents and build the Argv vector - writeResponseFile(SS); - buildArgvForResponseFile(Argv); + SmallVector<const char *, 128> Argv; + Argv.push_back(getExecutable()); + Argv.append(getArguments().begin(), getArguments().end()); Argv.push_back(nullptr); - SS.flush(); - - // Save the response file in the appropriate encoding - if (std::error_code EC = writeFileWithEncoding( - ResponseFile, RespContents, Creator.getResponseFileEncoding())) { - if (ErrMsg) - *ErrMsg = EC.message(); - if (ExecutionFailed) - *ExecutionFailed = true; - return -1; + + // This flag simply indicates that the program couldn't start, which isn't + // applicable here. + if (ExecutionFailed) + *ExecutionFailed = false; + + llvm::CrashRecoveryContext CRC; + CRC.DumpStackAndCleanupOnFailure = true; + + const void *PrettyState = llvm::SavePrettyStackState(); + const Driver &D = getCreator().getToolChain().getDriver(); + + int R = 0; + // Enter ExecuteCC1Tool() instead of starting up a new process + if (!CRC.RunSafely([&]() { R = D.CC1Main(Argv); })) { + llvm::RestorePrettyStackState(PrettyState); + return CRC.RetCode; } + return R; +} - auto Args = llvm::toStringRefArray(Argv.data()); - return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects, - /*secondsToWait*/ 0, - /*memoryLimit*/ 0, ErrMsg, ExecutionFailed); +void CC1Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { + // We don't support set a new environment when calling into ExecuteCC1Tool() + llvm_unreachable( + "The CC1Command doesn't support changing the environment vars!"); } FallbackCommand::FallbackCommand(const Action &Source_, const Tool &Creator_, diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 4d924e072f57..04c401ff8801 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5979,6 +5979,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, // fails, so that the main compilation's fallback to cl.exe runs. C.addCommand(std::make_unique<ForceSuccessCommand>(JA, *this, Exec, CmdArgs, Inputs)); + } else if (D.CC1Main && !D.CCGenDiagnostics) { + // Invoke the CC1 directly in this process + C.addCommand( + std::make_unique<CC1Command>(JA, *this, Exec, CmdArgs, Inputs)); } else { C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs)); } diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt index 6f6121f06e56..2c6487e8c260 100644 --- a/clang/test/CMakeLists.txt +++ b/clang/test/CMakeLists.txt @@ -13,6 +13,7 @@ llvm_canonicalize_cmake_booleans( CLANG_BUILD_EXAMPLES CLANG_ENABLE_ARCMT CLANG_ENABLE_STATIC_ANALYZER + CLANG_SPAWN_CC1 ENABLE_BACKTRACES ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER LLVM_ENABLE_ZLIB diff --git a/clang/test/Driver/cc1-spawnprocess.c b/clang/test/Driver/cc1-spawnprocess.c new file mode 100644 index 000000000000..5b2129ad309e --- /dev/null +++ b/clang/test/Driver/cc1-spawnprocess.c @@ -0,0 +1,4 @@ +// RUN: env -u CLANG_SPAWN_CC1 %clang -c %s -o /dev/null +// RUN: env CLANG_SPAWN_CC1=0 %clang -c %s -o /dev/null +// RUN: env CLANG_SPAWN_CC1=1 %clang -c %s -o /dev/null +// RUN: env CLANG_SPAWN_CC1=test not %clang -c %s -o /dev/null diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c index 0a7cfdf11281..970b4e934e78 100644 --- a/clang/test/Driver/clang_f_opts.c +++ b/clang/test/Driver/clang_f_opts.c @@ -493,7 +493,7 @@ // RUN: %clang -target x86_64-unknown-none-none -### -fno-short-wchar -fshort-wchar %s 2>&1 | FileCheck -check-prefix=CHECK-WCHAR2 -check-prefix=DELIMITERS %s // Make sure we don't match the -NOT lines with the linker invocation. // Delimiters match the start of the cc1 and the start of the linker lines -// DELIMITERS: {{^ *"}} +// DELIMITERS: {{^ (\(in-process\)|")}} // CHECK-WCHAR1: -fwchar-type=int // CHECK-WCHAR1-NOT: -fwchar-type=short // CHECK-WCHAR2: -fwchar-type=short diff --git a/clang/test/Driver/fsanitize-blacklist.c b/clang/test/Driver/fsanitize-blacklist.c index 6878298e6752..2d3ef3f6eee7 100644 --- a/clang/test/Driver/fsanitize-blacklist.c +++ b/clang/test/Driver/fsanitize-blacklist.c @@ -6,7 +6,7 @@ // Make sure we don't match the -NOT lines with the linker invocation. // Delimiters match the start of the cc1 and the start of the linker lines // for fragile tests. -// DELIMITERS: {{^ *"}} +// DELIMITERS: {{^ (\(in-process\)|")}} // RUN: echo "fun:foo" > %t.good // RUN: echo "fun:bar" > %t.second diff --git a/clang/test/Driver/unknown-arg.c b/clang/test/Driver/unknown-arg.c index 074a57bb6f8e..0e3e5bf21440 100644 --- a/clang/test/Driver/unknown-arg.c +++ b/clang/test/Driver/unknown-arg.c @@ -54,7 +54,7 @@ // SILENT-NOT: warning: // CC1AS-DID-YOU-MEAN: error: unknown argument '-hell'; did you mean '-help'? // CC1AS-DID-YOU-MEAN: error: unknown argument '--version'; did you mean '-version'? -// UNKNOWN-INTEGRATED: error: unknown integrated tool 'asphalt'. Valid tools include '-cc1' and '-cc1as'. +// UNKNOWN-INTEGRATED: error: unknown integrated tool '-cc1asphalt'. Valid tools include '-cc1' and '-cc1as'. // RUN: %clang -S %s -o %t.s -Wunknown-to-clang-option 2>&1 | FileCheck --check-prefix=IGNORED %s diff --git a/clang/test/Driver/warning-options_pedantic.cpp b/clang/test/Driver/warning-options_pedantic.cpp index 4dbf92d12bac..f3c9969c6c51 100644 --- a/clang/test/Driver/warning-options_pedantic.cpp +++ b/clang/test/Driver/warning-options_pedantic.cpp @@ -1,6 +1,6 @@ // Make sure we don't match the -NOT lines with the linker invocation. // Delimiters match the start of the cc1 and the start of the linker lines -// DELIMITERS: {{^ *"}} +// DELIMITERS: {{^ (\(in-process\)|")}} // RUN: %clang -### -pedantic -no-pedantic %s 2>&1 | FileCheck -check-prefix=NO_PEDANTIC -check-prefix=DELIMITERS %s // RUN: %clang -### -pedantic -Wno-pedantic %s 2>&1 | FileCheck -check-prefix=PEDANTIC -check-prefix=DELIMITERS %s diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp index f1600490017d..4cdf8015b1bf 100644 --- a/clang/tools/driver/driver.cpp +++ b/clang/tools/driver/driver.cpp @@ -14,6 +14,7 @@ #include "clang/Driver/Driver.h" #include "clang/Basic/DiagnosticOptions.h" #include "clang/Basic/Stack.h" +#include "clang/Config/config.h" #include "clang/Driver/Compilation.h" #include "clang/Driver/DriverDiagnostic.h" #include "clang/Driver/Options.h" @@ -30,6 +31,7 @@ #include "llvm/Option/OptTable.h" #include "llvm/Option/Option.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Host.h" @@ -239,6 +241,8 @@ static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, *NumberSignPtr = '='; } +static int ExecuteCC1Tool(ArrayRef<const char *> argv); + static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); @@ -254,6 +258,27 @@ static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); if (TheDriver.CCLogDiagnostics) TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); + + // Whether the cc1 tool should be called inside the current process, or if we + // should spawn a new clang process (old behavior). + // Not having an additional process saves some execution time of Windows, + // and makes debugging easier. + bool UseNewCC1Process = CLANG_SPAWN_CC1; + + StringRef SpawnCC1Str = ::getenv("CLANG_SPAWN_CC1"); + if (!SpawnCC1Str.empty()) { + if (SpawnCC1Str != "0" && SpawnCC1Str != "1") { + llvm::errs() << "error: the value of the environment variable " + "CLANG_SPAWN_CC1 must be either 0 or 1.\n"; + ::exit(1); + } + UseNewCC1Process = SpawnCC1Str[0] - '0'; + } + if (!UseNewCC1Process) { + TheDriver.CC1Main = &ExecuteCC1Tool; + // Ensure the CC1Command actually catches cc1 crashes + llvm::CrashRecoveryContext::Enable(); + } } static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, @@ -303,13 +328,19 @@ static void SetInstallDir(SmallVectorImpl<const char *> &argv, TheDriver.setInstalledDir(InstalledPathParent); } -static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) { +static int ExecuteCC1Tool(ArrayRef<const char *> argv) { + // If we call the cc1 tool from the clangDriver library (through + // Driver::CC1Main), we need to cleanup the options usage count. The options + // are currently global, and they might have been used previously by the + // driver. + llvm::cl::ResetAllOptionOccurrences(); + StringRef Tool = argv[1]; void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") + if (Tool == "-cc1") return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") + if (Tool == "-cc1as") return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "gen-reproducer") + if (Tool == "-cc1gen-reproducer") return cc1gen_reproducer_main(argv.slice(2), argv[0], GetExecutablePathVP); // Reject unknown tools. @@ -379,7 +410,7 @@ int main(int argc_, const char **argv_) { auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); argv.resize(newEnd - argv.begin()); } - return ExecuteCC1Tool(argv, argv[1] + 4); + return ExecuteCC1Tool(argv); } bool CanonicalPrefixes = true; @@ -503,7 +534,7 @@ int main(int argc_, const char **argv_) { #ifdef _WIN32 // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be + // Once abnormal termination was caught, negative status should not be // propagated. if (Res < 0) Res = 1; _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
