cpsughrue created this revision.
cpsughrue added reviewers: jansvoboda11, Bigcheese.
Herald added a project: All.
cpsughrue requested review of this revision.
Herald added subscribers: cfe-commits, MaskRay.
Herald added a project: clang.
Initial commit for module build daemon. The title will be updated soon to
remove the [WIP] tag once I am done writing tests.
The goal of the commit: A source file that doesn't depend on any modules can
connect to the daemon, get scanned, find that there are no modules, then resume
building. If there are any modules in the dependency graph have the daemon emit
a warning that dependencies were detected but that the daemon cannot yet build
modules or update the cc1 command line.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D158496
Files:
clang/include/clang/Driver/Options.td
clang/include/clang/Frontend/FrontendOptions.h
clang/include/clang/Tooling/ModuleBuildDaemon/Protocol.h
clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
clang/lib/Driver/ToolChains/Clang.cpp
clang/lib/Tooling/CMakeLists.txt
clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
clang/lib/Tooling/ModuleBuildDaemon/Protocol.cpp
clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
clang/tools/driver/CMakeLists.txt
clang/tools/driver/cc1_main.cpp
clang/tools/driver/cc1modbuildd_main.cpp
clang/tools/driver/driver.cpp
Index: clang/tools/driver/driver.cpp
===================================================================
--- clang/tools/driver/driver.cpp
+++ clang/tools/driver/driver.cpp
@@ -213,6 +213,9 @@
extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
const char *Argv0, void *MainAddr,
const llvm::ToolContext &);
+#if LLVM_ON_UNIX
+extern int cc1modbuildd_main(ArrayRef<const char *> Argv);
+#endif
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
SmallVectorImpl<const char *> &ArgVector,
@@ -369,9 +372,20 @@
if (Tool == "-cc1gen-reproducer")
return cc1gen_reproducer_main(ArrayRef(ArgV).slice(2), ArgV[0],
GetExecutablePathVP, ToolContext);
- // Reject unknown tools.
+#if LLVM_ON_UNIX
+ if (Tool == "-cc1modbuildd")
+ return cc1modbuildd_main(ArrayRef(ArgV).slice(2));
+
+ // FIXME: Once cc1modbuildd becomes portable unify llvm::errs messages
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
- << "Valid tools include '-cc1' and '-cc1as'.\n";
+ << "Valid tools include '-cc1', '-cc1as', '-cc1gen-reproducer', "
+ << "and '-cc1modbuildd'.\n";
+#else
+ // Reject unknown tools
+ llvm::errs()
+ << "error: unknown integrated tool '" << Tool << "'. "
+ << "Valid tools include '-cc1', '-cc1as', and '-cc1gen-reproducer'.\n";
+#endif
return 1;
}
Index: clang/tools/driver/cc1modbuildd_main.cpp
===================================================================
--- /dev/null
+++ clang/tools/driver/cc1modbuildd_main.cpp
@@ -0,0 +1,265 @@
+//===------- cc1modbuildd_main.cpp - Clang CC1 Module Build Daemon --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/Protocol.h"
+#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/Threading.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
+
+// TODO: Make portable
+#if LLVM_ON_UNIX
+
+#include <errno.h>
+#include <fstream>
+#include <signal.h>
+#include <sstream>
+#include <stdbool.h>
+#include <string>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+using namespace llvm;
+using namespace clang;
+
+namespace {
+class ModuleBuildDaemonServer {
+public:
+ SmallString<128> BasePath;
+ SmallString<128> SocketPath;
+ SmallString<128> PidPath;
+
+ ModuleBuildDaemonServer(SmallString<128> Path)
+ : BasePath(Path), SocketPath(Path) {
+ llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
+ }
+
+ ~ModuleBuildDaemonServer() { Shutdown(SIGTERM); }
+
+ int Fork();
+ int Launch();
+ int Listen();
+ static llvm::Error Service(int Client);
+
+ void Shutdown(int signal) {
+ unlink(SocketPath.c_str());
+ shutdown(ListenSocketFD, SHUT_RD);
+ close(ListenSocketFD);
+ exit(EXIT_SUCCESS);
+ }
+
+private:
+ pid_t Pid = -1;
+ int ListenSocketFD = -1;
+};
+
+// Required to handle SIGTERM by calling Shutdown
+ModuleBuildDaemonServer *DaemonPtr = nullptr;
+void HandleSignal(int signal) {
+ if (DaemonPtr != nullptr) {
+ DaemonPtr->Shutdown(signal);
+ }
+}
+} // namespace
+
+// Forks and detaches process, creating module build daemon
+int ModuleBuildDaemonServer::Fork() {
+
+ pid_t pid = fork();
+
+ if (pid < 0) {
+ exit(EXIT_FAILURE);
+ }
+ if (pid > 0) {
+ exit(EXIT_SUCCESS);
+ }
+
+ Pid = getpid();
+
+ close(STDIN_FILENO);
+ close(STDOUT_FILENO);
+ close(STDERR_FILENO);
+
+ SmallString<128> STDOUT = BasePath;
+ llvm::sys::path::append(STDOUT, STDOUT_FILE_NAME);
+ freopen(STDOUT.c_str(), "a", stdout);
+
+ SmallString<128> STDERR = BasePath;
+ llvm::sys::path::append(STDERR, STDERR_FILE_NAME);
+ freopen(STDERR.c_str(), "a", stderr);
+
+ if (signal(SIGTERM, HandleSignal) == SIG_ERR) {
+ errs() << "failed to handle SIGTERM" << '\n';
+ exit(EXIT_FAILURE);
+ }
+ if (signal(SIGHUP, SIG_IGN) == SIG_ERR) {
+ errs() << "failed to ignore SIGHUP" << '\n';
+ exit(EXIT_FAILURE);
+ }
+ if (setsid() == -1) {
+ errs() << "setsid failed" << '\n';
+ exit(EXIT_FAILURE);
+ }
+
+ return EXIT_SUCCESS;
+}
+
+// Creates unix socket for IPC with module build daemon
+int ModuleBuildDaemonServer::Launch() {
+
+ // new socket
+ if ((ListenSocketFD = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
+ std::perror("Socket create error: ");
+ exit(EXIT_FAILURE);
+ }
+
+ struct sockaddr_un addr;
+ memset(&addr, 0, sizeof(struct sockaddr_un));
+ addr.sun_family = AF_UNIX;
+ strncpy(addr.sun_path, SocketPath.c_str(), sizeof(addr.sun_path) - 1);
+
+ // bind to local address
+ if (bind(ListenSocketFD, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
+
+ // If the socket address is already in use, exit because another module
+ // build daemon has successfully launched. When translation units are
+ // compiled in parallel, until the socket file is created, all clang
+ // invocations will spawn a module build daemon.
+ if (errno == EADDRINUSE) {
+ close(ListenSocketFD);
+ exit(EXIT_SUCCESS);
+ }
+ std::perror("Socket bind error: ");
+ exit(EXIT_FAILURE);
+ }
+
+ // set socket to accept incoming connection request
+ unsigned MaxBacklog = llvm::hardware_concurrency().compute_thread_count();
+ if (listen(ListenSocketFD, MaxBacklog) == -1) {
+ std::perror("Socket listen error: ");
+ exit(EXIT_FAILURE);
+ }
+
+ cc1modbuildd::ub_outs() << "daemon initialization complete!" << '\n';
+ return 0;
+}
+
+// Function submitted to thread pool with each client connection. Not
+// responsible for closing client connections
+llvm::Error ModuleBuildDaemonServer::Service(int Client) {
+
+ // Read buffer from client connection
+ Expected<std::unique_ptr<char[]>> MaybeClientBuffer =
+ cc1modbuildd::readFromSocket(Client);
+ if (!MaybeClientBuffer)
+ return std::move(MaybeClientBuffer.takeError());
+ std::unique_ptr<char[]> ClientBuffer = std::move(*MaybeClientBuffer);
+
+ // Convert buffer into SocketMsg
+ Expected<cc1modbuildd::SocketMsg> MaybeClientRequest =
+ cc1modbuildd::getSocketMsgFromBuffer(ClientBuffer.get());
+ if (!MaybeClientRequest)
+ return std::move(MaybeClientRequest.takeError());
+ cc1modbuildd::SocketMsg ClientRequest = std::move(*MaybeClientRequest);
+
+ // Handle HANDSHAKE
+ // Currently cc1 invocations do not wait for response to HANDSHAKE SocketMsg
+ if (ClientRequest.MsgAction == cc1modbuildd::ActionType::HANDSHAKE)
+ return llvm::Error::success();
+
+ // Handle REGISTER
+ if (ClientRequest.MsgAction == cc1modbuildd::ActionType::REGISTER) {
+ llvm::Error ScanErr = cc1modbuildd::scanTranslationUnit(ClientRequest);
+ if (ScanErr)
+ return std::move(ScanErr);
+
+ std::string ServerResponse = cc1modbuildd::getBufferFromSocketMsg(
+ {cc1modbuildd::ActionType::REGISTER,
+ cc1modbuildd::StatusType::SUCCESS});
+ // Unblocks cc1 invocation
+ llvm::Error WriteErr = cc1modbuildd::writeToSocket(ServerResponse, Client);
+ if (WriteErr)
+ return std::move(WriteErr);
+
+ return llvm::Error::success();
+ }
+
+ // Conditional should exist for each ActionType
+ llvm_unreachable("Unrecognized Action");
+}
+
+int ModuleBuildDaemonServer::Listen() {
+
+ llvm::ThreadPool Pool;
+ int Client;
+
+ while (true) {
+
+ if ((Client = accept(ListenSocketFD, NULL, NULL)) == -1) {
+ std::perror("Socket accept error: ");
+ continue;
+ }
+
+ // FIXME: Error messages will be over written as results are returned
+ std::shared_future<llvm::Error> result = Pool.async(Service, Client);
+ llvm::Error Err = std::move(const_cast<llvm::Error &>(result.get()));
+
+ if (Err) {
+ handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) {
+ errs() << "Error while scanning: " << EIB.message() << '\n';
+ });
+ }
+
+ close(Client);
+ }
+ return 0;
+}
+
+// Module build daemon is spawned with the following command line:
+//
+// clang -cc1modbuildd <path>
+//
+// <path> defines the location of all files created by the module build daemon
+// and should follow the format /path/to/dir. For example, `clang -cc1modbuildd
+// /tmp/` creates a socket file at `/tmp/mbd.sock`. /tmp is also valid.
+//
+// When module build daemons are spawned by cc1 invocations, <path> follows the
+// format /tmp/clang-<BLAKE3HashOfClangFullVersion>
+//
+int cc1modbuildd_main(ArrayRef<const char *> Argv) {
+
+ if (Argv.size() < 1) {
+ outs() << "spawning a module build daemon requies a command line format of "
+ "`clang -cc1modbuildd <path>`. <path> defines where the module "
+ "build daemon will create files"
+ << '\n';
+ return 1;
+ }
+
+ // TODO: Add check to confirm BasePath is of correct format
+ SmallString<128> BasePath(Argv[0]);
+ llvm::sys::fs::create_directories(BasePath);
+ ModuleBuildDaemonServer Daemon(BasePath);
+
+ // Used to handle signals
+ DaemonPtr = &Daemon;
+
+ Daemon.Fork();
+ Daemon.Launch();
+ Daemon.Listen();
+
+ return 0;
+}
+
+#endif // LLVM_ON_UNIX
\ No newline at end of file
Index: clang/tools/driver/cc1_main.cpp
===================================================================
--- clang/tools/driver/cc1_main.cpp
+++ clang/tools/driver/cc1_main.cpp
@@ -25,6 +25,7 @@
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/FrontendTool/Utils.h"
+#include "clang/Tooling/ModuleBuildDaemon/Protocol.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/LinkAllPasses.h"
@@ -45,6 +46,7 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include <cstdio>
+#include <filesystem>
#ifdef CLANG_HAVE_RLIMITS
#include <sys/resource.h>
@@ -243,6 +245,12 @@
return 1;
}
+#if LLVM_ON_UNIX
+ // handle module build daemon functionality if enabled
+ if (Clang->getInvocation().getFrontendOpts().ModuleBuildDaemon)
+ cc1modbuildd::updateCC1WithModuleBuildDaemon(*Clang, Argv0);
+#endif
+
// Execute the frontend actions.
{
llvm::TimeTraceScope TimeScope("ExecuteCompiler");
Index: clang/tools/driver/CMakeLists.txt
===================================================================
--- clang/tools/driver/CMakeLists.txt
+++ clang/tools/driver/CMakeLists.txt
@@ -28,6 +28,7 @@
cc1_main.cpp
cc1as_main.cpp
cc1gen_reproducer_main.cpp
+ cc1modbuildd_main.cpp
DEPENDS
intrinsics_gen
@@ -39,9 +40,11 @@
PRIVATE
clangBasic
clangCodeGen
+ clangDependencyScanning
clangDriver
clangFrontend
clangFrontendTool
+ clangModuleBuildDaemon
clangSerialization
)
Index: clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
===================================================================
--- /dev/null
+++ clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
@@ -0,0 +1,106 @@
+//===------------------------- SocketSupport.cpp --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h"
+#include "clang/Basic/Version.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Tooling/ModuleBuildDaemon/Protocol.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/BLAKE3.h"
+
+// TODO: Make portable
+#if LLVM_ON_UNIX
+
+#include <cerrno>
+#include <filesystem>
+#include <fstream>
+#include <signal.h>
+#include <spawn.h>
+#include <string>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+Expected<int> cc1modbuildd::createSocket() {
+ int FD;
+ if ((FD = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
+ std::string Msg = "socket create error: " + std::string(strerror(errno));
+ return createStringError(inconvertibleErrorCode(), Msg);
+ }
+ return FD;
+}
+
+Expected<int> cc1modbuildd::connectToSocket(StringRef SocketPath) {
+
+ Expected<int> MaybeFD = cc1modbuildd::createSocket();
+ if (!MaybeFD)
+ return std::move(MaybeFD.takeError());
+
+ int FD = std::move(*MaybeFD);
+
+ struct sockaddr_un Addr;
+ memset(&Addr, 0, sizeof(Addr));
+ Addr.sun_family = AF_UNIX;
+ strncpy(Addr.sun_path, SocketPath.str().c_str(), sizeof(Addr.sun_path) - 1);
+
+ if (connect(FD, (struct sockaddr *)&Addr, sizeof(Addr)) == -1) {
+ close(FD);
+ std::string msg = "socket connect error: " + std::string(strerror(errno));
+ return createStringError(inconvertibleErrorCode(), msg);
+ }
+ return FD;
+}
+
+Expected<int> cc1modbuildd::connectAndWriteToSocket(std::string Buffer,
+ StringRef SocketPath) {
+
+ Expected<int> MaybeConnectedFD = connectToSocket(SocketPath);
+ if (!MaybeConnectedFD)
+ return std::move(MaybeConnectedFD.takeError());
+
+ int ConnectedFD = std::move(*MaybeConnectedFD);
+ llvm::Error Err = writeToSocket(Buffer, ConnectedFD);
+ if (Err)
+ return std::move(Err);
+
+ return ConnectedFD;
+}
+
+Expected<std::unique_ptr<char[]>> cc1modbuildd::readFromSocket(int FD) {
+
+ std::unique_ptr<char[]> Buffer(new char[MAX_BUFFER]);
+ memset(Buffer.get(), 0, MAX_BUFFER);
+ int n = read(FD, Buffer.get(), MAX_BUFFER);
+
+ if (n < 0) {
+ std::string Msg = "socket read error: " + std::string(strerror(errno));
+ return llvm::make_error<StringError>(Msg, inconvertibleErrorCode());
+ }
+ if (n == 0)
+ return llvm::make_error<StringError>("EOF", inconvertibleErrorCode());
+
+ return Buffer;
+}
+
+llvm::Error cc1modbuildd::writeToSocket(std::string Buffer, int WriteFD) {
+
+ ssize_t MessageSize = static_cast<ssize_t>(Buffer.size());
+
+ if (write(WriteFD, Buffer.c_str(), Buffer.size()) != MessageSize) {
+ std::string Msg = "socket write error: " + std::string(strerror(errno));
+ return llvm::make_error<StringError>(Msg, inconvertibleErrorCode());
+ }
+ return llvm::Error::success();
+}
+
+#endif // LLVM_ON_UNIX
\ No newline at end of file
Index: clang/lib/Tooling/ModuleBuildDaemon/Protocol.cpp
===================================================================
--- /dev/null
+++ clang/lib/Tooling/ModuleBuildDaemon/Protocol.cpp
@@ -0,0 +1,277 @@
+//===---------------------------- Protocol.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/Protocol.h"
+#include "clang/Basic/Version.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
+#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/BLAKE3.h"
+
+// TODO: Make portable
+#if LLVM_ON_UNIX
+
+#include <cerrno>
+#include <filesystem>
+#include <fstream>
+#include <signal.h>
+#include <spawn.h>
+#include <string>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+using namespace clang;
+using namespace llvm;
+
+static std::string getWorkingDir() {
+ return std::filesystem::current_path().string();
+}
+
+raw_fd_ostream &cc1modbuildd::ub_outs() {
+ static raw_fd_ostream S(STDOUT_FILENO, false, true);
+ return S;
+}
+
+Expected<int> cc1modbuildd::connectToSocketAndHandshake(StringRef SocketPath) {
+
+ Expected<int> ConnectedFD = connectToSocket(SocketPath);
+ if (!ConnectedFD)
+ return std::move(ConnectedFD.takeError());
+
+ llvm::Error Err = attemptHandshake(std::move(*ConnectedFD));
+ if (Err)
+ return std::move(Err);
+
+ return ConnectedFD;
+}
+
+bool cc1modbuildd::daemonExists(StringRef BasePath) {
+
+ SmallString<128> SocketPath = BasePath;
+ llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
+
+ if (!llvm::sys::fs::exists(SocketPath))
+ return false;
+
+ Expected<int> ConnectedFD = connectToSocketAndHandshake(SocketPath);
+ if (ConnectedFD) {
+ close(std::move(*ConnectedFD));
+ return true;
+ }
+
+ consumeError(ConnectedFD.takeError());
+ return false;
+}
+
+llvm::Error cc1modbuildd::spawnModuleBuildDaemon(StringRef BasePath,
+ const char *Argv0) {
+ std::string BasePathStr = BasePath.str();
+ const char *Args[] = {Argv0, "-cc1modbuildd", BasePathStr.c_str(), nullptr};
+ pid_t pid;
+ int EC = posix_spawn(&pid, Args[0],
+ /*file_actions*/ nullptr,
+ /*spawnattr*/ nullptr, const_cast<char **>(Args),
+ /*envp*/ nullptr);
+ if (EC)
+ return createStringError(std::error_code(EC, std::generic_category()),
+ "failed to spawn module build daemon process");
+
+ return llvm::Error::success();
+}
+
+SmallString<128> cc1modbuildd::getBasePath() {
+ llvm::BLAKE3 Hash;
+ Hash.update(getClangFullVersion());
+ auto HashResult = Hash.final<sizeof(uint64_t)>();
+ uint64_t HashValue =
+ llvm::support::endian::read<uint64_t, llvm::support::native>(
+ HashResult.data());
+ std::string Key = toString(llvm::APInt(64, HashValue), 36, /*Signed*/ false);
+
+ // set paths
+ SmallString<128> BasePath;
+ llvm::sys::path::system_temp_directory(/*erasedOnReboot*/ true, BasePath);
+ llvm::sys::path::append(BasePath, "clang-" + Key);
+ return BasePath;
+}
+
+llvm::Error cc1modbuildd::attemptHandshake(int SocketFD) {
+
+ cc1modbuildd::SocketMsg Request{ActionType::HANDSHAKE, StatusType::REQUEST};
+ std::string Buffer = cc1modbuildd::getBufferFromSocketMsg(Request);
+
+ if (llvm::Error Err = writeToSocket(Buffer, SocketFD))
+ return std::move(Err);
+
+ return llvm::Error::success();
+}
+
+std::string cc1modbuildd::getBufferFromSocketMsg(SocketMsg SocketMsg) {
+
+ std::string Buffer;
+ llvm::raw_string_ostream OS(Buffer);
+ llvm::yaml::Output YamlOut(OS);
+
+ YamlOut << SocketMsg;
+ return Buffer;
+}
+
+Expected<cc1modbuildd::SocketMsg>
+cc1modbuildd::getSocketMsgFromBuffer(char *Buffer) {
+
+ SocketMsg ClientRequest;
+ llvm::yaml::Input YamlIn(Buffer);
+ YamlIn >> ClientRequest;
+
+ if (YamlIn.error()) {
+ std::string Msg = "Syntax or semantic error during YAML parsing";
+ return llvm::make_error<StringError>(Msg, inconvertibleErrorCode());
+ }
+
+ return ClientRequest;
+}
+
+llvm::Error cc1modbuildd::getModuleBuildDaemon(const char *Argv0,
+ StringRef BasePath) {
+
+ // If module build daemon already exist return success
+ if (cc1modbuildd::daemonExists(BasePath)) {
+ return llvm::Error::success();
+ }
+
+ if (llvm::Error Err = cc1modbuildd::spawnModuleBuildDaemon(BasePath, Argv0))
+ return std::move(Err);
+
+ sleep(10);
+
+ // Confirm that module build daemon was created
+ if (cc1modbuildd::daemonExists(BasePath))
+ return llvm::Error::success();
+
+ return llvm::make_error<StringError>(
+ "Module build daemon did not exist after spawn attempt",
+ inconvertibleErrorCode());
+}
+
+void cc1modbuildd::updateCC1WithModuleBuildDaemon(CompilerInstance &Clang,
+ const char *Argv0) {
+
+ SmallString<128> BasePath = cc1modbuildd::getBasePath();
+
+ llvm::Error DaemonErr = cc1modbuildd::getModuleBuildDaemon(Argv0, BasePath);
+ if (DaemonErr) {
+ handleAllErrors(std::move(DaemonErr), [&](ErrorInfoBase &EIB) {
+ errs() << "Connect to daemon failed: " << EIB.message() << "\n";
+ });
+ return;
+ }
+
+ llvm::Error RegisterErr =
+ cc1modbuildd::registerTranslationUnit(Clang, Argv0, BasePath);
+ if (RegisterErr) {
+ handleAllErrors(std::move(RegisterErr), [&](ErrorInfoBase &EIB) {
+ errs() << "Register translation unit failed: " << EIB.message() << "\n";
+ });
+ return;
+ }
+
+ return;
+}
+
+llvm::Error cc1modbuildd::registerTranslationUnit(CompilerInstance &Clang,
+ StringRef Argv0,
+ StringRef BasePath) {
+
+ std::vector<std::string> CC1Cmd = Clang.getInvocation().getCC1CommandLine();
+ CC1Cmd.insert(CC1Cmd.begin(), Argv0.str());
+ cc1modbuildd::SocketMsg Request{ActionType::REGISTER, StatusType::REQUEST,
+ getWorkingDir(), CC1Cmd};
+
+ std::string Buffer = getBufferFromSocketMsg(Request);
+
+ // FIXME: Should not need to append again here
+ SmallString<128> SocketPath = BasePath;
+ llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
+
+ Expected<int> MaybeServerFD = connectAndWriteToSocket(Buffer, SocketPath);
+ if (!MaybeServerFD)
+ return std::move(MaybeServerFD.takeError());
+
+ // Blocks cc1 invocation until module build daemon is done processing
+ // translation unit. Currently receives a SUCCESS message and returns
+ // llvm::Error::success() but will eventually recive updated cc1 command line
+ Expected<std::unique_ptr<char[]>> MaybeResponseBuffer =
+ readFromSocket(std::move(*MaybeServerFD));
+ if (!MaybeResponseBuffer)
+ return std::move(MaybeResponseBuffer.takeError());
+
+ // Wait for response from module build daemon
+ Expected<SocketMsg> MaybeResponse =
+ getSocketMsgFromBuffer(std::move(*MaybeResponseBuffer).get());
+ if (!MaybeResponse)
+ return std::move(MaybeResponse.takeError());
+ SocketMsg ServerResponse = std::move(*MaybeResponse);
+
+ assert(ServerResponse.MsgAction == ActionType::REGISTER &&
+ "ActionType should only be REGISTER");
+ if (ServerResponse.MsgStatus == StatusType::SUCCESS)
+ return llvm::Error::success();
+
+ return llvm::make_error<StringError>(
+ "Daemon failed to processes registered translation unit",
+ inconvertibleErrorCode());
+}
+
+llvm::Error cc1modbuildd::scanTranslationUnit(SocketMsg Request) {
+
+ tooling::dependencies::DependencyScanningService Service(
+ tooling::dependencies::ScanningMode::DependencyDirectivesScan,
+ tooling::dependencies::ScanningOutputFormat::Full,
+ /*OptimizeArgs*/ false,
+ /*EagerLoadModules*/ false);
+
+ tooling::dependencies::DependencyScanningTool Tool(Service);
+
+ llvm::DenseSet<clang::tooling::dependencies::ModuleID> AlreadySeenModules;
+ auto LookupOutput = [&](const tooling::dependencies::ModuleID &MID,
+ tooling::dependencies::ModuleOutputKind MOK) {
+ return MID.ContextHash;
+ };
+
+ auto MaybeFile = Tool.getTranslationUnitDependencies(
+ Request.Argv0PlusCC1CommandLine.value(), Request.WorkingDirectory.value(),
+ AlreadySeenModules, LookupOutput);
+
+ if (!MaybeFile)
+ return std::move(MaybeFile.takeError());
+
+ tooling::dependencies::TranslationUnitDeps TUDeps = std::move(*MaybeFile);
+
+ // For now write dependencies to log file
+ for (auto const &Dep : TUDeps.FileDeps) {
+ cc1modbuildd::ub_outs() << Dep << '\n';
+ }
+
+ if (!TUDeps.ModuleGraph.empty())
+ errs() << "Warning: translation unit contained modules. Module build "
+ "daemon not yet able to build modules"
+ << '\n';
+
+ return llvm::Error::success();
+}
+
+#endif // LLVM_ON_UNIX
\ No newline at end of file
Index: clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
@@ -0,0 +1,8 @@
+set(LLVM_LINK_COMPONENTS
+ Support
+ )
+
+add_clang_library(clangModuleBuildDaemon
+ Protocol.cpp
+ SocketSupport.cpp
+ )
\ No newline at end of file
Index: clang/lib/Tooling/CMakeLists.txt
===================================================================
--- clang/lib/Tooling/CMakeLists.txt
+++ clang/lib/Tooling/CMakeLists.txt
@@ -13,6 +13,7 @@
add_subdirectory(Syntax)
add_subdirectory(DependencyScanning)
add_subdirectory(Transformer)
+add_subdirectory(ModuleBuildDaemon)
# Replace the last lib component of the current binary directory with include
string(FIND ${CMAKE_CURRENT_BINARY_DIR} "/lib/" PATH_LIB_START REVERSE)
Index: clang/lib/Driver/ToolChains/Clang.cpp
===================================================================
--- clang/lib/Driver/ToolChains/Clang.cpp
+++ clang/lib/Driver/ToolChains/Clang.cpp
@@ -5,7 +5,6 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-
#include "Clang.h"
#include "AMDGPU.h"
#include "Arch/AArch64.h"
@@ -3737,6 +3736,10 @@
Std->containsValue("c++latest") || Std->containsValue("gnu++latest"));
bool HaveModules = HaveStdCXXModules;
+ // -fmodule-build-daemon enables the module build daemon functionality
+ if (Args.hasArg(options::OPT_fmodule_build_daemon))
+ Args.AddLastArg(CmdArgs, options::OPT_fmodule_build_daemon);
+
// -fmodules enables the use of precompiled modules (off by default).
// Users can pass -fno-cxx-modules to turn off modules support for
// C++/Objective-C++ programs.
Index: clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
===================================================================
--- /dev/null
+++ clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
@@ -0,0 +1,31 @@
+//===-------------------------- SocketSupport.h ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETSUPPORT_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETSUPPORT_H
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
+
+using namespace clang;
+using namespace llvm;
+
+namespace cc1modbuildd {
+
+Expected<int> createSocket();
+Expected<int> connectToSocket(StringRef SocketPath);
+Expected<int> connectAndWriteToSocket(std::string Buffer, StringRef SocketPath);
+Expected<std::unique_ptr<char[]>> readFromSocket(int FD);
+llvm::Error writeToSocket(std::string Buffer, int WriteFD);
+
+} // namespace cc1modbuildd
+
+#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETSUPPORT_H
\ No newline at end of file
Index: clang/include/clang/Tooling/ModuleBuildDaemon/Protocol.h
===================================================================
--- /dev/null
+++ clang/include/clang/Tooling/ModuleBuildDaemon/Protocol.h
@@ -0,0 +1,109 @@
+//===---------------------------- Protocall.h -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_PROTOCAL_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_PROTOCAL_H
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
+
+#define MAX_BUFFER 4096
+#define SOCKET_FILE_NAME "mbd.sock"
+#define STDOUT_FILE_NAME "mbd.out"
+#define STDERR_FILE_NAME "mbd.err"
+
+using namespace clang;
+using namespace llvm;
+
+namespace cc1modbuildd {
+
+enum class ActionType { REGISTER, HANDSHAKE };
+enum class StatusType { REQUEST, SUCCESS, FAILURE };
+
+struct SocketMsg {
+ ActionType MsgAction;
+ StatusType MsgStatus;
+ std::optional<std::string> WorkingDirectory;
+ // First element needs to be path to compiler
+ std::optional<std::vector<std::string>> Argv0PlusCC1CommandLine;
+
+ SocketMsg() = default;
+
+ SocketMsg(ActionType Action, StatusType Status,
+ const std::optional<std::string> &CurrentWD,
+ const std::optional<std::vector<std::string>> &CommandLine)
+ : MsgAction(Action), MsgStatus(Status), WorkingDirectory(CurrentWD),
+ Argv0PlusCC1CommandLine(CommandLine) {}
+
+ SocketMsg(ActionType Action, StatusType Status)
+ : MsgAction(Action), MsgStatus(Status), WorkingDirectory(std::nullopt),
+ Argv0PlusCC1CommandLine(std::nullopt) {}
+};
+
+// Create unbuffered STDOUT stream so that any logging done by module build
+// daemon can be viewed without having to terminate the process
+raw_fd_ostream &ub_outs();
+
+SmallString<128> getBasePath();
+
+bool daemonExists(StringRef BasePath);
+
+std::string getBufferFromSocketMsg(SocketMsg Command);
+
+Expected<SocketMsg> getSocketMsgFromBuffer(char *Buffer);
+
+Expected<int> connectToSocketAndHandshake(StringRef SocketPath);
+
+llvm::Error attemptHandshake(int SocketFD);
+
+llvm::Error getModuleBuildDaemon(const char *Argv0, StringRef BasePath);
+
+llvm::Error spawnModuleBuildDaemon(StringRef BasePath, const char *Argv0);
+
+llvm::Error registerTranslationUnit(CompilerInstance &Clang, StringRef Argv0,
+ StringRef BasePath);
+
+// Work in progress. Eventually function will modify CC1 command line to include
+// path to modules already built by the daemon
+void updateCC1WithModuleBuildDaemon(CompilerInstance &Clang, const char *Argv0);
+
+llvm::Error scanTranslationUnit(SocketMsg Command);
+
+} // namespace cc1modbuildd
+
+template <>
+struct llvm::yaml::ScalarEnumerationTraits<cc1modbuildd::StatusType> {
+ static void enumeration(IO &io, cc1modbuildd::StatusType &value) {
+ io.enumCase(value, "REQUEST", cc1modbuildd::StatusType::REQUEST);
+ io.enumCase(value, "SUCCESS", cc1modbuildd::StatusType::SUCCESS);
+ io.enumCase(value, "FAILURE", cc1modbuildd::StatusType::FAILURE);
+ }
+};
+
+template <>
+struct llvm::yaml::ScalarEnumerationTraits<cc1modbuildd::ActionType> {
+ static void enumeration(IO &io, cc1modbuildd::ActionType &value) {
+ io.enumCase(value, "REGISTER", cc1modbuildd::ActionType::REGISTER);
+ io.enumCase(value, "HANDSHAKE", cc1modbuildd::ActionType::HANDSHAKE);
+ }
+};
+
+template <> struct llvm::yaml::MappingTraits<cc1modbuildd::SocketMsg> {
+ static void mapping(IO &io, cc1modbuildd::SocketMsg &info) {
+ io.mapRequired("Action", info.MsgAction);
+ io.mapRequired("Status", info.MsgStatus);
+ io.mapOptional("WorkingDirectory", info.WorkingDirectory);
+ io.mapOptional("FullCommandLine", info.Argv0PlusCC1CommandLine);
+ }
+};
+
+#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_PROTOCAL_H
Index: clang/include/clang/Frontend/FrontendOptions.h
===================================================================
--- clang/include/clang/Frontend/FrontendOptions.h
+++ clang/include/clang/Frontend/FrontendOptions.h
@@ -347,6 +347,9 @@
/// Whether to share the FileManager when building modules.
unsigned ModulesShareFileManager : 1;
+ /// Connect to module build daemon
+ unsigned ModuleBuildDaemon : 1;
+
CodeCompleteOptions CodeCompleteOpts;
/// Specifies the output format of the AST.
Index: clang/include/clang/Driver/Options.td
===================================================================
--- clang/include/clang/Driver/Options.td
+++ clang/include/clang/Driver/Options.td
@@ -2822,6 +2822,13 @@
NegFlag<SetFalse, [], [ClangOption], "Disallow">,
BothFlags<[], [ClangOption, CC1Option],
" __declspec as a keyword">>, Group<f_clang_Group>;
+
+def fmodule_build_daemon : Flag<["-"], "fmodule-build-daemon">, Group<f_Group>,
+ Flags<[NoXarchOption]>,
+ Visibility<[ClangOption, CC1Option]>,
+ HelpText<"Enables module build daemon functionality">,
+ MarshallingInfoFlag<FrontendOpts<"ModuleBuildDaemon">>;
+
def fmodules_cache_path : Joined<["-"], "fmodules-cache-path=">, Group<i_Group>,
Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option]>,
MetaVarName<"<directory>">,
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
