Author: Abhinav Kumar Date: 2025-08-08T08:54:34+03:00 New Revision: eccc6e22f81141691542f5dd5bbb7996e446a44f
URL: https://github.com/llvm/llvm-project/commit/eccc6e22f81141691542f5dd5bbb7996e446a44f DIFF: https://github.com/llvm/llvm-project/commit/eccc6e22f81141691542f5dd5bbb7996e446a44f.diff LOG: [clang-repl] Enable extending `launchExecutor` (#152562) This patch introduces the ability to customize the fork process with an external lambda function. This is useful for downstream clients where they want to do stream redirection. Added: clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp Modified: clang/include/clang/Interpreter/RemoteJITUtils.h clang/lib/Interpreter/RemoteJITUtils.cpp clang/unittests/Interpreter/CMakeLists.txt clang/unittests/Interpreter/InterpreterTest.cpp Removed: ################################################################################ diff --git a/clang/include/clang/Interpreter/RemoteJITUtils.h b/clang/include/clang/Interpreter/RemoteJITUtils.h index 8705a3b1f669d..bc71232a5cad8 100644 --- a/clang/include/clang/Interpreter/RemoteJITUtils.h +++ b/clang/include/clang/Interpreter/RemoteJITUtils.h @@ -26,7 +26,8 @@ llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>> launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory, - llvm::StringRef SlabAllocateSizeString); + llvm::StringRef SlabAllocateSizeString, + std::function<void()> CustomizeFork = nullptr); /// Create a JITLinkExecutor that connects to the given network address /// through a TCP socket. A valid NetworkAddress provides hostname and port, @@ -35,4 +36,13 @@ llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>> connectTCPSocket(llvm::StringRef NetworkAddress, bool UseSharedMemory, llvm::StringRef SlabAllocateSizeString); +#ifdef LLVM_ON_UNIX +/// Returns PID of last launched executor. +pid_t getLastLaunchedExecutorPID(); + +/// Returns PID of nth launched executor. +/// 1-based indexing. +pid_t getNthLaunchedExecutorPID(int n); +#endif + #endif // LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H diff --git a/clang/lib/Interpreter/RemoteJITUtils.cpp b/clang/lib/Interpreter/RemoteJITUtils.cpp index c0e663b764785..c100f46931b6d 100644 --- a/clang/lib/Interpreter/RemoteJITUtils.cpp +++ b/clang/lib/Interpreter/RemoteJITUtils.cpp @@ -33,6 +33,10 @@ using namespace llvm; using namespace llvm::orc; +#if LLVM_ON_UNIX +static std::vector<pid_t> LaunchedExecutorPID; +#endif + Expected<uint64_t> getSlabAllocSize(StringRef SizeString) { SizeString = SizeString.trim(); @@ -89,9 +93,14 @@ createSharedMemoryManager(SimpleRemoteEPC &SREPC, SlabSize, SREPC, SAs); } +// Launches an out-of-process executor for remote JIT. The calling program can +// provide a CustomizeFork callback, which allows it to run custom code in the +// child process before exec. This enables sending custom setup or code to be +// executed in the child (out-of-process) executor. Expected<std::unique_ptr<SimpleRemoteEPC>> launchExecutor(StringRef ExecutablePath, bool UseSharedMemory, - llvm::StringRef SlabAllocateSizeString) { + llvm::StringRef SlabAllocateSizeString, + std::function<void()> CustomizeFork) { #ifndef LLVM_ON_UNIX // FIXME: Add support for Windows. return make_error<StringError>("-" + ExecutablePath + @@ -134,6 +143,9 @@ launchExecutor(StringRef ExecutablePath, bool UseSharedMemory, close(ToExecutor[WriteEnd]); close(FromExecutor[ReadEnd]); + if (CustomizeFork) + CustomizeFork(); + // Execute the child process. std::unique_ptr<char[]> ExecutorPath, FDSpecifier; { @@ -158,6 +170,8 @@ launchExecutor(StringRef ExecutablePath, bool UseSharedMemory, } // else we're the parent... + LaunchedExecutorPID.push_back(ChildPID); + // Close the child ends of the pipes close(ToExecutor[ReadEnd]); close(FromExecutor[WriteEnd]); @@ -265,3 +279,18 @@ connectTCPSocket(StringRef NetworkAddress, bool UseSharedMemory, std::move(S), *SockFD, *SockFD); #endif } + +#if LLVM_ON_UNIX + +pid_t getLastLaunchedExecutorPID() { + if (!LaunchedExecutorPID.size()) + return -1; + return LaunchedExecutorPID.back(); +} + +pid_t getNthLaunchedExecutorPID(int n) { + if (n - 1 < 0 || n - 1 >= static_cast<int>(LaunchedExecutorPID.size())) + return -1; + return LaunchedExecutorPID.at(n - 1); +} +#endif \ No newline at end of file diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt index 1dda9024075a1..24f65cab95d7a 100644 --- a/clang/unittests/Interpreter/CMakeLists.txt +++ b/clang/unittests/Interpreter/CMakeLists.txt @@ -1,10 +1,22 @@ -add_distinct_clang_unittest(ClangReplInterpreterTests +set(CLANG_REPL_TEST_SOURCES IncrementalCompilerBuilderTest.cpp IncrementalProcessingTest.cpp InterpreterTest.cpp InterpreterExtensionsTest.cpp CodeCompletionTest.cpp +) +if(TARGET compiler-rt) + list(APPEND CLANG_REPL_TEST_SOURCES + OutOfProcessInterpreterTests.cpp + ) + message(STATUS "Compiler-RT found, enabling out of process JIT tests") +endif() + +add_distinct_clang_unittest(ClangReplInterpreterTests + ${CLANG_REPL_TEST_SOURCES} + + PARTIAL_SOURCES_INTENDED EXPORT_SYMBOLS CLANG_LIBS @@ -26,6 +38,14 @@ add_distinct_clang_unittest(ClangReplInterpreterTests TargetParser ) +if(TARGET compiler-rt) + add_dependencies(ClangReplInterpreterTests + llvm-jitlink-executor + compiler-rt + ) + message(STATUS "Adding dependency on compiler-rt for out of process JIT tests") +endif() + # Exceptions on Windows are not yet supported. if(NOT WIN32) add_subdirectory(ExceptionTests) diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp index 768058b954d49..5d49089dada87 100644 --- a/clang/unittests/Interpreter/InterpreterTest.cpp +++ b/clang/unittests/Interpreter/InterpreterTest.cpp @@ -15,12 +15,17 @@ #include "clang/AST/Decl.h" #include "clang/AST/DeclGroup.h" #include "clang/AST/Mangle.h" +#include "clang/Basic/Version.h" +#include "clang/Config/config.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Interpreter/Interpreter.h" +#include "clang/Interpreter/RemoteJITUtils.h" #include "clang/Interpreter/Value.h" #include "clang/Sema/Lookup.h" #include "clang/Sema/Sema.h" +#include "llvm/Support/Error.h" +#include "llvm/TargetParser/Host.h" #include "llvm/TargetParser/Host.h" @@ -34,6 +39,12 @@ int Global = 42; REPL_EXTERNAL_VISIBILITY int getGlobal() { return Global; } REPL_EXTERNAL_VISIBILITY void setGlobal(int val) { Global = val; } +#ifdef _WIN32 +#define STDIN_FILENO 0 +#define STDOUT_FILENO 1 +#define STDERR_FILENO 2 +#endif + namespace { class InterpreterTest : public InterpreterTestBase { diff --git a/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp new file mode 100644 index 0000000000000..4d5ef5c70d135 --- /dev/null +++ b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp @@ -0,0 +1,148 @@ +//===- unittests/Interpreter/OutOfProcessInterpreterTest.cpp --- Interpreter +// tests when Out-of-Process ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Unit tests for Clang's Interpreter library. +// +//===----------------------------------------------------------------------===// + +#include "InterpreterTestFixture.h" + +#include "clang/AST/Decl.h" +#include "clang/AST/DeclGroup.h" +#include "clang/AST/Mangle.h" +#include "clang/Basic/Version.h" +#include "clang/Config/config.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Interpreter/Interpreter.h" +#include "clang/Interpreter/RemoteJITUtils.h" +#include "clang/Interpreter/Value.h" +#include "clang/Sema/Lookup.h" +#include "clang/Sema/Sema.h" +#include "llvm/Support/Error.h" +#include "llvm/TargetParser/Host.h" + +#include "llvm/TargetParser/Host.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace clang; + +llvm::ExitOnError ExitOnError; + +#ifdef _WIN32 +#define STDIN_FILENO 0 +#define STDOUT_FILENO 1 +#define STDERR_FILENO 2 +#endif + +namespace { + +using Args = std::vector<const char *>; + +static void removePathComponent(unsigned N, llvm::SmallString<256> &Path) { + for (unsigned i = 0; i < N; ++i) + llvm::sys::path::remove_filename(Path); +} + +static std::string getExecutorPath() { + llvm::SmallString<256> ExecutorPath(llvm::sys::fs::getMainExecutable( + nullptr, reinterpret_cast<void *>(&getExecutorPath))); + removePathComponent(5, ExecutorPath); + llvm::sys::path::append(ExecutorPath, "bin", "llvm-jitlink-executor"); + return ExecutorPath.str().str(); +} + +static std::string getOrcRuntimePath() { + llvm::SmallString<256> RuntimePath(llvm::sys::fs::getMainExecutable( + nullptr, reinterpret_cast<void *>(&getOrcRuntimePath))); + removePathComponent(5, RuntimePath); + llvm::sys::path::append(RuntimePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang", + CLANG_VERSION_MAJOR_STRING, "lib"); + + llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); + if (SystemTriple.isOSBinFormatMachO()) { + llvm::sys::path::append(RuntimePath, "darwin", "liborc_rt_osx.a"); + } else if (SystemTriple.isOSBinFormatELF()) { + llvm::sys::path::append(RuntimePath, "x86_64-unknown-linux-gnu", + "liborc_rt.a"); + } + + return RuntimePath.str().str(); +} + +static std::unique_ptr<Interpreter> +createInterpreterWithRemoteExecution(const Args &ExtraArgs = {}, + DiagnosticConsumer *Client = nullptr) { + Args ClangArgs = {"-Xclang", "-emit-llvm-only"}; + llvm::append_range(ClangArgs, ExtraArgs); + auto CB = clang::IncrementalCompilerBuilder(); + CB.SetCompilerArgs(ClangArgs); + auto CI = cantFail(CB.CreateCpp()); + if (Client) + CI->getDiagnostics().setClient(Client, /*ShouldOwnClient=*/false); + + std::unique_ptr<llvm::orc::LLJITBuilder> JB; + + llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); + + if ((SystemTriple.isOSBinFormatELF() || SystemTriple.isOSBinFormatMachO())) { + std::string OOPExecutor = getExecutorPath(); + std::string OrcRuntimePath = getOrcRuntimePath(); + bool UseSharedMemory = false; + std::string SlabAllocateSizeString = ""; + std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC; + EPC = ExitOnError(launchExecutor(OOPExecutor, UseSharedMemory, + SlabAllocateSizeString, + [=] { // Lambda defined inline + auto redirect = [](int from, int to) { + if (from != to) { + dup2(from, to); + close(from); + } + }; + + redirect(0, STDIN_FILENO); + redirect(1, STDOUT_FILENO); + redirect(2, STDERR_FILENO); + + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + })); + if (EPC) { + CB.SetTargetTriple(EPC->getTargetTriple().getTriple()); + JB = ExitOnError(clang::Interpreter::createLLJITBuilder(std::move(EPC), + OrcRuntimePath)); + } + } + + return cantFail(clang::Interpreter::create(std::move(CI), std::move(JB))); +} + +static size_t DeclsSize(TranslationUnitDecl *PTUDecl) { + return std::distance(PTUDecl->decls().begin(), PTUDecl->decls().end()); +} + +TEST_F(InterpreterTestBase, SanityWithRemoteExecution) { + if (!HostSupportsJIT()) + GTEST_SKIP(); + + std::unique_ptr<Interpreter> Interp = createInterpreterWithRemoteExecution(); + + using PTU = PartialTranslationUnit; + + PTU &R1(cantFail(Interp->Parse("void g(); void g() {}"))); + EXPECT_EQ(2U, DeclsSize(R1.TUPart)); + + PTU &R2(cantFail(Interp->Parse("int i;"))); + EXPECT_EQ(1U, DeclsSize(R2.TUPart)); +} + +} // end anonymous namespace _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
