Author: Abhinav Kumar Date: 2025-09-13T10:01:31+03:00 New Revision: c642e2aa61c430ae597b0bd08e924339292e30e9
URL: https://github.com/llvm/llvm-project/commit/c642e2aa61c430ae597b0bd08e924339292e30e9 DIFF: https://github.com/llvm/llvm-project/commit/c642e2aa61c430ae597b0bd08e924339292e30e9.diff LOG: [clang-repl] Add support for running custom code in Remote JIT executor (#157358) Introduce a custom lambda mechanism that allows injecting user-defined code into the Remote JIT’s executor. --------- Co-authored-by: kr-2003 <kumar.kr.abhi...@gmail.com> Added: clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp Modified: clang/include/clang/Interpreter/Interpreter.h clang/lib/Interpreter/IncrementalExecutor.cpp clang/lib/Interpreter/IncrementalExecutor.h clang/lib/Interpreter/Interpreter.cpp clang/unittests/Interpreter/CMakeLists.txt Removed: ################################################################################ diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index fcc270a17001e..078d70b3b1749 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -135,13 +135,15 @@ class Interpreter { std::string OrcRuntimePath = ""; /// PID of the out-of-process JIT executor. uint32_t ExecutorPID = 0; + /// Custom lambda to be executed inside child process/executor + std::function<void()> CustomizeFork = nullptr; /// An optional code model to provide to the JITTargetMachineBuilder std::optional<llvm::CodeModel::Model> CM = std::nullopt; JITConfig() : IsOutOfProcess(false), OOPExecutor(""), OOPExecutorConnect(""), UseSharedMemory(false), SlabAllocateSize(0), OrcRuntimePath(""), - ExecutorPID(0), CM(std::nullopt) {} + ExecutorPID(0), CustomizeFork(nullptr), CM(std::nullopt) {} }; protected: diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index b0eb7d0e9f072..45620fcd358c8 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -172,7 +172,8 @@ createSharedMemoryManager(llvm::orc::SimpleRemoteEPC &SREPC, llvm::Expected<std::pair<std::unique_ptr<llvm::orc::SimpleRemoteEPC>, uint32_t>> IncrementalExecutor::launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory, - unsigned SlabAllocateSize) { + unsigned SlabAllocateSize, + std::function<void()> CustomizeFork) { #ifndef LLVM_ON_UNIX // FIXME: Add support for Windows. return llvm::make_error<llvm::StringError>( @@ -215,6 +216,9 @@ IncrementalExecutor::launchExecutor(llvm::StringRef ExecutablePath, close(ToExecutor[WriteEnd]); close(FromExecutor[ReadEnd]); + if (CustomizeFork) + CustomizeFork(); + // Execute the child process. std::unique_ptr<char[]> ExecutorPath, FDSpecifier; { diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h index d091535166770..bb1ec33452515 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.h +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -79,7 +79,8 @@ class IncrementalExecutor { static llvm::Expected< std::pair<std::unique_ptr<llvm::orc::SimpleRemoteEPC>, uint32_t>> launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory, - unsigned SlabAllocateSize); + unsigned SlabAllocateSize, + std::function<void()> CustomizeFork = nullptr); #if LLVM_ON_UNIX && LLVM_ENABLE_THREADS static llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>> diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 84f1c363b5f6f..07c170a63ce82 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -355,7 +355,8 @@ Interpreter::outOfProcessJITBuilder(JITConfig Config) { if (!Config.OOPExecutor.empty()) { // Launch an out-of-process executor locally in a child process. auto ResultOrErr = IncrementalExecutor::launchExecutor( - Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize); + Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize, + Config.CustomizeFork); if (!ResultOrErr) return ResultOrErr.takeError(); childPid = ResultOrErr->second; diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt index db9f80d9f53fe..7b8dcfc9b0546 100644 --- a/clang/unittests/Interpreter/CMakeLists.txt +++ b/clang/unittests/Interpreter/CMakeLists.txt @@ -29,12 +29,25 @@ set(CLANG_LIBS_TO_LINK ) endif() -add_distinct_clang_unittest(ClangReplInterpreterTests +set(CLANG_REPL_TEST_SOURCES IncrementalCompilerBuilderTest.cpp IncrementalProcessingTest.cpp InterpreterTest.cpp InterpreterExtensionsTest.cpp CodeCompletionTest.cpp +) + +if(TARGET compiler-rt) + list(APPEND CLANG_REPL_TEST_SOURCES + OutOfProcessInterpreterTests.cpp + ) + message(STATUS "Compiler-RT found, enabling out of process JIT tests") +endif() + +add_distinct_clang_unittest(ClangReplInterpreterTests + ${CLANG_REPL_TEST_SOURCES} + + PARTIAL_SOURCES_INTENDED EXPORT_SYMBOLS @@ -48,6 +61,14 @@ add_distinct_clang_unittest(ClangReplInterpreterTests ${LLVM_COMPONENTS_TO_LINK} ) +if(TARGET compiler-rt) + add_dependencies(ClangReplInterpreterTests + llvm-jitlink-executor + compiler-rt + ) + message(STATUS "Adding dependency on compiler-rt for out of process JIT tests") +endif() + if(EMSCRIPTEN) # Without the above you try to link to LLVMSupport twice, and end # up with a duplicate symbol error when creating the main module diff --git a/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp new file mode 100644 index 0000000000000..704ddc37e642e --- /dev/null +++ b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp @@ -0,0 +1,203 @@ +//===- unittests/Interpreter/OutOfProcessInterpreterTest.cpp --- Interpreter +// tests when Out-of-Process ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Unit tests for Clang's Interpreter library. +// +//===----------------------------------------------------------------------===// + +#include "InterpreterTestFixture.h" +#include "clang/AST/Decl.h" +#include "clang/AST/DeclGroup.h" +#include "clang/AST/Mangle.h" +#include "clang/Basic/Version.h" +#include "clang/Config/config.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Interpreter/Interpreter.h" +#include "clang/Interpreter/Value.h" +#include "clang/Sema/Lookup.h" +#include "clang/Sema/Sema.h" +#include "llvm/Support/Error.h" +#include "llvm/TargetParser/Host.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include <memory> +#include <signal.h> +#include <sstream> +#include <unistd.h> + +using namespace clang; + +llvm::ExitOnError ExitOnError; + +namespace { + +using Args = std::vector<const char *>; + +struct FileDeleter { + void operator()(FILE *f) { + if (f) + fclose(f); + } +}; + +struct IOContext { + std::unique_ptr<FILE, FileDeleter> stdin_file; + std::unique_ptr<FILE, FileDeleter> stdout_file; + std::unique_ptr<FILE, FileDeleter> stderr_file; + + bool initializeTempFiles() { + stdin_file.reset(tmpfile()); + stdout_file.reset(tmpfile()); + stderr_file.reset(tmpfile()); + return stdin_file && stdout_file && stderr_file; + } + + std::string readStdoutContent() { + if (!stdout_file) + return ""; + rewind(stdout_file.get()); + std::ostringstream content; + char buffer[1024]; + size_t bytes_read; + while ((bytes_read = fread(buffer, 1, sizeof(buffer), stdout_file.get())) > + 0) { + content.write(buffer, bytes_read); + } + return content.str(); + } + + std::string readStderrContent() { + if (!stderr_file) + return ""; + rewind(stderr_file.get()); + std::ostringstream content; + char buffer[1024]; + size_t bytes_read; + while ((bytes_read = fread(buffer, 1, sizeof(buffer), stderr_file.get())) > + 0) { + content.write(buffer, bytes_read); + } + return content.str(); + } +}; + +static void removePathComponent(unsigned N, llvm::SmallString<256> &Path) { + for (unsigned i = 0; i < N; ++i) + llvm::sys::path::remove_filename(Path); +} + +static std::string getExecutorPath() { + llvm::SmallString<256> ExecutorPath(llvm::sys::fs::getMainExecutable( + nullptr, reinterpret_cast<void *>(&getExecutorPath))); + removePathComponent(5, ExecutorPath); + llvm::sys::path::append(ExecutorPath, "bin", "llvm-jitlink-executor"); + return ExecutorPath.str().str(); +} + +static std::string getOrcRuntimePath() { + llvm::SmallString<256> RuntimePath(llvm::sys::fs::getMainExecutable( + nullptr, reinterpret_cast<void *>(&getOrcRuntimePath))); + removePathComponent(5, RuntimePath); + llvm::sys::path::append(RuntimePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang", + CLANG_VERSION_MAJOR_STRING, "lib"); + + llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); + if (SystemTriple.isOSBinFormatMachO()) { + llvm::sys::path::append(RuntimePath, "darwin", "liborc_rt_osx.a"); + } else if (SystemTriple.isOSBinFormatELF()) { + llvm::sys::path::append(RuntimePath, "x86_64-unknown-linux-gnu", + "liborc_rt.a"); + } + return RuntimePath.str().str(); +} + +static std::unique_ptr<Interpreter> +createInterpreterWithRemoteExecution(std::shared_ptr<IOContext> io_ctx, + const Args &ExtraArgs = {}) { + Args ClangArgs = {"-Xclang", "-emit-llvm-only"}; + llvm::append_range(ClangArgs, ExtraArgs); + auto CB = clang::IncrementalCompilerBuilder(); + CB.SetCompilerArgs(ClangArgs); + auto CI = cantFail(CB.CreateCpp()); + + clang::Interpreter::JITConfig Config; + llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); + + if (SystemTriple.isOSBinFormatELF() || SystemTriple.isOSBinFormatMachO()) { + Config.IsOutOfProcess = true; + Config.OOPExecutor = getExecutorPath(); + Config.UseSharedMemory = false; + Config.SlabAllocateSize = 0; + Config.OrcRuntimePath = getOrcRuntimePath(); + + int stdin_fd = fileno(io_ctx->stdin_file.get()); + int stdout_fd = fileno(io_ctx->stdout_file.get()); + int stderr_fd = fileno(io_ctx->stderr_file.get()); + + Config.CustomizeFork = [=] { + auto redirect = [](int from, int to) { + if (from != to) { + dup2(from, to); + close(from); + } + }; + + redirect(stdin_fd, STDIN_FILENO); + redirect(stdout_fd, STDOUT_FILENO); + redirect(stderr_fd, STDERR_FILENO); + + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + + printf("CustomizeFork executed\n"); + fflush(stdout); + }; + } + + return cantFail(clang::Interpreter::create(std::move(CI), Config)); +} + +static size_t DeclsSize(TranslationUnitDecl *PTUDecl) { + return std::distance(PTUDecl->decls().begin(), PTUDecl->decls().end()); +} + +TEST_F(InterpreterTestBase, SanityWithRemoteExecution) { + if (!HostSupportsJIT()) + GTEST_SKIP(); + + std::string OrcRuntimePath = getOrcRuntimePath(); + std::string ExecutorPath = getExecutorPath(); + + if (!llvm::sys::fs::exists(OrcRuntimePath) || + !llvm::sys::fs::exists(ExecutorPath)) + GTEST_SKIP(); + + auto io_ctx = std::make_shared<IOContext>(); + ASSERT_TRUE(io_ctx->initializeTempFiles()); + + std::unique_ptr<Interpreter> Interp = + createInterpreterWithRemoteExecution(io_ctx); + ASSERT_TRUE(Interp); + + using PTU = PartialTranslationUnit; + PTU &R1(cantFail(Interp->Parse("void g(); void g() {}"))); + EXPECT_EQ(2U, DeclsSize(R1.TUPart)); + + PTU &R2(cantFail(Interp->Parse("int i = 42;"))); + EXPECT_EQ(1U, DeclsSize(R2.TUPart)); + + std::string captured_stdout = io_ctx->readStdoutContent(); + std::string captured_stderr = io_ctx->readStderrContent(); + + EXPECT_TRUE(captured_stdout.find("CustomizeFork executed") != + std::string::npos); +} + +} // end anonymous namespace \ No newline at end of file _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
