On Wednesday, February 14, 2018 7:17:10 PM CET Andres Freund wrote:
> Hi,
>
> On 2018-02-07 06:54:05 -0800, Andres Freund wrote:
> > I've pushed v10.0. The big (and pretty painful to make) change is that
> > now all the LLVM specific code lives in src/backend/jit/llvm, which is
> > built as a shared library which is loaded on demand.
> >
> > The layout is now as follows:
> >
> > src/backend/jit/jit.c:
> > Part of JITing always linked into the server. Supports loading the
> > LLVM using JIT library.
> >
> > src/backend/jit/llvm/
> >
> > Infrastructure:
> > llvmjit.c:
> > General code generation and optimization infrastructure
> >
> > llvmjit_error.cpp, llvmjit_wrap.cpp:
> > Error / backward compat wrappers
> >
> > llvmjit_inline.cpp:
> > Cross module inlining support
> >
> > Code-Gen:
> > llvmjit_expr.c
> >
> > Expression compilation
> >
> > llvmjit_deform.c
> >
> > Deform compilation
>
> I've pushed a revised version that hopefully should address Jeff's
> wish/need of being able to experiment with this out of core. There's now
> a "jit_provider" PGC_POSTMASTER GUC that's by default set to
> "llvmjit". llvmjit.so is the .so implementing JIT using LLVM. It fills a
> set of callbacks via
> extern void _PG_jit_provider_init(JitProviderCallbacks *cb);
> which can also be implemented by any other potential provider.
>
> The other two biggest changes are that I've added a README
> https://git.postgresql.org/gitweb/?p=users/andresfreund/postgres.git;a=blob;
> f=src/backend/jit/README;hb=jit and that I've revised the configure support
> so it does more error
> checks, and moved it into config/llvm.m4.
>
> There's a larger smattering of small changes too.
>
> I'm pretty happy with how the separation of core / shlib looks now. I'm
> planning to work on cleaning and then pushing some of the preliminary
> patches (fixed tupledesc, grouping) over the next few days.
>
> Greetings,
>
> Andres Freund
Hi
Here are the LLVM4 and LLVM3.9 compatibility patches.
Successfully built, and executed some silly queries with JIT forced to make
sure it worked.
Pierre
>From c856a5db2f0ba34ba7c230a65f60277ae0e7347f Mon Sep 17 00:00:00 2001
From: Pierre <pierre.ducroq...@people-doc.com>
Date: Fri, 2 Feb 2018 09:11:55 +0100
Subject: [PATCH 1/8] Add support for LLVM4 in llvmjit.c
Signed-off-by: Pierre Ducroquet <pina...@pinaraf.info>
---
src/backend/jit/llvm/llvmjit.c | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 7a96ece0f7..7557dc9a19 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -222,13 +222,20 @@ llvm_get_function(LLVMJitContext *context, const char *funcname)
addr = 0;
if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, mangled))
- elog(ERROR, "failed to lookup symbol");
+ elog(ERROR, "failed to lookup symbol %s", mangled);
if (addr)
return (void *) addr;
}
#else
+#if LLVM_VERSION_MAJOR < 5
+ if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, mangled)))
+ return (void *) addr;
+ if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, mangled)))
+ return (void *) addr;
+ elog(ERROR, "failed to lookup symbol %s for %s", mangled, funcname);
+#else
if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, mangled))
elog(ERROR, "failed to lookup symbol");
if (addr)
@@ -237,6 +244,8 @@ llvm_get_function(LLVMJitContext *context, const char *funcname)
elog(ERROR, "failed to lookup symbol");
if (addr)
return (void *) addr;
+#endif // LLVM_VERSION_MAJOR
+
#endif
elog(ERROR, "failed to JIT: %s", funcname);
@@ -374,11 +383,18 @@ llvm_compile_module(LLVMJitContext *context)
* faster instruction selection mechanism is used.
*/
{
- LLVMSharedModuleRef smod;
instr_time tb, ta;
/* emit the code */
INSTR_TIME_SET_CURRENT(ta);
+#if LLVM_VERSION_MAJOR < 5
+ orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
+ llvm_resolve_symbol, NULL);
+ // It seems there is no error return from that function in LLVM < 5.
+#else
+ LLVMSharedModuleRef smod;
+
+ LLVMSharedModuleRef smod;
smod = LLVMOrcMakeSharedModule(context->module);
if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, smod,
llvm_resolve_symbol, NULL))
@@ -386,6 +402,7 @@ llvm_compile_module(LLVMJitContext *context)
elog(ERROR, "failed to jit module");
}
LLVMOrcDisposeSharedModuleRef(smod);
+#endif
INSTR_TIME_SET_CURRENT(tb);
INSTR_TIME_SUBTRACT(tb, ta);
ereport(DEBUG1, (errmsg("time to emit: %.3fs",
--
2.16.1
>From a44378f05c33a40c485f26e5f007614100c70fe7 Mon Sep 17 00:00:00 2001
From: Pierre <pierre.ducroq...@people-doc.com>
Date: Fri, 2 Feb 2018 09:13:40 +0100
Subject: [PATCH 2/8] Add LLVM4 support in llvmjit_error.cpp
Signed-off-by: Pierre Ducroquet <pina...@pinaraf.info>
---
src/backend/jit/llvm/llvmjit_error.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/src/backend/jit/llvm/llvmjit_error.cpp b/src/backend/jit/llvm/llvmjit_error.cpp
index 625ba2d25d..1c78bd956d 100644
--- a/src/backend/jit/llvm/llvmjit_error.cpp
+++ b/src/backend/jit/llvm/llvmjit_error.cpp
@@ -56,7 +56,9 @@ llvm_enter_fatal_on_oom(void)
if (fatal_new_handler_depth == 0)
{
old_new_handler = std::set_new_handler(fatal_system_new_handler);
+#if LLVM_VERSION_MAJOR > 4
llvm::install_bad_alloc_error_handler(fatal_llvm_new_handler);
+#endif
llvm::install_fatal_error_handler(fatal_llvm_error_handler);
}
fatal_new_handler_depth++;
@@ -72,7 +74,9 @@ llvm_leave_fatal_on_oom(void)
if (fatal_new_handler_depth == 0)
{
std::set_new_handler(old_new_handler);
+#if LLVM_VERSION_MAJOR > 4
llvm::remove_bad_alloc_error_handler();
+#endif
llvm::remove_fatal_error_handler();
}
}
@@ -87,7 +91,9 @@ llvm_reset_after_error(void)
if (fatal_new_handler_depth != 0)
{
std::set_new_handler(old_new_handler);
+#if LLVM_VERSION_MAJOR > 4
llvm::remove_bad_alloc_error_handler();
+#endif
llvm::remove_fatal_error_handler();
}
fatal_new_handler_depth = 0;
--
2.16.1
>From 9360b80327863d3ccaa93b8acfdb85bbe07730ce Mon Sep 17 00:00:00 2001
From: Pierre <pierre.ducroq...@people-doc.com>
Date: Fri, 2 Feb 2018 09:23:56 +0100
Subject: [PATCH 3/8] Add LLVM4 support in llvmjit_inline.cpp
Signed-off-by: Pierre Ducroquet <pina...@pinaraf.info>
---
src/backend/jit/llvm/llvmjit_inline.cpp | 36 +++++++++++++++++++++++++++++++--
1 file changed, 34 insertions(+), 2 deletions(-)
diff --git a/src/backend/jit/llvm/llvmjit_inline.cpp b/src/backend/jit/llvm/llvmjit_inline.cpp
index 7d0d18b43d..4fa0e5ab64 100644
--- a/src/backend/jit/llvm/llvmjit_inline.cpp
+++ b/src/backend/jit/llvm/llvmjit_inline.cpp
@@ -100,6 +100,13 @@ llvm_inline(LLVMModuleRef M)
llvm_execute_inline_plan(mod, globalsToInline.get());
}
+#if LLVM_VERSION_MAJOR < 5
+bool operator!(const llvm::ValueInfo &vi) {
+ return !( (vi.Kind == llvm::ValueInfo::VI_GUID && vi.TheValue.Id)
+ || (vi.Kind == llvm::ValueInfo::VI_Value && vi.TheValue.GV));
+}
+#endif
+
/*
* Build information necessary for inlining external function references in
* mod.
@@ -146,7 +153,14 @@ llvm_build_inline_plan(llvm::Module *mod)
if (threshold == -1)
continue;
+#if LLVM_VERSION_MAJOR > 4
llvm::ValueInfo funcVI = llvm_index->getValueInfo(funcGUID);
+#else
+ const llvm::const_gvsummary_iterator &I = llvm_index->findGlobalValueSummaryList(funcGUID);
+ if (I == llvm_index->end())
+ continue;
+ llvm::ValueInfo funcVI = llvm::ValueInfo(I->first);
+#endif
/* if index doesn't know function, we don't have a body, continue */
if (!funcVI)
@@ -157,7 +171,12 @@ llvm_build_inline_plan(llvm::Module *mod)
* look up module(s), check if function actually is defined (there
* could be hash conflicts).
*/
+#if LLVM_VERSION_MAJOR > 4
for (const auto &gvs : funcVI.getSummaryList())
+#else
+ auto it_gvs = llvm_index->findGlobalValueSummaryList(funcVI.getGUID());
+ for (const auto &gvs: it_gvs->second)
+#endif
{
const llvm::FunctionSummary *fs;
llvm::StringRef modPath = gvs->modulePath();
@@ -318,9 +337,14 @@ llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
}
+#if LLVM_VERSION_MAJOR > 4
+#define IRMOVE_PARAMS , /*IsPerformingImport=*/false
+#else
+#define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
+#endif
if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
- [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {},
- /*IsPerformingImport=*/false))
+ [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
+ IRMOVE_PARAMS))
elog(ERROR, "function import failed with linker error");
}
}
@@ -619,9 +643,17 @@ llvm_load_index(void)
elog(ERROR, "failed to open %s: %s", subpath,
EC.message().c_str());
llvm::MemoryBufferRef ref(*MBOrErr.get().get());
+#if LLVM_VERSION_MAJOR > 4
llvm::Error e = llvm::readModuleSummaryIndex(ref, *index, 0);
if (e)
elog(ERROR, "could not load summary at %s", subpath);
+#else
+ std::unique_ptr<llvm::ModuleSummaryIndex> subindex = std::move(llvm::getModuleSummaryIndex(ref).get());
+ if (!subindex)
+ elog(ERROR, "could not load summary at %s", subpath);
+ else
+ index->mergeFrom(std::move(subindex), 0);
+#endif
}
}
--
2.16.1
>From 4f0c2dc0dea6b3fb09b09451095f2225bde8458c Mon Sep 17 00:00:00 2001
From: Pierre <pierre.ducroq...@people-doc.com>
Date: Fri, 2 Feb 2018 10:34:09 +0100
Subject: [PATCH 4/8] Don't emit bitcode depending on an LLVM 5+ function
Signed-off-by: Pierre Ducroquet <pina...@pinaraf.info>
---
src/backend/jit/llvm/llvmjit_expr.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index a06319b1b6..4b3c5367e5 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -170,7 +170,11 @@ get_LifetimeEnd(LLVMModuleRef mod)
LLVMTypeRef sig;
LLVMValueRef fn;
LLVMTypeRef param_types[2];
+#if LLVM_VERSION_MAJOR > 4
const char *nm = "llvm.lifetime.end.p0i8";
+#else
+ const char *nm = "llvm.lifetime.end";
+#endif
fn = LLVMGetNamedFunction(mod, nm);
if (fn)
--
2.16.1
>From 9f0993d739d1df43e792a9523d1da3b941d67472 Mon Sep 17 00:00:00 2001
From: Pierre Ducroquet <pina...@pinaraf.info>
Date: Wed, 7 Feb 2018 20:23:43 +0100
Subject: [PATCH 6/8] Ignore LLVM .bc files
Signed-off-by: Pierre Ducroquet <pina...@pinaraf.info>
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index a59e3da3be..794e35b73c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
# Global excludes across all subdirectories
*.o
*.obj
+*.bc
*.so
*.so.[0-9]
*.so.[0-9].[0-9]
--
2.16.1
>From b6107de417e9654c7f78aa5945a7b78812dcfa96 Mon Sep 17 00:00:00 2001
From: Pierre Ducroquet <pina...@pinaraf.info>
Date: Wed, 7 Feb 2018 20:22:37 +0100
Subject: [PATCH 5/8] Fix warning
Signed-off-by: Pierre Ducroquet <pina...@pinaraf.info>
---
src/backend/jit/llvm/llvmjit_error.cpp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/backend/jit/llvm/llvmjit_error.cpp b/src/backend/jit/llvm/llvmjit_error.cpp
index 1c78bd956d..623a70a7cc 100644
--- a/src/backend/jit/llvm/llvmjit_error.cpp
+++ b/src/backend/jit/llvm/llvmjit_error.cpp
@@ -28,7 +28,9 @@ static int fatal_new_handler_depth = 0;
static std::new_handler old_new_handler = NULL;
static void fatal_system_new_handler(void);
+#if LLVM_VERSION_MAJOR > 4
static void fatal_llvm_new_handler(void *user_data, const std::string& reason, bool gen_crash_diag);
+#endif
static void fatal_llvm_error_handler(void *user_data, const std::string& reason, bool gen_crash_diag);
@@ -114,6 +116,7 @@ fatal_system_new_handler(void)
errdetail("while in LLVM")));
}
+#if LLVM_VERSION_MAJOR > 4
static void
fatal_llvm_new_handler(void *user_data,
const std::string& reason,
@@ -124,6 +127,7 @@ fatal_llvm_new_handler(void *user_data,
errmsg("out of memory"),
errdetail("while in LLVM: %s", reason.c_str())));
}
+#endif
static void
fatal_llvm_error_handler(void *user_data,
--
2.16.1
>From 9f60304697c559fd3cce7fc91ed81af7ea37b7e0 Mon Sep 17 00:00:00 2001
From: Pierre <pierre.ducroq...@people-doc.com>
Date: Fri, 2 Feb 2018 11:29:45 +0100
Subject: [PATCH 7/8] Fix building with LLVM 3.9
Signed-off-by: Pierre Ducroquet <pina...@pinaraf.info>
---
src/backend/jit/llvm/llvmjit_inline.cpp | 23 +++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/src/backend/jit/llvm/llvmjit_inline.cpp b/src/backend/jit/llvm/llvmjit_inline.cpp
index 4fa0e5ab64..8cfac20565 100644
--- a/src/backend/jit/llvm/llvmjit_inline.cpp
+++ b/src/backend/jit/llvm/llvmjit_inline.cpp
@@ -37,7 +37,12 @@ extern "C"
#include <llvm/ADT/StringSet.h>
#include <llvm/ADT/StringMap.h>
#include <llvm/Analysis/ModuleSummaryAnalysis.h>
+#if LLVM_VERSION_MAJOR > 3
#include <llvm/Bitcode/BitcodeReader.h>
+#else
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/Support/Error.h"
+#endif
#include <llvm/IR/CallSite.h>
#include <llvm/IR/DebugInfo.h>
#include <llvm/IR/IntrinsicInst.h>
@@ -100,7 +105,12 @@ llvm_inline(LLVMModuleRef M)
llvm_execute_inline_plan(mod, globalsToInline.get());
}
-#if LLVM_VERSION_MAJOR < 5
+#if LLVM_VERSION_MAJOR < 4
+bool operator!(const llvm::ValueInfo &vi) {
+ return !( (vi.Kind == llvm::ValueInfo::VI_GUID && vi.TheValue.Id)
+ || (vi.Kind == llvm::ValueInfo::VI_Value && vi.TheValue.V));
+}
+#elif LLVM_VERSION_MAJOR < 5
bool operator!(const llvm::ValueInfo &vi) {
return !( (vi.Kind == llvm::ValueInfo::VI_GUID && vi.TheValue.Id)
|| (vi.Kind == llvm::ValueInfo::VI_Value && vi.TheValue.GV));
@@ -188,12 +198,15 @@ llvm_build_inline_plan(llvm::Module *mod)
funcName.data(),
modPath.data());
+// XXX Missing in LLVM < 4.0 ?
+#if LLVM_VERSION_MAJOR > 3
if (gvs->notEligibleToImport())
{
elog(DEBUG1, "uneligible to import %s due to summary",
funcName.data());
continue;
}
+#endif
if ((int) fs->instCount() > threshold)
{
@@ -339,8 +352,10 @@ llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
#if LLVM_VERSION_MAJOR > 4
#define IRMOVE_PARAMS , /*IsPerformingImport=*/false
-#else
+#elif LLVM_VERSION_MAJOR > 3
#define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
+#else
+#define IRMOVE_PARAMS
#endif
if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
[](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
@@ -648,7 +663,11 @@ llvm_load_index(void)
if (e)
elog(ERROR, "could not load summary at %s", subpath);
#else
+#if LLVM_VERSION_MAJOR > 3
std::unique_ptr<llvm::ModuleSummaryIndex> subindex = std::move(llvm::getModuleSummaryIndex(ref).get());
+#else
+ std::unique_ptr<llvm::ModuleSummaryIndex> subindex = std::move(llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {}).get());
+#endif
if (!subindex)
elog(ERROR, "could not load summary at %s", subpath);
else
--
2.16.1
>From 504c311fc18ce2ef04e22144a21fd3af13b1ab04 Mon Sep 17 00:00:00 2001
From: Pierre <pierre.ducroq...@people-doc.com>
Date: Fri, 2 Feb 2018 11:29:57 +0100
Subject: [PATCH 8/8] Fix segfault with LLVM 3.9
Signed-off-by: Pierre Ducroquet <pina...@pinaraf.info>
---
src/backend/jit/llvm/llvmjit.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 7557dc9a19..b99408b468 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -455,12 +455,12 @@ llvm_session_initialize(void)
cpu = LLVMGetHostCPUName();
llvm_opt0_targetmachine =
- LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, NULL,
+ LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, "",
LLVMCodeGenLevelNone,
LLVMRelocDefault,
LLVMCodeModelJITDefault);
llvm_opt3_targetmachine =
- LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, NULL,
+ LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, "",
LLVMCodeGenLevelAggressive,
LLVMRelocDefault,
LLVMCodeModelJITDefault);
--
2.16.1
