Repository: incubator-impala Updated Branches: refs/heads/master f0e79314f -> 1e63ff843
IMPALA-5860: upgrade to LLVM 3.9.1 LLVM made a few API changes: * Misc minor changes to function and type signatures * The CloneFunction() API changed semantics (http://reviews.llvm.org/D18628) Needed to fix a few new clang-tidy warnings. Testing: Ran core and ASAN tests. Perf: Ran single node TPC-H and targeted perf with scale factor 60. Both improved on average. +----------+-----------------------+---------+------------+------------+----------------+ | Workload | File Format | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) | +----------+-----------------------+---------+------------+------------+----------------+ | TPCH(60) | parquet / none / none | 17.82 | -5.01% | 11.64 | -4.23% | +----------+-----------------------+---------+------------+------------+----------------+ +----------+----------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+ | Workload | Query | File Format | Avg(s) | Base Avg(s) | Delta(Avg) | StdDev(%) | Base StdDev(%) | Num Clients | Iters | +----------+----------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+ | TPCH(60) | TPCH-Q1 | parquet / none / none | 27.97 | 27.59 | +1.36% | 0.39% | 0.41% | 1 | 5 | | TPCH(60) | TPCH-Q20 | parquet / none / none | 5.81 | 5.78 | +0.44% | 0.73% | 0.21% | 1 | 5 | | TPCH(60) | TPCH-Q21 | parquet / none / none | 62.98 | 62.98 | +0.01% | 5.56% | 1.07% | 1 | 5 | | TPCH(60) | TPCH-Q15 | parquet / none / none | 8.45 | 8.46 | -0.20% | 0.40% | 0.38% | 1 | 5 | | TPCH(60) | TPCH-Q4 | parquet / none / none | 5.57 | 5.59 | -0.41% | 0.43% | 0.80% | 1 | 5 | | TPCH(60) | TPCH-Q6 | parquet / none / none | 3.16 | 3.17 | -0.45% | 0.78% | 1.70% | 1 | 5 | | TPCH(60) | TPCH-Q5 | parquet / none / none | 7.41 | 7.47 | -0.92% | 0.71% | 1.06% | 1 | 5 | | TPCH(60) | TPCH-Q9 | parquet / none / none | 33.45 | 33.78 | -0.99% | 1.15% | 0.85% | 1 | 5 | | TPCH(60) | TPCH-Q11 | parquet / none / none | 2.00 | 2.03 | -1.34% | 1.71% | 2.24% | 1 | 5 | | TPCH(60) | TPCH-Q2 | parquet / none / none | 4.71 | 4.79 | -1.60% | 1.49% | 1.95% | 1 | 5 | | TPCH(60) | TPCH-Q18 | parquet / none / none | 46.48 | 47.71 | -2.58% | 1.04% | 0.38% | 1 | 5 | | TPCH(60) | TPCH-Q14 | parquet / none / none | 5.85 | 6.02 | -2.84% | 0.44% | 0.70% | 1 | 5 | | TPCH(60) | TPCH-Q22 | parquet / none / none | 6.51 | 6.76 | -3.71% | 2.29% | 2.42% | 1 | 5 | | TPCH(60) | TPCH-Q19 | parquet / none / none | 7.27 | 7.63 | -4.69% | 1.33% | 0.78% | 1 | 5 | | TPCH(60) | TPCH-Q10 | parquet / none / none | 13.19 | 13.84 | -4.73% | 0.42% | 1.44% | 1 | 5 | | TPCH(60) | TPCH-Q13 | parquet / none / none | 21.95 | 23.12 | -5.03% | 0.25% | 1.19% | 1 | 5 | | TPCH(60) | TPCH-Q16 | parquet / none / none | 5.29 | 5.57 | -5.04% | 0.85% | 0.78% | 1 | 5 | | TPCH(60) | TPCH-Q7 | parquet / none / none | 42.05 | 44.33 | -5.16% | 2.07% | 2.28% | 1 | 5 | | TPCH(60) | TPCH-Q12 | parquet / none / none | 19.77 | 21.00 | -5.87% | 8.14% | 5.09% | 1 | 5 | | TPCH(60) | TPCH-Q3 | parquet / none / none | 11.46 | 12.32 | -6.94% | 0.76% | 0.53% | 1 | 5 | | TPCH(60) | TPCH-Q17 | parquet / none / none | 40.09 | 49.28 | -18.64% | 2.09% | 0.67% | 1 | 5 | | TPCH(60) | TPCH-Q8 | parquet / none / none | 10.63 | 13.47 | I -21.08% | * 12.34% * | * 21.09% * | 1 | 5 | +----------+----------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+ +-------------------+-----------------------+---------+------------+------------+----------------+ | Workload | File Format | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) | +-------------------+-----------------------+---------+------------+------------+----------------+ | TARGETED-PERF(60) | parquet / none / none | 22.38 | -1.24% | 4.17 | +0.81% | +-------------------+-----------------------+---------+------------+------------+----------------+ +-------------------+--------------------------------------------------------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+ | Workload | Query | File Format | Avg(s) | Base Avg(s) | Delta(Avg) | StdDev(%) | Base StdDev(%) | Num Clients | Iters | +-------------------+--------------------------------------------------------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+ | TARGETED-PERF(60) | primitive_conjunct_ordering_1 | parquet / none / none | 0.12 | 0.10 | R +22.38% | 0.81% | * 27.26% * | 1 | 5 | | TARGETED-PERF(60) | primitive_groupby_decimal_highndv | parquet / none / none | 29.86 | 25.46 | +17.31% | 6.18% | 3.83% | 1 | 5 | | TARGETED-PERF(60) | PERF_LIMIT-Q1 | parquet / none / none | 0.01 | 0.01 | +13.41% | * 15.35% * | 2.95% | 1 | 5 | | TARGETED-PERF(60) | primitive_filter_bigint_non_selective | parquet / none / none | 0.88 | 0.82 | +7.17% | 9.52% | 3.59% | 1 | 5 | | TARGETED-PERF(60) | primitive_filter_decimal_non_selective | parquet / none / none | 1.48 | 1.41 | +4.94% | 4.23% | 1.86% | 1 | 5 | | TARGETED-PERF(60) | primitive_small_join_1 | parquet / none / none | 0.18 | 0.18 | +4.26% | * 11.92% * | 2.43% | 1 | 5 | | TARGETED-PERF(60) | primitive_broadcast_join_3 | parquet / none / none | 7.29 | 7.03 | +3.77% | 5.98% | 9.35% | 1 | 5 | | TARGETED-PERF(60) | primitive_exchange_broadcast | parquet / none / none | 38.41 | 37.02 | +3.77% | 8.59% | 1.31% | 1 | 5 | | TARGETED-PERF(60) | PERF_AGG-Q6 | parquet / none / none | 1.93 | 1.89 | +2.14% | 2.22% | 1.75% | 1 | 5 | | TARGETED-PERF(60) | primitive_conjunct_ordering_2 | parquet / none / none | 7.26 | 7.17 | +1.29% | 2.28% | 4.54% | 1 | 5 | | TARGETED-PERF(60) | PERF_STRING-Q1 | parquet / none / none | 2.79 | 2.75 | +1.28% | 0.52% | 0.76% | 1 | 5 | | TARGETED-PERF(60) | PERF_STRING-Q3 | parquet / none / none | 3.51 | 3.47 | +1.01% | 0.63% | 0.57% | 1 | 5 | | TARGETED-PERF(60) | primitive_filter_string_selective | parquet / none / none | 1.05 | 1.04 | +0.76% | 3.03% | 2.40% | 1 | 5 | | TARGETED-PERF(60) | primitive_orderby_bigint | parquet / none / none | 4.88 | 4.84 | +0.75% | 0.58% | 0.97% | 1 | 5 | | TARGETED-PERF(60) | primitive_top-n_all | parquet / none / none | 38.56 | 38.28 | +0.73% | 0.20% | 0.24% | 1 | 5 | | TARGETED-PERF(60) | primitive_orderby_all | parquet / none / none | 25.68 | 25.54 | +0.55% | 0.27% | 0.40% | 1 | 5 | | TARGETED-PERF(60) | primitive_shuffle_join_union_all_with_groupby | parquet / none / none | 54.02 | 53.74 | +0.53% | 0.35% | 0.23% | 1 | 5 | | TARGETED-PERF(60) | PERF_STRING-Q5 | parquet / none / none | 4.28 | 4.26 | +0.43% | 0.68% | 0.47% | 1 | 5 | | TARGETED-PERF(60) | primitive_empty_build_join_1 | parquet / none / none | 16.25 | 16.19 | +0.42% | 0.33% | 0.42% | 1 | 5 | | TARGETED-PERF(60) | primitive_groupby_bigint_highndv | parquet / none / none | 32.49 | 32.36 | +0.42% | 0.23% | 0.88% | 1 | 5 | | TARGETED-PERF(60) | PERF_AGG-Q1 | parquet / none / none | 2.22 | 2.21 | +0.34% | 1.82% | 1.88% | 1 | 5 | | TARGETED-PERF(60) | primitive_groupby_bigint_pk | parquet / none / none | 112.73 | 112.50 | +0.21% | 0.75% | 0.99% | 1 | 5 | | TARGETED-PERF(60) | PERF_STRING-Q4 | parquet / none / none | 3.52 | 3.51 | +0.13% | 0.58% | 0.65% | 1 | 5 | | TARGETED-PERF(60) | PERF_STRING-Q2 | parquet / none / none | 3.06 | 3.06 | +0.03% | 0.69% | 0.76% | 1 | 5 | | TARGETED-PERF(60) | primitive_filter_decimal_selective | parquet / none / none | 1.20 | 1.20 | -0.01% | 2.35% | 1.24% | 1 | 5 | | TARGETED-PERF(60) | primitive_broadcast_join_2 | parquet / none / none | 4.27 | 4.27 | -0.03% | 0.52% | 0.48% | 1 | 5 | | TARGETED-PERF(60) | primitive_groupby_decimal_lowndv.test | parquet / none / none | 3.87 | 3.87 | -0.07% | 1.69% | 1.63% | 1 | 5 | | TARGETED-PERF(60) | PERF_AGG-Q7 | parquet / none / none | 1.92 | 1.93 | -0.28% | 2.33% | 1.94% | 1 | 5 | | TARGETED-PERF(60) | PERF_AGG-Q5 | parquet / none / none | 0.48 | 0.48 | -0.28% | 0.59% | 0.53% | 1 | 5 | | TARGETED-PERF(60) | PERF_AGG-Q4 | parquet / none / none | 17.48 | 17.53 | -0.30% | 0.43% | 0.58% | 1 | 5 | | TARGETED-PERF(60) | PERF_STRING-Q7 | parquet / none / none | 7.87 | 7.90 | -0.35% | 0.67% | 0.55% | 1 | 5 | | TARGETED-PERF(60) | primitive_exchange_shuffle | parquet / none / none | 74.25 | 74.53 | -0.37% | 0.57% | 0.36% | 1 | 5 | | TARGETED-PERF(60) | primitive_groupby_bigint_lowndv | parquet / none / none | 3.81 | 3.82 | -0.42% | 1.51% | 1.10% | 1 | 5 | | TARGETED-PERF(60) | PERF_AGG-Q2 | parquet / none / none | 9.93 | 10.00 | -0.67% | 0.77% | 0.67% | 1 | 5 | | TARGETED-PERF(60) | primitive_filter_string_like | parquet / none / none | 14.63 | 14.74 | -0.72% | 0.24% | 0.02% | 1 | 5 | | TARGETED-PERF(60) | primitive_conjunct_ordering_4 | parquet / none / none | 0.23 | 0.23 | -0.82% | 0.59% | 1.31% | 1 | 5 | | TARGETED-PERF(60) | PERF_STRING-Q6 | parquet / none / none | 9.87 | 10.03 | -1.55% | 0.39% | 0.22% | 1 | 5 | | TARGETED-PERF(60) | primitive_shuffle_join_one_to_many_string_with_groupby | parquet / none / none | 262.13 | 268.18 | -2.26% | 0.31% | 0.27% | 1 | 5 | | TARGETED-PERF(60) | primitive_filter_string_non_selective | parquet / none / none | 1.23 | 1.26 | -2.26% | 1.72% | 2.15% | 1 | 5 | | TARGETED-PERF(60) | primitive_broadcast_join_1 | parquet / none / none | 2.04 | 2.09 | -2.54% | 0.31% | 2.88% | 1 | 5 | | TARGETED-PERF(60) | primitive_conjunct_ordering_3 | parquet / none / none | 0.13 | 0.13 | -3.13% | 0.73% | 2.50% | 1 | 5 | | TARGETED-PERF(60) | primitive_filter_bigint_selective | parquet / none / none | 0.12 | 0.12 | -3.15% | 1.03% | 1.73% | 1 | 5 | | TARGETED-PERF(60) | primitive_conjunct_ordering_5 | parquet / none / none | 14.11 | 14.60 | -3.33% | 2.03% | 2.43% | 1 | 5 | | TARGETED-PERF(60) | PERF_AGG-Q3 | parquet / none / none | 8.28 | 8.64 | -4.17% | 0.79% | 1.08% | 1 | 5 | | TARGETED-PERF(60) | primitive_long_predicate | parquet / none / none | 215.27 | 227.90 | -5.54% | 0.06% | 0.08% | 1 | 5 | | TARGETED-PERF(60) | primitive_topn_bigint | parquet / none / none | 4.48 | 4.81 | -6.90% | 8.50% | * 15.79% * | 1 | 5 | | TARGETED-PERF(60) | primitive_filter_in_predicate | parquet / none / none | 1.84 | 1.99 | -7.51% | 3.98% | 5.29% | 1 | 5 | +-------------------+--------------------------------------------------------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+ Change-Id: Ida873ddb15e393b0bd37486db24add8a32f43ad0 Reviewed-on: http://gerrit.cloudera.org:8080/7974 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/1e63ff84 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/1e63ff84 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/1e63ff84 Branch: refs/heads/master Commit: 1e63ff8431baa17da0785406dccc60eb8038be93 Parents: f0e7931 Author: Tim Armstrong <[email protected]> Authored: Wed Aug 30 12:20:12 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Mon Sep 18 19:35:28 2017 +0000 ---------------------------------------------------------------------- .clang-tidy | 5 +++++ be/src/benchmarks/atoi-benchmark.cc | 4 ++-- be/src/benchmarks/scheduler-benchmark.cc | 3 ++- be/src/codegen/codegen-symbol-emitter.cc | 11 ++++++----- be/src/codegen/instruction-counter-test.cc | 6 ++++-- be/src/codegen/llvm-codegen.cc | 22 +++++++++++++--------- be/src/codegen/llvm-codegen.h | 3 +-- be/src/common/compiler-util.h | 2 +- be/src/udf/udf.cc | 9 +++++---- bin/bootstrap_toolchain.py | 2 +- bin/impala-config.sh | 10 +++++----- 11 files changed, 45 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/.clang-tidy ---------------------------------------------------------------------- diff --git a/.clang-tidy b/.clang-tidy index fcf5922..54f095c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -24,9 +24,12 @@ Checks: "-*,clang*,\ -clang-analyzer-core.uninitialized.ArraySubscript,\ -clang-analyzer-core.uninitialized.Assign,\ -clang-analyzer-core.uninitialized.Branch,\ +-clang-analyzer-cplusplus.NewDeleteLeaks,\ -clang-analyzer-deadcode.DeadStores,\ +-clang-analyzer-optin.performance.Padding,\ -clang-analyzer-unix.Malloc,\ -clang-analyzer-unix.MallocSizeof,\ +-clang-diagnostic-c++1z-extensions,\ -clang-diagnostic-c++98*,\ -clang-diagnostic-cast-align,\ -clang-diagnostic-class-varargs,\ @@ -53,10 +56,12 @@ Checks: "-*,clang*,\ -clang-diagnostic-padded,\ -clang-diagnostic-return-type-c-linkage,\ -clang-diagnostic-shadow,\ +-clang-diagnostic-shadow-field-in-constructor,\ -clang-diagnostic-shorten-64-to-32,\ -clang-diagnostic-sign-compare,\ -clang-diagnostic-sign-conversion,\ -clang-diagnostic-switch-enum,\ +-clang-diagnostic-undefined-func-template,\ -clang-diagnostic-undefined-reinterpret-cast,\ -clang-diagnostic-unreachable-code,\ -clang-diagnostic-unreachable-code-return,\ http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/be/src/benchmarks/atoi-benchmark.cc ---------------------------------------------------------------------- diff --git a/be/src/benchmarks/atoi-benchmark.cc b/be/src/benchmarks/atoi-benchmark.cc index 134ece4..eb0756e 100644 --- a/be/src/benchmarks/atoi-benchmark.cc +++ b/be/src/benchmarks/atoi-benchmark.cc @@ -108,7 +108,7 @@ inline int32_t AtoiUnrolled(char* s, int len) { bool negative = false; switch (*s) { case '-': negative = true; - case '+': --len, ++s; + case '+': --len; ++s; } switch(len) { @@ -141,7 +141,7 @@ inline int32_t AtoiCased(char* s, int len) { bool negative = false; switch (*s) { case '-': negative = true; - case '+': --len, ++s; + case '+': --len; ++s; } switch(len) { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/be/src/benchmarks/scheduler-benchmark.cc ---------------------------------------------------------------------- diff --git a/be/src/benchmarks/scheduler-benchmark.cc b/be/src/benchmarks/scheduler-benchmark.cc index df9e981..7149dde 100644 --- a/be/src/benchmarks/scheduler-benchmark.cc +++ b/be/src/benchmarks/scheduler-benchmark.cc @@ -122,7 +122,8 @@ void BenchmarkFunction(int num_iterations, void* data) { TestCtx* test_ctx = static_cast<TestCtx*>(data); for (int i = 0; i < num_iterations; ++i) { test_ctx->result->Reset(); - test_ctx->scheduler_wrapper->Compute(test_ctx->result.get()); + Status status = test_ctx->scheduler_wrapper->Compute(test_ctx->result.get()); + if (!status.ok()) LOG(FATAL) << status.GetDetail(); } } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/be/src/codegen/codegen-symbol-emitter.cc ---------------------------------------------------------------------- diff --git a/be/src/codegen/codegen-symbol-emitter.cc b/be/src/codegen/codegen-symbol-emitter.cc index 02008c7..a272f64 100644 --- a/be/src/codegen/codegen-symbol-emitter.cc +++ b/be/src/codegen/codegen-symbol-emitter.cc @@ -95,15 +95,16 @@ void CodegenSymbolEmitter::NotifyFreeingObject(const ObjectFile &obj) { void CodegenSymbolEmitter::ProcessSymbol(DIContext* debug_ctx, const SymbolRef& symbol, uint64_t size, vector<PerfMapEntry>* perf_map_entries, ofstream& asm_file) { - if (symbol.getType() != SymbolRef::ST_Function) return; + Expected<SymbolRef::Type> symType = symbol.getType(); + if (!symType || symType.get() != SymbolRef::ST_Function) return; - ErrorOr<StringRef> name_or_err = symbol.getName(); - ErrorOr<uint64_t> addr_or_err = symbol.getAddress(); + Expected<StringRef> name_or_err = symbol.getName(); + Expected<uint64_t> addr_or_err = symbol.getAddress(); if (!name_or_err || !addr_or_err) return; - uint64_t addr = *addr_or_err; + uint64_t addr = addr_or_err.get(); // Append id to symbol to disambiguate different instances of jitted functions. - string fn_symbol = Substitute("$0:$1", name_or_err->data(), id_); + string fn_symbol = Substitute("$0:$1", name_or_err.get().data(), id_); if (emit_perf_map_) { PerfMapEntry entry; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/be/src/codegen/instruction-counter-test.cc ---------------------------------------------------------------------- diff --git a/be/src/codegen/instruction-counter-test.cc b/be/src/codegen/instruction-counter-test.cc index 0bd4619..c40c551 100644 --- a/be/src/codegen/instruction-counter-test.cc +++ b/be/src/codegen/instruction-counter-test.cc @@ -34,6 +34,8 @@ using namespace llvm; namespace impala { class InstructionCounterTest : public testing:: Test { + protected: + LLVMContext context_; }; // IR output from CodegenMullAdd @@ -71,7 +73,7 @@ Module* CodegenMulAdd(LLVMContext* context) { } TEST_F(InstructionCounterTest, Count) { - Module* MulAddModule = CodegenMulAdd(&getGlobalContext()); + Module* MulAddModule = CodegenMulAdd(&context_); InstructionCounter* instruction_counter = new InstructionCounter(); instruction_counter->visit(*MulAddModule); instruction_counter->PrintCounters(); @@ -149,7 +151,7 @@ Module* CodegenGcd(LLVMContext* context) { } TEST_F(InstructionCounterTest, TestMemInstrCount) { - Module* GcdModule = CodegenGcd(&getGlobalContext()); + Module* GcdModule = CodegenGcd(&context_); InstructionCounter* instruction_counter = new InstructionCounter(); instruction_counter->visit(*GcdModule); std::cout << instruction_counter->PrintCounters(); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/be/src/codegen/llvm-codegen.cc ---------------------------------------------------------------------- diff --git a/be/src/codegen/llvm-codegen.cc b/be/src/codegen/llvm-codegen.cc index 2a8bb4b..6f8b156 100644 --- a/be/src/codegen/llvm-codegen.cc +++ b/be/src/codegen/llvm-codegen.cc @@ -20,6 +20,8 @@ #include <fstream> #include <iostream> #include <sstream> +#include <unordered_set> + #include <boost/algorithm/string.hpp> #include <boost/thread/mutex.hpp> #include <gutil/strings/substitute.h> @@ -251,9 +253,9 @@ Status LlvmCodeGen::LoadModuleFromMemory(unique_ptr<MemoryBuffer> module_ir_buf, string module_name, unique_ptr<Module>* module) { DCHECK(!module_name.empty()); SCOPED_TIMER(prepare_module_timer_); - ErrorOr<unique_ptr<Module>> tmp_module(NULL); COUNTER_ADD(module_bitcode_size_, module_ir_buf->getMemBufferRef().getBufferSize()); - tmp_module = getLazyBitcodeModule(std::move(module_ir_buf), context(), false); + ErrorOr<unique_ptr<Module>> tmp_module = + getLazyBitcodeModule(std::move(module_ir_buf), context(), false); if (!tmp_module) { stringstream ss; ss << "Could not parse module " << module_name << ": " << tmp_module.getError(); @@ -444,7 +446,7 @@ string LlvmCodeGen::GetIR(bool full_module) const { module_->print(stream, NULL); } else { for (int i = 0; i < codegend_functions_.size(); ++i) { - codegend_functions_[i]->print(stream, true); + codegend_functions_[i]->print(stream, nullptr, false, true); } } return str; @@ -970,9 +972,8 @@ Function* LlvmCodeGen::CloneFunction(Function* fn) { // GetFunction() to obtain the Function object. DCHECK(!fn->isMaterializable()); // CloneFunction() automatically gives the new function a unique name - Function* fn_clone = llvm::CloneFunction(fn, dummy_vmap, false); + Function* fn_clone = llvm::CloneFunction(fn, dummy_vmap); fn_clone->copyAttributesFrom(fn); - module_->getFunctionList().push_back(fn_clone); return fn_clone; } @@ -1117,13 +1118,16 @@ Status LlvmCodeGen::OptimizeModule() { // global dead code elimination pass. This causes all functions not registered to be // JIT'd to be marked as internal, and any internal functions that are not used are // deleted by DCE pass. This greatly decreases compile time by removing unused code. - vector<const char*> exported_fn_names; - for (int i = 0; i < fns_to_jit_compile_.size(); ++i) { - exported_fn_names.push_back(fns_to_jit_compile_[i].first->getName().data()); + unordered_set<string> exported_fn_names; + for (auto& entry : fns_to_jit_compile_) { + exported_fn_names.insert(entry.first->getName().str()); } unique_ptr<legacy::PassManager> module_pass_manager(new legacy::PassManager()); module_pass_manager->add(createTargetTransformInfoWrapperPass(target_analysis)); - module_pass_manager->add(createInternalizePass(exported_fn_names)); + module_pass_manager->add( + createInternalizePass([&exported_fn_names] (const GlobalValue &gv) { + return exported_fn_names.find(gv.getName().str()) != exported_fn_names.end(); + })); module_pass_manager->add(createGlobalDCEPass()); module_pass_manager->run(*module_); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/be/src/codegen/llvm-codegen.h ---------------------------------------------------------------------- diff --git a/be/src/codegen/llvm-codegen.h b/be/src/codegen/llvm-codegen.h index ea31372..8dce330 100644 --- a/be/src/codegen/llvm-codegen.h +++ b/be/src/codegen/llvm-codegen.h @@ -62,10 +62,9 @@ namespace llvm { class PassManager; } - template<bool B, typename T, typename I> + template<typename T, typename I> class IRBuilder; - template<bool preserveName> class IRBuilderDefaultInserter; } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/be/src/common/compiler-util.h ---------------------------------------------------------------------- diff --git a/be/src/common/compiler-util.h b/be/src/common/compiler-util.h index 592c08c..1acd73a 100644 --- a/be/src/common/compiler-util.h +++ b/be/src/common/compiler-util.h @@ -55,7 +55,7 @@ /// GCC 5+ and Clang 3.6+ support __has_cpp_attribute(). Always return false on compilers /// that don't know about __has_cpp_attribute(). -#if !defined(__GNUC__) || __GNUC__ >= 5 +#if defined(__clang__) || __GNUC__ >= 5 #define HAS_CPP_ATTRIBUTE(attr) __has_cpp_attribute(attr) #else #define HAS_CPP_ATTRIBUTE(attr) 0 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/be/src/udf/udf.cc ---------------------------------------------------------------------- diff --git a/be/src/udf/udf.cc b/be/src/udf/udf.cc index 212fd0c..82dd4d8 100644 --- a/be/src/udf/udf.cc +++ b/be/src/udf/udf.cc @@ -499,15 +499,16 @@ void FunctionContextImpl::SetNonConstantArgs(NonConstantArgsVector&& non_constan // Note: this function crashes LLVM's JIT in expr-test if it's xcompiled. Do not move to // expr-ir.cc. This could probably use further investigation. -StringVal::StringVal(FunctionContext* context, int len) noexcept : len(len), ptr(NULL) { - if (UNLIKELY(len > StringVal::MAX_LENGTH)) { +StringVal::StringVal(FunctionContext* context, int str_len) noexcept : len(str_len), + ptr(NULL) { + if (UNLIKELY(str_len > StringVal::MAX_LENGTH)) { context->SetError("String length larger than allowed limit of " "1 GB character data."); len = 0; is_null = true; } else { - ptr = context->impl()->AllocateLocal(len); - if (UNLIKELY(ptr == NULL && len > 0)) { + ptr = context->impl()->AllocateLocal(str_len); + if (UNLIKELY(ptr == NULL && str_len > 0)) { #ifndef IMPALA_UDF_SDK_BUILD assert(!context->impl()->state()->GetQueryStatus().ok()); #endif http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/bin/bootstrap_toolchain.py ---------------------------------------------------------------------- diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index 38b8d05..3a1f02d 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -352,7 +352,7 @@ if __name__ == "__main__": packages = ["avro", "binutils", "boost", "breakpad", "bzip2", "cmake", "crcutil", "flatbuffers", "gcc", "gflags", "glog", "gperftools", "gtest", "kudu", "libev", - "llvm", ("llvm", "3.8.0-asserts-p1"), "lz4", "openldap", "openssl", "protobuf", + "llvm", ("llvm", "3.9.1-asserts"), "lz4", "openldap", "openssl", "protobuf", "rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib"] bootstrap(toolchain_root, packages) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1e63ff84/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 74d888c..713e5fb 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -72,7 +72,7 @@ fi # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -export IMPALA_TOOLCHAIN_BUILD_ID=465-9a2affdeab +export IMPALA_TOOLCHAIN_BUILD_ID=468-0b4f08e601 # Versions of toolchain dependencies. # ----------------------------------- export IMPALA_AVRO_VERSION=1.7.4-p4 @@ -80,7 +80,7 @@ export IMPALA_BINUTILS_VERSION=2.26.1 export IMPALA_BOOST_VERSION=1.57.0-p3 export IMPALA_BREAKPAD_VERSION=1b704857f1e78a864e6942e613457e55f1aecb60-p3 export IMPALA_BZIP2_VERSION=1.0.6-p2 -export IMPALA_CMAKE_VERSION=3.2.3-p1 +export IMPALA_CMAKE_VERSION=3.8.2-p1 export IMPALA_CRCUTIL_VERSION=440ba7babeff77ffad992df3a10c767f184e946e-p1 export IMPALA_CYRUS_SASL_VERSION=2.1.23 export IMPALA_FLATBUFFERS_VERSION=1.6.0 @@ -90,11 +90,11 @@ export IMPALA_GLOG_VERSION=0.3.4-p2 export IMPALA_GPERFTOOLS_VERSION=2.5 export IMPALA_GTEST_VERSION=1.6.0 export IMPALA_LIBEV_VERSION=4.20 -export IMPALA_LLVM_VERSION=3.8.0-p1 -export IMPALA_LLVM_ASAN_VERSION=3.8.0-p1 +export IMPALA_LLVM_VERSION=3.9.1 +export IMPALA_LLVM_ASAN_VERSION=3.9.1 # Debug builds should use the release+asserts build to get additional coverage. # Don't use the LLVM debug build because the binaries are too large to distribute. -export IMPALA_LLVM_DEBUG_VERSION=3.8.0-asserts-p1 +export IMPALA_LLVM_DEBUG_VERSION=3.9.1-asserts export IMPALA_LZ4_VERSION=1.7.5 export IMPALA_OPENLDAP_VERSION=2.4.25 export IMPALA_OPENSSL_VERSION=1.0.2l
