This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 800a363 [chore] Prolog of hermetic build with GCC 11 and Clang 13.
(#7712)
800a363 is described below
commit 800a36343a2eddeb23442eb5fc8ae19a5b626c5a
Author: Amos Bird <[email protected]>
AuthorDate: Fri Jan 21 12:12:04 2022 +0800
[chore] Prolog of hermetic build with GCC 11 and Clang 13. (#7712)
Prepare to generate hermetic build using GCC 11 and Clang 13.
The ideal toolchain would be ldb toolchain generated by
[ldb_toolchain_gen.sh](https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.3/ldb_toolchain_gen.sh)
To kick off a clang build, set `DORIS_TOOLCHAIN=clang` before running any
build scripts.
---
CONTRIBUTING_CN.md | 4 +-
README.md | 2 +-
be/CMakeLists.txt | 162 ++++++++++++++++----
be/src/agent/topic_subscriber.h | 1 +
be/src/exec/olap_utils.h | 1 +
be/src/exprs/runtime_filter.cpp | 6 +-
be/src/exprs/runtime_filter.h | 2 +-
be/src/exprs/table_function/explode_split.cpp | 4 +-
be/src/glibc-compatibility/CMakeLists.txt | 11 ++
be/src/glibc-compatibility/glibc-compat-2.32.h | 50 +++++++
be/src/glibc-compatibility/glibc-compatibility.c | 9 +-
be/src/glibc-compatibility/musl/eventfd.c | 23 +++
be/src/glibc-compatibility/musl/getauxval.c | 49 +++++--
be/src/glibc-compatibility/musl/getrandom.c | 23 +++
be/src/glibc-compatibility/musl/sched_getcpu.c | 55 +++++++
be/src/glibc-compatibility/musl/signgam.c | 5 +
be/src/gutil/strings/util.cc | 5 +-
be/src/http/action/pprof_actions.cpp | 4 +-
be/src/olap/column_vector.cpp | 29 ++++
be/src/olap/column_vector.h | 29 ----
be/src/olap/rowset/segment_v2/page_handle.h | 2 +-
be/src/olap/types.h | 2 +-
be/src/runtime/dpp_sink_internal.cpp | 7 +-
be/src/runtime/large_int_value.h | 4 +
be/src/runtime/raw_value_ir.cpp | 1 +
be/src/udf/udf.cpp | 2 +-
be/src/util/bfd_parser.cpp | 4 +-
be/src/util/bitmap_value.h | 4 -
be/src/util/cidr.cpp | 4 +-
be/src/util/pprof_utils.cpp | 4 +-
be/src/util/simd/vstring_function.h | 11 +-
be/src/vec/common/exception.h | 1 +
be/src/vec/functions/function_grouping.h | 4 +-
.../segment_v2/frame_of_reference_page_test.cpp | 1 -
be/test/plugin/plugin_test/source/test.zip | Bin 2542556 -> 545991
bytes
be/test/plugin/plugin_test/source/test.zip.md5 | 2 +-
build.sh | 8 +-
contrib/udf/CMakeLists.txt | 9 --
docs/en/installing/compilation.md | 30 ++--
env.sh | 54 +++++--
gensrc/script/doris_builtins_functions.py | 2 +-
gensrc/script/gen_builtins_functions.py | 21 +--
gensrc/script/gen_functions.py | 8 +-
gensrc/script/gen_vector_functions.py | 8 +-
run-fe-ut.sh | 8 +-
thirdparty/build-thirdparty.sh | 163 +++++++++++++--------
thirdparty/download-thirdparty.sh | 39 +++--
thirdparty/patches/googletest-release-1.11.0.patch | 71 +++++++++
thirdparty/patches/libevent.patch | 11 ++
thirdparty/patches/libhdfs3-master.patch | 12 ++
thirdparty/patches/mysql-server-mysql-5.7.18.patch | 12 ++
thirdparty/patches/rocksdb-5.14.2.patch | 11 ++
thirdparty/vars.sh | 22 ++-
53 files changed, 769 insertions(+), 247 deletions(-)
diff --git a/CONTRIBUTING_CN.md b/CONTRIBUTING_CN.md
index ee1e9af..72f0839 100644
--- a/CONTRIBUTING_CN.md
+++ b/CONTRIBUTING_CN.md
@@ -1,4 +1,4 @@
-<!--
+<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -32,7 +32,7 @@ under the License.
初次来到 Doris 社区，您可以：
* 关注 Doris [Github 代码库](https://github.com/apache/incubator-doris)
-* 订阅我们的 [邮件列表](./subscribe-mail-list.md)；
+* 订阅我们的 [邮件列表](./subscribe-mail-list.md)；
* 加入 Doris 微信群(加微信号：morningman-cmy, 备注：加入Doris群) 随时提问；
* 进入 Doris 的 [Gitter](./gitter.md) 聊天室；
diff --git a/README.md b/README.md
index 47115d2..45f6c9a 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ The main advantages of Doris are the simplicity (of
developing, deploying and us
[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
-> **Note**
+> **Note**
> Some licenses of the third-party dependencies are not compatible with Apache
> 2.0 License. So you need to disable
some Doris features to be complied with Apache 2.0 License. For details, refer
to the `thirdparty/LICENSE.txt`
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 1bd3e49..320968f 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -17,15 +17,6 @@
cmake_minimum_required(VERSION 3.19.2)
-# set CMAKE_C_COMPILER, this must set before project command
-if (DEFINED ENV{DORIS_GCC_HOME})
- set(CMAKE_C_COMPILER "$ENV{DORIS_GCC_HOME}/bin/gcc")
- set(CMAKE_CXX_COMPILER "$ENV{DORIS_GCC_HOME}/bin/g++")
- set(GCC_HOME $ENV{DORIS_GCC_HOME})
-else()
- message(FATAL_ERROR "DORIS_GCC_HOME environment variable is not set")
-endif()
-
project(doris CXX C)
# Write compile_commands.json
@@ -52,6 +43,12 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)")
set (ARCH_PPC64LE 1)
endif ()
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+ set (COMPILER_GCC 1)
+elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+ set (COMPILER_CLANG 1)
+endif ()
+
option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries."
ON)
message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}")
@@ -95,8 +92,6 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
message(FATAL_ERROR "Need GCC version at least 7.3.0")
endif()
message(STATUS "GCC version is greater than 7.3.0, disable -Werror. Be
careful with compile warnings.")
-elseif (NOT APPLE)
- message(FATAL_ERROR "Compiler should be GNU")
endif()
set(PIC_LIB_PATH "${THIRDPARTY_DIR}")
@@ -299,6 +294,9 @@ set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib
add_library(minizip STATIC IMPORTED)
set_target_properties(minizip PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/libminizip.a)
+add_library(idn STATIC IMPORTED)
+set_target_properties(idn PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/libidn.a)
+
add_library(gsasl STATIC IMPORTED)
set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/libgsasl.a)
@@ -344,6 +342,106 @@ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}
-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFI
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED")
# Enable the cpu and heap profile of brpc
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBRPC_ENABLE_CPU_PROFILER")
+
+if (COMPILER_GCC)
+ # Avoid GCC 11 false alarm
+ #
https://stackoverflow.com/questions/67584073/gcc-11-false-array-subscript-is-partly-outside-array-bounds-warning
+ #
https://stackoverflow.com/questions/69426070/gcc-11-order-of-arguments-triggers-false-positive-wstringop-overflow-is-this-bu
+ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-array-bounds
-Wno-stringop-overread")
+endif ()
+
+if (COMPILER_CLANG)
+ # TODO: Fix all these warnings
+ add_compile_options(
+ -Wno-address-of-packed-member
+ -Wno-ambiguous-reversed-operator
+ -Wno-atomic-implicit-seq-cst
+ -Wno-cast-align
+ -Wno-cast-qual
+ -Wno-comma
+ -Wno-conditional-uninitialized
+ -Wno-constant-conversion
+ -Wno-covered-switch-default
+ -Wno-delete-abstract-non-virtual-dtor
+ -Wno-delete-non-abstract-non-virtual-dtor
+ -Wno-deprecated
+ -Wno-deprecated-copy-with-dtor
+ -Wno-deprecated-copy-with-user-provided-copy
+ -Wno-deprecated-copy-with-user-provided-dtor
+ -Wno-deprecated-declarations
+ -Wno-deprecated-this-capture
+ -Wno-documentation
+ -Wno-embedded-directive
+ -Wno-enum-compare-switch
+ -Wno-extra-semi
+ -Wno-extra-semi-stmt
+ -Wno-format-nonliteral
+ -Wno-frame-larger-than
+ -Wno-gnu-label-as-value
+ -Wno-gnu-redeclared-enum
+ -Wno-gnu-statement-expression
+ -Wno-gnu-zero-variadic-macro-arguments
+ -Wno-header-hygiene
+ -Wno-ignored-qualifiers
+ -Wno-implicit-const-int-float-conversion
+ -Wno-implicit-exception-spec-mismatch
+ -Wno-implicit-fallthrough
+ -Wno-inconsistent-missing-destructor-override
+ -Wno-inconsistent-missing-override
+ -Wno-instantiation-after-specialization
+ -Wno-macro-redefined
+ -Wno-mismatched-new-delete
+ -Wno-mismatched-tags
+ -Wno-missing-field-initializers
+ -Wno-missing-noreturn
+ -Wno-newline-eof
+ -Wno-non-virtual-dtor
+ -Wno-null-conversion
+ -Wno-null-dereference
+ -Wno-old-style-cast
+ -Wno-overloaded-virtual
+ -Wno-pedantic
+ -Wno-pessimizing-move
+ -Wno-pragma-once-outside-header
+ -Wno-reserved-identifier
+ -Wno-reserved-macro-identifier
+ -Wno-return-type
+ -Wno-shadow
+ -Wno-shadow-field
+ -Wno-shadow-field-in-constructor
+ -Wno-shadow-uncaptured-local
+ -Wno-shift-count-overflow
+ -Wno-sign-compare
+ -Wno-string-plus-int
+ -Wno-suggest-destructor-override
+ -Wno-suggest-override
+ -Wno-tautological-constant-out-of-range-compare
+ -Wno-tautological-unsigned-zero-compare
+ -Wno-undef
+ -Wno-uninitialized
+ -Wno-unknown-pragmas
+ -Wno-unknown-warning-option
+ -Wno-unreachable-code
+ -Wno-unreachable-code-break
+ -Wno-unreachable-code-return
+ -Wno-unused-but-set-variable
+ -Wno-unused-const-variable
+ -Wno-unused-exception-parameter
+ -Wno-unused-function
+ -Wno-unused-lambda-capture
+ -Wno-unused-local-typedef
+ -Wno-unused-macros
+ -Wno-unused-member-function
+ -Wno-unused-parameter
+ -Wno-unused-private-field
+ -Wno-unused-result
+ -Wno-unused-value
+ -Wno-unused-variable
+ -Wno-variadic-macros
+ -Wno-zero-as-null-pointer-constant
+ )
+endif ()
+
# https://github.com/boostorg/uuid/issues/92
# We need to avoid using SYS_getrandom system calls
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}
-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1")
@@ -436,6 +534,10 @@ add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H)
include_directories(
${SRC_DIR}/
${TEST_DIR}/
+)
+
+include_directories(
+ SYSTEM
${GENSRC_DIR}/
${THIRDPARTY_DIR}/include
${GPERFTOOLS_HOME}/include
@@ -498,6 +600,7 @@ set(COMMON_THIRDPARTY
lz4
libevent
libevent_pthreads
+ idn
gsasl
curl
${LIBZ}
@@ -532,13 +635,13 @@ set(COMMON_THIRDPARTY
if (${MAKE_TEST} STREQUAL "ON")
set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
- benchmark
+ benchmark
)
endif()
# thirdparties dependescies that can only run on X86 platform
set(X86_DEPENDENCIES
- ${COMMON_THIRDPARTY}
+ ${COMMON_THIRDPARTY}
hdfs3
xml2
lzma
@@ -590,31 +693,41 @@ if ("${CMAKE_BUILD_TYPE}" STREQUAL "BCC")
)
endif()
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+ set(ASAN_LIBS -static-libasan)
+ set(LSAN_LIBS -static-liblsan)
+ set(UBSAN_LIBS -static-libubsan tcmalloc)
+ set(TSAN_LIBS -static-libtsan)
+else ()
+ set(UBSAN_LIBS tcmalloc)
+endif ()
+
# Add sanitize static link flags or tcmalloc
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR "${CMAKE_BUILD_TYPE}" STREQUAL
"RELEASE" OR "${CMAKE_BUILD_TYPE}" STREQUAL "BCC")
set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} tcmalloc)
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ASAN")
- set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -static-libasan)
+ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${ASAN_LIBS})
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "LSAN")
- set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -static-liblsan)
+ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${LSAN_LIBS})
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
- set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -static-libubsan tcmalloc)
+ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${UBSAN_LIBS})
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
- set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -static-libtsan)
+ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${TSAN_LIBS})
add_definitions("-DTHREAD_SANITIZER")
else()
message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
endif()
-set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
- -lrt -l:libbfd.a -liberty -lc -lm -ldl -pthread
-)
-
+# NOTE(amos): This should come before -lc -lm to interpose symbols correctly.
if (GLIBC_COMPATIBILITY)
add_subdirectory(${SRC_DIR}/glibc-compatibility)
- set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} glibc-compatibility)
+ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} glibc-compatibility-explicit
glibc-compatibility)
endif()
+set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
+ -lrt -l:libbfd.a -liberty -lc -lm -ldl -pthread
+)
+
# Set libraries for test
set (TEST_LINK_LIBS ${DORIS_LINK_LIBS}
${WL_START_GROUP}
@@ -735,8 +848,7 @@ install(FILES
${BASE_DIR}/../conf/odbcinst.ini
DESTINATION ${OUTPUT_DIR}/conf)
-
get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY
INCLUDE_DIRECTORIES)
foreach(dir ${dirs})
- message(STATUS "dir='${dir}'")
- endforeach()
+ message(STATUS "dir='${dir}'")
+endforeach()
diff --git a/be/src/agent/topic_subscriber.h b/be/src/agent/topic_subscriber.h
index 2f5ed7f..2c5d32f 100644
--- a/be/src/agent/topic_subscriber.h
+++ b/be/src/agent/topic_subscriber.h
@@ -19,6 +19,7 @@
#define DORIS_BE_SRC_AGENT_TOPIC_SUBSCRIBER_H
#include <map>
+#include <mutex>
#include <shared_mutex>
#include <thread>
diff --git a/be/src/exec/olap_utils.h b/be/src/exec/olap_utils.h
index 129bd2c..525c688 100644
--- a/be/src/exec/olap_utils.h
+++ b/be/src/exec/olap_utils.h
@@ -76,6 +76,7 @@ inline CompareLargeFunc get_compare_func(PrimitiveType type) {
default:
DCHECK(false) << "Unsupported Compare type";
}
+ __builtin_unreachable();
}
static const char* NEGATIVE_INFINITY = "-oo";
diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 3cf45c4..8e0e55e 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -942,11 +942,11 @@ Status IRuntimeFilter::init_with_desc(const
TRuntimeFilterDesc* desc, const TQue
}
Status IRuntimeFilter::serialize(PMergeFilterRequest* request, void** data,
int* len) {
- return _serialize(request, data, len);
+ return serialize_impl(request, data, len);
}
Status IRuntimeFilter::serialize(PPublishFilterRequest* request, void** data,
int* len) {
- return _serialize(request, data, len);
+ return serialize_impl(request, data, len);
}
Status IRuntimeFilter::create_wrapper(const MergeRuntimeFilterParams* param,
MemTracker* tracker,
@@ -1034,7 +1034,7 @@ void batch_copy(PInFilter* filter,
HybridSetBase::IteratorBase* it,
}
template <class T>
-Status IRuntimeFilter::_serialize(T* request, void** data, int* len) {
+Status IRuntimeFilter::serialize_impl(T* request, void** data, int* len) {
request->set_filter_type(get_type(_runtime_filter_type));
if (_runtime_filter_type == RuntimeFilterType::IN_FILTER) {
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index c5e597a..60a3c68 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -221,7 +221,7 @@ protected:
void to_protobuf(PMinMaxFilter* filter);
template <class T>
- Status _serialize(T* request, void** data, int* len);
+ Status serialize_impl(T* request, void** data, int* len);
template <class T>
static Status _create_wrapper(const T* param, MemTracker* tracker,
ObjectPool* pool,
diff --git a/be/src/exprs/table_function/explode_split.cpp
b/be/src/exprs/table_function/explode_split.cpp
index 959dad4..b1a0fe7 100644
--- a/be/src/exprs/table_function/explode_split.cpp
+++ b/be/src/exprs/table_function/explode_split.cpp
@@ -40,7 +40,7 @@ Status ExplodeSplitTableFunction::open() {
// check if the delimiter argument(the 2nd arg) is constant.
// if yes, cache it
if (fn_ctx->is_arg_constant(1)) {
- _is_delimiter_constant = true;
+ _is_delimiter_constant = true;
StringVal* delimiter =
reinterpret_cast<StringVal*>(fn_ctx->get_constant_arg(1));
_const_delimter = StringPiece((char*) delimiter->ptr, delimiter->len);
}
@@ -65,7 +65,7 @@ Status ExplodeSplitTableFunction::process(TupleRow*
tuple_row) {
StringVal delimiter =
_expr_context->root()->get_child(1)->get_string_val(_expr_context, tuple_row);
_backup = strings::Split(StringPiece((char*) text.ptr, text.len),
StringPiece((char*) delimiter.ptr, delimiter.len));
}
- for (const std::string str : _backup) {
+ for (const std::string & str : _backup) {
_data.emplace_back(str);
}
_cur_size = _backup.size();
diff --git a/be/src/glibc-compatibility/CMakeLists.txt
b/be/src/glibc-compatibility/CMakeLists.txt
index 40dd6fd..bd2e520 100644
--- a/be/src/glibc-compatibility/CMakeLists.txt
+++ b/be/src/glibc-compatibility/CMakeLists.txt
@@ -48,9 +48,19 @@ if (GLIBC_COMPATIBILITY)
list(APPEND glibc_compatibility_sources musl/getentropy.c)
endif()
+ list(REMOVE_ITEM glibc_compatibility_sources musl/getrandom.c
memcpy_wrapper.c)
+
# Need to omit frame pointers to match the performance of glibc
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer")
+ # NOTE(amos): getrandom is hard to interpose since we build thirdparty deps
+ # without glibc-compatibility. Also sanitizers might generate getrandom
+ # libcalls. Workaround: Use object file so that linker will always take a
+ # look at its symbol table.
+ # NOTE(amos): sanitizers might generate memcpy references that are too
late to
+ # refer. Let's also extract memcpy definitions explicitly to avoid UNDEF
GLIBC 2.14.
+ add_library(glibc-compatibility-explicit OBJECT musl/getrandom.c
memcpy_wrapper.c)
+ target_compile_options(glibc-compatibility-explicit PRIVATE -fPIC)
add_library(glibc-compatibility STATIC ${glibc_compatibility_sources})
if (COMPILER_CLANG)
@@ -60,6 +70,7 @@ if (GLIBC_COMPATIBILITY)
endif ()
target_include_directories(glibc-compatibility PRIVATE
${musl_arch_include_dir})
+ target_include_directories(glibc-compatibility-explicit PRIVATE
${musl_arch_include_dir})
message (STATUS "Some symbols from glibc will be replaced for
compatibility")
else()
diff --git a/be/src/glibc-compatibility/glibc-compat-2.32.h
b/be/src/glibc-compatibility/glibc-compat-2.32.h
new file mode 100644
index 0000000..53ed34d
--- /dev/null
+++ b/be/src/glibc-compatibility/glibc-compat-2.32.h
@@ -0,0 +1,50 @@
+/// In glibc 2.32 new version of some symbols had been added [1]:
+///
+/// $ nm -D clickhouse | fgrep -e @GLIBC_2.32
+/// U pthread_getattr_np@GLIBC_2.32
+/// U pthread_sigmask@GLIBC_2.32
+///
+/// [1]: https://www.spinics.net/lists/fedora-devel/msg273044.html
+///
+/// Right now ubuntu 20.04 is used as official image for building
+/// ClickHouse, however once it will be switched someone may not be happy
+/// with that fact that he/she cannot use official binaries anymore because
+/// they have glibc < 2.32.
+///
+/// To avoid this dependency, let's force previous version of those
+/// symbols from glibc.
+///
+/// Also note, that the following approach had been tested:
+/// a) -Wl,--wrap -- but it goes into endless recursion whey you try to do
+/// something like this:
+///
+/// int __pthread_getattr_np_compact(pthread_t thread, pthread_attr_t
*attr);
+/// GLIBC_COMPAT_SYMBOL(__pthread_getattr_np_compact, pthread_getattr_np)
+/// int __pthread_getattr_np_compact(pthread_t thread, pthread_attr_t
*attr);
+/// int __wrap_pthread_getattr_np(pthread_t thread, pthread_attr_t *attr)
+/// {
+/// return __pthread_getattr_np_compact(thread, attr);
+/// }
+///
+/// int __pthread_sigmask_compact(int how, const sigset_t *set, sigset_t
*oldset);
+/// GLIBC_COMPAT_SYMBOL(__pthread_sigmask_compact, pthread_sigmask)
+/// int __pthread_sigmask_compact(int how, const sigset_t *set, sigset_t
*oldset);
+/// int __wrap_pthread_sigmask(int how, const sigset_t *set, sigset_t
*oldset)
+/// {
+/// return __pthread_sigmask_compact(how, set, oldset);
+/// }
+///
+/// b) -Wl,--defsym -- same problems (and you cannot use version of symbol with
+/// version in the expression)
+/// c) this approach -- simply add this file with -include directive.
+
+#if defined(__amd64__)
+#define GLIBC_COMPAT_SYMBOL(func) __asm__(".symver " #func "," #func
"@GLIBC_2.2.5");
+#elif defined(__aarch64__)
+#define GLIBC_COMPAT_SYMBOL(func) __asm__(".symver " #func "," #func
"@GLIBC_2.17");
+#else
+#error Your platform is not supported.
+#endif
+
+GLIBC_COMPAT_SYMBOL(pthread_sigmask)
+GLIBC_COMPAT_SYMBOL(pthread_getattr_np)
diff --git a/be/src/glibc-compatibility/glibc-compatibility.c
b/be/src/glibc-compatibility/glibc-compatibility.c
index d4bb739..e3f62b7 100644
--- a/be/src/glibc-compatibility/glibc-compatibility.c
+++ b/be/src/glibc-compatibility/glibc-compatibility.c
@@ -8,13 +8,6 @@
extern "C" {
#endif
-#include <pthread.h>
-
-size_t __pthread_get_minstack(const pthread_attr_t * attr)
-{
- return 1048576; /// This is a guess. Don't sure it is correct.
-}
-
#include <signal.h>
#include <unistd.h>
#include <string.h>
@@ -141,6 +134,8 @@ int __open_2(const char *path, int oflag)
}
+#include <pthread.h>
+
/// No-ops.
int pthread_setname_np(pthread_t thread, const char *name) { return 0; }
int pthread_getname_np(pthread_t thread, char *name, size_t len) { name[0] =
'\0'; return 0; };
diff --git a/be/src/glibc-compatibility/musl/eventfd.c
b/be/src/glibc-compatibility/musl/eventfd.c
new file mode 100644
index 0000000..68e489c
--- /dev/null
+++ b/be/src/glibc-compatibility/musl/eventfd.c
@@ -0,0 +1,23 @@
+#include <sys/eventfd.h>
+#include <unistd.h>
+#include <errno.h>
+#include "syscall.h"
+
+int eventfd(unsigned int count, int flags)
+{
+ int r = __syscall(SYS_eventfd2, count, flags);
+#ifdef SYS_eventfd
+ if (r==-ENOSYS && !flags) r = __syscall(SYS_eventfd, count);
+#endif
+ return __syscall_ret(r);
+}
+
+int eventfd_read(int fd, eventfd_t *value)
+{
+ return (sizeof(*value) == read(fd, value, sizeof(*value))) ? 0 : -1;
+}
+
+int eventfd_write(int fd, eventfd_t value)
+{
+ return (sizeof(value) == write(fd, &value, sizeof(value))) ? 0 : -1;
+}
diff --git a/be/src/glibc-compatibility/musl/getauxval.c
b/be/src/glibc-compatibility/musl/getauxval.c
index a429273..dad7aa9 100644
--- a/be/src/glibc-compatibility/musl/getauxval.c
+++ b/be/src/glibc-compatibility/musl/getauxval.c
@@ -1,4 +1,5 @@
#include <sys/auxv.h>
+#include "atomic.h"
#include <unistd.h> // __environ
#include <errno.h>
@@ -17,18 +18,7 @@ static size_t __find_auxv(unsigned long type)
return (size_t) -1;
}
-__attribute__((constructor)) static void __auxv_init()
-{
- size_t i;
- for (i = 0; __environ[i]; i++);
- __auxv = (unsigned long *) (__environ + i + 1);
-
- size_t secure_idx = __find_auxv(AT_SECURE);
- if (secure_idx != ((size_t) -1))
- __auxv_secure = __auxv[secure_idx];
-}
-
-unsigned long getauxval(unsigned long type)
+unsigned long __getauxval(unsigned long type)
{
if (type == AT_SECURE)
return __auxv_secure;
@@ -43,3 +33,38 @@ unsigned long getauxval(unsigned long type)
errno = ENOENT;
return 0;
}
+
+static void * volatile getauxval_func;
+
+static unsigned long __auxv_init(unsigned long type)
+{
+ if (!__environ)
+ {
+ // __environ is not initialized yet so we can't initialize __auxv
right now.
+ // That's normally occurred only when getauxval() is called from some
sanitizer's internal code.
+ errno = ENOENT;
+ return 0;
+ }
+
+ // Initialize __auxv and __auxv_secure.
+ size_t i;
+ for (i = 0; __environ[i]; i++);
+ __auxv = (unsigned long *) (__environ + i + 1);
+
+ size_t secure_idx = __find_auxv(AT_SECURE);
+ if (secure_idx != ((size_t) -1))
+ __auxv_secure = __auxv[secure_idx];
+
+ // Now we've initialized __auxv, next time getauxval() will only call
__get_auxval().
+ a_cas_p(&getauxval_func, (void *)__auxv_init, (void *)__getauxval);
+
+ return __getauxval(type);
+}
+
+// First time getauxval() will call __auxv_init().
+static void * volatile getauxval_func = (void *)__auxv_init;
+
+unsigned long getauxval(unsigned long type)
+{
+ return ((unsigned long (*)(unsigned long))getauxval_func)(type);
+}
diff --git a/be/src/glibc-compatibility/musl/getrandom.c
b/be/src/glibc-compatibility/musl/getrandom.c
new file mode 100644
index 0000000..32364ee
--- /dev/null
+++ b/be/src/glibc-compatibility/musl/getrandom.c
@@ -0,0 +1,23 @@
+/** We have to replace glibc getrandom only when glibc version is higher than
2.25.
+ * In previous versions of glibc this function doesn't exist
+ * and old kernels may be missing SYS_getrandom syscall.
+ */
+#include <features.h>
+#if defined(__GLIBC__) && __GLIBC__ >= 2
+# define GLIBC_MINOR __GLIBC_MINOR__
+#elif defined (__GNU_LIBRARY__) && __GNU_LIBRARY__ >= 2
+# define GLIBC_MINOR __GNU_LIBRARY_MINOR__
+#endif
+
+#if defined(GLIBC_MINOR) && GLIBC_MINOR >= 25
+
+#include <unistd.h>
+#include <syscall.h>
+#include "syscall.h"
+
+ssize_t getrandom(void *buf, size_t buflen, unsigned flags)
+{
+ /// There was cancellable syscall (syscall_cp), but I don't care too.
+ return syscall(SYS_getrandom, buf, buflen, flags);
+}
+#endif
diff --git a/be/src/glibc-compatibility/musl/sched_getcpu.c
b/be/src/glibc-compatibility/musl/sched_getcpu.c
new file mode 100644
index 0000000..f290f01
--- /dev/null
+++ b/be/src/glibc-compatibility/musl/sched_getcpu.c
@@ -0,0 +1,55 @@
+#define _GNU_SOURCE
+#include <errno.h>
+#include <sched.h>
+#include "syscall.h"
+#include "atomic.h"
+
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+#include <sanitizer/msan_interface.h>
+#endif
+#endif
+
+#ifdef VDSO_GETCPU_SYM
+
+static void *volatile vdso_func;
+
+typedef long (*getcpu_f)(unsigned *, unsigned *, void *);
+
+static long getcpu_init(unsigned *cpu, unsigned *node, void *unused)
+{
+ void *p = __vdsosym(VDSO_GETCPU_VER, VDSO_GETCPU_SYM);
+ getcpu_f f = (getcpu_f)p;
+ a_cas_p(&vdso_func, (void *)getcpu_init, p);
+ return f ? f(cpu, node, unused) : -ENOSYS;
+}
+
+static void *volatile vdso_func = (void *)getcpu_init;
+
+#endif
+
+int sched_getcpu(void)
+{
+ int r;
+ unsigned cpu = 0;
+
+#ifdef VDSO_GETCPU_SYM
+ getcpu_f f = (getcpu_f)vdso_func;
+ if (f) {
+ r = f(&cpu, 0, 0);
+ if (!r) return cpu;
+ if (r != -ENOSYS) return __syscall_ret(r);
+ }
+#endif
+
+ r = __syscall(SYS_getcpu, &cpu, 0, 0);
+ if (!r) {
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+ __msan_unpoison(&cpu, sizeof(cpu));
+#endif
+#endif
+ return cpu;
+ }
+ return __syscall_ret(r);
+}
diff --git a/be/src/glibc-compatibility/musl/signgam.c
b/be/src/glibc-compatibility/musl/signgam.c
new file mode 100644
index 0000000..d2147fb
--- /dev/null
+++ b/be/src/glibc-compatibility/musl/signgam.c
@@ -0,0 +1,5 @@
+#include <math.h>
+#include "libm.h"
+
+int __signgam = 0;
+extern __typeof(__signgam) signgam __attribute__((__weak__,
__alias__("__signgam")));
diff --git a/be/src/gutil/strings/util.cc b/be/src/gutil/strings/util.cc
index aa3bcd3..57751ce 100644
--- a/be/src/gutil/strings/util.cc
+++ b/be/src/gutil/strings/util.cc
@@ -697,8 +697,9 @@ char* strndup_with_new(const char* the_string, int
max_length) {
str_len = max_length;
}
auto result = new char[str_len + 1];
- result[max_length] = '\0'; // terminate the string because strncpy might
not
- return strncpy(result, the_string, str_len);
+ result[max_length] = '\0'; // truncated string might not have \0 at end
+ memcpy(result, the_string, str_len);
+ return result;
}
// ----------------------------------------------------------------------
diff --git a/be/src/http/action/pprof_actions.cpp
b/be/src/http/action/pprof_actions.cpp
index 099d084..cf7b95c 100644
--- a/be/src/http/action/pprof_actions.cpp
+++ b/be/src/http/action/pprof_actions.cpp
@@ -225,7 +225,9 @@ void CmdlineAction::handle(HttpRequest* req) {
return;
}
char buf[1024];
- fscanf(fp, "%s ", buf);
+ // Ignore unused return value
+ if (fscanf(fp, "%s ", buf))
+ ;
fclose(fp);
std::string str = buf;
diff --git a/be/src/olap/column_vector.cpp b/be/src/olap/column_vector.cpp
index 7e3cf30..8672339 100644
--- a/be/src/olap/column_vector.cpp
+++ b/be/src/olap/column_vector.cpp
@@ -253,4 +253,33 @@ void DataBuffer<T>::resize(size_t new_size) {
current_size = new_size;
}
+template class DataBuffer<bool>;
+template class DataBuffer<int8_t>;
+template class DataBuffer<int16_t>;
+template class DataBuffer<int32_t>;
+template class DataBuffer<uint32_t>;
+template class DataBuffer<int64_t>;
+template class DataBuffer<uint64_t>;
+template class DataBuffer<int128_t>;
+template class DataBuffer<float>;
+template class DataBuffer<double>;
+template class DataBuffer<decimal12_t>;
+template class DataBuffer<uint24_t>;
+template class DataBuffer<Slice>;
+template class DataBuffer<CollectionValue>;
+
+template class ScalarColumnVectorBatch<bool>;
+template class ScalarColumnVectorBatch<int8_t>;
+template class ScalarColumnVectorBatch<int16_t>;
+template class ScalarColumnVectorBatch<int32_t>;
+template class ScalarColumnVectorBatch<uint32_t>;
+template class ScalarColumnVectorBatch<int64_t>;
+template class ScalarColumnVectorBatch<uint64_t>;
+template class ScalarColumnVectorBatch<int128_t>;
+template class ScalarColumnVectorBatch<float>;
+template class ScalarColumnVectorBatch<double>;
+template class ScalarColumnVectorBatch<decimal12_t>;
+template class ScalarColumnVectorBatch<uint24_t>;
+template class ScalarColumnVectorBatch<Slice>;
+
} // namespace doris
diff --git a/be/src/olap/column_vector.h b/be/src/olap/column_vector.h
index ca5b6e1..bc01596 100644
--- a/be/src/olap/column_vector.h
+++ b/be/src/olap/column_vector.h
@@ -51,21 +51,6 @@ public:
void resize(size_t _size);
};
-template class DataBuffer<bool>;
-template class DataBuffer<int8_t>;
-template class DataBuffer<int16_t>;
-template class DataBuffer<int32_t>;
-template class DataBuffer<uint32_t>;
-template class DataBuffer<int64_t>;
-template class DataBuffer<uint64_t>;
-template class DataBuffer<int128_t>;
-template class DataBuffer<float>;
-template class DataBuffer<double>;
-template class DataBuffer<decimal12_t>;
-template class DataBuffer<uint24_t>;
-template class DataBuffer<Slice>;
-template class DataBuffer<CollectionValue>;
-
// struct that contains column data(null bitmap), data array in sub class.
class ColumnVectorBatch {
public:
@@ -267,18 +252,4 @@ private:
std::unique_ptr<ScalarColumnVectorBatch<uint32_t>> _offsets;
};
-template class ScalarColumnVectorBatch<bool>;
-template class ScalarColumnVectorBatch<int8_t>;
-template class ScalarColumnVectorBatch<int16_t>;
-template class ScalarColumnVectorBatch<int32_t>;
-template class ScalarColumnVectorBatch<uint32_t>;
-template class ScalarColumnVectorBatch<int64_t>;
-template class ScalarColumnVectorBatch<uint64_t>;
-template class ScalarColumnVectorBatch<int128_t>;
-template class ScalarColumnVectorBatch<float>;
-template class ScalarColumnVectorBatch<double>;
-template class ScalarColumnVectorBatch<decimal12_t>;
-template class ScalarColumnVectorBatch<uint24_t>;
-template class ScalarColumnVectorBatch<Slice>;
-
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/page_handle.h
b/be/src/olap/rowset/segment_v2/page_handle.h
index 67526a5..ecb5410 100644
--- a/be/src/olap/rowset/segment_v2/page_handle.h
+++ b/be/src/olap/rowset/segment_v2/page_handle.h
@@ -59,7 +59,7 @@ public:
}
~PageHandle() {
- if (_is_data_owner) {
+ if (_is_data_owner && _data.size > 0) {
delete[] _data.data;
}
}
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 4dddf08..ec59a92 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -420,7 +420,7 @@ struct CppTypeTraits<OLAP_FIELD_TYPE_UNSIGNED_BIGINT> {
template <>
struct CppTypeTraits<OLAP_FIELD_TYPE_LARGEINT> {
using CppType = int128_t;
- using UnsignedCppType = unsigned int128_t;
+ using UnsignedCppType = uint128_t;
};
template <>
struct CppTypeTraits<OLAP_FIELD_TYPE_FLOAT> {
diff --git a/be/src/runtime/dpp_sink_internal.cpp
b/be/src/runtime/dpp_sink_internal.cpp
index ad17202..79a1c82 100644
--- a/be/src/runtime/dpp_sink_internal.cpp
+++ b/be/src/runtime/dpp_sink_internal.cpp
@@ -31,12 +31,7 @@ namespace doris {
PartRangeKey PartRangeKey::_s_pos_infinite(1);
PartRangeKey PartRangeKey::_s_neg_infinite(-1);
-PartRange PartRange::_s_all_range = {
- ._start_key = PartRangeKey::neg_infinite(),
- ._end_key = PartRangeKey::pos_infinite(),
- ._include_start_key = true,
- ._include_end_key = true,
-};
+PartRange PartRange::_s_all_range(PartRangeKey::neg_infinite(),
PartRangeKey::pos_infinite(), true, true);
RollupSchema::RollupSchema() {}
diff --git a/be/src/runtime/large_int_value.h b/be/src/runtime/large_int_value.h
index 926ec93..a7389d3 100644
--- a/be/src/runtime/large_int_value.h
+++ b/be/src/runtime/large_int_value.h
@@ -54,4 +54,8 @@ std::size_t hash_value(LargeIntValue const& value);
} // namespace doris
+// Thirdparty printers like gtest needs operator<< to be exported into global
namespace, so that ADL will work.
+inline std::ostream& operator<<(std::ostream& os, __int128 const& value) {
return doris::operator<<(os, value); }
+inline std::istream& operator>>(std::istream& is, __int128& value) { return
doris::operator>>(is, value); }
+
#endif
diff --git a/be/src/runtime/raw_value_ir.cpp b/be/src/runtime/raw_value_ir.cpp
index 2837575..d6fef33 100644
--- a/be/src/runtime/raw_value_ir.cpp
+++ b/be/src/runtime/raw_value_ir.cpp
@@ -16,6 +16,7 @@
// under the License.
#include "runtime/raw_value.h"
+#include "runtime/string_value.hpp"
#include "util/types.h"
namespace doris {
diff --git a/be/src/udf/udf.cpp b/be/src/udf/udf.cpp
index 269bd88..ce88fd6 100644
--- a/be/src/udf/udf.cpp
+++ b/be/src/udf/udf.cpp
@@ -50,7 +50,7 @@ public:
return reinterpret_cast<uint8_t*>(realloc(ptr, byte_size));
}
- void free(uint8_t* ptr) { free(ptr); }
+ void free(uint8_t* ptr) { ::free(ptr); }
};
class RuntimeState {
diff --git a/be/src/util/bfd_parser.cpp b/be/src/util/bfd_parser.cpp
index 751fc31..88b9c0d 100644
--- a/be/src/util/bfd_parser.cpp
+++ b/be/src/util/bfd_parser.cpp
@@ -102,7 +102,9 @@ BfdParser* BfdParser::create() {
}
char prog_name[1024];
- fscanf(file, "%s ", prog_name);
+ // Ignore unused return value
+ if (fscanf(file, "%s ", prog_name))
+ ;
fclose(file);
std::unique_ptr<BfdParser> parser(new BfdParser(prog_name));
if (parser->parse()) {
diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h
index bdb124b..1ac8a9d 100644
--- a/be/src/util/bitmap_value.h
+++ b/be/src/util/bitmap_value.h
@@ -1740,10 +1740,6 @@ private:
// }
class BitmapValueIterator {
public:
- BitmapValueIterator()
- : _bitmap(BitmapValue()) {
- }
-
BitmapValueIterator(const BitmapValue& bitmap, bool end = false)
: _bitmap(bitmap), _end(end) {
diff --git a/be/src/util/cidr.cpp b/be/src/util/cidr.cpp
index 9351d7a..e222b78 100644
--- a/be/src/util/cidr.cpp
+++ b/be/src/util/cidr.cpp
@@ -58,7 +58,9 @@ bool CIDR::reset(const std::string& cidr_str) {
if ((errno == ERANGE && (mask_length == LONG_MAX || mask_length ==
LONG_MIN)) ||
(errno != 0 && mask_length == 0)) {
char errmsg[64];
- strerror_r(errno, errmsg, 64);
+ // Ignore unused return value
+ if (strerror_r(errno, errmsg, 64))
+ ;
LOG(WARNING) << "wrong CIDR mask format. network=" << cidr_str
<< ", mask_length=" << mask_length << ", errno=" << errno
<< ", errmsg=" << errmsg;
diff --git a/be/src/util/pprof_utils.cpp b/be/src/util/pprof_utils.cpp
index e14436d..b1d0e05 100644
--- a/be/src/util/pprof_utils.cpp
+++ b/be/src/util/pprof_utils.cpp
@@ -66,7 +66,9 @@ Status PprofUtils::get_self_cmdline(std::string* cmd) {
return Status::InternalError("Unable to open file:
/proc/self/cmdline");
}
char buf[1024];
- fscanf(fp, "%s ", buf);
+ // Ignore unused return value
+ if (fscanf(fp, "%s ", buf))
+ ;
fclose(fp);
*cmd = buf;
return Status::OK();
diff --git a/be/src/util/simd/vstring_function.h
b/be/src/util/simd/vstring_function.h
index 5ed4fb0..c4b268f 100644
--- a/be/src/util/simd/vstring_function.h
+++ b/be/src/util/simd/vstring_function.h
@@ -96,15 +96,12 @@ public:
while (end - begin + 1 >= REGISTER_SIZE) {
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const
__m128i *>(str.ptr + begin));
const auto v_against_pattern = _mm_cmpeq_epi8(v_haystack, pattern);
- const auto mask = _mm_movemask_epi8(v_against_pattern);
- const auto offset = __builtin_ctz(mask ^ 0xffff);
- /// means not found
- if (offset == 0)
- {
- return StringVal(str.ptr + begin, end - begin + 1);
- } else if (offset > REGISTER_SIZE) {
+ const auto mask = _mm_movemask_epi8(v_against_pattern) ^ 0xffff;
+ /// zero means not found
+ if (mask == 0) {
begin += REGISTER_SIZE;
} else {
+ const auto offset = __builtin_ctz(mask);
begin += offset;
return StringVal(str.ptr + begin, end - begin + 1);
}
diff --git a/be/src/vec/common/exception.h b/be/src/vec/common/exception.h
index 3259134..c30fd01 100644
--- a/be/src/vec/common/exception.h
+++ b/be/src/vec/common/exception.h
@@ -24,6 +24,7 @@
#include <boost/stacktrace.hpp>
#include <cerrno>
#include <memory>
+#include <optional>
#include <stdexcept>
#include <vector>
diff --git a/be/src/vec/functions/function_grouping.h
b/be/src/vec/functions/function_grouping.h
index 6bdaa7e..23875af 100644
--- a/be/src/vec/functions/function_grouping.h
+++ b/be/src/vec/functions/function_grouping.h
@@ -52,7 +52,7 @@ public:
const ColumnWithTypeAndName& src_column =
block.get_by_position(arguments[0]);
const ColumnWithTypeAndName& rel_column =
block.get_by_position(result);
if (!src_column.column)
- return Status::InternalError("Illegal column " +
src_column.column->get_name() + " of first argument of function " + name);
+ return Status::InternalError("Illegal column " + src_column.name +
" of first argument of function " + name);
DCHECK(src_column.type->is_nullable() == true);
MutableColumnPtr res_column = rel_column.type->create_column();
@@ -76,7 +76,7 @@ public:
const ColumnWithTypeAndName& src_column =
block.get_by_position(arguments[0]);
const ColumnWithTypeAndName& rel_column =
block.get_by_position(result);
if (!src_column.column)
- return Status::InternalError("Illegal column " +
src_column.column->get_name() + " of first argument of function " + name);
+ return Status::InternalError("Illegal column " + src_column.name +
" of first argument of function " + name);
DCHECK(src_column.type->is_nullable() == true);
MutableColumnPtr res_column = rel_column.type->create_column();
diff --git a/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
b/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
index 1cadab5..f25ff55 100644
--- a/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
+++ b/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
@@ -30,7 +30,6 @@
using doris::segment_v2::PageBuilderOptions;
using doris::segment_v2::PageDecoderOptions;
-using doris::operator<<;
namespace doris {
class FrameOfReferencePageTest : public testing::Test {
diff --git a/be/test/plugin/plugin_test/source/test.zip
b/be/test/plugin/plugin_test/source/test.zip
index b43fb58..5c625c1 100644
Binary files a/be/test/plugin/plugin_test/source/test.zip and
b/be/test/plugin/plugin_test/source/test.zip differ
diff --git a/be/test/plugin/plugin_test/source/test.zip.md5
b/be/test/plugin/plugin_test/source/test.zip.md5
index 73c492c..801d385 100755
--- a/be/test/plugin/plugin_test/source/test.zip.md5
+++ b/be/test/plugin/plugin_test/source/test.zip.md5
@@ -1 +1 @@
-c68ae6cce50da90a5fd8a818d4e85c96 test.zip
+726cac15e3326403285c19c66fe2c44b test.zip
diff --git a/build.sh b/build.sh
index a5ccaae..1b5670e 100755
--- a/build.sh
+++ b/build.sh
@@ -18,7 +18,7 @@
##############################################################
# This script is used to compile Apache Doris(incubating).
-# Usage:
+# Usage:
# sh build.sh --help
# Eg:
# sh build.sh build all
@@ -238,7 +238,7 @@ function build_ui() {
# check NPM env here, not in env.sh.
# Because UI should be considered a non-essential component at runtime.
# Only when the compilation is required, check the relevant compilation
environment.
- NPM=npm
+ NPM=npm
if ! ${NPM} --version; then
echo "Error: npm is not found"
exit 1
@@ -253,7 +253,7 @@ function build_ui() {
ui_dist=${DORIS_HOME}/ui/dist/
if [[ ! -z ${CUSTOM_UI_DIST} ]]; then
ui_dist=${CUSTOM_UI_DIST}
- else
+ else
cd ${DORIS_HOME}/ui
${NPM} install
${NPM} run build
@@ -265,7 +265,7 @@ function build_ui() {
}
# FE UI must be built before building FE
-if [ ${BUILD_UI} -eq 1 ] ; then
+if [ ${BUILD_UI} -eq 1 ] ; then
build_ui
fi
diff --git a/contrib/udf/CMakeLists.txt b/contrib/udf/CMakeLists.txt
index 30ce621..6b347f8 100644
--- a/contrib/udf/CMakeLists.txt
+++ b/contrib/udf/CMakeLists.txt
@@ -16,15 +16,6 @@
# under the License.
cmake_minimum_required(VERSION 3.19.2)
-
-# set CMAKE_C_COMPILER, this must set before project command
-if (DEFINED ENV{DORIS_GCC_HOME})
- set(CMAKE_C_COMPILER "$ENV{DORIS_GCC_HOME}/bin/gcc")
- set(CMAKE_CXX_COMPILER "$ENV{DORIS_GCC_HOME}/bin/g++")
- set(GCC_HOME $ENV{DORIS_GCC_HOME})
-else()
- message(FATAL_ERROR "DORIS_GCC_HOME environment variable is not set")
-endif()
project(doris_udf)
diff --git a/docs/en/installing/compilation.md
b/docs/en/installing/compilation.md
index aa58e14..a8c6131 100644
--- a/docs/en/installing/compilation.md
+++ b/docs/en/installing/compilation.md
@@ -5,7 +5,7 @@
}
---
-<!--
+<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -67,15 +67,15 @@ This document focuses on how to code Doris through source
code.
> 3. From docker image of build-env-1.3.1, both OpenJDK 8 and OpenJDK 11 are
> included, and OpenJDK 11 is used for compilation by default. Please make
> sure that the JDK version used for compiling is the same as the JDK version
> used at runtime, otherwise it may cause unexpected operation errors. You can
> use the following command to switch the default JDK version in container:
>
> Switch to JDK 8:
->
+>
> ```
> $ alternatives --set java java-1.8.0-openjdk.x86_64
> $ alternatives --set javac java-1.8.0-openjdk.x86_64
> $ export JAVA_HOME=/usr/lib/jvm/java-1.8.0
> ```
->
+>
> Switch to JDK 11:
->
+>
> ```
> $ alternatives --set java java-11-openjdk.x86_64
> $ alternatives --set javac java-11-openjdk.x86_64
@@ -117,7 +117,7 @@ This document focuses on how to code Doris through source
code.
> `sh build.sh --clean --be --fe --ui`
>
> This is because from build-env-for-0.15.0, we upgraded thrift (0.9 ->
0.13), you need to use the --clean command to force the use of the new version
of thrift to generate code files, otherwise incompatible code will appear.
-
+
After compilation, the output file is in the `output/` directory.
### Self-compiling Development Environment Mirror
@@ -133,30 +133,30 @@ You can try to compile Doris directly in your own Linux
environment.
* Before commit
[ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f)
will use the dependencies as follows:
`GCC 7.3+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+
Bison 3.0+`
-
+
If you are using Ubuntu 16.04 or newer, you can use the following
command to install the dependencies
-
+
`sudo apt-get install build-essential openjdk-8-jdk maven cmake byacc
flex automake libtool-bin bison binutils-dev libiberty-dev zip unzip
libncurses5-dev curl git ninja-build python autopoint pkg-config`
-
+
If you are using CentOS you can use the following command to install
the dependencies
-
+
`sudo yum groupinstall 'Development Tools' && sudo yum install maven
cmake byacc flex automake libtool bison binutils-devel zip unzip ncurses-devel
curl git wget python2 glibc-static libstdc++-static java-1.8.0-openjdk`
-
+
* After commit
[ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f)
will use the dependencies as follows:
`GCC 10+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake
3.19.2+ Bison 3.0+`
-
+
If you are using Ubuntu 16.04 or newer, you can use the following
command to install the dependencies
-
+
```
sudo apt install build-essential openjdk-8-jdk maven cmake byacc flex
automake libtool-bin bison binutils-dev libiberty-dev zip unzip libncurses5-dev
curl git ninja-build python
sudo add-apt-repository ppa:ubuntu-toolchain-r/ppa
sudo apt update
- sudo apt install gcc-10 g++-10
+ sudo apt install gcc-10 g++-10
sudo apt-get install autoconf automake libtool autopoint
```
If you are using CentOS you can use the following command to install
the dependencies
-
+
```
sudo yum groupinstall 'Development Tools' && sudo yum install maven
cmake byacc flex automake libtool bison binutils-devel zip unzip ncurses-devel
curl git wget python2 glibc-static libstdc++-static java-1.8.0-openjdk
sudo yum install centos-release-scl
@@ -174,7 +174,7 @@ You can try to compile Doris directly in your own Linux
environment.
```
After installation, set environment variables `PATH`, `JAVA_HOME`, etc.
Doris 0.14.0 will use gcc7 env to compile.
-
+
2. Compile Doris
```
diff --git a/env.sh b/env.sh
index bf64801..439de9e 100755
--- a/env.sh
+++ b/env.sh
@@ -51,22 +51,52 @@ if ! ${PYTHON} --version; then
fi
fi
-# set GCC HOME
-if [[ -z ${DORIS_GCC_HOME} ]]; then
- export DORIS_GCC_HOME=$(dirname $(which gcc))/..
+if [[ -z ${DORIS_TOOLCHAIN} ]]; then
+ DORIS_TOOLCHAIN=gcc
fi
-gcc_ver=$(${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion)
-required_ver="7.3.0"
-if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" =
"$required_ver" ]]; then
- echo "Error: GCC version (${gcc_ver}) must be greater than or equal to
${required_ver}"
+if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then
+ # set GCC HOME
+ if [[ -z ${DORIS_GCC_HOME} ]]; then
+ DORIS_GCC_HOME=$(dirname $(which gcc))/..
+ export DORIS_GCC_HOME
+ fi
+
+ gcc_ver=$(${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion)
+ required_ver="7.3.0"
+ if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)"
= "$required_ver" ]]; then
+ echo "Error: GCC version (${gcc_ver}) must be greater than or equal to
${required_ver}"
+ exit 1
+ fi
+ export CC=${DORIS_GCC_HOME}/bin/gcc
+ export CXX=${DORIS_GCC_HOME}/bin/g++
+ if test -x ${DORIS_GCC_HOME}/bin/ld; then
+ export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/
+ fi
+elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then
+ # set CLANG HOME
+ if [[ -z ${DORIS_CLANG_HOME} ]]; then
+ DORIS_CLANG_HOME=$(dirname $(which clang))/..
+ export DORIS_CLANG_HOME
+ fi
+
+ clang_ver=$(${DORIS_CLANG_HOME}/bin/clang -dumpfullversion -dumpversion)
+ required_ver="13.0.0"
+ if [[ ! "$(printf '%s\n' "$required_ver" "$clang_ver" | sort -V | head
-n1)" = "$required_ver" ]]; then
+ echo "Error: CLANG version (${clang_ver}) must be greater than or
equal to ${required_ver}"
+ exit 1
+ fi
+ export CC=${DORIS_CLANG_HOME}/bin/clang
+ export CXX=${DORIS_CLANG_HOME}/bin/clang++
+ if test -x ${DORIS_CLANG_HOME}/bin/ld.lld; then
+ export DORIS_BIN_UTILS=${DORIS_CLANG_HOME}/bin/
+ fi
+else
+ echo "Error: unknown DORIS_TOOLCHAIN=${DORIS_TOOLCHAIN}, currently only
'gcc' and 'clang' are supported"
exit 1
fi
-# find binutils
-if test -x ${DORIS_GCC_HOME}/bin/ld; then
- export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/
-else
+if [ -z "$DORIS_BIN_UTILS" ]; then
export DORIS_BIN_UTILS=/usr/bin/
fi
@@ -144,3 +174,5 @@ fi
export GENERATOR
export BUILD_SYSTEM
+
+export
PKG_CONFIG_PATH=${DORIS_HOME}/thirdparty/installed/lib64/pkgconfig:$PKG_CONFIG_PATH
diff --git a/gensrc/script/doris_builtins_functions.py
b/gensrc/script/doris_builtins_functions.py
index ae7dc11..d1c0100 100755
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -1118,7 +1118,7 @@ visible_functions = [
[['json_quote'], 'VARCHAR', ['VARCHAR'],
'_ZN5doris13JsonFunctions10json_quoteEPN9doris_udf15FunctionContextERKNS1_9StringValE',
'', '', '', ''],
-
+
#hll function
[['hll_cardinality'], 'BIGINT', ['VARCHAR'],
'_ZN5doris12HllFunctions15hll_cardinalityEPN9doris_udf15FunctionContextERKNS1_9StringValE',
diff --git a/gensrc/script/gen_builtins_functions.py
b/gensrc/script/gen_builtins_functions.py
index 91a3876..2aca007 100755
--- a/gensrc/script/gen_builtins_functions.py
+++ b/gensrc/script/gen_builtins_functions.py
@@ -173,16 +173,17 @@ def generate_fe_registry_init(filename):
java_registry_file.write(java_registry_epilogue)
java_registry_file.close()
+if __name__ == "__main__":
-# Read the function metadata inputs
-for function in doris_builtins_functions.visible_functions:
- add_function(function, True)
-for function in doris_builtins_functions.invisible_functions:
- add_function(function, False)
-
-if not os.path.exists(FE_PATH):
- os.makedirs(FE_PATH)
-
-generate_fe_registry_init(FE_PATH + "ScalarBuiltins.java")
+ try:
+ os.makedirs(FE_PATH)
+ except FileExistsError:
+ pass
+ # Read the function metadata inputs
+ for function in doris_builtins_functions.visible_functions:
+ add_function(function, True)
+ for function in doris_builtins_functions.invisible_functions:
+ add_function(function, False)
+ generate_fe_registry_init(FE_PATH + "ScalarBuiltins.java")
diff --git a/gensrc/script/gen_functions.py b/gensrc/script/gen_functions.py
index 5173ad9..5e462af 100755
--- a/gensrc/script/gen_functions.py
+++ b/gensrc/script/gen_functions.py
@@ -630,8 +630,6 @@ header_template = string.Template("\
static void* ${fn_signature}(Expr* e, TupleRow* row);\n")
BE_PATH = "../gen_cpp/opcode/"
-if not os.path.exists(BE_PATH):
- os.makedirs(BE_PATH)
def initialize_sub(op, return_type, arg_types):
"""
@@ -654,6 +652,12 @@ def initialize_sub(op, return_type, arg_types):
return sub
if __name__ == "__main__":
+
+ try:
+ os.makedirs(BE_PATH)
+ except FileExistsError:
+ pass
+
h_file = open(BE_PATH + 'functions.h', 'w')
cc_file = open(BE_PATH + 'functions.cc', 'w')
python_file = open('generated_functions.py', 'w')
diff --git a/gensrc/script/gen_vector_functions.py
b/gensrc/script/gen_vector_functions.py
index 0f3231d..ea734ff 100755
--- a/gensrc/script/gen_vector_functions.py
+++ b/gensrc/script/gen_vector_functions.py
@@ -458,8 +458,6 @@ header_template = string.Template("\
Expr* e, VectorizedRowBatch* batch);\n")
BE_PATH = "../gen_cpp/opcode/"
-if not os.path.exists(BE_PATH):
- os.makedirs(BE_PATH)
def initialize_sub(op, return_type, arg_types):
"""
@@ -481,6 +479,12 @@ def initialize_sub(op, return_type, arg_types):
return sub
if __name__ == "__main__":
+
+ try:
+ os.makedirs(BE_PATH)
+ except FileExistsError:
+ pass
+
h_file = open(BE_PATH + 'vector-functions.h', 'w')
cc_file = open(BE_PATH + 'vector-functions.cc', 'w')
python_file = open('generated_vector_functions.py', 'w')
diff --git a/run-fe-ut.sh b/run-fe-ut.sh
index c93023b..eb603a4 100755
--- a/run-fe-ut.sh
+++ b/run-fe-ut.sh
@@ -63,7 +63,7 @@ if [ $# == 1 ] ; then
else
RUN=0
COVERAGE=0
- while true; do
+ while true; do
case "$1" in
--coverage) COVERAGE=1 ; shift ;;
--run) RUN=1 ; shift ;;
@@ -98,8 +98,8 @@ else
# sh run-fe-ut.sh --run org.apache.doris.utframe.Demo
# sh run-fe-ut.sh --run
org.apache.doris.utframe.Demo#testCreateDbAndTable+test2
${MVN_CMD} test -DfailIfNoTests=false -D test=$1
- else
+ else
echo "Run Frontend UT"
- ${MVN_CMD} test -DfailIfNoTests=false
- fi
+ ${MVN_CMD} test -DfailIfNoTests=false
+ fi
fi
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index cedec49..c2307a0 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -105,14 +105,26 @@ ${TP_DIR}/download-thirdparty.sh
export LD_LIBRARY_PATH=$TP_DIR/installed/lib:$LD_LIBRARY_PATH
-# set COMPILER
-if [[ ! -z ${DORIS_GCC_HOME} ]]; then
- export CC=${DORIS_GCC_HOME}/bin/gcc
- export CPP=${DORIS_GCC_HOME}/bin/cpp
- export CXX=${DORIS_GCC_HOME}/bin/g++
-else
- echo "DORIS_GCC_HOME environment variable is not set"
- exit 1
+# toolchain specific warning options and settings
+if [[ "$CC" == *gcc ]]
+then
+ warning_uninitialized=-Wno-maybe-uninitialized
+ warning_stringop_truncation=-Wno-stringop-truncation
+ warning_class_memaccess=-Wno-class-memaccess
+ warning_array_parameter=-Wno-array-parameter
+ boost_toolset=gcc
+elif [[ "$CC" == *clang ]]
+then
+ warning_uninitialized=-Wno-uninitialized
+ warning_shadow=-Wno-shadow
+ warning_dangling_gsl=-Wno-dangling-gsl
+ warning_unused_but_set_variable=-Wno-unused-but-set-variable
+ warning_defaulted_function_deleted=-Wno-defaulted-function-deleted
+ warning_reserved_identifier=-Wno-reserved-identifier
+ warning_suggest_override="-Wno-suggest-override
-Wno-suggest-destructor-override"
+ warning_option_ignored=-Wno-option-ignored
+ boost_toolset=clang
+ libhdfs_cxx17=-std=c++1z
fi
# prepare installed prefix
@@ -162,6 +174,7 @@ check_prerequest "libtoolize --version" "libtool"
# sudo apt-get install bison
# sudo yum install bison
+# necessary only when compiling be
#check_prerequest "bison --version" "bison"
#########################
@@ -228,8 +241,11 @@ build_openssl() {
LDFLAGS="-L${TP_LIB_DIR}" \
CFLAGS="-fPIC" \
LIBDIR="lib" \
- ./Configure --prefix=$TP_INSTALL_DIR --with-rand-seed=devrandom -zlib
-shared ${OPENSSL_PLATFORM}
- make -j $PARALLEL && make install_sw
+ ./Configure --prefix=$TP_INSTALL_DIR --with-rand-seed=devrandom -shared
${OPENSSL_PLATFORM}
+ # NOTE(amos): Never use '&&' to concat commands as it will eat error code
+ # See https://mywiki.wooledge.org/BashFAQ/105 for more detail.
+ make -j $PARALLEL
+ make install_sw
# NOTE(zc): remove this dynamic library files to make libcurl static link.
# If I don't remove this files, I don't known how to make libcurl link
static library
if [ -f $TP_INSTALL_DIR/lib64/libcrypto.so ]; then
@@ -246,11 +262,12 @@ build_thrift() {
cd $TP_SOURCE_DIR/$THRIFT_SOURCE
echo ${TP_LIB_DIR}
- ./configure CPPFLAGS="-I${TP_INCLUDE_DIR}" LDFLAGS="-L${TP_LIB_DIR}
-static-libstdc++ -static-libgcc" LIBS="-lcrypto -ldl -lssl" CFLAGS="-fPIC" \
+ # NOTE(amos): libtool discard -static. --static works.
+ ./configure CPPFLAGS="-I${TP_INCLUDE_DIR}" LDFLAGS="-L${TP_LIB_DIR}
--static" LIBS="-lcrypto -ldl -lssl" CFLAGS="-fPIC" \
--prefix=$TP_INSTALL_DIR --docdir=$TP_INSTALL_DIR/doc --enable-static
--disable-shared --disable-tests \
--disable-tutorial --without-qt4 --without-qt5 --without-csharp
--without-erlang --without-nodejs \
--without-lua --without-perl --without-php --without-php_extension
--without-dart --without-ruby \
- --without-haskell --without-go --without-haxe --without-d --without-python
-without-java --with-cpp \
+ --without-haskell --without-go --without-haxe --without-d --without-python
-without-java -without-rs --with-cpp \
--with-libevent=$TP_INSTALL_DIR --with-boost=$TP_INSTALL_DIR
--with-openssl=$TP_INSTALL_DIR
if [ -f compiler/cpp/thrifty.hh ];then
@@ -265,10 +282,12 @@ build_protobuf() {
check_if_source_exist $PROTOBUF_SOURCE
cd $TP_SOURCE_DIR/$PROTOBUF_SOURCE
rm -fr gmock
+ # NOTE(amos): -Wl,--undefined=pthread_create force searching for pthread
symbols.
+ # See https://stackoverflow.com/a/65348893/1329147 for detailed
explanation.
mkdir gmock && cd gmock && tar xf ${TP_SOURCE_DIR}/${GTEST_NAME} \
&& mv ${GTEST_SOURCE} gtest && cd $TP_SOURCE_DIR/$PROTOBUF_SOURCE &&
./autogen.sh
CXXFLAGS="-fPIC -O2 -I${TP_INCLUDE_DIR}" \
- LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
+ LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc
-Wl,--undefined=pthread_create" \
./configure --prefix=${TP_INSTALL_DIR} --disable-shared --enable-static
--with-zlib=${TP_INSTALL_DIR}/include
cd src
sed -i 's/^AM_LDFLAGS\(.*\)$/AM_LDFLAGS\1 -all-static/' Makefile
@@ -311,8 +330,8 @@ build_gtest() {
cd $TP_SOURCE_DIR/$GTEST_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
rm -rf CMakeCache.txt CMakeFiles/
- ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
- -DCMAKE_POSITION_INDEPENDENT_CODE=On ../
+ ${CMAKE_CMD} ../ -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_POSITION_INDEPENDENT_CODE=On
+ # -DCMAKE_CXX_FLAGS="$warning_uninitialized"
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -426,13 +445,14 @@ build_curl() {
check_if_source_exist $CURL_SOURCE
cd $TP_SOURCE_DIR/$CURL_SOURCE
- CPPFLAGS="-I${TP_INCLUDE_DIR}" \
- LDFLAGS="-L${TP_LIB_DIR}" LIBS="-lcrypto -lssl -lcrypto -ldl" \
+ CPPFLAGS="-I${TP_INCLUDE_DIR} -DNGHTTP2_STATICLIB" \
+ LDFLAGS="-L${TP_LIB_DIR}" LIBS="-lcrypto -lssl -lcrypto -ldl -static" \
CFLAGS="-fPIC" \
+ PKG_CONFIG="pkg-config --static" \
./configure --prefix=$TP_INSTALL_DIR --disable-shared --enable-static \
--without-librtmp --with-ssl=${TP_INSTALL_DIR} --without-libidn2
--disable-ldap --enable-ipv6 \
- --without-libssh2
- make -j $PARALLEL && make install
+ --without-libssh2 --without-brotli
+ make curl_LDFLAGS=-all-static -j $PARALLEL && make
curl_LDFLAGS=-all-static install
}
# re2
@@ -449,8 +469,10 @@ build_boost() {
check_if_source_exist $BOOST_SOURCE
cd $TP_SOURCE_DIR/$BOOST_SOURCE
- ./bootstrap.sh --prefix=$TP_INSTALL_DIR
- ./b2 link=static runtime-link=static -j $PARALLEL --without-mpi
--without-graph --without-graph_parallel --without-python cxxflags="-std=c++11
-g -fPIC -I$TP_INCLUDE_DIR -L$TP_LIB_DIR" install
+ CXXFLAGS="-static" \
+ ./bootstrap.sh --prefix=$TP_INSTALL_DIR --with-toolset=$boost_toolset
+ # -q: Fail at first error
+ ./b2 -q link=static runtime-link=static -j $PARALLEL --without-mpi
--without-graph --without-graph_parallel --without-python cxxflags="-std=c++11
-g -fPIC -I$TP_INCLUDE_DIR -L$TP_LIB_DIR" install
}
# mysql
@@ -466,11 +488,13 @@ build_mysql() {
cp -rf $TP_SOURCE_DIR/$BOOST_SOURCE ./
fi
- ${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_BOOST=`pwd`/$BOOST_SOURCE
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/mysql/ \
- -DCMAKE_INCLUDE_PATH=$TP_INCLUDE_DIR -DWITHOUT_SERVER=1
-DWITH_ZLIB=$TP_INSTALL_DIR \
+ CFLAGS="-static -pthread -lrt" CXXFLAGS="-static -pthread -lrt" \
+ ${CMAKE_CMD} -G "${GENERATOR}" ../ -DCMAKE_LINK_SEARCH_END_STATIC=1 \
+ -DWITH_BOOST=`pwd`/$BOOST_SOURCE
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/mysql/ \
+ -DCMAKE_INCLUDE_PATH=$TP_INCLUDE_DIR -DWITHOUT_SERVER=1 -DWITH_ZLIB=1
-DZLIB_ROOT=$TP_INSTALL_DIR \
-DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O3 -g -fabi-version=2
-fno-omit-frame-pointer -fno-strict-aliasing -std=gnu++11" \
- -DDISABLE_SHARED=1 -DBUILD_SHARED_LIBS=0
-DZLIB_LIBRARY=$TP_INSTALL_DIR/lib/libz.a
- ${BUILD_SYSTEM} -j $PARALLEL mysqlclient
+ -DDISABLE_SHARED=1 -DBUILD_SHARED_LIBS=0
-DZLIB_LIBRARY=$TP_INSTALL_DIR/lib/libz.a -DENABLE_DTRACE=0
+ ${BUILD_SYSTEM} -v -j $PARALLEL mysqlclient
# copy headers manually
rm -rf ../../../installed/include/mysql/
@@ -506,7 +530,7 @@ build_brpc() {
${CMAKE_CMD} -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-DCMAKE_LIBRARY_PATH=$TP_INSTALL_DIR/lib64 \
-DBRPC_WITH_GLOG=ON -DWITH_GLOG=ON
-DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a -DGLOG_LIB=$TP_INSTALL_DIR/lib
\
- -DGFLAGS_INCLUDE_DIR=$TP_INSTALL_DIR/include
-DGLOG_LIB=$TP_INSTALL_DIR/lib/libglog.a
-DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \
+ -DGFLAGS_INCLUDE_PATH=$TP_INSTALL_DIR/include
-DGLOG_LIB=$TP_INSTALL_DIR/lib/libglog.a
-DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \
-DPROTOBUF_PROTOC_EXECUTABLE=$TP_INSTALL_DIR/bin/protoc ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -517,8 +541,12 @@ build_rocksdb() {
cd $TP_SOURCE_DIR/$ROCKSDB_SOURCE
- CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I
${TP_INCLUDE_DIR}/lz4" CXXFLAGS="-fPIC -Wno-deprecated-copy
-Wno-stringop-truncation -Wno-pessimizing-move" LDFLAGS="-static-libstdc++
-static-libgcc" \
- PORTABLE=1 make USE_RTTI=1 -j $PARALLEL static_lib
+ # -Wno-range-loop-construct gcc-11
+ CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I
${TP_INCLUDE_DIR}/lz4" \
+ CXXFLAGS="-fPIC -Wno-deprecated-copy $warning_stringop_truncation
$warning_shadow $warning_dangling_gsl \
+ $warning_defaulted_function_deleted $warning_unused_but_set_variable
-Wno-pessimizing-move -Wno-range-loop-construct" \
+ LDFLAGS="-static-libstdc++ -static-libgcc" \
+ PORTABLE=1 make USE_RTTI=1 -j $PARALLEL static_lib
cp librocksdb.a ../../installed/lib/librocksdb.a
cp -r include/rocksdb ../../installed/include/
}
@@ -566,9 +594,9 @@ build_flatbuffers() {
cd $TP_SOURCE_DIR/$FLATBUFFERS_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
rm -rf CMakeCache.txt CMakeFiles/
- CXXFLAGS="-fPIC -Wno-class-memaccess" \
+ CXXFLAGS="-fPIC $warning_class_memaccess" \
LDFLAGS="-static-libstdc++ -static-libgcc" \
- ${CMAKE_CMD} -G "${GENERATOR}" ..
+ ${CMAKE_CMD} -G "${GENERATOR}" -DFLATBUFFERS_BUILD_TESTS=OFF ..
${BUILD_SYSTEM} -j $PARALLEL
cp flatc ../../../installed/bin/flatc
cp -r ../include/flatbuffers ../../../installed/include/flatbuffers
@@ -588,30 +616,27 @@ build_arrow() {
export ARROW_Thrift_URL=${TP_SOURCE_DIR}/${THRIFT_NAME}
export ARROW_SNAPPY_URL=${TP_SOURCE_DIR}/${SNAPPY_NAME}
export ARROW_ZLIB_URL=${TP_SOURCE_DIR}/${ZLIB_NAME}
- export LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
+ LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
${CMAKE_CMD} -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON
-DARROW_BUILD_SHARED=OFF \
- -DARROW_BUILD_STATIC=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_LZ4=ON \
+ -DARROW_BUILD_STATIC=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_LZ4=ON
-DARROW_USE_GLOG=ON \
-DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON
-DARROW_JSON=ON \
-DARROW_WITH_UTF8PROC=OFF -DARROW_WITH_RE2=OFF \
- -DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR \
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-DCMAKE_INSTALL_LIBDIR=lib64 \
- -DARROW_BOOST_USE_SHARED=OFF -DARROW_GFLAGS_USE_SHARED=OFF \
- -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=$TP_INSTALL_DIR \
- -DPARQUET_ARROW_LINKAGE=static \
+ -DARROW_BOOST_USE_SHARED=OFF \
+ -DARROW_GFLAGS_USE_SHARED=OFF \
-Dgflags_ROOT=$TP_INSTALL_DIR \
- -DSnappy_ROOT=$TP_INSTALL_DIR \
-DGLOG_ROOT=$TP_INSTALL_DIR \
- -DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a \
- -DGLOG_LIB=$TP_INSTALL_DIR/lib/libglog.a \
- -DLZ4_ROOT=$TP_INSTALL_DIR \
- -DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR/include/lz4 \
- -DZSTD_ROOT=$TP_INSTALL_DIR \
-DZLIB_LIBRARY=$TP_INSTALL_DIR/lib/libz.a
-DZLIB_INCLUDE_DIR=$TP_INSTALL_DIR/include \
- -DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a \
- -DBoost_NO_BOOST_CMAKE=ON \
- -DBOOST_ROOT=$TP_INSTALL_DIR \
+ -DRapidJSON_ROOT=$TP_INSTALL_DIR \
+ -DBrotli_SOURCE=BUNDLED \
+ -DLZ4_LIB=$TP_INSTALL_DIR/lib/liblz4.a
-DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR/include/lz4 \
+ -DLz4_SOURCE=SYSTEM \
+ -DZSTD_LIB=$TP_INSTALL_DIR/lib/libzstd.a
-DZSTD_INCLUDE_DIR=$TP_INSTALL_DIR/include \
+ -Dzstd_SOURCE=SYSTEM \
+ -DSnappy_LIB=$TP_INSTALL_DIR/lib/libsnappy.a
-DSnappy_INCLUDE_DIR=$TP_INSTALL_DIR/include \
+ -DSnappy_SOURCE=SYSTEM \
-DThrift_ROOT=$TP_INSTALL_DIR ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
@@ -668,7 +693,7 @@ build_bitshuffle() {
fi
tmp_obj=bitshuffle_${arch}_tmp.o
dst_obj=bitshuffle_${arch}.o
- ${CC:-$DORIS_GCC_HOME/bin/gcc} $EXTRA_CFLAGS $arch_flag -std=c99
-I$PREFIX/include/lz4/ -O3 -DNDEBUG -fPIC -c \
+ $CC $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include/lz4/ -O3
-DNDEBUG -fPIC -c \
"src/bitshuffle_core.c" \
"src/bitshuffle.c" \
"src/iochain.c"
@@ -744,7 +769,7 @@ build_orc() {
cd $TP_SOURCE_DIR/$ORC_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
rm -rf CMakeCache.txt CMakeFiles/
- CXXFLAGS="-O3 -Wno-array-bounds" \
+ CXXFLAGS="-O3 -Wno-array-bounds $warning_reserved_identifier
$warning_suggest_override" \
${CMAKE_CMD} -G "${GENERATOR}" ../ -DBUILD_JAVA=OFF \
-DPROTOBUF_HOME=$TP_INSTALL_DIR \
-DSNAPPY_HOME=$TP_INSTALL_DIR \
@@ -766,8 +791,8 @@ build_orc() {
build_cctz() {
check_if_source_exist $CCTZ_SOURCE
cd $TP_SOURCE_DIR/$CCTZ_SOURCE
- export PREFIX=$TP_INSTALL_DIR
- make -j $PARALLEL && make install
+ PREFIX=$TP_INSTALL_DIR make -j $PARALLEL
+ PREFIX=$TP_INSTALL_DIR make install
}
# all js and csss related
@@ -801,7 +826,8 @@ build_aws_c_common() {
check_if_source_exist $AWS_C_COMMON_SOURCE
cd $TP_SOURCE_DIR/$AWS_C_COMMON_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
+ cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_uninitialized
$warning_option_ignored"
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -811,7 +837,7 @@ build_aws_c_event_stream() {
cd $TP_SOURCE_DIR/$AWS_C_EVENT_STREAM_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
- -DBUILD_TESTING=OFF
+ -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_option_ignored"
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -820,7 +846,8 @@ build_aws_checksums() {
check_if_source_exist $AWS_CHECKSUMS_SOURCE
cd $TP_SOURCE_DIR/$AWS_CHECKSUMS_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
+ cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_option_ignored"
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -829,7 +856,8 @@ build_aws_c_io() {
check_if_source_exist $AWS_C_IO_SOURCE
cd $TP_SOURCE_DIR/$AWS_C_IO_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
+ cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_option_ignored"
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -838,7 +866,8 @@ build_aws_s2n() {
check_if_source_exist $AWS_S2N_SOURCE
cd $TP_SOURCE_DIR/$AWS_S2N_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
+ cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
+ -DCMAKE_C_FLAGS="$warning_array_parameter"
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -847,7 +876,8 @@ build_aws_c_cal() {
check_if_source_exist $AWS_C_CAL_SOURCE
cd $TP_SOURCE_DIR/$AWS_C_CAL_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
+ cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_option_ignored"
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -856,9 +886,11 @@ build_aws_sdk() {
check_if_source_exist $AWS_SDK_SOURCE
cd $TP_SOURCE_DIR/$AWS_SDK_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
+ # -Wno-nonnull gcc-11
$CMAKE_CMD -G "${GENERATOR}" .. -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-DBUILD_DEPS=OFF -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR
-DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF \
- -DCMAKE_MODULE_PATH=$TP_INSTALL_DIR/lib64/cmake -DBUILD_ONLY="s3"
+ -DCMAKE_MODULE_PATH=$TP_INSTALL_DIR/lib64/cmake -DBUILD_ONLY="s3" \
+ -DCMAKE_CXX_FLAGS="-Wno-nonnull"
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -887,12 +919,21 @@ build_xml2() {
make -j $PARALLEL && make install
}
+# idn
+build_idn() {
+ check_if_source_exist $IDN_SOURCE
+ cd $TP_SOURCE_DIR/$IDN_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+ ../configure --prefix=$TP_INSTALL_DIR --enable-shared=no --with-pic
+ make -j $PARALLEL && make install
+}
+
# gsasl
build_gsasl() {
check_if_source_exist $GSASL_SOURCE
cd $TP_SOURCE_DIR/$GSASL_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- ../configure --prefix=$TP_INSTALL_DIR --enable-shared=no --with-pic
+ ../configure --prefix=$TP_INSTALL_DIR --enable-shared=no --with-pic
--with-libidn-prefix=$TP_INSTALL_DIR
make -j $PARALLEL && make install
}
@@ -902,7 +943,8 @@ build_hdfs3() {
cd $TP_SOURCE_DIR/$HDFS3_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
../bootstrap --dependency=$TP_INSTALL_DIR --prefix=$TP_INSTALL_DIR
- make -j $PARALLEL && make install
+ make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
+ make install
}
# benchmark
@@ -912,7 +954,9 @@ build_benchmark() {
cd $TP_SOURCE_DIR/$BENCHMARK_SOURCE
cmake -E make_directory "build"
- CXXFLAGS="-lresolv -pthread -lrt" cmake -E chdir "build" cmake
-DBENCHMARK_ENABLE_GTEST_TESTS=OFF -DCMAKE_BUILD_TYPE=Release ../
+ # NOTE(amos): -DHAVE_STD_REGEX=1 avoid runtime checks as it will fail when
compiling with non-standard toolchain
+ CXXFLAGS="-lresolv -pthread -lrt" cmake -E chdir "build" \
+ cmake ../ -DBENCHMARK_ENABLE_GTEST_TESTS=OFF
-DBENCHMARK_ENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DHAVE_STD_REGEX=1
cmake --build "build" --config Release
mkdir $TP_INCLUDE_DIR/benchmark
@@ -976,6 +1020,7 @@ build_aws_sdk
build_js_and_css
build_lzma
build_xml2
+build_idn
build_gsasl
build_hdfs3
build_benchmark
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index 9748580..bd5f65d 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -142,7 +142,7 @@ do
URL="${REPOSITORY_URL}/${!NAME}"
download_func ${!NAME} ${URL} $TP_SOURCE_DIR ${!MD5SUM}
if [ "$?x" == "0x" ]; then
- #try to download from home
+ #try to download from home
URL=$TP_ARCH"_DOWNLOAD"
download_func ${!NAME} ${!URL} $TP_SOURCE_DIR ${!MD5SUM}
if [ "$?x" == "0x" ]; then
@@ -199,7 +199,7 @@ do
exit 1
fi
elif [[ "${!NAME}" =~ $SUFFIX_ZIP ]]; then
- if ! $UNZIP_CMD -qq "$TP_SOURCE_DIR/${!NAME}" -d
"$TP_SOURCE_DIR/"; then
+ if ! $UNZIP_CMD -o -qq "$TP_SOURCE_DIR/${!NAME}" -d
"$TP_SOURCE_DIR/"; then
echo "Failed to unzip ${!NAME}"
exit 1
fi
@@ -224,14 +224,23 @@ echo "===== Patching thirdparty archives..."
###################################################################################
PATCHED_MARK="patched_mark"
- # glog patch
- cd $TP_SOURCE_DIR/$GLOG_SOURCE
- if [ ! -f $PATCHED_MARK ]; then
- patch -p1 < $TP_PATCH_DIR/glog-0.4.0.patch
- touch $PATCHED_MARK
- fi
- cd -
- echo "Finished patching $GLOG_SOURCE"
+# glog patch
+cd $TP_SOURCE_DIR/$GLOG_SOURCE
+if [ ! -f $PATCHED_MARK ]; then
+ patch -p1 < $TP_PATCH_DIR/glog-0.4.0.patch
+ touch $PATCHED_MARK
+fi
+cd -
+echo "Finished patching $GLOG_SOURCE"
+
+# gtest patch
+cd $TP_SOURCE_DIR/$GTEST_SOURCE
+if [ ! -f $PATCHED_MARK ]; then
+ patch -p1 < $TP_PATCH_DIR/googletest-release-1.11.0.patch
+ touch $PATCHED_MARK
+fi
+cd -
+echo "Finished patching $GTEST_SOURCE"
# mysql patch
cd $TP_SOURCE_DIR/$MYSQL_SOURCE
@@ -281,3 +290,13 @@ if [ $AWS_C_CAL_SOURCE == "aws-c-cal-0.4.5" ]; then
fi
echo "Finished patching $AWS_C_CAL_SOURCE"
+# rocksdb patch to fix compile error
+if [ $ROCKSDB_SOURCE == "rocksdb-5.14.2" ]; then
+ cd $TP_SOURCE_DIR/$ROCKSDB_SOURCE
+ if [ ! -f $PATCHED_MARK ]; then
+ patch -p1 < $TP_PATCH_DIR/rocksdb-5.14.2.patch
+ touch $PATCHED_MARK
+ fi
+ cd -
+fi
+echo "Finished patching $ROCKSDB_SOURCE"
diff --git a/thirdparty/patches/googletest-release-1.11.0.patch
b/thirdparty/patches/googletest-release-1.11.0.patch
new file mode 100644
index 0000000..9765267
--- /dev/null
+++ b/thirdparty/patches/googletest-release-1.11.0.patch
@@ -0,0 +1,71 @@
+diff -u a/googletest/include/gtest/gtest-printers.h
b/googletest/include/gtest/gtest-printers.h
+--- a/googletest/include/gtest/gtest-printers.h
++++ b/googletest/include/gtest/gtest-printers.h
+@@ -479,6 +479,12 @@ inline void PrintTo(char8_t c, ::std::ostream* os) {
+ }
+ #endif
+
++// gcc/clang __{u,}int128_t
++#if defined(__SIZEOF_INT128__)
++GTEST_API_ void PrintTo(__uint128_t v, ::std::ostream* os);
++GTEST_API_ void PrintTo(__int128_t v, ::std::ostream* os);
++#endif // __SIZEOF_INT128__
++
+ // Overloads for C strings.
+ GTEST_API_ void PrintTo(const char* s, ::std::ostream* os);
+ inline void PrintTo(char* s, ::std::ostream* os) {
+diff -u a/googletest/src/gtest-printers.cc b/googletest/src/gtest-printers.cc
+--- a/googletest/src/gtest-printers.cc
++++ b/googletest/src/gtest-printers.cc
+@@ -304,6 +304,51 @@ void PrintTo(char32_t c, ::std::ostream* os) {
+ << static_cast<uint32_t>(c);
+ }
+
++// gcc/clang __{u,}int128_t
++#if defined(__SIZEOF_INT128__)
++void PrintTo(__uint128_t v, ::std::ostream* os) {
++ if (v == 0) {
++ *os << "0";
++ return;
++ }
++
++ // Buffer large enough for ceil(log10(2^128))==39 and the null terminator
++ char buf[40];
++ char* p = buf + sizeof(buf);
++
++ // Some configurations have a __uint128_t, but no support for built in
++ // division. Do manual long division instead.
++
++ uint64_t high = static_cast<uint64_t>(v >> 64);
++ uint64_t low = static_cast<uint64_t>(v);
++
++ *--p = 0;
++ while (high != 0 || low != 0) {
++ uint64_t high_mod = high % 10;
++ high = high / 10;
++ // This is the long division algorithm specialized for a divisor of 10 and
++ // only two elements.
++ // Notable values:
++ // 2^64 / 10 == 1844674407370955161
++ // 2^64 % 10 == 6
++ const uint64_t carry = 6 * high_mod + low % 10;
++ low = low / 10 + high_mod * 1844674407370955161 + carry / 10;
++
++ char digit = static_cast<char>(carry % 10);
++ *--p = '0' + digit;
++ }
++ *os << p;
++}
++void PrintTo(__int128_t v, ::std::ostream* os) {
++ __uint128_t uv = static_cast<__uint128_t>(v);
++ if (v < 0) {
++ *os << "-";
++ uv = -uv;
++ }
++ PrintTo(uv, os);
++}
++#endif // __SIZEOF_INT128__
++
+ // Prints the given array of characters to the ostream. CharType must be
either
+ // char, char8_t, char16_t, char32_t, or wchar_t.
+ // The array starts at begin, the length is len, it may include '\0'
characters
diff --git a/thirdparty/patches/libevent.patch
b/thirdparty/patches/libevent.patch
index 43c4b5b..83e426d 100644
--- a/thirdparty/patches/libevent.patch
+++ b/thirdparty/patches/libevent.patch
@@ -132,3 +132,14 @@ diff -uprN a/include/event2/http_struct.h
b/include/event2/http_struct.h
};
#ifdef __cplusplus
+diff -uprN a/CMakeLists.txt b/CMakeLists.txt
+--- a/CMakeLists.txt 2020-07-05 20:02:46.000000000 +0800
++++ b/CMakeLists.txt 2022-01-10 13:29:32.912883436 +0800
+@@ -200,6 +200,6 @@ endif()
+ if (("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") OR (${CLANG}))
+ set(GNUC 1)
+ endif()
+-if (("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") OR (${CLANG}))
++if (("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") OR ("${CMAKE_C_SIMULATE_ID}"
STREQUAL "MSVC"))
+ set(MSVC 1)
+ endif()
diff --git a/thirdparty/patches/libhdfs3-master.patch
b/thirdparty/patches/libhdfs3-master.patch
index 52cd1ef..febf1c4 100644
--- a/thirdparty/patches/libhdfs3-master.patch
+++ b/thirdparty/patches/libhdfs3-master.patch
@@ -1,3 +1,15 @@
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+--- a/src/CMakeLists.txt 2021-09-23 22:03:55.000000000 +0800
++++ b/src/CMakeLists.txt 2022-01-18 00:58:22.411061469 +0800
+@@ -46,7 +46,7 @@ SET(HEADER
+ common/XmlConfig.h)
+
+ ADD_LIBRARY(libhdfs3-static STATIC ${libhdfs3_SOURCES}
${libhdfs3_PROTO_SOURCES} ${libhdfs3_PROTO_HEADERS})
+-ADD_LIBRARY(libhdfs3-shared SHARED ${libhdfs3_SOURCES}
${libhdfs3_PROTO_SOURCES} ${libhdfs3_PROTO_HEADERS})
++ADD_LIBRARY(libhdfs3-shared STATIC ${libhdfs3_SOURCES}
${libhdfs3_PROTO_SOURCES} ${libhdfs3_PROTO_HEADERS})
+
+ ADD_CUSTOM_COMMAND(
+ TARGET libhdfs3-shared libhdfs3-static
diff --git a/src/client/FileSystem.cpp b/src/client/FileSystem.cpp
index 6c347c7..6aec1a3 100644
--- a/src/client/FileSystem.cpp
diff --git a/thirdparty/patches/mysql-server-mysql-5.7.18.patch
b/thirdparty/patches/mysql-server-mysql-5.7.18.patch
index c179754..f7e87a3 100644
--- a/thirdparty/patches/mysql-server-mysql-5.7.18.patch
+++ b/thirdparty/patches/mysql-server-mysql-5.7.18.patch
@@ -1,3 +1,15 @@
+diff -uprN a/cmake/zlib.cmake b/cmake/zlib.cmake
+--- a/cmake/zlib.cmake 2022-01-18 00:21:44.993277439 +0800
++++ b/cmake/zlib.cmake 2022-01-18 00:21:21.943260690 +0800
+@@ -52,7 +52,7 @@ MACRO (MYSQL_CHECK_ZLIB_WITH_COMPRESS)
+ INCLUDE(FindZLIB)
+ IF(ZLIB_FOUND)
+ INCLUDE(CheckFunctionExists)
+- SET(CMAKE_REQUIRED_LIBRARIES z)
++ SET(CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES})
+ CHECK_FUNCTION_EXISTS(crc32 HAVE_CRC32)
+ CHECK_FUNCTION_EXISTS(compressBound HAVE_COMPRESSBOUND)
+ CHECK_FUNCTION_EXISTS(deflateBound HAVE_DEFLATEBOUND)
diff -uprN a/cmake/boost.cmake b/cmake/boost.cmake
--- a/cmake/boost.cmake 2021-09-28 17:56:01.656714647 +0800
+++ b/cmake/boost.cmake 2021-09-28 17:58:09.156339908 +0800
diff --git a/thirdparty/patches/rocksdb-5.14.2.patch
b/thirdparty/patches/rocksdb-5.14.2.patch
new file mode 100644
index 0000000..27f9bcb
--- /dev/null
+++ b/thirdparty/patches/rocksdb-5.14.2.patch
@@ -0,0 +1,11 @@
+--- a/util/channel.h 2022-01-10 03:06:56.580066258 +0800
++++ b/util/channel.h 2022-01-10 03:06:46.337071094 +0800
+@@ -60,7 +60,7 @@
+
+ private:
+ std::condition_variable cv_;
+- std::mutex lock_;
++ mutable std::mutex lock_;
+ std::queue<T> buffer_;
+ bool eof_;
+ };
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
old mode 100644
new mode 100755
index d166163..029a270
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -22,7 +22,7 @@
############################################################
###################################################
-# DO NOT change variables bellow unless you known
+# DO NOT change variables bellow unless you known
# what you are doing.
###################################################
@@ -88,10 +88,10 @@ GLOG_SOURCE=glog-0.4.0
GLOG_MD5SUM="0daea8785e6df922d7887755c3d100d0"
# gtest
-GTEST_DOWNLOAD="https://github.com/google/googletest/archive/release-1.10.0.tar.gz";
-GTEST_NAME=googletest-release-1.10.0.tar.gz
-GTEST_SOURCE=googletest-release-1.10.0
-GTEST_MD5SUM="ecd1fa65e7de707cd5c00bdac56022cd"
+GTEST_DOWNLOAD="https://github.com/google/googletest/archive/release-1.11.0.tar.gz";
+GTEST_NAME=googletest-release-1.11.0.tar.gz
+GTEST_SOURCE=googletest-release-1.11.0
+GTEST_MD5SUM="e8a8df240b6938bb6384155d4c37d937"
# snappy
SNAPPY_DOWNLOAD="https://github.com/google/snappy/archive/1.1.8.tar.gz";
@@ -108,7 +108,7 @@ GPERFTOOLS_MD5SUM="e340f1b247ff512119a2db98c1538dc1"
# zlib
ZLIB_DOWNLOAD="https://sourceforge.net/projects/libpng/files/zlib/1.2.11/zlib-1.2.11.tar.gz";
ZLIB_NAME=zlib-1.2.11.tar.gz
-ZLIB_SOURCE=zlib-1.2.11
+ZLIB_SOURCE=zlib-1.2.11
ZLIB_MD5SUM="1c9f62f0778697a09d36121ead88e08e"
# lz4
@@ -184,7 +184,7 @@ ROCKSDB_NAME=rocksdb-5.14.2.tar.gz
ROCKSDB_SOURCE=rocksdb-5.14.2
ROCKSDB_MD5SUM="b72720ea3b1e9ca9e4ed0febfef65b14"
-#cyrus-sasl
+# cyrus-sasl
CYRUS_SASL_DOWNLOAD="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-2.1.27/cyrus-sasl-2.1.27.tar.gz";
CYRUS_SASL_NAME=cyrus-sasl-2.1.27.tar.gz
CYRUS_SASL_SOURCE=cyrus-sasl-2.1.27
@@ -349,6 +349,12 @@ XML2_NAME="libxml2-v2.9.10.tar.gz"
XML2_SOURCE="libxml2-v2.9.10"
XML2_MD5SUM="b18faee9173c3378c910f6d7d1493115"
+# idn
+IDN_DOWNLOAD="https://ftp.gnu.org/gnu/libidn/libidn-1.38.tar.gz";
+IDN_NAME="libidn-1.38.tar.gz"
+IDN_SOURCE="libidn-1.38"
+IDN_MD5SUM="718ff3700dd71f830c592ebe97249193"
+
# gsasl
GSASL_DOWNLOAD="https://ftp.gnu.org/gnu/gsasl/libgsasl-1.10.0.tar.gz";
GSASL_NAME="libgsasl-1.10.0.tar.gz"
@@ -438,10 +444,10 @@ AWS_S2N
AWS_SDK
LZMA
XML2
+IDN
GSASL
HDFS3
LIBDIVIDE
PDQSORT
BENCHMARK
BREAKPAD"
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
