This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3ee000c [chore] support build with libc++ && add some build config
(#7903)
3ee000c is described below
commit 3ee000c13cde28e5ae073dd6b2257f59bfdd5c99
Author: Pxl <[email protected]>
AuthorDate: Sun Jan 30 16:47:22 2022 +0800
[chore] support build with libc++ && add some build config (#7903)
support LIBCPP/LDD/BUILD_META_TOOL for build.sh
---
be/CMakeLists.txt | 11 ++++++++++-
be/src/agent/cgroups_mgr.h | 2 --
be/src/common/configbase.cpp | 4 ++--
be/src/exprs/grouping_sets_functions.h | 2 --
be/src/exprs/json_functions.cpp | 20 ++++++++++---------
be/src/exprs/math_functions.cpp | 16 +++++++--------
be/src/gutil/strings/memutil.h | 2 --
be/src/gutil/strtoint.cc | 3 +--
be/src/gutil/strtoint.h | 3 +--
be/src/gutil/sysinfo.cc | 2 +-
be/src/http/http_parser.h | 2 +-
be/src/olap/delete_handler.h | 1 -
be/src/olap/olap_server.cpp | 6 +++++-
be/src/olap/tablet_schema.h | 1 -
be/src/runtime/data_stream_sender.cpp | 7 ++++---
be/src/udf/udf_ir.cpp | 7 +++++++
be/src/util/encryption_util.cpp | 7 ++-----
be/src/util/errno.cpp | 1 +
be/src/util/error_util.cc | 2 +-
be/src/util/error_util.h | 1 -
be/src/util/md5.h | 2 +-
be/src/util/mysql_load_error_hub.h | 1 +
be/src/util/string_parser.hpp | 35 +++++++++++++++++++--------------
be/src/util/string_util.h | 11 +++++++++++
be/src/util/utf8_check.cpp | 2 --
be/src/vec/functions/function_json.cpp | 6 +++---
be/src/vec/functions/modulo.cpp | 7 ++++---
be/src/vec/olap/vcollect_iterator.h | 9 +++++++++
be/src/vec/sink/vdata_stream_sender.cpp | 31 +++++++++++++++++------------
build.sh | 15 ++++++++++++++
run-be-ut.sh | 2 +-
31 files changed, 138 insertions(+), 83 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 74fc266..9a9f0f1 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -342,6 +342,15 @@ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}
-DBOOST_SYSTEM_NO_DEPRECATED")
# Enable the cpu and heap profile of brpc
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBRPC_ENABLE_CPU_PROFILER")
+if (USE_LDD)
+ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fuse-ld=lld")
+endif ()
+
+if (USE_LIBCPP AND COMPILER_CLANG)
+ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++ -lstdc++")
+ add_definitions(-DUSE_LIBCPP)
+endif()
+
if (COMPILER_GCC)
# Avoid GCC 11 false alarm
#
https://stackoverflow.com/questions/67584073/gcc-11-false-array-subscript-is-partly-outside-array-bounds-warning
@@ -760,7 +769,7 @@ add_subdirectory(${SRC_DIR}/runtime)
add_subdirectory(${SRC_DIR}/service)
add_subdirectory(${SRC_DIR}/udf)
-if (${MAKE_TEST} STREQUAL "OFF")
+if (${BUILD_META_TOOL} STREQUAL "ON")
add_subdirectory(${SRC_DIR}/tools)
endif()
diff --git a/be/src/agent/cgroups_mgr.h b/be/src/agent/cgroups_mgr.h
index 4c30f19..de46a47 100644
--- a/be/src/agent/cgroups_mgr.h
+++ b/be/src/agent/cgroups_mgr.h
@@ -18,8 +18,6 @@
#ifndef DORIS_BE_SRC_AGENT_CGROUPS_MGR_H
#define DORIS_BE_SRC_AGENT_CGROUPS_MGR_H
-#include <sys/types.h>
-
#include <cstdint>
#include <map>
#include <mutex>
diff --git a/be/src/common/configbase.cpp b/be/src/common/configbase.cpp
index 7593f76..3258da3 100644
--- a/be/src/common/configbase.cpp
+++ b/be/src/common/configbase.cpp
@@ -45,12 +45,12 @@ std::mutex mutable_string_config_lock;
// trim string
std::string& trim(std::string& s) {
// rtrim
- s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int,
int>(std::isspace)))
+ s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char c) { return
!std::isspace(c); })
.base(),
s.end());
// ltrim
s.erase(s.begin(),
- std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<int,
int>(std::isspace))));
+ std::find_if(s.begin(), s.end(), [](unsigned char c) { return
!std::isspace(c); }));
return s;
}
diff --git a/be/src/exprs/grouping_sets_functions.h
b/be/src/exprs/grouping_sets_functions.h
index 05bb807..524692b 100644
--- a/be/src/exprs/grouping_sets_functions.h
+++ b/be/src/exprs/grouping_sets_functions.h
@@ -21,8 +21,6 @@
namespace doris {
-class Expr;
-
class GroupingSetsFunctions {
public:
static void init();
diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp
index 55e11aa..bfe26c2 100644
--- a/be/src/exprs/json_functions.cpp
+++ b/be/src/exprs/json_functions.cpp
@@ -25,7 +25,6 @@
#include <sys/time.h>
#include <boost/algorithm/string.hpp>
-#include <boost/tokenizer.hpp>
#include <iomanip>
#include <sstream>
#include <string>
@@ -41,6 +40,7 @@
#include "runtime/string_value.h"
#include "runtime/tuple_row.h"
#include "udf/udf.h"
+#include "util/string_util.h"
namespace doris {
@@ -180,7 +180,9 @@ rapidjson::Value JsonFunctions::parse_str_with_flag(const
StringVal& arg, const
val.SetDouble(number);
} else if (*(flag.ptr + num) == '4' || *(flag.ptr + num) == '5') {
StringPiece str((char*)arg.ptr, arg.len);
- if (*(flag.ptr + num) == '4') str = str.substr(1, str.length() - 2);
+ if (*(flag.ptr + num) == '4') {
+ str = str.substr(1, str.length() - 2);
+ }
val.SetString(str.data(), str.length(), allocator);
} else {
DCHECK(false) << "parse json type error with unknown type";
@@ -310,24 +312,24 @@ rapidjson::Value*
JsonFunctions::get_json_object(FunctionContext* context,
// '$.text#abc.xyz' -> [$, text#abc, xyz]
// '$."text.abc".xyz' -> [$, text.abc, xyz]
// '$."text.abc"[1].xyz' -> [$, text.abc[1], xyz]
- JsonState* json_state;
+ JsonState* json_state = nullptr;
JsonState tmp_json_state;
+
#ifndef BE_TEST
- json_state =
reinterpret_cast<JsonState*>(context->get_function_state(FunctionContext::FRAGMENT_LOCAL));
+ json_state = reinterpret_cast<JsonState*>(
+ context->get_function_state(FunctionContext::FRAGMENT_LOCAL));
if (json_state == nullptr) {
json_state = &tmp_json_state;
}
if (json_state->json_paths.size() == 0) {
- boost::tokenizer<boost::escaped_list_separator<char>> tok(
- path_string, boost::escaped_list_separator<char>("\\", ".",
"\""));
+ auto tok = get_json_token(path_string);
std::vector<std::string> paths(tok.begin(), tok.end());
get_parsed_paths(paths, &json_state->json_paths);
}
#else
json_state = &tmp_json_state;
- boost::tokenizer<boost::escaped_list_separator<char>> tok(
- path_string, boost::escaped_list_separator<char>("\\", ".", "\""));
+ auto tok = get_json_token(path_string);
std::vector<std::string> paths(tok.begin(), tok.end());
get_parsed_paths(paths, &json_state->json_paths);
#endif
@@ -353,7 +355,7 @@ rapidjson::Value*
JsonFunctions::get_json_object(FunctionContext* context,
//rapidjson::Document document;
if (UNLIKELY(document->HasParseError())) {
VLOG_CRITICAL << "Error at offset " << document->GetErrorOffset()
<< ": "
- << GetParseError_En(document->GetParseError());
+ << GetParseError_En(document->GetParseError());
document->SetNull();
return document;
}
diff --git a/be/src/exprs/math_functions.cpp b/be/src/exprs/math_functions.cpp
index 00cf3bd..1bdd83f 100644
--- a/be/src/exprs/math_functions.cpp
+++ b/be/src/exprs/math_functions.cpp
@@ -27,8 +27,8 @@
#include "common/compiler_util.h"
#include "exprs/anyval_util.h"
#include "runtime/decimalv2_value.h"
-#include "util/string_parser.hpp"
#include "util/simd/vstring_function.h"
+#include "util/string_parser.hpp"
namespace doris {
@@ -113,7 +113,7 @@ DecimalV2Val MathFunctions::abs(FunctionContext* ctx, const
doris_udf::DecimalV2
if (UNLIKELY(val.val == MIN_INT128)) {
return DecimalV2Val::null();
} else {
- return DecimalV2Val(::abs(val.val));
+ return DecimalV2Val(val.val > 0 ? val.val : -val.val);
}
}
@@ -124,7 +124,7 @@ LargeIntVal MathFunctions::abs(FunctionContext* ctx, const
doris_udf::LargeIntVa
if (UNLIKELY(val.val == MIN_INT128)) {
return LargeIntVal::null();
} else {
- return LargeIntVal(::abs(val.val));
+ return LargeIntVal(val.val > 0 ? val.val : -val.val);
}
}
@@ -132,28 +132,28 @@ LargeIntVal MathFunctions::abs(FunctionContext* ctx,
const doris_udf::BigIntVal&
if (val.is_null) {
return LargeIntVal::null();
}
- return LargeIntVal(::abs(__int128(val.val)));
+ return LargeIntVal(__int128(std::abs(val.val)));
}
BigIntVal MathFunctions::abs(FunctionContext* ctx, const doris_udf::IntVal&
val) {
if (val.is_null) {
return BigIntVal::null();
}
- return BigIntVal(::abs(int64_t(val.val)));
+ return BigIntVal(int64_t(std::abs(val.val)));
}
IntVal MathFunctions::abs(FunctionContext* ctx, const doris_udf::SmallIntVal&
val) {
if (val.is_null) {
return IntVal::null();
}
- return IntVal(::abs(int32_t(val.val)));
+ return IntVal(int32_t(std::abs(val.val)));
}
SmallIntVal MathFunctions::abs(FunctionContext* ctx, const
doris_udf::TinyIntVal& val) {
if (val.is_null) {
return SmallIntVal::null();
}
- return SmallIntVal(::abs(int16_t(val.val)));
+ return SmallIntVal(int16_t(std::abs(val.val)));
}
// Generates a UDF that always calls FN() on the input val and returns it.
@@ -352,7 +352,7 @@ StringVal MathFunctions::hex_string(FunctionContext* ctx,
const StringVal& s) {
}
StringVal result = StringVal::create_temp_string_val(ctx, s.len * 2);
- simd::VStringFunctions::hex_encode(s.ptr, s.len, reinterpret_cast<char
*>(result.ptr));
+ simd::VStringFunctions::hex_encode(s.ptr, s.len,
reinterpret_cast<char*>(result.ptr));
return result;
}
diff --git a/be/src/gutil/strings/memutil.h b/be/src/gutil/strings/memutil.h
index 83425da..ee0eeb7 100644
--- a/be/src/gutil/strings/memutil.h
+++ b/be/src/gutil/strings/memutil.h
@@ -56,8 +56,6 @@
#include <stddef.h>
#include <string.h> // to get the POSIX mem*() routines
-#include "gutil/port.h" // disable some warnings on Windows
-
inline char* memcat(char* dest, size_t destlen, const char* src, size_t
srclen) {
return reinterpret_cast<char*>(memcpy(dest + destlen, src, srclen));
}
diff --git a/be/src/gutil/strtoint.cc b/be/src/gutil/strtoint.cc
index b3b711f..d0aeff5 100644
--- a/be/src/gutil/strtoint.cc
+++ b/be/src/gutil/strtoint.cc
@@ -7,8 +7,7 @@
#include "gutil/strtoint.h"
#include <errno.h>
-
-#include "gutil/port.h"
+#include <limits.h>
// Replacement strto[u]l functions that have identical overflow and underflow
// characteristics for both ILP-32 and LP-64 platforms, including errno
diff --git a/be/src/gutil/strtoint.h b/be/src/gutil/strtoint.h
index fb83903..c253b93 100644
--- a/be/src/gutil/strtoint.h
+++ b/be/src/gutil/strtoint.h
@@ -31,12 +31,11 @@
#define BASE_STRTOINT_H_
#include <stdlib.h> // For strtol* functions.
-
#include <string>
+
using std::string;
#include "gutil/integral_types.h"
#include "gutil/macros.h"
-#include "gutil/port.h"
// Adapter functions for handling overflow and errno.
int32 strto32_adapter(const char* nptr, char** endptr, int base);
diff --git a/be/src/gutil/sysinfo.cc b/be/src/gutil/sysinfo.cc
index ce43f5f..cfd325c 100644
--- a/be/src/gutil/sysinfo.cc
+++ b/be/src/gutil/sysinfo.cc
@@ -35,6 +35,7 @@
#include <ctype.h>
#include <fcntl.h> // for open()
#include <unistd.h> // for read()
+#include <strings.h>
#if defined __MACH__ // Mac OS X, almost certainly
#include <sys/sysctl.h> // how we figure out numcpu's on OS X
@@ -50,7 +51,6 @@
#endif
#include <glog/logging.h>
-
#include <algorithm>
#include <cerrno> // for errno
#include <cstdio> // for snprintf(), sscanf()
diff --git a/be/src/http/http_parser.h b/be/src/http/http_parser.h
index c211cae..6d4b1d4 100644
--- a/be/src/http/http_parser.h
+++ b/be/src/http/http_parser.h
@@ -19,7 +19,7 @@
#define DORIS_BE_SRC_COMMON_UTIL_HTTP_PARSER_H
#include <stdint.h>
-
+#include <stddef.h>
#include <ostream>
namespace doris {
diff --git a/be/src/olap/delete_handler.h b/be/src/olap/delete_handler.h
index 0f2693a..d95f0f9 100644
--- a/be/src/olap/delete_handler.h
+++ b/be/src/olap/delete_handler.h
@@ -26,7 +26,6 @@
#include "olap/block_column_predicate.h"
#include "olap/column_predicate.h"
#include "olap/olap_define.h"
-#include "olap/tablet_schema.h"
namespace doris {
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index fafbde5..7acca70 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -23,6 +23,7 @@
#include <cmath>
#include <ctime>
#include <string>
+#include <random>
#include "agent/cgroups_mgr.h"
#include "common/status.h"
@@ -412,7 +413,10 @@ std::vector<TabletSharedPtr>
StorageEngine::_generate_compaction_tasks(
std::vector<TabletSharedPtr> tablets_compaction;
uint32_t max_compaction_score = 0;
- std::random_shuffle(data_dirs.begin(), data_dirs.end());
+
+ std::random_device rd;
+ std::mt19937 g(rd());
+ std::shuffle(data_dirs.begin(), data_dirs.end(), g);
// Copy _tablet_submitted_xxx_compaction map so that we don't need to hold
_tablet_submitted_compaction_mutex
// when travesing the data dir
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index ffd8c34..becee2e 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -22,7 +22,6 @@
#include "gen_cpp/olap_file.pb.h"
#include "olap/olap_define.h"
-#include "olap/tablet_schema.h"
#include "olap/types.h"
namespace doris {
diff --git a/be/src/runtime/data_stream_sender.cpp
b/be/src/runtime/data_stream_sender.cpp
index efd9260..8c306de 100644
--- a/be/src/runtime/data_stream_sender.cpp
+++ b/be/src/runtime/data_stream_sender.cpp
@@ -22,6 +22,7 @@
#include <algorithm>
#include <iostream>
+#include <random>
#include <thread>
#include "common/config.h"
@@ -377,9 +378,9 @@ Status DataStreamSender::prepare(RuntimeState* state) {
state->instance_mem_tracker());
if (_part_type == TPartitionType::UNPARTITIONED || _part_type ==
TPartitionType::RANDOM) {
- // Randomize the order we open/transmit to channels to avoid
thundering herd problems.
- srand(reinterpret_cast<uint64_t>(this));
- random_shuffle(_channels.begin(), _channels.end());
+ std::random_device rd;
+ std::mt19937 g(rd());
+ shuffle(_channels.begin(), _channels.end(), g);
} else if (_part_type == TPartitionType::HASH_PARTITIONED ||
_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) {
RETURN_IF_ERROR(Expr::prepare(_partition_expr_ctxs, state, _row_desc,
_expr_mem_tracker));
diff --git a/be/src/udf/udf_ir.cpp b/be/src/udf/udf_ir.cpp
index a1ceb02..416a520 100644
--- a/be/src/udf/udf_ir.cpp
+++ b/be/src/udf/udf_ir.cpp
@@ -15,7 +15,14 @@
// specific language governing permissions and limitations
// under the License.
+#include <vector>
+
#include "udf/udf_internal.h"
+#include "udf/udf.h"
+
+namespace doris {
+class ColumnPtrWrapper;
+} // namespace doris
namespace doris_udf {
bool FunctionContext::is_arg_constant(int i) const {
diff --git a/be/src/util/encryption_util.cpp b/be/src/util/encryption_util.cpp
index 5854a7f..eb95cf5 100644
--- a/be/src/util/encryption_util.cpp
+++ b/be/src/util/encryption_util.cpp
@@ -17,14 +17,11 @@
#include "util/encryption_util.h"
-#include <openssl/aes.h>
#include <openssl/err.h>
#include <openssl/evp.h>
-
+#include <openssl/ossl_typ.h>
+#include <sys/types.h>
#include <cstring>
-#include <iostream>
-#include <memory>
-#include <string>
namespace doris {
diff --git a/be/src/util/errno.cpp b/be/src/util/errno.cpp
index df72495..3be386d 100644
--- a/be/src/util/errno.cpp
+++ b/be/src/util/errno.cpp
@@ -17,6 +17,7 @@
#include "util/errno.h"
+#include <features.h>
#include <cstring>
#include "gutil/dynamic_annotations.h"
diff --git a/be/src/util/error_util.cc b/be/src/util/error_util.cc
index a7c76fd..031bf8c 100644
--- a/be/src/util/error_util.cc
+++ b/be/src/util/error_util.cc
@@ -18,9 +18,9 @@
#include "util/error_util.h"
#include <errno.h>
-
#include <cstring>
#include <sstream>
+#include <vector>
using std::string;
using std::stringstream;
diff --git a/be/src/util/error_util.h b/be/src/util/error_util.h
index 4c6bfab..ff1e078 100644
--- a/be/src/util/error_util.h
+++ b/be/src/util/error_util.h
@@ -19,7 +19,6 @@
#define DORIS_BE_SRC_UTIL_ERROR_UTIL_H
#include <string>
-#include <vector>
// #include "gen-cpp/CatalogObjects_types.h"
// #include "gen-cpp/ErrorCodes_types.h"
diff --git a/be/src/util/md5.h b/be/src/util/md5.h
index 130fc1c..064040c 100644
--- a/be/src/util/md5.h
+++ b/be/src/util/md5.h
@@ -18,7 +18,7 @@
#pragma once
#include <openssl/md5.h>
-
+#include <stddef.h>
#include <string>
namespace doris {
diff --git a/be/src/util/mysql_load_error_hub.h
b/be/src/util/mysql_load_error_hub.h
index aaa3120..0b51ebb 100644
--- a/be/src/util/mysql_load_error_hub.h
+++ b/be/src/util/mysql_load_error_hub.h
@@ -18,6 +18,7 @@
#ifndef DORIS_BE_SRC_UTIL_MYSQL_LOAD_ERROR_HUB_H
#define DORIS_BE_SRC_UTIL_MYSQL_LOAD_ERROR_HUB_H
+#include <array>
#include <mutex>
#include <queue>
#include <sstream>
diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp
index cc1110c..cfb9a48 100644
--- a/be/src/util/string_parser.hpp
+++ b/be/src/util/string_parser.hpp
@@ -134,10 +134,9 @@ public:
return string_to_bool_internal(s + i, len - i, result);
}
- static inline __int128 string_to_decimal(const char* s, int len,
- int type_precision, int type_scale, ParseResult* result);
+ static inline __int128 string_to_decimal(const char* s, int len, int
type_precision,
+ int type_scale, ParseResult*
result);
-
template <typename T>
static Status split_string_to_map(const std::string& base, const T
element_separator,
const T key_value_separator,
@@ -146,23 +145,23 @@ public:
int key_end;
int val_pos;
int val_end;
-
+
while ((key_end = base.find(key_value_separator, key_pos)) !=
std::string::npos) {
- if ((val_pos = base.find_first_not_of(key_value_separator,
key_end))
- == std::string::npos) {
+ if ((val_pos = base.find_first_not_of(key_value_separator,
key_end)) ==
+ std::string::npos) {
break;
}
- if ((val_end = base.find(element_separator, val_pos)) ==
std::string::npos) {
+ if ((val_end = base.find(element_separator, val_pos)) ==
std::string::npos) {
val_end = base.size();
}
- result->insert(std::make_pair(base.substr(key_pos, key_end -
key_pos),
- base.substr(val_pos,
val_end - val_pos)));
+ result->insert(std::make_pair(base.substr(key_pos, key_end -
key_pos),
+ base.substr(val_pos, val_end -
val_pos)));
key_pos = val_end;
if (key_pos != std::string::npos) {
- ++key_pos;
+ ++key_pos;
}
}
-
+
return Status::OK();
}
private:
@@ -659,7 +658,9 @@ inline __int128 StringParser::get_scale_multiplier(int
scale) {
static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll *
10ll,
static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll *
100ll,
static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll *
1000ll};
- if (scale >= 0 && scale < 39) return values[scale];
+ if (scale >= 0 && scale < 39) {
+ return values[scale];
+ }
return -1; // Overflow
}
@@ -773,7 +774,9 @@ inline __int128 StringParser::string_to_decimal(const char*
s, int len,
}
// Ex: 0.001, at this point would have precision 1 and scale 3 since
leading zeros
// were ignored during previous parsing.
- if (scale > precision) precision = scale;
+ if (scale > precision) {
+ precision = scale;
+ }
// Microbenchmarks show that beyond this point, returning on parse failure
is slower
// than just letting the function run out.
@@ -783,13 +786,15 @@ inline __int128 StringParser::string_to_decimal(const
char* s, int len,
} else if (UNLIKELY(scale > type_scale)) {
*result = StringParser::PARSE_UNDERFLOW;
int shift = scale - type_scale;
- if (UNLIKELY(truncated_digit_count > 0)) shift -=
truncated_digit_count;
+ if (UNLIKELY(truncated_digit_count > 0)) {
+ shift -= truncated_digit_count;
+ }
if (shift > 0) {
__int128 divisor = get_scale_multiplier(shift);
if (LIKELY(divisor >= 0)) {
value /= divisor;
__int128 remainder = value % divisor;
- if (abs(remainder) >= (divisor >> 1)) {
+ if ((remainder > 0 ? remainder : -remainder) >= (divisor >>
1)) {
value += 1;
}
} else {
diff --git a/be/src/util/string_util.h b/be/src/util/string_util.h
index d5c7fe8..7fce0a8 100644
--- a/be/src/util/string_util.h
+++ b/be/src/util/string_util.h
@@ -20,6 +20,7 @@
#include <strings.h>
#include <algorithm>
+#include <boost/tokenizer.hpp>
#include <map>
#include <set>
#include <sstream>
@@ -129,4 +130,14 @@ template <class T>
using StringCaseUnorderedMap =
std::unordered_map<std::string, T, StringCaseHasher, StringCaseEqual>;
+inline auto get_json_token(const std::string_view& path_string) {
+#ifdef USE_LIBCPP
+ return boost::tokenizer<boost::escaped_list_separator<char>>(
+ std::string(path_string),
boost::escaped_list_separator<char>("\\", ".", "\""));
+#else
+ return boost::tokenizer<boost::escaped_list_separator<char>>(
+ path_string, boost::escaped_list_separator<char>("\\", ".", "\""));
+#endif
+}
+
} // namespace doris
diff --git a/be/src/util/utf8_check.cpp b/be/src/util/utf8_check.cpp
index 0d58c50..5355b90 100644
--- a/be/src/util/utf8_check.cpp
+++ b/be/src/util/utf8_check.cpp
@@ -9,8 +9,6 @@
#include "util/utf8_check.h"
-#include <cstdlib>
-
#if defined(__i386) || defined(__x86_64__)
#include "util/simdutf8check.h"
#elif defined(__aarch64__)
diff --git a/be/src/vec/functions/function_json.cpp
b/be/src/vec/functions/function_json.cpp
index fdfab43..dd2da73 100644
--- a/be/src/vec/functions/function_json.cpp
+++ b/be/src/vec/functions/function_json.cpp
@@ -18,10 +18,11 @@
#include <rapidjson/document.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
+
#include <boost/token_functions.hpp>
-#include <boost/tokenizer.hpp>
#include "exprs/json_functions.h"
+#include "util/string_util.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_string.h"
#include "vec/columns/column_vector.h"
@@ -185,8 +186,7 @@ rapidjson::Value* get_json_object(const std::string_view&
json_string,
std::vector<JsonPath>* parsed_paths;
std::vector<JsonPath> tmp_parsed_paths;
- boost::tokenizer<boost::escaped_list_separator<char>> tok(
- path_string, boost::escaped_list_separator<char>("\\", ".", "\""));
+ auto tok = get_json_token(path_string);
std::vector<std::string> paths(tok.begin(), tok.end());
get_parsed_paths(paths, &tmp_parsed_paths);
parsed_paths = &tmp_parsed_paths;
diff --git a/be/src/vec/functions/modulo.cpp b/be/src/vec/functions/modulo.cpp
index 94a02d0..e0d2e6d 100644
--- a/be/src/vec/functions/modulo.cpp
+++ b/be/src/vec/functions/modulo.cpp
@@ -39,7 +39,7 @@ struct ModuloImpl {
static inline Result apply(A a, B b, NullMap& null_map, size_t index) {
if constexpr (std::is_floating_point_v<Result>) {
null_map[index] = 0;
- return fmod(a, b);
+ return std::fmod((double)a, (double)b);
} else {
null_map[index] = b == 0;
return typename NumberTraits::ToInteger<A>::Type(a) %
@@ -47,7 +47,8 @@ struct ModuloImpl {
}
}
- static inline DecimalV2Value apply(DecimalV2Value a, DecimalV2Value b,
NullMap& null_map, size_t index) {
+ static inline DecimalV2Value apply(DecimalV2Value a, DecimalV2Value b,
NullMap& null_map,
+ size_t index) {
null_map[index] = b == DecimalV2Value(0);
return a % (b + DecimalV2Value(b == DecimalV2Value(0)));
}
@@ -64,7 +65,7 @@ struct ModuloByConstantImpl : BinaryOperationImplBase<A, B,
ModuloImpl<A, B>> {
static void vector_constant(const PaddedPODArray<A>& a, B b,
PaddedPODArray<ResultType>& c) {
// TODO: Support return NULL in the future
if (UNLIKELY(b == 0)) {
-// throw Exception("Division by zero",
TStatusCode::VEC_ILLEGAL_DIVISION);
+ // throw Exception("Division by zero",
TStatusCode::VEC_ILLEGAL_DIVISION);
memset(c.data(), 0, sizeof(ResultType) * c.size());
return;
}
diff --git a/be/src/vec/olap/vcollect_iterator.h
b/be/src/vec/olap/vcollect_iterator.h
index 6cf9d62..6c525be 100644
--- a/be/src/vec/olap/vcollect_iterator.h
+++ b/be/src/vec/olap/vcollect_iterator.h
@@ -17,7 +17,11 @@
#pragma once
+#ifdef USE_LIBCPP
+#include <queue>
+#else
#include <ext/pb_ds/priority_queue.hpp>
+#endif
#include "olap/olap_define.h"
#include "olap/reader.h"
@@ -106,8 +110,13 @@ private:
int _sequence;
};
+#ifdef USE_LIBCPP
+ using MergeHeap = std::priority_queue<LevelIterator*,
std::vector<LevelIterator*>,
+ LevelIteratorComparator>;
+#else
using MergeHeap = __gnu_pbds::priority_queue<LevelIterator*,
LevelIteratorComparator,
__gnu_pbds::pairing_heap_tag>;
+#endif
// Iterate from rowset reader. This Iterator usually like a leaf node
class Level0Iterator : public LevelIterator {
diff --git a/be/src/vec/sink/vdata_stream_sender.cpp
b/be/src/vec/sink/vdata_stream_sender.cpp
index 8afa66b..e3e081d 100644
--- a/be/src/vec/sink/vdata_stream_sender.cpp
+++ b/be/src/vec/sink/vdata_stream_sender.cpp
@@ -20,6 +20,8 @@
#include <fmt/format.h>
#include <fmt/ranges.h>
+#include <random>
+
#include "runtime/client_cache.h"
#include "runtime/dpp_sink_internal.h"
#include "runtime/exec_env.h"
@@ -69,11 +71,11 @@ Status VDataStreamSender::Channel::init(RuntimeState*
state) {
}
Status VDataStreamSender::Channel::send_current_block(bool eos) {
-// TODO: Now, local exchange will cause the performance problem is in a
multi-threaded scenario
-// so this feature is turned off here. We need to re-examine this logic
-// if (is_local()) {
-// return send_local_block(eos);
-// }
+ // TODO: Now, local exchange will cause the performance problem is in a
multi-threaded scenario
+ // so this feature is turned off here. We need to re-examine this logic
+ // if (is_local()) {
+ // return send_local_block(eos);
+ // }
{
SCOPED_TIMER(_parent->_serialize_batch_timer);
_pb_block.Clear();
@@ -184,8 +186,9 @@ Status VDataStreamSender::Channel::add_rows(Block* block,
const std::vector<int>
const int* begin = &rows[0];
while (row_wait_add > 0) {
- int row_add, max_add = batch_size - _mutable_block->rows();
- if (row_wait_add >= max_add) {
+ int row_add = 0;
+ int max_add = batch_size - _mutable_block->rows();
+ if (row_wait_add >= max_add) {
row_add = max_add;
} else {
row_add = row_wait_add;
@@ -343,9 +346,9 @@ Status VDataStreamSender::prepare(RuntimeState* state) {
state->instance_mem_tracker());
if (_part_type == TPartitionType::UNPARTITIONED || _part_type ==
TPartitionType::RANDOM) {
- // Randomize the order we open/transmit to channels to avoid
thundering herd problems.
- srand(reinterpret_cast<uint64_t>(this));
- random_shuffle(_channels.begin(), _channels.end());
+ std::random_device rd;
+ std::mt19937 g(rd());
+ shuffle(_channels.begin(), _channels.end(), g);
} else if (_part_type == TPartitionType::HASH_PARTITIONED ||
_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) {
RETURN_IF_ERROR(VExpr::prepare(_partition_expr_ctxs, state, _row_desc,
_expr_mem_tracker));
@@ -446,7 +449,7 @@ Status VDataStreamSender::send(RuntimeState* state, Block*
block) {
}
}
- // channel2rows' subscript means channel id
+ // channel2rows' subscript means channel id
std::vector<vectorized::UInt64> hash_vals(rows);
for (int i = 0; i < rows; i++) {
hash_vals[i] = siphashs[i].get64();
@@ -479,12 +482,14 @@ Status VDataStreamSender::send(RuntimeState* state,
Block* block) {
hash_vals[i] = RawValue::zlib_crc32(&INT_VALUE, INT_TYPE,
hash_vals[i]);
} else {
hash_vals[i] = RawValue::zlib_crc32(val.data, val.size,
-
_partition_expr_ctxs[j]->root()->type(), hash_vals[i]);
+
_partition_expr_ctxs[j]->root()->type(),
+ hash_vals[i]);
}
}
}
- RETURN_IF_ERROR(channel_add_rows(_channel_shared_ptrs, num_channels,
hash_vals, rows, block));
+ RETURN_IF_ERROR(
+ channel_add_rows(_channel_shared_ptrs, num_channels,
hash_vals, rows, block));
} else {
// Range partition
// 1. caculate range
diff --git a/build.sh b/build.sh
index df1f7ed..a4dd472 100755
--- a/build.sh
+++ b/build.sh
@@ -170,6 +170,15 @@ fi
if [[ -z ${WITH_LZO} ]]; then
WITH_LZO=OFF
fi
+if [[ -z ${USE_LIBCPP} ]]; then
+ USE_LIBCPP=OFF
+fi
+if [[ -z ${BUILD_META_TOOL} ]]; then
+ BUILD_META_TOOL=ON
+fi
+if [[ -z ${USE_LDD} ]]; then
+ USE_LDD=OFF
+fi
echo "Get params:
BUILD_BE -- $BUILD_BE
@@ -182,6 +191,9 @@ echo "Get params:
WITH_LZO -- $WITH_LZO
GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY
USE_AVX2 -- $USE_AVX2
+ USE_LIBCPP -- $USE_LIBCPP
+ BUILD_META_TOOL -- $BUILD_META_TOOL
+ USE_LDD -- $USE_LDD
"
# Clean and build generated code
@@ -211,6 +223,9 @@ if [ ${BUILD_BE} -eq 1 ] ; then
${CMAKE_USE_CCACHE} \
-DWITH_MYSQL=${WITH_MYSQL} \
-DWITH_LZO=${WITH_LZO} \
+ -DUSE_LIBCPP=${USE_LIBCPP} \
+ -DBUILD_META_TOOL=${BUILD_META_TOOL} \
+ -DUSE_LDD=${USE_LDD} \
-DUSE_AVX2=${USE_AVX2} \
-DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} ../
${BUILD_SYSTEM} -j ${PARALLEL}
diff --git a/run-be-ut.sh b/run-be-ut.sh
index 66b4b67..1aa420b 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -127,7 +127,7 @@ else
fi
cd ${CMAKE_BUILD_DIR}
-${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_MYSQL=OFF -DMAKE_TEST=ON
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
+${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_MYSQL=OFF -DMAKE_TEST=ON
-DBUILD_META_TOOL=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
-DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} ${CMAKE_USE_CCACHE}
${BUILD_SYSTEM} -j ${PARALLEL} $RUN_FILE
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
