This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 0ae1b9c70a1 [chore](remove code) Remove dragonbox related (#34528)
0ae1b9c70a1 is described below
commit 0ae1b9c70a11669562f187569470eccdb09b1a8d
Author: zhiqiang <[email protected]>
AuthorDate: Fri May 10 11:41:43 2024 +0800
[chore](remove code) Remove dragonbox related (#34528)
* Revert "[refactor](mysql result format) use new serde framework to tuple
convert (#25006)"
This reverts commit e5ef0aa6d439c3f9b1f1fe5bc89c9ea6a71d4019.
* run buildall
* MORE
* FIX
---
be/cmake/thirdparty.cmake | 2 -
be/src/gutil/CMakeLists.txt | 2 -
be/src/gutil/strings/numbers.cc | 16 ++----
be/src/gutil/strings/numbers.h | 4 +-
be/src/runtime/runtime_state.h | 5 --
be/src/util/mysql_row_buffer.cpp | 10 ++--
be/src/util/mysql_row_buffer.h | 4 --
be/src/vec/sink/vmysql_result_writer.cpp | 3 --
be/src/vec/sink/vmysql_result_writer.h | 2 -
be/test/util/mysql_row_buffer_test.cpp | 58 ----------------------
.../java/org/apache/doris/qe/SessionVariable.java | 8 ---
gensrc/thrift/PaloInternalService.thrift | 2 +-
.../mysql_tupleconvert_p0/mysql_tupleconvert.out | 10 ----
.../mysql_tupleconvert.groovy | 40 ---------------
thirdparty/build-thirdparty.sh | 16 ------
thirdparty/vars.sh | 7 ---
16 files changed, 12 insertions(+), 177 deletions(-)
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index e9fbdabee8b..78f28fe72dc 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -150,9 +150,7 @@ add_thirdparty(krb5)
add_thirdparty(com_err)
add_thirdparty(k5crypto)
add_thirdparty(gssapi_krb5)
-add_thirdparty(dragonbox_to_chars LIB64)
add_thirdparty(streamvbyte LIB64)
-target_include_directories(dragonbox_to_chars INTERFACE
"${THIRDPARTY_DIR}/include/dragonbox-1.1.3")
if (OS_MACOSX)
add_thirdparty(bfd)
diff --git a/be/src/gutil/CMakeLists.txt b/be/src/gutil/CMakeLists.txt
index 12c2330884f..bac9aec3037 100644
--- a/be/src/gutil/CMakeLists.txt
+++ b/be/src/gutil/CMakeLists.txt
@@ -57,8 +57,6 @@ add_library(Gutil STATIC ${SOURCE_FILES})
set_target_properties(Gutil PROPERTIES COMPILE_FLAGS "-funsigned-char
-Wno-deprecated")
target_compile_options(Gutil PRIVATE -Wno-char-subscripts
-Wno-implicit-fallthrough)
-target_link_libraries(Gutil PRIVATE dragonbox_to_chars)
-
# target_link_libraries(Gutil glog protobuf rt)
#set(GUTIL_LIBS
diff --git a/be/src/gutil/strings/numbers.cc b/be/src/gutil/strings/numbers.cc
index 2f0d5082396..d6b7295f413 100644
--- a/be/src/gutil/strings/numbers.cc
+++ b/be/src/gutil/strings/numbers.cc
@@ -28,7 +28,7 @@ using std::string;
#include <fmt/format.h>
#include "common/logging.h"
-#include "dragonbox/dragonbox_to_chars.h"
+
#include "gutil/gscoped_ptr.h"
#include "gutil/int128.h"
#include "gutil/integral_types.h"
@@ -1276,21 +1276,13 @@ int FloatToBuffer(float value, int width, char* buffer)
{
return snprintf_result;
}
-int FastDoubleToBuffer(double value, char* buffer, bool faster_float_convert) {
- if (faster_float_convert) {
- return jkj::dragonbox::to_chars_n(value, buffer) - buffer;
- }
-
- auto* end = fmt::format_to(buffer, FMT_COMPILE("{}"), value);
+int FastDoubleToBuffer(double value, char* buffer) {
+ auto end = fmt::format_to(buffer, FMT_COMPILE("{}"), value);
*end = '\0';
return end - buffer;
}
-int FastFloatToBuffer(float value, char* buffer, bool faster_float_convert) {
- if (faster_float_convert) {
- return jkj::dragonbox::to_chars_n(value, buffer) - buffer;
- }
-
+int FastFloatToBuffer(float value, char* buffer) {
auto* end = fmt::format_to(buffer, FMT_COMPILE("{}"), value);
*end = '\0';
return end - buffer;
diff --git a/be/src/gutil/strings/numbers.h b/be/src/gutil/strings/numbers.h
index f47982d75e5..291634133c4 100644
--- a/be/src/gutil/strings/numbers.h
+++ b/be/src/gutil/strings/numbers.h
@@ -443,8 +443,8 @@ int FloatToBuffer(float i, int width, char* buffer);
char* DoubleToBuffer(double i, char* buffer);
char* FloatToBuffer(float i, char* buffer);
-int FastDoubleToBuffer(double i, char* buffer, bool faster_float_convert =
false);
-int FastFloatToBuffer(float i, char* buffer, bool faster_float_convert =
false);
+int FastDoubleToBuffer(double i, char* buffer);
+int FastFloatToBuffer(float i, char* buffer);
// In practice, doubles should never need more than 24 bytes and floats
// should never need more than 14 (including null terminators), but we
// overestimate to be safe.
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 30402630463..abef8615b1e 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -182,11 +182,6 @@ public:
return _query_options.__isset.mysql_row_binary_format &&
_query_options.mysql_row_binary_format;
}
-
- bool enable_faster_float_convert() const {
- return _query_options.__isset.faster_float_convert &&
_query_options.faster_float_convert;
- }
-
Status query_status();
// Appends error to the _error_log if there is space
diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp
index b7b4a774a35..a15fa37a0f3 100644
--- a/be/src/util/mysql_row_buffer.cpp
+++ b/be/src/util/mysql_row_buffer.cpp
@@ -170,12 +170,12 @@ static char* add_largeint(int128_t data, char* pos, bool
dynamic_mode) {
}
template <typename T>
-char* add_float(T data, char* pos, bool dynamic_mode, bool
faster_float_convert = false) {
+char* add_float(T data, char* pos, bool dynamic_mode) {
int length = 0;
if constexpr (std::is_same_v<T, float>) {
- length = FastFloatToBuffer(data, pos + !dynamic_mode,
faster_float_convert);
+ length = FastFloatToBuffer(data, pos + !dynamic_mode);
} else if constexpr (std::is_same_v<T, double>) {
- length = FastDoubleToBuffer(data, pos + !dynamic_mode,
faster_float_convert);
+ length = FastDoubleToBuffer(data, pos + !dynamic_mode);
}
if (!dynamic_mode) {
int1store(pos++, length);
@@ -335,7 +335,7 @@ int MysqlRowBuffer<is_binary_format>::push_float(float
data) {
// 1 for string trail, 1 for length, 1 for sign, other for digits
reserve(3 + MAX_FLOAT_STR_LENGTH);
- _pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert);
+ _pos = add_float(data, _pos, _dynamic_mode);
return 0;
}
@@ -349,7 +349,7 @@ int MysqlRowBuffer<is_binary_format>::push_double(double
data) {
}
// 1 for string trail, 1 for length, 1 for sign, other for digits
reserve(3 + MAX_DOUBLE_STR_LENGTH);
- _pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert);
+ _pos = add_float(data, _pos, _dynamic_mode);
return 0;
}
diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h
index 69f84782df9..6f12fda1ecb 100644
--- a/be/src/util/mysql_row_buffer.h
+++ b/be/src/util/mysql_row_buffer.h
@@ -128,8 +128,6 @@ public:
*/
void close_dynamic_mode();
- void set_faster_float_convert(bool faster) { _faster_float_convert =
faster; }
-
private:
int reserve(int64_t size);
@@ -148,8 +146,6 @@ private:
uint64_t _len_pos;
uint32_t _field_pos = 0;
uint32_t _field_count = 0;
-
- bool _faster_float_convert = false;
};
} // namespace doris
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp
b/be/src/vec/sink/vmysql_result_writer.cpp
index 41f0682b1c3..68c777ab38b 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -89,8 +89,6 @@ Status
VMysqlResultWriter<is_binary_format>::init(RuntimeState* state) {
}
set_output_object_data(state->return_object_data_as_binary());
_is_dry_run = state->query_options().dry_run_query;
- _enable_faster_float_convert = state->enable_faster_float_convert();
-
return Status::OK();
}
@@ -127,7 +125,6 @@ Status VMysqlResultWriter<is_binary_format>::write(Block&
input_block) {
{
SCOPED_TIMER(_convert_tuple_timer);
MysqlRowBuffer<is_binary_format> row_buffer;
- row_buffer.set_faster_float_convert(_enable_faster_float_convert);
if constexpr (is_binary_format) {
row_buffer.start_binary_row(_output_vexpr_ctxs.size());
}
diff --git a/be/src/vec/sink/vmysql_result_writer.h
b/be/src/vec/sink/vmysql_result_writer.h
index 8227d09dcc1..0c63a41dd3d 100644
--- a/be/src/vec/sink/vmysql_result_writer.h
+++ b/be/src/vec/sink/vmysql_result_writer.h
@@ -89,8 +89,6 @@ private:
bool _is_dry_run = false;
uint64_t _bytes_sent = 0;
-
- bool _enable_faster_float_convert = false;
};
} // namespace vectorized
} // namespace doris
diff --git a/be/test/util/mysql_row_buffer_test.cpp
b/be/test/util/mysql_row_buffer_test.cpp
index 3e2380b6c11..cfe012fdcd5 100644
--- a/be/test/util/mysql_row_buffer_test.cpp
+++ b/be/test/util/mysql_row_buffer_test.cpp
@@ -118,62 +118,4 @@ TEST(MysqlRowBufferTest, dynamic_mode) {
EXPECT_EQ(0, strncmp(buf + 43, "test", 4));
}
-TEST(MysqlRowBufferTest /*unused*/, faster_float_convert /*unused*/) {
- MysqlRowBuffer mrb;
- mrb.set_faster_float_convert(true);
-
- mrb.push_float(0);
- mrb.push_float(1.0);
- mrb.push_float(-1.0);
- mrb.push_float(56.45);
- mrb.push_double(10.12);
-
- const char* buf = mrb.buf();
-
- // mem: size-data-size-data
- // 3-'0E0'-3-'1E0'-4-'-1E0'-7-'5.645E1'-7-'1.012E1'
- // 1b-3b---1b-3b---1b-4b----1b-7b-------1b-7b------
- // 0 1 4 5 8 9 13 14 21 22
-
- EXPECT_EQ(29, mrb.length());
-
- EXPECT_EQ(3, *((int8_t*)(buf + 0)));
- EXPECT_EQ(0, strncmp(buf + 1, "0E0", 3));
-
- EXPECT_EQ(3, *((int8_t*)(buf + 4)));
- EXPECT_EQ(0, strncmp(buf + 5, "1E0", 3));
-
- EXPECT_EQ(4, *((int8_t*)(buf + 8)));
- EXPECT_EQ(0, strncmp(buf + 9, "-1E0", 4));
-
- EXPECT_EQ(7, *((int8_t*)(buf + 13)));
- EXPECT_EQ(0, strncmp(buf + 14, "5.645E1", 7));
-
- EXPECT_EQ(7, *((int8_t*)(buf + 21)));
- EXPECT_EQ(0, strncmp(buf + 22, "1.012E1", 7));
-}
-
-TEST(MysqlRowBufferTest /*unused*/, faster_float_convert_dynamic /*unused*/) {
- MysqlRowBuffer mrb;
- mrb.set_faster_float_convert(true);
- mrb.open_dynamic_mode();
-
- mrb.push_float(0);
- mrb.push_float(1.0);
- mrb.push_float(-1.0);
- mrb.push_float(56.45);
- mrb.push_double(10.12);
-
- const char* buf = mrb.buf();
-
- EXPECT_EQ(33, mrb.length());
- EXPECT_EQ(254, *((uint8_t*)(buf)));
-
- EXPECT_EQ(0, strncmp(buf + 9, "0E0", 3));
- EXPECT_EQ(0, strncmp(buf + 12, "1E0", 3));
- EXPECT_EQ(0, strncmp(buf + 15, "-1E0", 4));
- EXPECT_EQ(0, strncmp(buf + 19, "5.645E1", 7));
- EXPECT_EQ(0, strncmp(buf + 26, "1.012E1", 7));
-}
-
} // namespace doris
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index eab4c0f7608..cfa39932491 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -477,8 +477,6 @@ public class SessionVariable implements Serializable,
Writable {
public static final String AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD =
"auto_analyze_table_width_threshold";
- public static final String FASTER_FLOAT_CONVERT = "faster_float_convert";
-
public static final String ENABLE_DECIMAL256 = "enable_decimal256";
public static final String STATS_INSERT_MERGE_ITEM_COUNT =
"stats_insert_merge_item_count";
@@ -1560,10 +1558,6 @@ public class SessionVariable implements Serializable,
Writable {
flag = VariableMgr.GLOBAL)
public String autoAnalyzeEndTime = "23:59:59";
- @VariableMgr.VarAttr(name = FASTER_FLOAT_CONVERT,
- description = {"是否启用更快的浮点数转换算法，注意会影响输出格式", "Set true to enable
faster float pointer number convert"})
- public boolean fasterFloatConvert = false;
-
@VariableMgr.VarAttr(name = IGNORE_RUNTIME_FILTER_IDS,
description = {"在IGNORE_RUNTIME_FILTER_IDS列表中的runtime
filter将不会被生成",
"the runtime filter id in IGNORE_RUNTIME_FILTER_IDS list
will not be generated"})
@@ -3181,8 +3175,6 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setInvertedIndexConjunctionOptThreshold(invertedIndexConjunctionOptThreshold);
tResult.setInvertedIndexMaxExpansions(invertedIndexMaxExpansions);
- tResult.setFasterFloatConvert(fasterFloatConvert);
-
tResult.setEnableDecimal256(enableNereidsPlanner && enableDecimal256);
tResult.setSkipMissingVersion(skipMissingVersion);
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 2f7886e10c2..97a50dae06c 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -248,7 +248,7 @@ struct TQueryOptions {
85: optional bool enable_page_cache = false;
86: optional i32 analyze_timeout = 43200;
- 87: optional bool faster_float_convert = false;
+ 87: optional bool faster_float_convert = false; // deprecated
88: optional bool enable_decimal256 = false;
diff --git a/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out
b/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out
deleted file mode 100644
index c27a9d00b53..00000000000
--- a/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out
+++ /dev/null
@@ -1,10 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !select --
-1 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]
-
--- !select --
-1 0.33 3.1415925 [3.3E-1, 6.7E-1] [3.1415926E0,
8.78787878E-1]
-
--- !select --
-1 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]
-2 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]
diff --git
a/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy
b/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy
deleted file mode 100644
index 51e47cfd243..00000000000
--- a/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy
+++ /dev/null
@@ -1,40 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("mysql_tuple_convert_test") {
-
- testTable = "doule_faster_convert_test"
- sql "DROP TABLE IF EXISTS ${testTable}"
- sql """
- CREATE TABLE IF NOT EXISTS ${testTable} (
- `col_0` INT,
- `col_1` DOUBLE,
- `col_2` FLOAT,
- `col_3` ARRAY<FLOAT>,
- `col_4` ARRAY<DOUBLE>)
- ENGINE=OLAP DUPLICATE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`)
BUCKETS 1
- PROPERTIES("replication_num" = "1");
- """
- sql """INSERT INTO ${testTable} VALUES (1, 0.33, 3.1415926, [0.33, 0.67],
[3.1415926, 0.878787878]);"""
- qt_select """SELECT /*+SET_VAR(faster_float_convert=false)*/ * FROM
${testTable};"""
- qt_select """SELECT /*+SET_VAR(faster_float_convert=true)*/ * FROM
${testTable};"""
- // make sure we can convert number from string to value
- sql """INSERT INTO ${testTable} SELECT 2, col_1, col_2, col_3, col_4 from
${testTable} where col_0 = 1;"""
- qt_select """SELECT * FROM ${testTable} ORDER BY col_0;"""
- sql """SET faster_float_convert=false;"""
- sql "DROP TABLE IF EXISTS ${testTable}"
-}
\ No newline at end of file
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index ad8dbc28e93..f17af7571bb 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1682,21 +1682,6 @@ build_hadoop_libs() {
find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type l -exec cp -P {}
"${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \;
}
-# dragonbox
-build_dragonbox() {
- check_if_source_exist "${DRAGONBOX_SOURCE}"
- cd "${TP_SOURCE_DIR}/${DRAGONBOX_SOURCE}"
-
- rm -rf "${BUILD_DIR}"
- mkdir -p "${BUILD_DIR}"
- cd "${BUILD_DIR}"
-
- "${CMAKE_CMD}" -G "${GENERATOR}"
-DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DDRAGONBOX_INSTALL_TO_CHARS=ON ..
-
- "${BUILD_SYSTEM}" -j "${PARALLEL}"
- "${BUILD_SYSTEM}" install
-}
-
# AvxToNeon
build_avx2neon() {
check_if_source_exist "${AVX2NEON_SOURCE}"
@@ -1854,7 +1839,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
concurrentqueue
fast_float
libunwind
- dragonbox
avx2neon
libdeflate
streamvbyte
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index ffbfe1e384d..ff8092dba66 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -466,12 +466,6 @@ HADOOP_LIBS_NAME="hadoop-3.3.6.1-for-doris.tar.gz"
HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.6.1-for-doris"
HADOOP_LIBS_MD5SUM="8a4455b07f5df2bb10a2718c957972b8"
-# libdragonbox for faster double/float to string
-DRAGONBOX_DOWNLOAD="https://github.com/jk-jeon/dragonbox/archive/refs/tags/1.1.3.tar.gz";
-DRAGONBOX_NAME=dragonbox-1.1.3.tar.gz
-DRAGONBOX_SOURCE=dragonbox-1.1.3
-DRAGONBOX_MD5SUM="889dc00db9612c6949a4ccf8115e0e6a"
-
# AvxToNeon
AVX2NEON_DOWNLOAD="https://github.com/kunpengcompute/AvxToNeon/archive/refs/tags/v1.0.0.tar.gz";
AVX2NEON_NAME=v1.0.0.tar.gz
@@ -582,7 +576,6 @@ export TP_ARCHIVES=(
'CONCURRENTQUEUE'
'FAST_FLOAT'
'HADOOP_LIBS'
- 'DRAGONBOX'
'AVX2NEON'
'LIBDEFLATE'
'STREAMVBYTE'
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
