Copilot commented on code in PR #60004:
URL: https://github.com/apache/doris/pull/60004#discussion_r2702569805
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(test_data_file,
orig_line_count, 18,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
Review Comment:
The test methods contain hardcoded absolute file paths (e.g.,
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt"). These paths are
machine-specific and will fail on other development or CI environments.
Consider using relative paths from the project root, or environment variables,
or generate test data programmatically within the test itself.
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
Review Comment:
The test methods contain hardcoded absolute file paths (e.g.,
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt"). These paths
are machine-specific and will fail on other development or CI environments.
Consider using relative paths from the project root, or environment variables,
or generate test data programmatically within the test itself.
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -27,6 +27,7 @@
#include "vec/columns/column_vector.h"
#include "vec/columns/common_column_test.h"
#include "vec/core/field.h"
+#include "vec/json/json_parser.h"
Review Comment:
The include "vec/json/json_parser.h" is not used anywhere in this file.
Remove unused includes to keep the code clean and reduce compilation
dependencies.
```suggestion
```
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(test_data_file,
orig_line_count, 18,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL32>(test_data_file,
orig_line_count, 9,
+ 4, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt";
Review Comment:
The test methods contain hardcoded absolute file paths (e.g.,
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt"). These paths
are machine-specific and will fail on other development or CI environments.
Consider using relative paths from the project root, or environment variables,
or generate test data programmatically within the test itself.
##########
be/src/util/string_parser.cpp:
##########
@@ -160,51 +156,69 @@ typename PrimitiveTypeTraits<P>::CppType::NativeType
StringParser::string_to_dec
// whose max value is std::numeric_limits<int32_t>::max() - 4,
// so int_part_count will be in range of int32_t,
// and int_part_count + exponent will be in range of int64_t
- int64_t tmp_actual_int_part_count = int_part_count + exponent;
- if (tmp_actual_int_part_count > std::numeric_limits<int>::max() ||
- tmp_actual_int_part_count < std::numeric_limits<int>::min()) {
- *result = StringParser::PARSE_OVERFLOW;
+ int64_t tmp_result_int_part_digit_count = int_part_count + exponent;
+ if (tmp_result_int_part_digit_count > std::numeric_limits<int>::max() ||
+ tmp_result_int_part_digit_count < std::numeric_limits<int>::min()) {
+ *result = is_negative ? StringParser::PARSE_UNDERFLOW :
StringParser::PARSE_OVERFLOW;
return 0;
}
- int actual_int_part_count = tmp_actual_int_part_count;
+ int result_int_part_digit_count = tmp_result_int_part_digit_count;
int actual_frac_part_count = 0;
int digit_index = 0;
- if (actual_int_part_count >= 0) {
- int max_index = std::min(actual_int_part_count, total_digit_count);
+ if (result_int_part_digit_count >= 0) {
+ int max_index = std::min(found_dot ? (result_int_part_digit_count +
+ ((int_part_count > 0 && exponent
> 0) ? 1 : 0))
+ : result_int_part_digit_count,
+ end_digit_index);
+ max_index = (max_index == std::numeric_limits<int>::min() ?
end_digit_index : max_index);
// skip zero number
- for (; digit_index != max_index && digits[digit_index] == 0;
++digit_index) {
+ for (; digit_index != max_index && s[digit_index] == '0';
++digit_index) {
}
// test 0.00, .00, 0.{00...}e2147483647
// 0.00000e2147483647
- if (max_index - digit_index > type_precision - type_scale) {
+ if (digit_index != max_index &&
+ (result_int_part_digit_count - digit_index > type_precision -
type_scale)) {
*result = is_negative ? StringParser::PARSE_UNDERFLOW :
StringParser::PARSE_OVERFLOW;
return 0;
}
// get int part number
for (; digit_index != max_index; ++digit_index) {
- int_part_number = int_part_number * 10 + digits[digit_index];
+ if (UNLIKELY(s[digit_index] == '.')) {
+ continue;
+ }
+ int_part_number = int_part_number * 10 + (s[digit_index] - '0');
}
- if (digit_index != actual_int_part_count) {
- int_part_number *= get_scale_multiplier<T>(actual_int_part_count -
digit_index);
+ auto total_significant_digit_count = i - ((found_dot && int_part_count
> 0) ? 1 : 0);
+ if (result_int_part_digit_count > total_significant_digit_count) {
+ int_part_number *=
get_scale_multiplier<T>(result_int_part_digit_count -
+
total_significant_digit_count);
Review Comment:
The variable name 'total_significant_digit_count' is misleading. Based on
the calculation 'i - ((found_dot && int_part_count > 0) ? 1 : 0)', this
represents the total count of characters processed (including digits and
potentially a decimal point), not just significant digits. Leading zeros were
already skipped earlier in the function. A more accurate name would be
'total_digit_char_count' or 'digits_processed_count' to clarify what this value
represents.
```suggestion
auto total_digit_char_count = i - ((found_dot && int_part_count > 0)
? 1 : 0);
if (result_int_part_digit_count > total_digit_char_count) {
int_part_number *=
get_scale_multiplier<T>(result_int_part_digit_count -
total_digit_char_count);
```
##########
be/src/util/string_parser.cpp:
##########
@@ -109,6 +104,7 @@ typename PrimitiveTypeTraits<P>::CppType::NativeType
StringParser::string_to_dec
}
// parse exponent if any
int64_t exponent = 0;
+ auto end_digit_index = i;
Review Comment:
The variable name 'end_digit_index' is misleading. This variable stores the
index where digit parsing stopped (where 'e'/'E' was found or end of string),
but it may include the position of a decimal point '.'. A more accurate name
would be 'digit_section_end_index' or 'parse_end_index' to clarify that it
marks the end of the numeric section before the exponent.
```suggestion
auto parse_end_index = i;
```
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(test_data_file,
orig_line_count, 18,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL32>(test_data_file,
orig_line_count, 9,
+ 4, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMALV2>(test_data_file,
orig_line_count, 27,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_strict_mode_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000;
+
+ std::cout << "\n=== Testing with strict_mode=false ===\n";
+ perf_test_string_to_decimal_strict<PrimitiveType::TYPE_DECIMAL128I>(
+ test_data_file, orig_line_count, 38, 19, false, false);
+
+ std::cout << "\n=== Testing with strict_mode=true ===\n";
+ perf_test_string_to_decimal_strict<PrimitiveType::TYPE_DECIMAL128I>(
+ test_data_file, orig_line_count, 38, 19, false, true);
+}
+
+// Comparison test: String vs Direct decimal parsing
+TEST_F(FunctionCastToDecimalPerfTest, test_string_parse_comparison) {
+ std::vector<std::string> file_contents;
+
read_file_to_vector_optimized("/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt",
Review Comment:
The test methods contain hardcoded absolute file paths (e.g.,
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt"). These paths
are machine-specific and will fail on other development or CI environments.
Consider using relative paths from the project root, or environment variables,
or generate test data programmatically within the test itself.
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(test_data_file,
orig_line_count, 18,
+ 9, false);
Review Comment:
Type mismatch in function call. The method 'perf_test_string_to_decimal'
declared on line 81 expects a 'MutableColumnPtr' as the first parameter, but
here it's being called with 'test_data_file' which is a 'std::string'. This
code will fail to compile. Either the method signature needs an overload that
accepts a string filename, or the test needs to load the data first into a
column like the other test cases do.
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
Review Comment:
The test methods contain hardcoded absolute file paths (e.g.,
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt"). These paths
are machine-specific and will fail on other development or CI environments.
Consider using relative paths from the project root, or environment variables,
or generate test data programmatically within the test itself.
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(test_data_file,
orig_line_count, 18,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL32>(test_data_file,
orig_line_count, 9,
+ 4, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMALV2>(test_data_file,
orig_line_count, 27,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_strict_mode_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000;
+
+ std::cout << "\n=== Testing with strict_mode=false ===\n";
+ perf_test_string_to_decimal_strict<PrimitiveType::TYPE_DECIMAL128I>(
+ test_data_file, orig_line_count, 38, 19, false, false);
+
+ std::cout << "\n=== Testing with strict_mode=true ===\n";
+ perf_test_string_to_decimal_strict<PrimitiveType::TYPE_DECIMAL128I>(
+ test_data_file, orig_line_count, 38, 19, false, true);
Review Comment:
Type mismatch in function call. The method 'perf_test_string_to_decimal'
declared on line 81 expects a 'MutableColumnPtr' as the first parameter, but
here it's being called with 'test_data_file' which is a 'std::string'. This
code will fail to compile. Either the method signature needs an overload that
accepts a string filename, or the test needs to load the data first into a
column like the other test cases do.
```suggestion
// Create string column and load data from file
auto dt_from = std::make_shared<DataTypeString>();
auto column_orig = dt_from->create_column();
{
MutableColumns columns;
columns.push_back(column_orig->get_ptr());
DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
load_columns_data_from_file(columns, serde, ';', {0},
test_data_file);
EXPECT_EQ(column_orig->size(), orig_line_count);
}
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(column_orig,
orig_line_count, 18, 9,
false);
}
TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
size_t orig_line_count = 10000000;
// Create string column and load data from file
auto dt_from = std::make_shared<DataTypeString>();
auto column_orig = dt_from->create_column();
{
MutableColumns columns;
columns.push_back(column_orig->get_ptr());
DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
load_columns_data_from_file(columns, serde, ';', {0},
test_data_file);
EXPECT_EQ(column_orig->size(), orig_line_count);
}
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL32>(column_orig,
orig_line_count, 9, 4,
false);
}
TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) {
std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt";
size_t orig_line_count = 10000000;
// Create string column and load data from file
auto dt_from = std::make_shared<DataTypeString>();
auto column_orig = dt_from->create_column();
{
MutableColumns columns;
columns.push_back(column_orig->get_ptr());
DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
load_columns_data_from_file(columns, serde, ';', {0},
test_data_file);
EXPECT_EQ(column_orig->size(), orig_line_count);
}
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMALV2>(column_orig,
orig_line_count, 27, 9,
false);
}
TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_strict_mode_perf) {
std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
size_t orig_line_count = 10000000;
// Create string column and load data from file
auto dt_from = std::make_shared<DataTypeString>();
auto column_orig = dt_from->create_column();
{
MutableColumns columns;
columns.push_back(column_orig->get_ptr());
DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
load_columns_data_from_file(columns, serde, ';', {0},
test_data_file);
EXPECT_EQ(column_orig->size(), orig_line_count);
}
std::cout << "\n=== Testing with strict_mode=false ===\n";
perf_test_string_to_decimal_strict<PrimitiveType::TYPE_DECIMAL128I>(
column_orig, orig_line_count, 38, 19, false, false);
std::cout << "\n=== Testing with strict_mode=true ===\n";
perf_test_string_to_decimal_strict<PrimitiveType::TYPE_DECIMAL128I>(
column_orig, orig_line_count, 38, 19, false, true);
```
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(test_data_file,
orig_line_count, 18,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL32>(test_data_file,
orig_line_count, 9,
+ 4, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMALV2>(test_data_file,
orig_line_count, 27,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_strict_mode_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
Review Comment:
The test methods contain hardcoded absolute file paths (e.g.,
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt"). These paths
are machine-specific and will fail on other development or CI environments.
Consider using relative paths from the project root, or environment variables,
or generate test data programmatically within the test itself.
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
Review Comment:
The test methods contain hardcoded absolute file paths (e.g.,
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt"). These paths
are machine-specific and will fail on other development or CI environments.
Consider using relative paths from the project root, or environment variables,
or generate test data programmatically within the test itself.
##########
be/src/util/string_parser.cpp:
##########
@@ -160,51 +156,69 @@ typename PrimitiveTypeTraits<P>::CppType::NativeType
StringParser::string_to_dec
// whose max value is std::numeric_limits<int32_t>::max() - 4,
// so int_part_count will be in range of int32_t,
// and int_part_count + exponent will be in range of int64_t
- int64_t tmp_actual_int_part_count = int_part_count + exponent;
- if (tmp_actual_int_part_count > std::numeric_limits<int>::max() ||
- tmp_actual_int_part_count < std::numeric_limits<int>::min()) {
- *result = StringParser::PARSE_OVERFLOW;
+ int64_t tmp_result_int_part_digit_count = int_part_count + exponent;
+ if (tmp_result_int_part_digit_count > std::numeric_limits<int>::max() ||
+ tmp_result_int_part_digit_count < std::numeric_limits<int>::min()) {
+ *result = is_negative ? StringParser::PARSE_UNDERFLOW :
StringParser::PARSE_OVERFLOW;
return 0;
}
- int actual_int_part_count = tmp_actual_int_part_count;
+ int result_int_part_digit_count = tmp_result_int_part_digit_count;
int actual_frac_part_count = 0;
int digit_index = 0;
- if (actual_int_part_count >= 0) {
- int max_index = std::min(actual_int_part_count, total_digit_count);
+ if (result_int_part_digit_count >= 0) {
+ int max_index = std::min(found_dot ? (result_int_part_digit_count +
+ ((int_part_count > 0 && exponent
> 0) ? 1 : 0))
+ : result_int_part_digit_count,
+ end_digit_index);
Review Comment:
The calculation on lines 169-172 is complex and difficult to understand. The
expression adds a conditional offset '((int_part_count > 0 && exponent > 0) ? 1
: 0)' when 'found_dot' is true, but the purpose of this offset is not
immediately clear. Consider adding a comment explaining why this adjustment is
needed and providing an example case where it applies (e.g., "Adjusts for the
decimal point position when both integer digits and positive exponent exist").
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(test_data_file,
orig_line_count, 18,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL32>(test_data_file,
orig_line_count, 9,
+ 4, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMALV2>(test_data_file,
orig_line_count, 27,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_strict_mode_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000;
+
+ std::cout << "\n=== Testing with strict_mode=false ===\n";
+ perf_test_string_to_decimal_strict<PrimitiveType::TYPE_DECIMAL128I>(
+ test_data_file, orig_line_count, 38, 19, false, false);
+
+ std::cout << "\n=== Testing with strict_mode=true ===\n";
+ perf_test_string_to_decimal_strict<PrimitiveType::TYPE_DECIMAL128I>(
+ test_data_file, orig_line_count, 38, 19, false, true);
+}
+
+// Comparison test: String vs Direct decimal parsing
+TEST_F(FunctionCastToDecimalPerfTest, test_string_parse_comparison) {
+ std::vector<std::string> file_contents;
+
read_file_to_vector_optimized("/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt",
+ file_contents);
+
+ size_t line_count = file_contents.size();
+ std::cout << "Read " << line_count << " lines" << std::endl;
+
+ // Test 1: Direct StringParser
+ int64_t duration_ns_parser = 0;
+ StringParser::ParseResult result;
+ {
+ SCOPED_RAW_TIMER(&duration_ns_parser);
+ for (size_t i = 0; i != line_count; ++i) {
+ auto decimal_value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ file_contents[i].c_str(), file_contents[i].size(), 38, 19,
&result);
+ (void)decimal_value;
+ }
+ }
+ std::cout << "Direct StringParser time: "
+ << PrettyPrinter::print(duration_ns_parser, TUnit::TIME_NS) <<
"\n";
+
+ // Test 2: Cast function (already tested in other tests)
+ std::cout << "\nFor cast function performance, see
test_string_to_decimal128v3_perf\n";
+}
+ */
Review Comment:
These test cases are commented out with a block comment (/* */). If these
tests are no longer needed, they should be removed entirely. If they're
intended for future use or debugging, consider using a feature flag or test
attribute to disable them instead of commenting them out, which makes the
codebase harder to maintain.
##########
be/src/util/string_parser.cpp:
##########
@@ -160,51 +156,69 @@ typename PrimitiveTypeTraits<P>::CppType::NativeType
StringParser::string_to_dec
// whose max value is std::numeric_limits<int32_t>::max() - 4,
// so int_part_count will be in range of int32_t,
// and int_part_count + exponent will be in range of int64_t
- int64_t tmp_actual_int_part_count = int_part_count + exponent;
- if (tmp_actual_int_part_count > std::numeric_limits<int>::max() ||
- tmp_actual_int_part_count < std::numeric_limits<int>::min()) {
- *result = StringParser::PARSE_OVERFLOW;
+ int64_t tmp_result_int_part_digit_count = int_part_count + exponent;
+ if (tmp_result_int_part_digit_count > std::numeric_limits<int>::max() ||
+ tmp_result_int_part_digit_count < std::numeric_limits<int>::min()) {
+ *result = is_negative ? StringParser::PARSE_UNDERFLOW :
StringParser::PARSE_OVERFLOW;
return 0;
}
- int actual_int_part_count = tmp_actual_int_part_count;
+ int result_int_part_digit_count = tmp_result_int_part_digit_count;
int actual_frac_part_count = 0;
int digit_index = 0;
- if (actual_int_part_count >= 0) {
- int max_index = std::min(actual_int_part_count, total_digit_count);
+ if (result_int_part_digit_count >= 0) {
+ int max_index = std::min(found_dot ? (result_int_part_digit_count +
+ ((int_part_count > 0 && exponent
> 0) ? 1 : 0))
+ : result_int_part_digit_count,
+ end_digit_index);
+ max_index = (max_index == std::numeric_limits<int>::min() ?
end_digit_index : max_index);
// skip zero number
- for (; digit_index != max_index && digits[digit_index] == 0;
++digit_index) {
+ for (; digit_index != max_index && s[digit_index] == '0';
++digit_index) {
}
// test 0.00, .00, 0.{00...}e2147483647
// 0.00000e2147483647
- if (max_index - digit_index > type_precision - type_scale) {
+ if (digit_index != max_index &&
+ (result_int_part_digit_count - digit_index > type_precision -
type_scale)) {
*result = is_negative ? StringParser::PARSE_UNDERFLOW :
StringParser::PARSE_OVERFLOW;
return 0;
}
// get int part number
for (; digit_index != max_index; ++digit_index) {
- int_part_number = int_part_number * 10 + digits[digit_index];
+ if (UNLIKELY(s[digit_index] == '.')) {
+ continue;
+ }
+ int_part_number = int_part_number * 10 + (s[digit_index] - '0');
}
- if (digit_index != actual_int_part_count) {
- int_part_number *= get_scale_multiplier<T>(actual_int_part_count -
digit_index);
+ auto total_significant_digit_count = i - ((found_dot && int_part_count
> 0) ? 1 : 0);
Review Comment:
Incorrect variable used in calculation. At this point in the code, the
variable 'i' has been modified during exponent parsing (see lines 111, 118,
126) and now points to the end of the string or past the exponent section. The
calculation should use 'end_digit_index' instead of 'i', which correctly stores
the position where the digit section ended (before the exponent). Using 'i'
here will result in incorrect calculation of the total significant digit count.
```suggestion
auto total_significant_digit_count =
end_digit_index - ((found_dot && int_part_count > 0) ? 1 :
0);
```
##########
be/src/util/string_parser.cpp:
##########
@@ -160,51 +156,69 @@ typename PrimitiveTypeTraits<P>::CppType::NativeType
StringParser::string_to_dec
// whose max value is std::numeric_limits<int32_t>::max() - 4,
// so int_part_count will be in range of int32_t,
// and int_part_count + exponent will be in range of int64_t
- int64_t tmp_actual_int_part_count = int_part_count + exponent;
- if (tmp_actual_int_part_count > std::numeric_limits<int>::max() ||
- tmp_actual_int_part_count < std::numeric_limits<int>::min()) {
- *result = StringParser::PARSE_OVERFLOW;
+ int64_t tmp_result_int_part_digit_count = int_part_count + exponent;
+ if (tmp_result_int_part_digit_count > std::numeric_limits<int>::max() ||
+ tmp_result_int_part_digit_count < std::numeric_limits<int>::min()) {
+ *result = is_negative ? StringParser::PARSE_UNDERFLOW :
StringParser::PARSE_OVERFLOW;
return 0;
}
- int actual_int_part_count = tmp_actual_int_part_count;
+ int result_int_part_digit_count = tmp_result_int_part_digit_count;
int actual_frac_part_count = 0;
int digit_index = 0;
- if (actual_int_part_count >= 0) {
- int max_index = std::min(actual_int_part_count, total_digit_count);
+ if (result_int_part_digit_count >= 0) {
+ int max_index = std::min(found_dot ? (result_int_part_digit_count +
+ ((int_part_count > 0 && exponent
> 0) ? 1 : 0))
+ : result_int_part_digit_count,
+ end_digit_index);
Review Comment:
The check for 'std::numeric_limits<int>::min()' seems suspicious. The
condition checks if 'max_index == std::numeric_limits<int>::min()', which would
only occur if both 'result_int_part_digit_count' and 'end_digit_index' are at
their minimum values due to overflow in the ternary operation. This appears to
be defensive coding for an edge case, but the logic is unclear. Consider adding
a comment explaining when this condition would be true and why setting
max_index to end_digit_index is the correct behavior in that case.
```suggestion
end_digit_index);
// Defensive check: under normal circumstances max_index is never
INT_MIN, because
// result_int_part_digit_count is clamped to the range of int above
and end_digit_index
// is derived from the parsed string length. If max_index were ever
INT_MIN (e.g. due to
// unexpected overflow or undefined behavior in intermediate
computations), fall back to
// using end_digit_index as a safe upper bound for digit iteration.
```
##########
be/test/vec/function/cast/cast_to_decimal128_perf.cpp:
##########
@@ -100,6 +203,141 @@ void read_file_to_vector_optimized(const std::string&
filename, std::vector<std:
lines.push_back(std::move(line)); // Move instead of copy
}
}
+// Test cases
+/*
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128_tmp) {
+ {
+ // std::string s("1234567890.123456");
+ // std::string
s("0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e2147483647");
+ // std::string s("1234567890.123456e7");
+ // std::string s("1.5");
+ // std::string s("+0000.004e1");
+ // std::string s("+00099999999999999999999999999999999999.904e1");
+ std::string s("+000900000000000000000000000000000000000.004e1");
+ int type_precision = 38;
+ int type_scale = 1;
+ StringParser::ParseResult result = StringParser::PARSE_FAILURE;
+ auto value =
StringParser::string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ s.c_str(), s.size(), type_precision, type_scale, &result);
+ std::cout << "Parsed decimal128 value: " << value << ", ParseResult: "
<< result
+ << std::endl;
+ }
+}
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, false);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest,
test_string_to_decimal128v3_nullable_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt";
+ size_t orig_line_count = 10000000; // 10 million lines
+ // Create string column and load data from file
+ auto dt_from = std::make_shared<DataTypeString>();
+ auto column_orig = dt_from->create_column();
+
+ {
+ MutableColumns columns;
+ columns.push_back(column_orig->get_ptr());
+ DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
+ load_columns_data_from_file(columns, serde, ';', {0}, test_data_file);
+ EXPECT_EQ(column_orig->size(), orig_line_count);
+ }
+
+ int test_rounds = 10;
+ int64_t total_duration_ns = 0;
+ for (int i = 0; i < test_rounds; ++i) {
+ std::cout << "\n--- Test Round " << (i + 1) << " ---\n";
+ total_duration_ns +=
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL128I>(
+ column_orig, orig_line_count, 38, 19, true);
+ }
+ auto avg_duration_ns = total_duration_ns / test_rounds;
+ std::cout << fmt::format("\nAverage time over {} rounds: {}\n",
test_rounds,
+ PrettyPrinter::print(avg_duration_ns,
TUnit::TIME_NS));
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(test_data_file,
orig_line_count, 18,
+ 9, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL32>(test_data_file,
orig_line_count, 9,
+ 4, false);
+}
+
+TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) {
+ std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt";
+ size_t orig_line_count = 10000000;
+ perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMALV2>(test_data_file,
orig_line_count, 27,
Review Comment:
Type mismatch in function call. The method 'perf_test_string_to_decimal'
declared on line 81 expects a 'MutableColumnPtr' as the first parameter, but
here it's being called with 'test_data_file' which is a 'std::string'. This
code will fail to compile. Either the method signature needs an overload that
accepts a string filename, or the test needs to load the data first into a
column like the other test cases do.
```suggestion
// Create string column and load data from file
auto dt_from = std::make_shared<DataTypeString>();
auto column_orig = dt_from->create_column();
{
MutableColumns columns;
columns.push_back(column_orig->get_ptr());
DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
load_columns_data_from_file(columns, serde, ';', {0},
test_data_file);
EXPECT_EQ(column_orig->size(), orig_line_count);
}
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL64>(column_orig,
orig_line_count, 18,
9, false);
}
TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) {
std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt";
size_t orig_line_count = 10000000;
// Create string column and load data from file
auto dt_from = std::make_shared<DataTypeString>();
auto column_orig = dt_from->create_column();
{
MutableColumns columns;
columns.push_back(column_orig->get_ptr());
DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
load_columns_data_from_file(columns, serde, ';', {0},
test_data_file);
EXPECT_EQ(column_orig->size(), orig_line_count);
}
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMAL32>(column_orig,
orig_line_count, 9,
4, false);
}
TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) {
std::string test_data_file =
"/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt";
size_t orig_line_count = 10000000;
// Create string column and load data from file
auto dt_from = std::make_shared<DataTypeString>();
auto column_orig = dt_from->create_column();
{
MutableColumns columns;
columns.push_back(column_orig->get_ptr());
DataTypeSerDeSPtrs serde = {dt_from->get_serde()};
load_columns_data_from_file(columns, serde, ';', {0},
test_data_file);
EXPECT_EQ(column_orig->size(), orig_line_count);
}
perf_test_string_to_decimal<PrimitiveType::TYPE_DECIMALV2>(column_orig,
orig_line_count, 27,
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
