This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new eb2834a38c4 [fix](datetime) fix datetime rounding on BE (#32075)
(#32359)
eb2834a38c4 is described below
commit eb2834a38c473e48719bde8b2becb68cad7110a3
Author: zhiqiang <[email protected]>
AuthorDate: Thu Mar 21 19:03:54 2024 +0800
[fix](datetime) fix datetime rounding on BE (#32075) (#32359)
---
be/src/exec/tablet_info.cpp | 6 +-
be/src/vec/data_types/data_type_time_v2.cpp | 4 +-
be/src/vec/data_types/data_type_time_v2.h | 5 +-
be/src/vec/runtime/vdatetime_value.cpp | 84 +++--
be/test/vec/data_types/datetime_round_test.cpp | 421 +++++++++++++++++++++++++
be/test/vec/data_types/from_string_test.cpp | 15 +-
be/test/vec/exprs/vexpr_test.cpp | 11 +-
7 files changed, 501 insertions(+), 45 deletions(-)
diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp
index ac99ea63abb..af91938d354 100644
--- a/be/src/exec/tablet_info.cpp
+++ b/be/src/exec/tablet_info.cpp
@@ -417,9 +417,11 @@ Status
VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr,
}
column->insert_data(reinterpret_cast<const char*>(&dt), 0);
} else if
(TypeDescriptor::from_thrift(t_expr.type).is_datetime_v2_type()) {
- vectorized::DateV2Value<doris::vectorized::DateTimeV2ValueType> dt;
+ vectorized::DateV2Value<vectorized::DateTimeV2ValueType> dt;
+ const int32_t scale =
+ t_expr.type.types.empty() ? -1 :
t_expr.type.types.front().scalar_type.scale;
if (!dt.from_date_str(t_expr.date_literal.value.c_str(),
- t_expr.date_literal.value.size())) {
+ t_expr.date_literal.value.size(), scale)) {
std::stringstream ss;
ss << "invalid date literal in partition column, date=" <<
t_expr.date_literal;
return Status::InternalError(ss.str());
diff --git a/be/src/vec/data_types/data_type_time_v2.cpp
b/be/src/vec/data_types/data_type_time_v2.cpp
index c4b397f6e70..c92201ee08d 100644
--- a/be/src/vec/data_types/data_type_time_v2.cpp
+++ b/be/src/vec/data_types/data_type_time_v2.cpp
@@ -151,9 +151,9 @@ void DataTypeDateTimeV2::to_string(const IColumn& column,
size_t row_num,
}
Status DataTypeDateTimeV2::from_string(ReadBuffer& rb, IColumn* column) const {
- auto* column_data = static_cast<ColumnUInt64*>(column);
+ auto* column_data = assert_cast<ColumnUInt64*>(column);
UInt64 val = 0;
- if (!read_datetime_v2_text_impl<UInt64>(val, rb)) {
+ if (!read_datetime_v2_text_impl<UInt64>(val, rb, _scale)) {
return Status::InvalidArgument("parse date fail, string: '{}'",
std::string(rb.position(),
rb.count()).c_str());
}
diff --git a/be/src/vec/data_types/data_type_time_v2.h
b/be/src/vec/data_types/data_type_time_v2.h
index e309773183c..c141e3fb547 100644
--- a/be/src/vec/data_types/data_type_time_v2.h
+++ b/be/src/vec/data_types/data_type_time_v2.h
@@ -135,7 +135,10 @@ public:
Field get_field(const TExprNode& node) const override {
DateV2Value<DateTimeV2ValueType> value;
- if (value.from_date_str(node.date_literal.value.c_str(),
node.date_literal.value.size())) {
+ const int32_t scale =
+ node.type.types.empty() ? -1 :
node.type.types.front().scalar_type.scale;
+ if (value.from_date_str(node.date_literal.value.c_str(),
node.date_literal.value.size(),
+ scale)) {
return value.to_date_int_val();
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
diff --git a/be/src/vec/runtime/vdatetime_value.cpp
b/be/src/vec/runtime/vdatetime_value.cpp
index 04f374a5ec4..5815ec85250 100644
--- a/be/src/vec/runtime/vdatetime_value.cpp
+++ b/be/src/vec/runtime/vdatetime_value.cpp
@@ -1990,7 +1990,6 @@ bool DateV2Value<T>::from_date_str_base(const char*
date_str, int len, int scale
const static int allow_space_mask = 4 | 64;
uint32_t date_val[MAX_DATE_PARTS] = {0};
int32_t date_len[MAX_DATE_PARTS] = {0};
- bool carry_bits[MAX_DATE_PARTS] = {false};
// Skip space character
while (ptr < end && isspace(*ptr)) {
@@ -2024,46 +2023,72 @@ bool DateV2Value<T>::from_date_str_base(const char*
date_str, int len, int scale
int field_idx = 0;
int field_len = year_len;
long sec_offset = 0;
+
while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) {
const char* start = ptr;
int temp_val = 0;
bool scan_to_delim = (!is_interval_format) && (field_idx != 6);
- while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) {
// field_len <= 6
- temp_val = temp_val * 10 + (*ptr++ - '0');
+ while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) {
// field_len <= 7
+ temp_val = temp_val * 10 + (*ptr - '0');
+ ptr++;
}
+
if (field_idx == 6) {
- // Microsecond
- const auto ms_part = ptr - start;
- temp_val *= int_exp10(std::max(0L, 6 - ms_part));
if constexpr (is_datetime) {
+ // round of microseconds
+ // 1. normalize to 7 digits for rounding
+ // 2. rounding
+ // 3. nomalize to 6 digits for storage
if (scale >= 0) {
- if (scale == 6 && ms_part >= 6) {
- if (ptr <= end && isdigit(*ptr) && *ptr >= '5') {
- temp_val += 1;
- }
+ // do normalization
+ const auto ms_digit_count = ptr - start;
+ const auto normalizer = int_exp10(std::abs(7 -
ms_digit_count));
+ temp_val *= normalizer;
+
+ // check round
+ const auto rounder = int_exp10(std::abs(7 - scale));
+ const auto reminder = temp_val % rounder;
+ temp_val -= reminder;
+
+ if (reminder >= 5 * normalizer) {
+ temp_val += rounder;
+ }
+
+ // truncate to 6 digits
+ if (temp_val == int_exp10(7)) {
+ temp_val = 0;
+ sec_offset += 1;
} else {
- const int divisor = int_exp10(6 - scale);
- int remainder = temp_val % divisor;
- temp_val /= divisor;
- if (scale < 6 && std::abs(remainder) >= (divisor >>
1)) {
- temp_val += 1;
- }
- temp_val *= divisor;
- if (temp_val == 1000000L) {
- temp_val = 0;
- date_val[field_idx - 1] += 1;
- carry_bits[field_idx] = true;
- }
+ temp_val /= 10;
}
}
+
+ // move ptr to start of timezone or end
+ while (ptr < end && isdigit(*ptr)) {
+ ptr++;
+ }
+ } else {
+ // Microsecond
+ const auto ms_part = ptr - start;
+ temp_val *= int_exp10(std::max(0L, 6 - ms_part));
}
}
+
// Impossible
if (temp_val > 999999L) {
return false;
}
+
date_val[field_idx] = temp_val;
- date_len[field_idx] = ptr - start;
+
+ if (field_idx == 6) {
+ // select cast("2020-01-01 12:00:00.12345" as Datetime(4))
+ // ptr - start will be 5, but scale is 4
+ date_len[field_idx] = std::min(static_cast<int>(ptr - start),
scale);
+ } else {
+ date_len[field_idx] = ptr - start;
+ }
+
field_len = 2;
if (ptr == end) {
@@ -2110,7 +2135,14 @@ bool DateV2Value<T>::from_date_str_base(const char*
date_str, int len, int scale
if (field_idx == 5) {
if (*ptr == '.') {
ptr++;
- field_len = 6;
+ // for datetime, we need to discard the fraction part
+ // that beyond the scale + 1, and scale + 1 digit will
+ // be used to round the fraction part
+ if constexpr (is_datetime) {
+ field_len = std::min(7, scale + 1);
+ } else {
+ field_len = 6;
+ }
} else if (isdigit(*ptr)) {
field_idx++;
break;
@@ -2132,6 +2164,7 @@ bool DateV2Value<T>::from_date_str_base(const char*
date_str, int len, int scale
}
field_idx++;
}
+
int num_field = field_idx;
if (!is_interval_format) {
year_len = date_len[0];
@@ -2154,11 +2187,12 @@ bool DateV2Value<T>::from_date_str_base(const char*
date_str, int len, int scale
if (is_invalid(date_val[0], date_val[1], date_val[2], 0, 0, 0, 0)) {
return false;
}
- format_datetime(date_val, carry_bits);
+
if (!check_range_and_set_time(date_val[0], date_val[1], date_val[2],
date_val[3], date_val[4],
date_val[5], date_val[6])) {
return false;
}
+
return sec_offset ? date_add_interval<TimeUnit::SECOND>(
TimeInterval {TimeUnit::SECOND, sec_offset,
false})
: true;
diff --git a/be/test/vec/data_types/datetime_round_test.cpp
b/be/test/vec/data_types/datetime_round_test.cpp
new file mode 100644
index 00000000000..48dfe3fcb70
--- /dev/null
+++ b/be/test/vec/data_types/datetime_round_test.cpp
@@ -0,0 +1,421 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/Exprs_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gmock/gmock-more-matchers.h>
+#include <gtest/gtest.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <map>
+#include <memory>
+#include <string>
+#include <string_view>
+
+#include "util/mysql_row_buffer.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h" // UInt32
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_time_v2.h"
+#include "vec/io/reader_buffer.h"
+
+using namespace doris;
+using namespace doris::vectorized;
+
+static void from_string_checker(UInt32 scale, const std::string& rounding,
+ const std::string& expected) {
+ // DataTypeDateTimeV2
+ std::shared_ptr<const DataTypeDateTimeV2> datetime_ptr =
+ std::dynamic_pointer_cast<const DataTypeDateTimeV2>(
+ DataTypeFactory::instance().create_data_type(
+ doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2, 0,
scale));
+
+ // constructor of ReadBuffer is not const(which seems not reasonable),
+ // so we need to cast away const
+ ReadBuffer rb(const_cast<char*>(rounding.c_str()), rounding.size());
+ ColumnUInt64::MutablePtr column = ColumnUInt64::create(0);
+ // DataTypeDateTimeV2::from_string
+ auto rt = datetime_ptr->from_string(rb, &(*column));
+ EXPECT_TRUE(rt.ok());
+ EXPECT_EQ(rt.msg(), ""); // so that we can print error msg if failed
+ EXPECT_EQ(datetime_ptr->to_string(*column, 0), expected);
+};
+
+static void from_thrift_checker(UInt32 scale, const String& input, const
String& expected) {
+ TScalarType tscale_type;
+ tscale_type.type = TPrimitiveType::DATETIMEV2;
+ tscale_type.precision = 18;
+ tscale_type.scale = scale;
+
+ TTypeNode type_node;
+ type_node.type = TTypeNodeType::SCALAR;
+ type_node.scalar_type = tscale_type;
+
+ TTypeDesc type_desc;
+ type_desc.types.push_back(type_node);
+
+ TDateLiteral date_literal;
+ date_literal.value = input;
+
+ TExprNode expr_node;
+ expr_node.node_type = TExprNodeType::DATE_LITERAL;
+ expr_node.type = type_desc;
+ expr_node.date_literal = date_literal;
+
+ std::shared_ptr<const DataTypeDateTimeV2> datetime_ptr =
+ std::dynamic_pointer_cast<const DataTypeDateTimeV2>(
+ DataTypeFactory::instance().create_data_type(
+ doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2, 0,
scale));
+
+ auto field = datetime_ptr->get_field(expr_node);
+ uint64_t value = 0;
+ // = datetime_ptr->get_storage_field_type();
+ EXPECT_EQ(field.try_get(value), true);
+ auto column = datetime_ptr->create_column_const(1, field);
+ EXPECT_EQ(datetime_ptr->to_string(*column, 1), expected);
+}
+
+static void serialization_checker(UInt32 scale, const std::string& input,
+ const std::string& expected) {
+ std::shared_ptr<const DataTypeDateTimeV2> datetime_ptr =
+ std::dynamic_pointer_cast<const DataTypeDateTimeV2>(
+ DataTypeFactory::instance().create_data_type(
+ doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2, 0,
scale));
+
+ ReadBuffer rb(const_cast<char*>(input.c_str()), input.size());
+ ColumnUInt64::MutablePtr column = ColumnUInt64::create(0);
+ auto rt = datetime_ptr->from_string(rb, &(*column));
+ EXPECT_TRUE(rt.ok());
+ auto serde =
std::dynamic_pointer_cast<DataTypeDateTimeV2SerDe>(datetime_ptr->get_serde());
+ MysqlRowBuffer<false> mysql_rb;
+ rt = serde->write_column_to_mysql(*column, mysql_rb, 0, false);
+ EXPECT_TRUE(rt.ok());
+ auto elem_size = static_cast<uint8_t>(*mysql_rb.buf());
+ if (elem_size != expected.size()) {
+ std::cerr << "Left size " << elem_size << " right size " <<
expected.size() << " left str "
+ << std::string_view(mysql_rb.buf() + 1, elem_size) << "
right str " << expected
+ << std::endl;
+ ASSERT_TRUE(false); // terminate ut
+ }
+ EXPECT_EQ(std::string_view(mysql_rb.buf() + 1, expected.size()), expected);
+}
+
+static std::multimap<size_t /*scale*/,
+ std::multimap<std::string /*input*/, std::string
/*expected*/>>
+ scale_with_input_and_expected = {
+ {0, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00"}}},
+ {0, {{"2020-01-01 12:00:00.0", "2020-01-01 12:00:00"}}},
+ {0, {{"2020-01-01 12:00:00.5", "2020-01-01 12:00:01"}}},
+ {0, {{"2020-01-01 12:00:00.49", "2020-01-01 12:00:00"}}},
+ {0, {{"2020-01-01 12:00:00.00", "2020-01-01 12:00:00"}}},
+ {0, {{"2020-01-01 12:00:00.999999999999", "2020-01-01
12:00:01"}}},
+ {0, {{"9999-12-31 23:59:59", "9999-12-31 23:59:59"}}},
+ // normal cast, no rounding
+ {1, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.0"}}},
+ {1, {{"2020-01-01 12:00:00.0", "2020-01-01 12:00:00.0"}}},
+ {1, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.2", "2020-01-01 12:00:00.2"}}},
+ {1, {{"2020-01-01 12:00:00.3", "2020-01-01 12:00:00.3"}}},
+ {1, {{"2020-01-01 12:00:00.4", "2020-01-01 12:00:00.4"}}},
+ {1, {{"2020-01-01 12:00:00.5", "2020-01-01 12:00:00.5"}}},
+ {1, {{"2020-01-01 12:00:00.6", "2020-01-01 12:00:00.6"}}},
+ {1, {{"2020-01-01 12:00:00.7", "2020-01-01 12:00:00.7"}}},
+ {1, {{"2020-01-01 12:00:00.8", "2020-01-01 12:00:00.8"}}},
+ {1, {{"2020-01-01 12:00:00.9", "2020-01-01 12:00:00.9"}}},
+ // round test
+ {1, {{"2020-01-01 12:00:00.10", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.11", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.12", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.13", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.14", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.15", "2020-01-01 12:00:00.2"}}},
+ {1, {{"2020-01-01 12:00:00.16", "2020-01-01 12:00:00.2"}}},
+ {1, {{"2020-01-01 12:00:00.17", "2020-01-01 12:00:00.2"}}},
+ {1, {{"2020-01-01 12:00:00.18", "2020-01-01 12:00:00.2"}}},
+ {1, {{"2020-01-01 12:00:00.19", "2020-01-01 12:00:00.2"}}},
+ {1, {{"2020-01-01 12:00:00.90", "2020-01-01 12:00:00.9"}}},
+ {1, {{"2020-01-01 12:00:00.91", "2020-01-01 12:00:00.9"}}},
+ {1, {{"2020-01-01 12:00:00.92", "2020-01-01 12:00:00.9"}}},
+ {1, {{"2020-01-01 12:00:00.93", "2020-01-01 12:00:00.9"}}},
+ {1, {{"2020-01-01 12:00:00.94", "2020-01-01 12:00:00.9"}}},
+ {1, {{"2020-01-01 12:00:00.95", "2020-01-01 12:00:01.0"}}},
+ {1, {{"2020-01-01 12:00:00.96", "2020-01-01 12:00:01.0"}}},
+ {1, {{"2020-01-01 12:00:00.97", "2020-01-01 12:00:01.0"}}},
+ {1, {{"2020-01-01 12:00:00.98", "2020-01-01 12:00:01.0"}}},
+ {1, {{"2020-01-01 12:00:00.99", "2020-01-01 12:00:01.0"}}},
+ // make sure we are doing truncate instead of round up from
last digit
+ {1, {{"2020-01-01 12:00:00.140", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.141", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.142", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.143", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.144", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.145", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.146", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.147", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.148", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.149", "2020-01-01 12:00:00.1"}}},
+ {1, {{"2020-01-01 12:00:00.150", "2020-01-01 12:00:00.2"}}},
+ {1, {{"9999-12-31 23:59:59.9", "9999-12-31 23:59:59.9"}}},
+ // normal cast, no rounding
+ {2, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.00"}}},
+ {2, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.10"}}},
+ {2, {{"2020-01-01 12:00:00.00", "2020-01-01 12:00:00.00"}}},
+ {2, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.01"}}},
+ {2, {{"2020-01-01 12:00:00.02", "2020-01-01 12:00:00.02"}}},
+ {2, {{"2020-01-01 12:00:00.03", "2020-01-01 12:00:00.03"}}},
+ {2, {{"2020-01-01 12:00:00.04", "2020-01-01 12:00:00.04"}}},
+ {2, {{"2020-01-01 12:00:00.05", "2020-01-01 12:00:00.05"}}},
+ {2, {{"2020-01-01 12:00:00.06", "2020-01-01 12:00:00.06"}}},
+ {2, {{"2020-01-01 12:00:00.07", "2020-01-01 12:00:00.07"}}},
+ {2, {{"2020-01-01 12:00:00.08", "2020-01-01 12:00:00.08"}}},
+ {2, {{"2020-01-01 12:00:00.09", "2020-01-01 12:00:00.09"}}},
+ {2, {{"2020-01-01 12:00:00.12", "2020-01-01 12:00:00.12"}}},
+ // rounding
+ {2, {{"2020-01-01 12:00:00.990", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.991", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.992", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.993", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.994", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.995", "2020-01-01 12:00:01.00"}}},
+ {2, {{"2020-01-01 12:00:00.996", "2020-01-01 12:00:01.00"}}},
+ {2, {{"2020-01-01 12:00:00.997", "2020-01-01 12:00:01.00"}}},
+ {2, {{"2020-01-01 12:00:00.998", "2020-01-01 12:00:01.00"}}},
+ {2, {{"2020-01-01 12:00:00.999", "2020-01-01 12:00:01.00"}}},
+ // make sure we are doing truncate instead of round from last
digit
+ {2, {{"2020-01-01 12:00:00.9940", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9941", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9942", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9943", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9944", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9945", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9946", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9947", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9948", "2020-01-01 12:00:00.99"}}},
+ {2, {{"2020-01-01 12:00:00.9949", "2020-01-01 12:00:00.99"}}},
+ {2, {{"9999-12-31 23:59:59.99", "9999-12-31 23:59:59.99"}}},
+ {3, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.000"}}},
+ {3, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.100"}}},
+ {3, {{"2020-01-01 12:00:00.00", "2020-01-01 12:00:00.000"}}},
+ {3, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.010"}}},
+ {3, {{"2020-01-01 12:00:00.001", "2020-01-01 12:00:00.001"}}},
+ {3, {{"2020-01-01 12:00:00.002", "2020-01-01 12:00:00.002"}}},
+ {3, {{"2020-01-01 12:00:00.003", "2020-01-01 12:00:00.003"}}},
+ {3, {{"2020-01-01 12:00:00.004", "2020-01-01 12:00:00.004"}}},
+ {3, {{"2020-01-01 12:00:00.005", "2020-01-01 12:00:00.005"}}},
+ {3, {{"2020-01-01 12:00:00.006", "2020-01-01 12:00:00.006"}}},
+ {3, {{"2020-01-01 12:00:00.007", "2020-01-01 12:00:00.007"}}},
+ {3, {{"2020-01-01 12:00:00.008", "2020-01-01 12:00:00.008"}}},
+ {3, {{"2020-01-01 12:00:00.009", "2020-01-01 12:00:00.009"}}},
+ {3, {{"2020-01-01 12:00:00.123", "2020-01-01 12:00:00.123"}}},
+ {3, {{"2020-01-01 12:00:00.999", "2020-01-01 12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.123", "2020-01-01 12:00:00.123"}}},
+ // rounding
+ {3, {{"2020-01-01 12:00:00.9990", "2020-01-01 12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.9991", "2020-01-01 12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.9992", "2020-01-01 12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.9993", "2020-01-01 12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.9994", "2020-01-01 12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.9995", "2020-01-01 12:00:01.000"}}},
+ {3, {{"2020-01-01 12:00:00.9996", "2020-01-01 12:00:01.000"}}},
+ {3, {{"2020-01-01 12:00:00.9997", "2020-01-01 12:00:01.000"}}},
+ {3, {{"2020-01-01 12:00:00.9998", "2020-01-01 12:00:01.000"}}},
+ {3, {{"2020-01-01 12:00:00.9999", "2020-01-01 12:00:01.000"}}},
+ // make sure we are doing truncate instead of round from last
digit
+ {3, {{"2020-01-01 12:00:00.99940", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99941", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99942", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99943", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99944", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99945", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99946", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99947", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99948", "2020-01-01
12:00:00.999"}}},
+ {3, {{"2020-01-01 12:00:00.99949", "2020-01-01
12:00:00.999"}}},
+ {3, {{"9999-12-31 23:59:59.999", "9999-12-31 23:59:59.999"}}},
+ // normal cast, no rounding
+ {4, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.0000"}}},
+ {4, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.1000"}}},
+ {4, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.0100"}}},
+ {4, {{"2020-01-01 12:00:00.001", "2020-01-01 12:00:00.0010"}}},
+ {4, {{"2020-01-01 12:00:00.0001", "2020-01-01
12:00:00.0001"}}},
+ {4, {{"2020-01-01 12:00:00.0002", "2020-01-01
12:00:00.0002"}}},
+ {4, {{"2020-01-01 12:00:00.0003", "2020-01-01
12:00:00.0003"}}},
+ {4, {{"2020-01-01 12:00:00.0004", "2020-01-01
12:00:00.0004"}}},
+ {4, {{"2020-01-01 12:00:00.0005", "2020-01-01
12:00:00.0005"}}},
+ {4, {{"2020-01-01 12:00:00.0006", "2020-01-01
12:00:00.0006"}}},
+ {4, {{"2020-01-01 12:00:00.0007", "2020-01-01
12:00:00.0007"}}},
+ {4, {{"2020-01-01 12:00:00.0008", "2020-01-01
12:00:00.0008"}}},
+ {4, {{"2020-01-01 12:00:00.0009", "2020-01-01
12:00:00.0009"}}},
+ {4, {{"2020-01-01 12:00:00.1234", "2020-01-01
12:00:00.1234"}}},
+ {4, {{"2020-01-01 12:00:00.9999", "2020-01-01
12:00:00.9999"}}},
+ // rounding
+ {4, {{"2020-01-01 12:00:00.99990", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.99991", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.99992", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.99993", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.99994", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.99995", "2020-01-01
12:00:01.0000"}}},
+ {4, {{"2020-01-01 12:00:00.99996", "2020-01-01
12:00:01.0000"}}},
+ {4, {{"2020-01-01 12:00:00.99997", "2020-01-01
12:00:01.0000"}}},
+ {4, {{"2020-01-01 12:00:00.99998", "2020-01-01
12:00:01.0000"}}},
+ {4, {{"2020-01-01 12:00:00.99999", "2020-01-01
12:00:01.0000"}}},
+ // make sure we are doing truncate instead of round from last
digit
+ {4, {{"2020-01-01 12:00:00.999940", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999941", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999942", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999943", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999944", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999945", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999946", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999947", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999948", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"2020-01-01 12:00:00.999949", "2020-01-01
12:00:00.9999"}}},
+ {4, {{"9999-12-31 23:59:59.9999", "9999-12-31
23:59:59.9999"}}},
+ {5, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.00000"}}},
+ {5, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.10000"}}},
+ {5, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.01000"}}},
+ {5, {{"2020-01-01 12:00:00.001", "2020-01-01
12:00:00.00100"}}},
+ {5, {{"2020-01-01 12:00:00.0001", "2020-01-01
12:00:00.00010"}}},
+ {5, {{"2020-01-01 12:00:00.00001", "2020-01-01
12:00:00.00001"}}},
+ {5, {{"2020-01-01 12:00:00.00002", "2020-01-01
12:00:00.00002"}}},
+ {5, {{"2020-01-01 12:00:00.00003", "2020-01-01
12:00:00.00003"}}},
+ {5, {{"2020-01-01 12:00:00.00004", "2020-01-01
12:00:00.00004"}}},
+ {5, {{"2020-01-01 12:00:00.00005", "2020-01-01
12:00:00.00005"}}},
+ {5, {{"2020-01-01 12:00:00.00006", "2020-01-01
12:00:00.00006"}}},
+ {5, {{"2020-01-01 12:00:00.00007", "2020-01-01
12:00:00.00007"}}},
+ {5, {{"2020-01-01 12:00:00.00008", "2020-01-01
12:00:00.00008"}}},
+ {5, {{"2020-01-01 12:00:00.00009", "2020-01-01
12:00:00.00009"}}},
+ {5, {{"2020-01-01 12:00:00.12345", "2020-01-01
12:00:00.12345"}}},
+ {5, {{"2020-01-01 12:00:00.99999", "2020-01-01
12:00:00.99999"}}},
+ // rounding
+ {5, {{"2020-01-01 12:00:00.999990", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.999991", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.999992", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.999993", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.999994", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.999995", "2020-01-01
12:00:01.00000"}}},
+ {5, {{"2020-01-01 12:00:00.999996", "2020-01-01
12:00:01.00000"}}},
+ {5, {{"2020-01-01 12:00:00.999997", "2020-01-01
12:00:01.00000"}}},
+ {5, {{"2020-01-01 12:00:00.999998", "2020-01-01
12:00:01.00000"}}},
+ {5, {{"2020-01-01 12:00:00.999999", "2020-01-01
12:00:01.00000"}}},
+ // make sure we are doing truncate instead of round from last
digit
+ {5, {{"2020-01-01 12:00:00.9999940", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999941", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999942", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999943", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999944", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999945", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999946", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999947", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999948", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"2020-01-01 12:00:00.9999949", "2020-01-01
12:00:00.99999"}}},
+ {5, {{"9999-12-31 23:59:59.99999", "9999-12-31
23:59:59.99999"}}},
+ // normal cast, no rounding
+ {6, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.000000"}}},
+ {6, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.100000"}}},
+ {6, {{"2020-01-01 12:00:00.01", "2020-01-01
12:00:00.010000"}}},
+ {6, {{"2020-01-01 12:00:00.001", "2020-01-01
12:00:00.001000"}}},
+ {6, {{"2020-01-01 12:00:00.0001", "2020-01-01
12:00:00.000100"}}},
+ {6, {{"2020-01-01 12:00:00.00001", "2020-01-01
12:00:00.000010"}}},
+ {6, {{"2020-01-01 12:00:00.000001", "2020-01-01
12:00:00.000001"}}},
+ {6, {{"2020-01-01 12:00:00.000002", "2020-01-01
12:00:00.000002"}}},
+ {6, {{"2020-01-01 12:00:00.000003", "2020-01-01
12:00:00.000003"}}},
+ {6, {{"2020-01-01 12:00:00.000004", "2020-01-01
12:00:00.000004"}}},
+ {6, {{"2020-01-01 12:00:00.000005", "2020-01-01
12:00:00.000005"}}},
+ {6, {{"2020-01-01 12:00:00.000006", "2020-01-01
12:00:00.000006"}}},
+ {6, {{"2020-01-01 12:00:00.000007", "2020-01-01
12:00:00.000007"}}},
+ {6, {{"2020-01-01 12:00:00.000008", "2020-01-01
12:00:00.000008"}}},
+ {6, {{"2020-01-01 12:00:00.000009", "2020-01-01
12:00:00.000009"}}},
+ {6, {{"2020-01-01 12:00:00.123456", "2020-01-01
12:00:00.123456"}}},
+ {6, {{"2020-01-01 12:00:00.999999", "2020-01-01
12:00:00.999999"}}},
+ // rounding
+ {6, {{"2020-01-01 12:00:00.9999990", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.9999991", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.9999992", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.9999993", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.9999994", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.9999995", "2020-01-01
12:00:01.000000"}}},
+ {6, {{"2020-01-01 12:00:00.9999996", "2020-01-01
12:00:01.000000"}}},
+ {6, {{"2020-01-01 12:00:00.9999997", "2020-01-01
12:00:01.000000"}}},
+ {6, {{"2020-01-01 12:00:00.9999998", "2020-01-01
12:00:01.000000"}}},
+ {6, {{"2020-01-01 12:00:00.9999999", "2020-01-01
12:00:01.000000"}}},
+ // make sure we are doing truncate instead of round from last
digit
+ {6, {{"2020-01-01 12:00:00.99999940", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999941", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999942", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999943", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999944", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999945", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999946", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999947", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999948", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"2020-01-01 12:00:00.99999949", "2020-01-01
12:00:00.999999"}}},
+ {6, {{"9999-12-31 23:59:59.999999", "9999-12-31
23:59:59.999999"}}},
+ //
+ {0, {{"2024-02-29 23:59:59.9", "2024-03-01 00:00:00"}}},
+ {1, {{"2024-02-29 23:59:59.99", "2024-03-01 00:00:00.0"}}},
+ {2, {{"2024-02-29 23:59:59.999", "2024-03-01 00:00:00.00"}}},
+ {3, {{"2024-02-29 23:59:59.9999", "2024-03-01 00:00:00.000"}}},
+ {4, {{"2024-02-29 23:59:59.99999", "2024-03-01
00:00:00.0000"}}},
+ {5, {{"2024-02-29 23:59:59.999999", "2024-03-01
00:00:00.00000"}}},
+ {6, {{"2024-02-29 23:59:59.9999999", "2024-03-01
00:00:00.000000"}}},
+ //
+ {0, {{"2025-02-28 23:59:59.9999999", "2025-03-01 00:00:00"}}},
+ {1, {{"2025-02-28 23:59:59.9999999", "2025-03-01
00:00:00.0"}}},
+ {2, {{"2025-02-28 23:59:59.9999999", "2025-03-01
00:00:00.00"}}},
+ {3, {{"2025-02-28 23:59:59.9999999", "2025-03-01
00:00:00.000"}}},
+ {4, {{"2025-02-28 23:59:59.9999999", "2025-03-01
00:00:00.0000"}}},
+ {5, {{"2025-02-28 23:59:59.9999999", "2025-03-01
00:00:00.00000"}}},
+ {6, {{"2025-02-28 23:59:59.9999999", "2025-03-01
00:00:00.000000"}}},
+
+};
+
+namespace doris::vectorized {
+// // make sure micro-seconds part of datetime has correct round behaviour
+TEST(DatetimeRountTest, test_datetime_round_behaviour_basic) {
+ {
+ for (auto itr : scale_with_input_and_expected) {
+ for (const auto& [input, expected] : itr.second) {
+ from_string_checker(itr.first, input, expected);
+ }
+ }
+ }
+}
+
+TEST(DatetimeRountTest, test_datetime_round_behaviour_get_field) {
+ {
+ for (auto itr : scale_with_input_and_expected) {
+ for (const auto& [input, expected] : itr.second) {
+ from_thrift_checker(itr.first, input, expected);
+ }
+ }
+ }
+}
+
+TEST(DatetimeRountTest, test_datetime_round_behaviour_serialize) {
+ {
+ for (auto itr : scale_with_input_and_expected) {
+ for (const auto& [input, expected] : itr.second) {
+ serialization_checker(itr.first, input, expected);
+ }
+ }
+ }
+}
+} // namespace doris::vectorized
diff --git a/be/test/vec/data_types/from_string_test.cpp
b/be/test/vec/data_types/from_string_test.cpp
index e3d3b5bd5d6..beeac5d96d8 100644
--- a/be/test/vec/data_types/from_string_test.cpp
+++ b/be/test/vec/data_types/from_string_test.cpp
@@ -16,6 +16,7 @@
// under the License.
#include "gtest/gtest_pred_impl.h"
+#include "olap/olap_common.h"
#include "olap/types.h" // for TypeInfo
#include "olap/wrapper_field.h"
#include "vec/columns/column.h"
@@ -225,7 +226,13 @@ TEST(FromStringTest, ScalaWrapperFieldVsDataType) {
};
for (auto pair : date_scala_field_types) {
auto type = pair.first;
- DataTypePtr data_type_ptr =
DataTypeFactory::instance().create_data_type(type, 0, 0);
+ DataTypePtr data_type_ptr = nullptr;
+ if (type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2) {
+ data_type_ptr =
DataTypeFactory::instance().create_data_type(type, 0, 6);
+ } else {
+ data_type_ptr =
DataTypeFactory::instance().create_data_type(type, 0, 0);
+ }
+
std::cout << "this type is " << data_type_ptr->get_name() << ": "
<< fmt::format("{}", type) << std::endl;
@@ -262,12 +269,6 @@ TEST(FromStringTest, ScalaWrapperFieldVsDataType) {
std::cout << "min(" << min_s << ") with datat_ype_str:" << min_s_d
<< std::endl;
std::cout << "max(" << max_s << ") with datat_ype_str:" << max_s_d
<< std::endl;
std::cout << "rand(" << rand_date << ") with datat_type_str:" <<
rand_s_d << std::endl;
- if (FieldType::OLAP_FIELD_TYPE_DATETIMEV2 == type) {
- // field to_string : %Y-%m-%d %H:%i:%s.%f vs data type
to_string %Y-%m-%d %H:%i:%s
- min_s = min_s.substr(0, min_s.find_last_of('.'));
- max_s = max_s.substr(0, max_s.find_last_of('.'));
- rand_date = rand_date.substr(0, rand_date.find_last_of('.'));
- }
// min wrapper field date to_string in macOS and linux system has
different result
// macOs equals with data type to_string(0000-01-01), but in
linux is (0-01-01)
if (FieldType::OLAP_FIELD_TYPE_DATE == type ||
diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp
index 8c90debda6b..3fe5c60f6b1 100644
--- a/be/test/vec/exprs/vexpr_test.cpp
+++ b/be/test/vec/exprs/vexpr_test.cpp
@@ -480,21 +480,16 @@ TEST(TEST_VEXPR, LITERALTEST) {
uint8_t hour = 9;
uint8_t minute = 12;
uint8_t second = 46;
- uint32_t microsecond = 999999;
+ uint32_t microsecond = 999999; // target scale is 4, so the
microsecond will be rounded up
DateV2Value<DateTimeV2ValueType> datetime_v2;
datetime_v2.set_time(year, month, day, hour, minute, second,
microsecond);
std::string date = datetime_v2.debug_string();
- __uint64_t dt;
- memcpy(&dt, &datetime_v2, sizeof(__uint64_t));
VLiteral literal(create_literal<TYPE_DATETIMEV2, std::string>(date,
4));
Block block;
int ret = -1;
- literal.execute(nullptr, &block, &ret);
- auto ctn = block.safe_get_by_position(ret);
- auto v = (*ctn.column)[0].get<__uint64_t>();
- EXPECT_EQ(v, dt);
- EXPECT_EQ("1997-11-18 09:12:46.9999", literal.value());
+ EXPECT_TRUE(literal.execute(nullptr, &block, &ret).ok());
+ EXPECT_EQ("1997-11-18 09:12:47.0000", literal.value());
}
// date
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
