This is an automated email from the ASF dual-hosted git repository.
zclll pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 14c7745b962 [Refactor](cast) Remove old cast to datelike type codes
(#55887)
14c7745b962 is described below
commit 14c7745b96200890d800a6baec3271f18860956a
Author: zclllyybb <[email protected]>
AuthorDate: Fri Sep 19 10:36:14 2025 +0800
[Refactor](cast) Remove old cast to datelike type codes (#55887)
btw, fixed some wrong flexiable update code about wrong usage of
datetime to string when generate default value of column.
---
be/src/olap/partial_update_info.cpp | 6 +-
be/src/olap/types.h | 36 +-
be/src/olap/uint24.h | 17 +-
be/src/util/asan_util.h | 15 +-
be/src/vec/runtime/vdatetime_value.cpp | 442 ++-----------------
be/src/vec/runtime/vdatetime_value.h | 14 +-
be/test/vec/data_types/data_type_time_v2_test.cpp | 53 ++-
be/test/vec/data_types/from_string_test.cpp | 9 +-
.../data_types/serde/data_type_serde_text_test.cpp | 474 ++++++---------------
be/test/vec/exprs/vexpr_test.cpp | 15 +-
.../utils/arrow_column_to_doris_column_test.cpp | 7 +-
11 files changed, 265 insertions(+), 823 deletions(-)
diff --git a/be/src/olap/partial_update_info.cpp
b/be/src/olap/partial_update_info.cpp
index f4dd4529713..5e6082c5c13 100644
--- a/be/src/olap/partial_update_info.cpp
+++ b/be/src/olap/partial_update_info.cpp
@@ -264,19 +264,19 @@ void
PartialUpdateInfo::_generate_default_values_for_missing_cids(
if (pos == std::string::npos) {
DateV2Value<DateTimeV2ValueType> dtv;
dtv.from_unixtime(timestamp_ms / 1000, timezone);
- default_value = dtv.debug_string();
+ default_value = dtv.to_string();
} else {
int precision =
std::stoi(column.default_value().substr(pos + 1));
DateV2Value<DateTimeV2ValueType> dtv;
dtv.from_unixtime(timestamp_ms / 1000, nano_seconds,
timezone, precision);
- default_value = dtv.debug_string();
+ default_value = dtv.to_string();
}
} else if (UNLIKELY(column.type() ==
FieldType::OLAP_FIELD_TYPE_DATEV2 &&
to_lower(column.default_value()).find(to_lower("CURRENT_DATE")) !=
std::string::npos)) {
DateV2Value<DateV2ValueType> dv;
dv.from_unixtime(timestamp_ms / 1000, timezone);
- default_value = dv.debug_string();
+ default_value = dv.to_string();
} else if (UNLIKELY(column.type() ==
FieldType::OLAP_FIELD_TYPE_BITMAP &&
to_lower(column.default_value()).find(to_lower("BITMAP_EMPTY")) !=
std::string::npos)) {
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 4de5f5f0566..7ff538418ae 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -1224,15 +1224,11 @@ struct
FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>
return Status::OK();
}
static std::string to_string(const void* src) {
+ // uint24_t
CppType tmp = *reinterpret_cast<const CppType*>(src);
DateV2Value<DateV2ValueType> value =
binary_cast<CppType, DateV2Value<DateV2ValueType>>(tmp);
- std::string format = "%Y-%m-%d";
- std::string res;
- res.resize(12 + SAFE_FORMAT_STRING_MARGIN);
- value.to_format_string_conservative(format.c_str(), format.size(),
res.data(),
- 12 + SAFE_FORMAT_STRING_MARGIN);
- return res;
+ return value.to_string();
}
static void set_to_max(void* buf) {
@@ -1262,16 +1258,12 @@ struct
FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>
return Status::OK();
}
+ // only used in Field so we dont know the scale, use max scale 6 as default
static std::string to_string(const void* src) {
CppType tmp = *reinterpret_cast<const CppType*>(src);
DateV2Value<DateTimeV2ValueType> value =
binary_cast<CppType, DateV2Value<DateTimeV2ValueType>>(tmp);
- std::string format = "%Y-%m-%d %H:%i:%s.%f";
- std::string res;
- res.resize(30 + SAFE_FORMAT_STRING_MARGIN);
- value.to_format_string_conservative(format.c_str(), format.size(),
res.data(),
- 30 + SAFE_FORMAT_STRING_MARGIN);
- return res;
+ return value.to_string(6);
}
static void set_to_max(void* buf) {
@@ -1306,26 +1298,24 @@ struct
FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIME>
return Status::OK();
}
static std::string to_string(const void* src) {
- tm time_tm;
CppType tmp = *reinterpret_cast<const CppType*>(src);
CppType part1 = (tmp / 1000000L);
CppType part2 = (tmp - part1 * 1000000L);
- time_tm.tm_year = static_cast<int>((part1 / 10000L) % 10000) - 1900;
- time_tm.tm_mon = static_cast<int>((part1 / 100) % 100) - 1;
- time_tm.tm_mday = static_cast<int>(part1 % 100);
+ int year = (part1 / 10000L) % 10000;
+ int mon = (part1 / 100) % 100;
+ int mday = part1 % 100;
- time_tm.tm_hour = static_cast<int>((part2 / 10000L) % 10000);
- time_tm.tm_min = static_cast<int>((part2 / 100) % 100);
- time_tm.tm_sec = static_cast<int>(part2 % 100);
+ int hour = (part2 / 10000L) % 10000;
+ int min = (part2 / 100) % 100;
+ int sec = part2 % 100;
- char buf[20] = {'\0'};
- strftime(buf, 20, "%Y-%m-%d %H:%M:%S", &time_tm);
- return std::string(buf);
+ return fmt::format(FMT_COMPILE("{:04d}-{:02d}-{:02d}
{:02d}:{:02d}:{:02d}"), year, mon,
+ mday, hour, min, sec);
}
static void set_to_max(void* buf) {
- // 设置为最大时间，其含义为：9999-12-31 23:59:59
+ // 9999-12-31 23:59:59
*reinterpret_cast<CppType*>(buf) = 99991231235959L;
}
static void set_to_min(void* buf) { *reinterpret_cast<CppType*>(buf) =
101000000; }
diff --git a/be/src/olap/uint24.h b/be/src/olap/uint24.h
index e04ccae8c52..f2fc77ef7ab 100644
--- a/be/src/olap/uint24.h
+++ b/be/src/olap/uint24.h
@@ -17,6 +17,8 @@
#pragma once
+#include <fmt/compile.h>
+
#include <cstdint>
#include <cstring>
#include <iostream>
@@ -127,15 +129,12 @@ public:
}
std::string to_string() const {
- tm time_tm;
int value = *reinterpret_cast<const uint24_t*>(data);
- memset(&time_tm, 0, sizeof(time_tm));
- time_tm.tm_mday = static_cast<int>(value & 31);
- time_tm.tm_mon = static_cast<int>(value >> 5 & 15) - 1;
- time_tm.tm_year = static_cast<int>(value >> 9) - 1900;
- char buf[20] = {'\0'};
- strftime(buf, sizeof(buf), "%Y-%m-%d", &time_tm);
- return std::string(buf);
+ int mday = value & 31;
+ int mon = value >> 5 & 15;
+ int year = value >> 9;
+
+ return fmt::format(FMT_COMPILE("{:04d}-{:02d}-{:02d}"), year, mon,
mday);
}
const uint8_t* get_data() const { return data; }
@@ -144,7 +143,7 @@ private:
uint8_t data[3];
} __attribute__((packed));
-static_assert(std::is_trivial<uint24_t>::value, "uint24_t should be a POD
type");
+static_assert(std::is_trivial_v<uint24_t>, "uint24_t should be a POD type");
inline std::ostream& operator<<(std::ostream& os, const uint24_t& val) {
os << val.to_string();
diff --git a/be/src/util/asan_util.h b/be/src/util/asan_util.h
index d9aec88c0b9..702c84e75b8 100644
--- a/be/src/util/asan_util.h
+++ b/be/src/util/asan_util.h
@@ -22,17 +22,20 @@
class AsanPoisonGuard {
#ifdef ADDRESS_SANITIZER
public:
- // Poison the memory region to prevent accidental access
- // during the lifetime of this object.
+ // Poison the memory region to prevent accidental access during the
lifetime of this object.
AsanPoisonGuard(const void* start, size_t len) : start(start), len(len) {
- ASAN_POISON_MEMORY_REGION(start, len);
+ //FIXME: now it may cause some false-positive, need to find a way to
fix it. maybe ASAN_UNPOISON_MEMORY_REGION
+ // didn't clean the same memory as ASAN_POISON_MEMORY_REGION
+ // ASAN_POISON_MEMORY_REGION(start, len);
}
// Unpoison the memory region when this object goes out of scope.
- ~AsanPoisonGuard() { ASAN_UNPOISON_MEMORY_REGION(start, len); }
+ ~AsanPoisonGuard() {
+ // ASAN_UNPOISON_MEMORY_REGION(start, len);
+ }
private:
- const void* start;
- size_t len;
+ const void* start [[maybe_unused]];
+ size_t len [[maybe_unused]];
#else
public:
// No-op for platforms without ASAN_DEFINE_REGION_MACROS
diff --git a/be/src/vec/runtime/vdatetime_value.cpp
b/be/src/vec/runtime/vdatetime_value.cpp
index d480c3d04fa..d8d6531c3f0 100644
--- a/be/src/vec/runtime/vdatetime_value.cpp
+++ b/be/src/vec/runtime/vdatetime_value.cpp
@@ -30,7 +30,6 @@
#include <algorithm>
#include <chrono> // IWYU pragma: keep
// IWYU pragma: no_include <bits/std_abs.h>
-#include <cmath>
#include <cstdint>
#include <string_view>
@@ -40,6 +39,10 @@
#include "common/status.h"
#include "util/timezone_utils.h"
#include "vec/common/int_exp.h"
+#include "vec/functions/cast/cast_parameters.h"
+#include "vec/functions/cast/cast_to_date_or_datetime_impl.hpp"
+#include "vec/functions/cast/cast_to_datetimev2_impl.hpp"
+#include "vec/functions/cast/cast_to_datev2_impl.hpp"
namespace doris {
#include "common/compile_check_avoid_begin.h"
@@ -61,15 +64,6 @@ static bool check_space(char ch) {
return UNLIKELY(ch == ' ' || (ch >= 9 && ch <= 13));
}
-static bool check_date_punct(char ch) {
- return UNLIKELY(!(isdigit(ch) || isalpha(ch)));
-}
-
-static bool time_zone_begins(const char* ptr, const char* end) {
- return *ptr == '+' || (*ptr == '-' && ptr + 3 < end && *(ptr + 3) == ':')
||
- (isalpha(*ptr) && *ptr != 'T');
-}
-
bool VecDateTimeValue::check_range(uint32_t year, uint32_t month, uint32_t
day, uint32_t hour,
uint32_t minute, uint32_t second, uint16_t
type) {
bool time = hour > (type == TIME_TIME ? TIME_MAX_HOUR : 23) || minute > 59
|| second > 59;
@@ -90,174 +84,26 @@ bool VecDateTimeValue::check_date(uint32_t year, uint32_t
month, uint32_t day) {
return false;
}
-// The interval format is that with no delimiters
-// YYYY-MM-DD HH-MM-DD.FFFFFF AM in default format
-// 0 1 2 3 4 5 6 7
bool VecDateTimeValue::from_date_str(const char* date_str, size_t len) {
- return from_date_str_base(date_str, len, nullptr);
+ vectorized::CastParameters params;
+ if (type() == TIME_DATE) {
+ return
vectorized::CastToDateOrDatetime::from_string_non_strict_mode<false>(
+ {date_str, len}, *this, nullptr, params);
+ } else {
+ return
vectorized::CastToDateOrDatetime::from_string_non_strict_mode<true>(
+ {date_str, len}, *this, nullptr, params);
+ }
}
-//parse timezone to get offset
bool VecDateTimeValue::from_date_str(const char* date_str, size_t len,
const cctz::time_zone& local_time_zone) {
- return from_date_str_base(date_str, len, &local_time_zone);
-}
-
-bool VecDateTimeValue::from_date_str_base(const char* date_str, size_t len,
- const cctz::time_zone*
local_time_zone) {
- const char* ptr = date_str;
- const char* end = date_str + len;
- // ONLY 2, 6 can follow by a space
- const static int allow_space_mask = 4 | 64;
- const static int MAX_DATE_PARTS = 8;
- uint32_t date_val[MAX_DATE_PARTS];
- int32_t date_len[MAX_DATE_PARTS];
-
- _neg = false;
- // Skip space character
- while (ptr < end && check_space(*ptr)) {
- ptr++;
- }
- if (ptr == end || !isdigit(*ptr)) {
- return false;
- }
- // Fix year length
- const char* pos = ptr;
- while (pos < end && (isdigit(*pos) || *pos == 'T')) {
- pos++;
- }
- int year_len = 4;
- int digits = pos - ptr;
- bool is_interval_format = false;
- bool has_bar = false;
-
- // Compatible with MySQL.
- // For YYYYMMDD/YYYYMMDDHHMMSS is 4 digits years
- if (pos == end || *pos == '.') {
- if (digits == 4 || digits == 8 || digits >= 14) {
- year_len = 4;
- } else {
- year_len = 2;
- }
- is_interval_format = true;
- }
-
- int field_idx = 0;
- int field_len = year_len;
- long sec_offset = 0;
- while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS - 1) {
- const char* start = ptr;
- int temp_val = 0;
- bool scan_to_delim = (!is_interval_format) && (field_idx != 6);
- while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) {
- temp_val = temp_val * 10 + (*ptr++ - '0');
- }
- // Impossible
- if (temp_val > 999999L) {
- return false;
- }
- date_val[field_idx] = temp_val;
- date_len[field_idx] = ptr - start;
- field_len = 2;
-
- if (ptr == end) {
- field_idx++;
- break;
- }
-
- // timezone
- if (UNLIKELY((field_idx > 2 ||
- !has_bar) /*dont treat xxxx-xx-xx:xx:xx as
xxxx-xx(-xx:xx:xx)*/
- && time_zone_begins(ptr, end))) {
- if (local_time_zone == nullptr) {
- return false;
- }
- auto get_tz_offset = [&](const std::string& str_tz,
- const cctz::time_zone* local_time_zone)
-> long {
- cctz::time_zone given_tz {};
- if (!TimezoneUtils::find_cctz_time_zone(str_tz, given_tz)) {
- throw Exception {ErrorCode::INVALID_ARGUMENT, ""};
- }
- auto given = cctz::convert(cctz::civil_second {}, given_tz);
- auto local = cctz::convert(cctz::civil_second {},
*local_time_zone);
- // these two values is absolute time. so they are negative.
need to use (-local) - (-given)
- return std::chrono::duration_cast<std::chrono::seconds>(given
- local).count();
- };
- try {
- sec_offset = get_tz_offset(std::string {ptr, end},
- local_time_zone); // use the whole
remain string
- } catch ([[maybe_unused]] Exception& e) {
- return false; // invalid format
- }
- field_idx++;
- break;
- }
-
- if (field_idx == 2 && *ptr == 'T') {
- // YYYYMMDDTHHMMDD, skip 'T' and continue
- ptr++;
- field_idx++;
- continue;
- }
-
- // Second part
- if (field_idx == 5) {
- if (*ptr == '.') {
- ptr++;
- field_len = 6;
- } else if (isdigit(*ptr)) {
- field_idx++;
- break;
- }
- field_idx++;
- continue;
- }
- // escape separator
- while (ptr < end && (check_date_punct(*ptr) || check_space(*ptr))) {
- if (check_space(*ptr)) {
- if (((1 << field_idx) & allow_space_mask) == 0) {
- return false;
- }
- }
- if (*ptr == '-') {
- has_bar = true;
- }
- ptr++;
- }
- field_idx++;
- }
- int num_field = field_idx;
- if (num_field <= 3) {
- _type = TIME_DATE;
+ vectorized::CastParameters params;
+ if (type() == TIME_DATE) {
+ return
vectorized::CastToDateOrDatetime::from_string_non_strict_mode<false>(
+ {date_str, len}, *this, &local_time_zone, params);
} else {
- _type = TIME_DATETIME;
- }
- if (!is_interval_format) {
- year_len = date_len[0];
- }
- for (; field_idx < MAX_DATE_PARTS; ++field_idx) {
- date_len[field_idx] = 0;
- date_val[field_idx] = 0;
- }
-
- if (year_len == 2) {
- if (date_val[0] < YY_PART_YEAR) {
- date_val[0] += 2000;
- } else {
- date_val[0] += 1900;
- }
+ return
vectorized::CastToDateOrDatetime::from_string_non_strict_mode<true>(
+ {date_str, len}, *this, &local_time_zone, params);
}
-
- if (num_field < 3) {
- return false;
- }
- if (!check_range_and_set_time(date_val[0], date_val[1], date_val[2],
date_val[3], date_val[4],
- date_val[5], _type)) {
- return false;
- }
- return sec_offset ? date_add_interval<TimeUnit::SECOND>(TimeInterval {
- TimeUnit::SECOND,
static_cast<uint64_t>(std::abs(sec_offset)),
- sec_offset < 0})
- : true;
}
// [0, 101) invalid
@@ -1940,251 +1786,31 @@ void DateV2Value<T>::format_datetime(uint32_t*
date_val, bool* carry_bits) const
}
}
-// The interval format is that with no delimiters
-// YYYY-MM-DD HH-MM-DD.FFFFFF AM in default format
-// 0 1 2 3 4 5 6 7
template <typename T>
bool DateV2Value<T>::from_date_str(const char* date_str, size_t len, int scale
/* = -1*/,
bool convert_zero) {
- return from_date_str_base(date_str, len, scale, nullptr, convert_zero);
+ vectorized::CastParameters params;
+ if constexpr (is_datetime) {
+ return
vectorized::CastToDatetimeV2::from_string_non_strict_mode({date_str, len},
*this,
+
nullptr, scale, params);
+ } else {
+ return
vectorized::CastToDateV2::from_string_non_strict_mode({date_str, len}, *this,
+ nullptr,
params);
+ }
}
+
template <typename T>
bool DateV2Value<T>::from_date_str(const char* date_str, size_t len,
const cctz::time_zone& local_time_zone, int
scale /* = -1*/,
bool convert_zero) {
- return from_date_str_base(date_str, len, scale, &local_time_zone,
convert_zero);
-}
-// if local_time_zone is null, only be able to parse time without timezone
-template <typename T>
-bool DateV2Value<T>::from_date_str_base(const char* date_str, size_t len, int
scale,
- const cctz::time_zone*
local_time_zone, bool convert_zero) {
- const char* ptr = date_str;
- const char* end = date_str + len;
- // ONLY 2, 6 can follow by a space
- const static int allow_space_mask = 4 | 64;
- uint32_t date_val[MAX_DATE_PARTS] = {0};
- int32_t date_len[MAX_DATE_PARTS] = {0};
-
- // Skip space character
- while (ptr < end && check_space(*ptr)) {
- ptr++;
- }
- if (ptr == end || !isdigit(*ptr)) {
- return false;
- }
- // Fix year length
- const char* pos = ptr;
- while (pos < end && (isdigit(*pos) || *pos == 'T')) {
- pos++;
- }
- int year_len = 4;
- int digits = pos - ptr;
- bool is_interval_format = false;
- bool has_bar = false;
-
- // Compatible with MySQL.
- // For YYYYMMDD/YYYYMMDDHHMMSS is 4 digits years
- if (pos == end || *pos == '.' ||
- time_zone_begins(pos, end)) { // no delimeter until ./Asia/Z/GMT...
- if (digits == 4 || digits == 8 || digits >= 14) {
- year_len = 4;
- } else {
- year_len = 2;
- }
- is_interval_format = true;
- }
-
- int field_idx = 0;
- int field_len = year_len;
- int sec_offset = 0;
- bool need_use_timezone = false;
-
- while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) {
- const char* start = ptr;
- int temp_val = 0;
- bool scan_to_delim = (!is_interval_format) && (field_idx != 6);
- while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) {
// field_len <= 7
- temp_val = temp_val * 10 + (*ptr - '0');
- ptr++;
- }
-
- if (ptr == start) {
- return false;
- }
-
- if (field_idx == 6) {
- if constexpr (is_datetime) {
- // round of microseconds
- // 1. normalize to 7 digits for rounding
- // 2. rounding
- // 3. nomalize to 6 digits for storage
- if (scale >= 0) {
- // do normalization
- const auto ms_digit_count = ptr - start;
- const auto normalizer = int_exp10(std::abs(7 -
ms_digit_count));
- temp_val *= normalizer;
-
- // check round
- const auto rounder = int_exp10(std::abs(7 - scale));
- const auto reminder = temp_val % rounder;
- temp_val -= reminder;
-
- if (reminder >= 5 * normalizer) {
- temp_val += rounder;
- }
-
- // truncate to 6 digits
- if (temp_val == int_exp10(7)) {
- temp_val = 0;
- sec_offset += 1;
- } else {
- temp_val /= 10;
- }
- }
-
- // move ptr to start of timezone or end
- while (ptr < end && isdigit(*ptr)) {
- ptr++;
- }
- } else {
- // Microsecond
- const auto ms_part = ptr - start;
- temp_val *= int_exp10(std::max(0L, 6 - ms_part));
- }
- }
-
- // Impossible
- if (temp_val > 999999L) {
- return false;
- }
-
- date_val[field_idx] = temp_val;
-
- if (field_idx == 6) {
- // select cast("2020-01-01 12:00:00.12345" as Datetime(4))
- // ptr - start will be 5, but scale is 4
- date_len[field_idx] = std::min(static_cast<int>(ptr - start),
scale);
- } else {
- date_len[field_idx] = ptr - start;
- }
-
- field_len = 2;
-
- if (ptr == end) {
- field_idx++;
- break;
- }
-
- // timezone
- if (UNLIKELY((field_idx > 2 ||
- !has_bar) /*dont treat xxxx-xx-xx:xx:xx as
xxxx-xx(-xx:xx:xx)*/
- && time_zone_begins(ptr, end))) {
- if (local_time_zone == nullptr) {
- return false;
- }
- need_use_timezone = true;
- field_idx++;
- break;
- }
-
- if (field_idx == 2 && *ptr == 'T') {
- // YYYYMMDDTHHMMDD, skip 'T' and continue
- ptr++;
- field_idx++;
- continue;
- }
-
- // Second part
- if (field_idx == 5) {
- if (*ptr == '.') {
- ptr++;
- // for datetime, we need to discard the fraction part
- // that beyond the scale + 1, and scale + 1 digit will
- // be used to round the fraction part
- if constexpr (is_datetime) {
- field_len = std::min(7, scale + 1);
- } else {
- field_len = 6;
- }
- } else if (isdigit(*ptr)) {
- field_idx++;
- break;
- }
- field_idx++;
- continue;
- }
- // escape separator
- while (ptr < end && (check_date_punct(*ptr) || check_space(*ptr))) {
- if (check_space(*ptr)) {
- if (((1 << field_idx) & allow_space_mask) == 0) {
- return false;
- }
- }
- if (*ptr == '-') {
- has_bar = true;
- }
- ptr++;
- }
- field_idx++;
- }
-
- int num_field = field_idx;
- if (!is_interval_format) {
- year_len = date_len[0];
- }
- for (; field_idx < MAX_DATE_PARTS; ++field_idx) {
- date_val[field_idx] = 0;
- }
-
- if (year_len == 2) {
- if (date_val[0] < YY_PART_YEAR) {
- date_val[0] += 2000;
- } else {
- date_val[0] += 1900;
- }
- }
-
- if (num_field < 3) {
- return false;
- }
- if (is_invalid(date_val[0], date_val[1], date_val[2], 0, 0, 0, 0)) {
- if (date_val[0] == 0 && date_val[1] == 0 && date_val[2] == 0 &&
convert_zero) {
- date_val[1] = 1;
- date_val[2] = 1;
- } else {
- return false;
- }
+ vectorized::CastParameters params;
+ if constexpr (is_datetime) {
+ return vectorized::CastToDatetimeV2::from_string_non_strict_mode(
+ {date_str, len}, *this, &local_time_zone, scale, params);
+ } else {
+ return
vectorized::CastToDateV2::from_string_non_strict_mode({date_str, len}, *this,
+
&local_time_zone, params);
}
-
- if (need_use_timezone) {
- cctz::time_zone given_tz {};
- if (!TimezoneUtils::find_cctz_time_zone(std::string {ptr, end},
given_tz)) {
- return false; // invalid format
- }
- if (is_invalid(date_val[0], date_val[1], date_val[2], date_val[3],
date_val[4], date_val[5],
- date_val[6])) {
- return false;
- }
- // will carring on the bits in cctz::civil_second. if day is 70, will
carry to month.
- cctz::civil_second cs {date_val[0], date_val[1], date_val[2],
- date_val[3], date_val[4], date_val[5]};
-
- auto given = cctz::convert(cs, given_tz);
- auto local = cctz::convert(given, *local_time_zone);
- date_val[0] = local.year();
- date_val[1] = local.month();
- date_val[2] = local.day();
- date_val[3] = local.hour();
- date_val[4] = local.minute();
- date_val[5] = local.second();
- }
-
- return check_range_and_set_time(date_val[0], date_val[1], date_val[2],
date_val[3], date_val[4],
- date_val[5], date_val[6]) &&
- (sec_offset ? date_add_interval<TimeUnit::SECOND>(TimeInterval {
- TimeUnit::SECOND,
static_cast<uint64_t>(std::abs(sec_offset)),
- sec_offset < 0})
- : true);
}
template <typename T>
diff --git a/be/src/vec/runtime/vdatetime_value.h
b/be/src/vec/runtime/vdatetime_value.h
index 26929f091e9..b2020ca81bd 100644
--- a/be/src/vec/runtime/vdatetime_value.h
+++ b/be/src/vec/runtime/vdatetime_value.h
@@ -773,9 +773,6 @@ private:
char* to_date_buffer(char* to) const;
char* to_time_buffer(char* to) const;
- bool from_date_str_base(const char* date_str, size_t len,
- const cctz::time_zone* local_time_zone);
-
int64_t to_date_int64() const;
int64_t to_time_int64() const;
@@ -927,7 +924,15 @@ public:
// DATETIME: format 'YYYY-MM-DD hh:mm:ss.xxxxxx'
int32_t to_buffer(char* buffer, int scale = -1) const;
+ // to_string with buffer will append '\0' at the end
char* to_string(char* to, int scale = -1) const;
+ // to_string return std::string will NOT append '\0' at the end
+ std::string to_string(int scale = -1) const {
+ std::string buf(40, 0);
+ int len = to_buffer(buf.data(), scale);
+ buf.resize(len);
+ return buf;
+ }
// Return true if range or date is invalid
static bool is_invalid(uint32_t year, uint32_t month, uint32_t day,
uint8_t hour,
@@ -1440,9 +1445,6 @@ private:
const uint8_t& day, uint8_t mode, uint16_t*
to_year,
bool disable_lut = false);
- bool from_date_str_base(const char* date_str, size_t len, int scale,
- const cctz::time_zone* local_time_zone, bool
convert_zero);
-
// Used to construct from int value
int64_t standardize_timevalue(int64_t value);
diff --git a/be/test/vec/data_types/data_type_time_v2_test.cpp
b/be/test/vec/data_types/data_type_time_v2_test.cpp
index 146d77ce9d7..4d1d0d1b90b 100644
--- a/be/test/vec/data_types/data_type_time_v2_test.cpp
+++ b/be/test/vec/data_types/data_type_time_v2_test.cpp
@@ -122,6 +122,7 @@ TEST_F(DataTypeDateTimeV2Test, simple_func_test) {
EXPECT_THROW(create_datetimev2(7), Exception);
EXPECT_THROW(DataTypeTimeV2(7), Exception);
}
+
TEST_F(DataTypeDateTimeV2Test, get_default) {
EXPECT_EQ(dt_datetime_v2_0.get_default(),
Field::create_field<TYPE_DATETIMEV2>(0UL));
EXPECT_EQ(dt_datetime_v2_5.get_default(),
Field::create_field<TYPE_DATETIMEV2>(0UL));
@@ -129,7 +130,9 @@ TEST_F(DataTypeDateTimeV2Test, get_default) {
EXPECT_EQ(dt_date_v2.get_default(), Field::create_field<TYPE_DATEV2>(0UL));
EXPECT_EQ(dt_time_v2_6.get_default(),
Field::create_field<TYPE_TIMEV2>(0.0));
}
+
TEST_F(DataTypeDateTimeV2Test, get_field) {
+ config::allow_zero_date = true;
{
TExprNode expr_node;
expr_node.date_literal.value = "abc";
@@ -146,8 +149,8 @@ TEST_F(DataTypeDateTimeV2Test, get_field) {
expr_node.date_literal.value = " ";
EXPECT_THROW(dt_date_v2.get_field(expr_node), Exception);
- expr_node.date_literal.value = "0000-00-00";
- EXPECT_THROW(dt_date_v2.get_field(expr_node), Exception);
+ expr_node.date_literal.value = "0000-00-00"; // allow_zero_date
+ EXPECT_NO_THROW(dt_date_v2.get_field(expr_node));
// invalid year
expr_node.date_literal.value = "10000-12-15";
@@ -164,21 +167,18 @@ TEST_F(DataTypeDateTimeV2Test, get_field) {
expr_node.date_literal.value = "2025-04-31";
EXPECT_THROW(dt_date_v2.get_field(expr_node), Exception);
- /*
- // TODO: currently the following cases are OK,
- // check if its' as expected
- // trailing invalid chars for date???
expr_node.date_literal.value = "2025-01-01x";
EXPECT_THROW(dt_date_v2.get_field(expr_node), Exception);
expr_node.date_literal.value = "2025-01-01 x";
EXPECT_THROW(dt_date_v2.get_field(expr_node), Exception);
- // invalid hour for date???
expr_node.date_literal.value = "2025-01-01 25:00:00";
EXPECT_THROW(dt_date_v2.get_field(expr_node), Exception);
- */
}
{
TExprNode expr_node;
+ TTypeNode type_node = TTypeNode();
+ type_node.type = TTypeNodeType::SCALAR;
+
expr_node.date_literal.value = "abc";
EXPECT_THROW(dt_datetime_v2_0.get_field(expr_node), Exception);
@@ -193,8 +193,8 @@ TEST_F(DataTypeDateTimeV2Test, get_field) {
expr_node.date_literal.value = " ";
EXPECT_THROW(dt_datetime_v2_0.get_field(expr_node), Exception);
- expr_node.date_literal.value = "0000-00-00";
- EXPECT_THROW(dt_datetime_v2_0.get_field(expr_node), Exception);
+ expr_node.date_literal.value = "0000-00-00"; // allow_zero_date
+ EXPECT_NO_THROW(dt_datetime_v2_0.get_field(expr_node));
// invalid year
expr_node.date_literal.value = "10000-13-15";
@@ -211,31 +211,44 @@ TEST_F(DataTypeDateTimeV2Test, get_field) {
expr_node.date_literal.value = "2025-04-31";
EXPECT_THROW(dt_datetime_v2_0.get_field(expr_node), Exception);
- // invalid microsecond
+ // invalid microsecond?
+ type_node.scalar_type.type = TPrimitiveType::DATETIMEV2;
+ type_node.scalar_type.scale = 0;
+ expr_node.type.types.push_back(type_node);
+
expr_node.date_literal.value = "0000-01-01 00:00:00.1";
- EXPECT_THROW(dt_datetime_v2_0.get_field(expr_node), Exception);
+ EXPECT_NO_THROW(dt_datetime_v2_0.get_field(expr_node));
expr_node.date_literal.value = "0000-01-01 00:00:00.999999";
- EXPECT_THROW(dt_datetime_v2_0.get_field(expr_node), Exception);
+ EXPECT_NO_THROW(dt_datetime_v2_0.get_field(expr_node));
expr_node.date_literal.value = "2021-12-30 12:23:34.1";
- EXPECT_THROW(dt_datetime_v2_0.get_field(expr_node), Exception);
+ EXPECT_NO_THROW(dt_datetime_v2_0.get_field(expr_node));
expr_node.date_literal.value = "9999-12-31 23:59:59.999999";
EXPECT_THROW(dt_datetime_v2_0.get_field(expr_node), Exception);
+ expr_node.type.types.clear();
+ type_node.scalar_type.type = TPrimitiveType::DATETIMEV2;
+ type_node.scalar_type.scale = 5;
+ expr_node.type.types.push_back(type_node);
expr_node.date_literal.value = "0000-01-01 00:00:00.100000";
- EXPECT_THROW(dt_datetime_v2_5.get_field(expr_node), Exception);
+ EXPECT_NO_THROW(dt_datetime_v2_5.get_field(expr_node));
expr_node.date_literal.value = "2021-12-30 12:23:34.100000";
- EXPECT_THROW(dt_datetime_v2_5.get_field(expr_node), Exception);
+ EXPECT_NO_THROW(dt_datetime_v2_5.get_field(expr_node));
expr_node.date_literal.value = "9999-12-31 23:59:59.999999";
EXPECT_THROW(dt_datetime_v2_5.get_field(expr_node), Exception);
+ expr_node.type.types.clear();
+ type_node.scalar_type.type = TPrimitiveType::DATETIMEV2;
+ type_node.scalar_type.scale = 0;
+ expr_node.type.types.push_back(type_node);
expr_node.date_literal.value = "0000-01-01 00:00:00.1000000";
- EXPECT_THROW(dt_datetime_v2_6.get_field(expr_node), Exception);
+ EXPECT_NO_THROW(dt_datetime_v2_6.get_field(expr_node));
expr_node.date_literal.value = "2021-12-30 12:23:34.1000000";
- EXPECT_THROW(dt_datetime_v2_6.get_field(expr_node), Exception);
+ EXPECT_NO_THROW(dt_datetime_v2_6.get_field(expr_node));
expr_node.date_literal.value = "2021-12-30 12:23:34.9999999";
- EXPECT_THROW(dt_datetime_v2_6.get_field(expr_node), Exception);
+ EXPECT_NO_THROW(dt_datetime_v2_6.get_field(expr_node));
expr_node.date_literal.value = "9999-12-31 23:59:59.9999999";
EXPECT_THROW(dt_datetime_v2_6.get_field(expr_node), Exception);
+ expr_node.type.types.clear();
}
{
TExprNode expr_node;
@@ -655,7 +668,9 @@ TEST_F(DataTypeDateTimeV2Test, get_field) {
EXPECT_EQ(date_value.second(), 59);
EXPECT_EQ(date_value.microsecond(), 0);
}
+ config::allow_zero_date = false;
}
+
TEST_F(DataTypeDateTimeV2Test, ser_deser) {
auto test_func = [](auto& dt, const auto& column, int be_exec_version) {
std::cout << "test serialize/deserialize datatype " <<
dt.get_family_name()
diff --git a/be/test/vec/data_types/from_string_test.cpp
b/be/test/vec/data_types/from_string_test.cpp
index 2eea3736ca4..d684b371e87 100644
--- a/be/test/vec/data_types/from_string_test.cpp
+++ b/be/test/vec/data_types/from_string_test.cpp
@@ -35,9 +35,8 @@ TEST(FromStringTest, ScalaWrapperFieldVsDataType) {
// arithmetic scala field types
{
// fieldType, test_string, expect_wrapper_field_string,
expect_data_type_string
- typedef std::tuple<FieldType, std::vector<std::string>,
std::vector<std::string>,
- std::vector<std::string>>
- FieldType_RandStr;
+ using FieldType_RandStr = std::tuple<FieldType,
std::vector<std::string>,
+ std::vector<std::string>,
std::vector<std::string>>;
std::vector<FieldType_RandStr> arithmetic_scala_field_types = {
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_BOOL, {"0", "1",
"-9"},
{"0", "1", "1"}, {"0", "1", ""}),
@@ -213,7 +212,7 @@ TEST(FromStringTest, ScalaWrapperFieldVsDataType) {
// date and datetime type
{
- typedef std::pair<FieldType, std::string> FieldType_RandStr;
+ using FieldType_RandStr = std::pair<FieldType, std::string>;
std::vector<FieldType_RandStr> date_scala_field_types = {
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATE,
"2020-01-01"),
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATEV2,
"2020-01-01"),
@@ -284,7 +283,7 @@ TEST(FromStringTest, ScalaWrapperFieldVsDataType) {
// ipv4 and ipv6 type
{
- typedef std::pair<FieldType, std::string> FieldType_RandStr;
+ using FieldType_RandStr = std::pair<FieldType, std::string>;
std::vector<FieldType_RandStr> ip_scala_field_types = {
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_IPV4, "0.0.0.0"),
// min case
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_IPV4,
"127.0.0.1"), // rand case
diff --git a/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
index cf8cdc2bfaa..0e6792d53f8 100644
--- a/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
@@ -36,8 +36,8 @@ TEST(TextSerde, ScalaDataTypeSerdeTextTest) {
// arithmetic scala field types
{
// fieldType, test_string, expect_string
- typedef std::tuple<FieldType, std::vector<std::string>,
std::vector<std::string>>
- FieldType_RandStr;
+ using FieldType_RandStr =
+ std::tuple<FieldType, std::vector<std::string>,
std::vector<std::string>>;
std::vector<FieldType_RandStr> arithmetic_scala_field_types = {
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_BOOL, {"0", "1",
"-1"},
{"0", "1", ""}),
@@ -261,7 +261,7 @@ TEST(TextSerde, ScalaDataTypeSerdeTextTest) {
// ipv4 and ipv6
{
- typedef std::pair<FieldType, std::string> FieldType_RandStr;
+ using FieldType_RandStr = std::pair<FieldType, std::string>;
std::vector<FieldType_RandStr> ip_scala_field_types = {
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_IPV4,
"127.0.0.1"),
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_IPV6,
"2405:9800:9800:66::2")};
@@ -352,16 +352,13 @@ TEST(TextSerde, ScalaDataTypeSerdeTextTest) {
}
}
-// test for array and map
-
// test for array and map
TEST(TextSerde, ComplexTypeSerdeTextTest) {
// array-scala
{
// nested type,test string, expect
string(option.converted_from_string=false),expect
string(option.converted_from_string=true)
- typedef std::tuple<FieldType, std::vector<std::string>,
std::vector<std::string>,
- std::vector<std::string>>
- FieldType_RandStr;
+ using FieldType_RandStr = std::tuple<FieldType,
std::vector<std::string>,
+ std::vector<std::string>,
std::vector<std::string>>;
std::vector<FieldType_RandStr> nested_field_types = {
FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_BOOL,
@@ -390,49 +387,43 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
"[2343.12345465746, 2.22507e-308, null,
2.22507e-308]"}),
FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_STRING,
- {"[\"hello\", \"world\"]", "['a', 'b', 'c']",
+ {R"(["hello", "world"])", "['a', 'b', 'c']",
"[\"42\",1412341,true,42.43,3.40282e+38+1,alpha:beta:gamma,Earth#42:"
"Control#86:Bob#31,17:true:Abe "
"Linkedin,BLUE,\"\\N\",\"\u0001\u0002\u0003,\\u0001bc\"]",
-
"[\"heeeee\",null,\"null\",\"\\N\",null,\"sssssssss\"]"},
+ R"(["heeeee",null,"null","\N",null,"sssssssss"])"},
// last :
["42",1412341,true,42.43,3.40282e+38+1,alpha:beta:gamma,Earth#42:Control#86:Bob#31,17:true:Abe
Linkedin,BLUE,"\N",",\u0001bc"]
- {"[\"hello\", \"world\"]", "[\"a\", \"b\", \"c\"]",
+ {R"(["hello", "world"])", R"(["a", "b", "c"])",
"[\"42\", \"1412341\", \"true\", \"42.43\",
\"3.40282e+38+1\", "
- "\"alpha:beta:gamma\", "
- "\"Earth#42:Control#86:Bob#31\", \"17:true:Abe
Linkedin\", \"BLUE\", "
- "\"\\N\", "
- "\"\x1\x2\x3,\\u0001bc\"]",
- "[\"heeeee\", null, \"null\", \"\\N\", null,
\"sssssssss\"]"},
- {"[\"hello\", \"world\"]", "[\"a\", \"b\", \"c\"]",
+ "\"alpha:beta:gamma\",
\"Earth#42:Control#86:Bob#31\", \"17:true:Abe "
+ "Linkedin\", \"BLUE\", \"\\N\",
\"\x1\x2\x3,\\u0001bc\"]",
+ R"(["heeeee", null, "null", "\N", null,
"sssssssss"])"},
+ {R"(["hello", "world"])", R"(["a", "b", "c"])",
"[\"42\", \"1412341\", \"true\", \"42.43\",
\"3.40282e+38+1\", "
- "\"alpha:beta:gamma\", "
- "\"Earth#42:Control#86:Bob#31\", \"17:true:Abe
Linkedin\", \"BLUE\", "
- "\"\\N\", "
- "\"\x1\x2\x3,\\u0001bc\"]",
- "[\"heeeee\", null, \"null\", \"\\N\", null,
\"sssssssss\"]"}),
+ "\"alpha:beta:gamma\",
\"Earth#42:Control#86:Bob#31\", \"17:true:Abe "
+ "Linkedin\", \"BLUE\", \"\\N\",
\"\x1\x2\x3,\\u0001bc\"]",
+ R"(["heeeee", null, "null", "\N", null,
"sssssssss"])"}),
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATE,
- {"[\\\"2022-07-13\\\",\"2022-07-13
12:30:00\"]",
+ {R"([\"2022-07-13\","2022-07-13 12:30:00"])",
"[2022-07-13 12:30:00, \"2022-07-13\"]",
"[2022-07-13 12:30:00.000, 2022-07-13]"},
- {"[null, \"2022-07-13\"]", "[\"2022-07-13\",
\"2022-07-13\"]",
- "[\"2022-07-13\", \"2022-07-13\"]"},
- {"[null, \"2022-07-13\"]", "[\"2022-07-13\",
\"2022-07-13\"]",
- "[\"2022-07-13\", \"2022-07-13\"]"}),
+ {"[null, \"2022-07-13\"]", R"(["2022-07-13",
"2022-07-13"])",
+ R"(["2022-07-13", "2022-07-13"])"},
+ {"[null, \"2022-07-13\"]", R"(["2022-07-13",
"2022-07-13"])",
+ R"(["2022-07-13", "2022-07-13"])"}),
FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_DATETIME,
{
- "[\"2022-07-13\",\"2022-07-13 12:30:00\"]",
+ R"(["2022-07-13","2022-07-13 12:30:00"])",
"[2022-07-13 12:30:00, \"2022-07-13\",
2022-07-13 12:30:00.0000]",
"\\N",
"[null,null,null]",
},
- {"[\"2022-07-13 00:00:00\", \"2022-07-13 12:30:00\"]",
- "[\"2022-07-13 12:30:00\", \"2022-07-13 00:00:00\",
\"2022-07-13 "
- "12:30:00\"]",
+ {R"(["2022-07-13 00:00:00", "2022-07-13 12:30:00"])",
+ R"(["2022-07-13 12:30:00", "2022-07-13 00:00:00",
"2022-07-13 12:30:00"])",
"\\N", "[null, null, null]"},
- {"[\"2022-07-13 00:00:00\", \"2022-07-13 12:30:00\"]",
- "[\"2022-07-13 12:30:00\", \"2022-07-13 00:00:00\",
\"2022-07-13 "
- "12:30:00\"]",
+ {R"(["2022-07-13 00:00:00", "2022-07-13 12:30:00"])",
+ R"(["2022-07-13 12:30:00", "2022-07-13 00:00:00",
"2022-07-13 12:30:00"])",
"\\N", "[null, null, null]"}),
FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_DECIMAL,
@@ -440,9 +431,7 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
"[012345678901234567.012345678,123456789012345678.01234567, "
"12345678901234567.0123456779,12345678901234567.01234567791,"
"1234567890123456789.01234567]",
-
"[\"012345678901234567.012345678\",\"123456789012345678.01234567\", "
- "\"12345678901234567.0123456779\", "
-
"\"12345678901234567.01234567791\",\"1234567890123456789.01234567\"]",
+
R"(["012345678901234567.012345678","123456789012345678.01234567",
"12345678901234567.0123456779",
"12345678901234567.01234567791","1234567890123456789.01234567"])",
"[\\1234567890123456789.01234567\\]"},
{"[4.000000000, 5.500000000, 6.670000000]",
"[12345678901234567.012345678,
123456789012345678.012345670, "
@@ -482,31 +471,31 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
std::string rand_str = std::get<1>(type_pair)[i];
std::string expect_str = std::get<2>(type_pair)[i];
std::string expect_str_1 = std::get<3>(type_pair)[i];
- std::cout << "rand_str:" << rand_str << std::endl;
- std::cout << "expect_str:" << expect_str << std::endl;
- std::cout << "expect_str_can_format_from_string:" <<
expect_str_1 << std::endl;
+ // std::cout << "rand_str:" << rand_str << std::endl;
+ // std::cout << "expect_str:" << expect_str << std::endl;
+ // std::cout << "expect_str_can_format_from_string:" <<
expect_str_1 << std::endl;
{
Slice slice(rand_str.data(), rand_str.size());
formatOptions.converted_from_string = false;
Status st = serde->deserialize_one_cell_from_json(*col,
slice, formatOptions);
if (expect_str == "[]") {
- if (st.ok()) {
- auto& item_column = assert_cast<ColumnNullable&>(
- assert_cast<ColumnArray&>(
-
assert_cast<ColumnNullable&>(*col).get_nested_column())
- .get_data());
- for (auto ix = 0; ix < item_column.size(); ++ix) {
- if (item_column.is_null_at(ix)) {
- std::cout << "idx null:" << ix <<
std::endl;
- } else {
- std::cout << "idx:" <<
item_column.get_data_at(ix).to_string()
- << std::endl;
- }
- }
- } else {
- EXPECT_EQ(st.ok(), false);
- std::cout << st.to_json() << std::endl;
- }
+ // if (st.ok()) {
+ // auto& item_column =
assert_cast<ColumnNullable&>(
+ // assert_cast<ColumnArray&>(
+ //
assert_cast<ColumnNullable&>(*col).get_nested_column())
+ // .get_data());
+ // for (auto ix = 0; ix < item_column.size();
++ix) {
+ // if (item_column.is_null_at(ix)) {
+ // std::cout << "idx null:" << ix <<
std::endl;
+ // } else {
+ // std::cout << "idx:" <<
item_column.get_data_at(ix).to_string()
+ // << std::endl;
+ // }
+ // }
+ // } else {
+ // EXPECT_EQ(st.ok(), false);
+ // std::cout << st.to_json() << std::endl;
+ // }
} else {
EXPECT_EQ(st.ok(), true);
auto ser_col = ColumnString::create();
@@ -517,7 +506,7 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
EXPECT_EQ(st.ok(), true);
buffer_writer.commit();
StringRef rand_s_d = ser_col->get_data_at(0);
- std::cout << "test : " << rand_s_d << std::endl;
+ // std::cout << "test : " << rand_s_d << std::endl;
EXPECT_EQ(expect_str, rand_s_d.to_string());
}
}
@@ -533,14 +522,14 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
formatOptions);
EXPECT_EQ(status.ok(), true);
buffer_writer.commit();
- StringRef rand_s_d = ser_col->get_data_at(0);
- std::cout << "test from string: " << rand_s_d << std::endl;
+ // StringRef rand_s_d = ser_col->get_data_at(0);
+ // std::cout << "test from string: " << rand_s_d <<
std::endl;
// EXPECT_EQ(expect_str,
rand_s_d.to_string());
}
{
formatOptions.converted_from_string = true;
- std::cout << "======== change " <<
formatOptions.converted_from_string
- << " with rand_str: " << rand_str << std::endl;
+ // std::cout << "======== change " <<
formatOptions.converted_from_string
+ // << " with rand_str: " << rand_str <<
std::endl;
Slice slice(rand_str.data(), rand_str.size());
Status st =
serde_1->deserialize_one_cell_from_json(*col3,
slice, formatOptions);
@@ -567,31 +556,25 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
// map-scala-scala
{
// nested key type , nested value type, test string , expect string
- typedef std::tuple<FieldType, FieldType, std::vector<std::string>,
std::vector<std::string>>
- FieldType_RandStr;
+ using FieldType_RandStr = std::tuple<FieldType, FieldType,
std::vector<std::string>,
+ std::vector<std::string>>;
std::vector<FieldType_RandStr> nested_field_types = {
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_BOOL,
FieldType::OLAP_FIELD_TYPE_STRING,
- {"{1: \"amory is 7\", 0: \" doris be better
\", -1: \"wrong,\"}",
- "{\"1\": \"amory is 7\", \"0\": 1}"},
- {"{1:\"amory is 7\", 0:\" doris be better
\", null:\"wrong,\"}",
- "{null:\"amory is 7\", null:\"1\"}"}),
+ {R"({1: "amory is 7", 0: " doris be better
", -1: "wrong,"})",
+ R"({"1": "amory is 7", "0": 1})"},
+ {R"({1:"amory is 7", 0:" doris be better ",
null:"wrong,"})",
+ R"({null:"amory is 7", null:"1"})"}),
FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_STRING,
FieldType::OLAP_FIELD_TYPE_DOUBLE,
- {"{\" ,.amory\": 111.2343, \"\": 112., 'dggs': 13.14 ,
null: 12.2222222, "
- ": null\\}",
+ {R"({" ,.amory": 111.2343, "": 112., 'dggs': 13.14 ,
null: 12.2222222, : null\})",
"{\"\": null, null: 12.44}", "{{}}", "{{}", "}}",
"{}, {}", "\\N",
"{null:null,\"null\":null}",
- "{\"hello "
-
"world\":0.2222222,\"hello2\":null,null:1111.1,\"null\":null,\"null\":"
- "null,\"null\":0.1}"},
- {"{\" ,.amory\":111.2343, \"\":112, \"dggs\":13.14, "
- "null:12.2222222, \"\":null}",
+ R"({"hello
world":0.2222222,"hello2":null,null:1111.1,"null":null,"null":null,"null":0.1})"},
+ {R"({" ,.amory":111.2343, "":112, "dggs":13.14,
null:12.2222222, "":null})",
"{\"\":null, null:12.44}", "{}", "{}", "\\N", "{}",
"\\N",
"{null:null, \"null\":null}",
- "{\"hello world\":0.2222222, \"hello2\":null,
null:1111.1, "
- "\"null\":null, \"null\":null, "
- "\"null\":0.1}"}),
+ R"({"hello world":0.2222222, "hello2":null,
null:1111.1, "null":null, "null":null, "null":0.1})"}),
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_FLOAT,
FieldType::OLAP_FIELD_TYPE_DOUBLE,
{"{0.33: 3.1415926,3.1415926: 22}", "{3.14,
15926: 22}", "{3.14}",
@@ -601,38 +584,27 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
"{null:null, 67.6:67.7}", "{null:null,
null:1, 1:null}"}),
FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_DATE,
FieldType::OLAP_FIELD_TYPE_DATETIME,
- {"{\"2022-07-13\": \"2022-07-13 12:30:00\",
\"2022-07-13 12:30:00\": "
- "2022-07-13 "
- "12:30:00, \"2022-07-13 12:30:00.000\": 2022-07-13
12:30:00.000, null: "
- "null, "
- "\"2022-07-13\":'2022-07-13 12:30:00'}",
- // escaped char ':'
- "{2022-07-13 12\\:30\\:00: \"2022-07-13\", 2022-07-13
12\\:30\\:00.000: "
- "\"2022-07-13 12:30:00.000\",
\"2022-07-13\":\'$2022-07-13 12:30:00\'}",
+ {R"({"2022-07-13": "2022-07-13 12:30:00", "2022-07-13
12:30:00": 2022-07-13 12:30:00, "2022-07-13 12:30:00.000": 2022-07-13
12:30:00.000, null: null, "2022-07-13":'2022-07-13 12:30:00'})",
+ //FIXME: need support \ in cell?
+ // R"({2022-07-13 12\:30\:00: "2022-07-13",
2022-07-13 12\:30\:00.000: "2022-07-13 12:30:00.000", "2022-07-13":'$2022-07-13
12:30:00'})",
"\\N"},
- {"{\"2022-07-13\":\"2022-07-13 12:30:00\",
\"2022-07-13\":\"2022-07-13 "
- "12:30:00\", "
- "\"2022-07-13\":\"2022-07-13 12:30:00\", null:null, "
- "\"2022-07-13\":\"2022-07-13 12:30:00\"}",
- "{\"2022-07-13\":\"2022-07-13 00:00:00\",
\"2022-07-13\":\"2022-07-13 "
- "12:30:00\", "
- "\"2022-07-13\":null}",
+ {R"({"2022-07-13":"2022-07-13 12:30:00",
"2022-07-13":"2022-07-13 12:30:00", "2022-07-13":"2022-07-13 12:30:00",
null:null, "2022-07-13":"2022-07-13 12:30:00"})",
+ // R"({"2022-07-13":"2022-07-13 00:00:00",
"2022-07-13":"2022-07-13 12:30:00", "2022-07-13":null})",
"\\N"}),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATETIME,
- FieldType::OLAP_FIELD_TYPE_DECIMAL,
- {"{\"2022-07-13 12:30:00\": 12.45675432,
\"2022-07-13\": "
- "12.45675432, null: null}",
- "{\"$2022-07-13 12:30:00\":
\"12.45675432\"}",
- "{\"2022-07-13 12\\:30\\:00\":12.45675432, "
- "\"2022-07-13#12:30:00\": 12.45675432}",
- "{\"2022-07-13
12\\:30\\:00.0000\":12.45675432, null:12.34}"},
- {"{\"2022-07-13 12:30:00\":12.456754320,
\"2022-07-13 "
- "00:00:00\":12.456754320, "
- "null:null}",
- "{null:null}",
- "{\"2022-07-13 12:30:00\":12.456754320,
\"2022-07-13 "
- "12:30:00\":12.456754320}",
- "{\"2022-07-13 12:30:00\":12.456754320,
null:12.340000000}"}),
+ FieldType_RandStr(
+ FieldType::OLAP_FIELD_TYPE_DATETIME,
FieldType::OLAP_FIELD_TYPE_DECIMAL,
+ {
+ R"({"2022-07-13 12:30:00": 12.45675432,
"2022-07-13": 12.45675432, null: null})",
+ R"({"$2022-07-13 12:30:00": "12.45675432"})",
+ // R"({"2022-07-13 12\:30\:00":12.45675432,
"2022-07-13#12:30:00": 12.45675432})",
+ // R"({"2022-07-13
12\:30\:00.0000":12.45675432, null:12.34})"
+ },
+ {
+ R"({"2022-07-13 12:30:00":12.456754320,
"2022-07-13 00:00:00":12.456754320, null:null})",
+ "{null:null}",
+ // R"({"2022-07-13 12:30:00":12.456754320,
"2022-07-13 12:30:00":12.456754320})",
+ // "{\"2022-07-13 12:30:00\":12.456754320,
null:12.340000000}"
+ }),
};
for (auto type_pair : nested_field_types) {
@@ -660,13 +632,15 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
for (int i = 0; i < std::get<2>(type_pair).size(); ++i) {
std::string rand_str = std::get<2>(type_pair)[i];
std::string expect_str = std::get<3>(type_pair)[i];
+ std::cout << "\n\n----------------- the " << i << " case
-----------------"
+ << std::endl;
std::cout << "rand_str:" << rand_str << std::endl;
std::cout << "expect_str:" << expect_str << std::endl;
{
auto col = map_data_type_ptr->create_column();
Slice slice(rand_str.data(), rand_str.size());
Status st = serde->deserialize_one_cell_from_json(*col,
slice, formatOptions);
- std::cout << st.to_json() << std::endl;
+ // std::cout << st.to_json() << std::endl;
if (expect_str.empty()) {
EXPECT_FALSE(st.ok());
continue;
@@ -700,35 +674,37 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
}
// option with converted_with_string true
- typedef std::tuple<FieldType, FieldType, std::vector<std::string>,
std::vector<std::string>>
- FieldType_RandStr;
+ using FieldType_RandStr = std::tuple<FieldType, FieldType,
std::vector<std::string>,
+ std::vector<std::string>>;
std::vector<FieldType_RandStr> field_types = {
FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_DATE,
FieldType::OLAP_FIELD_TYPE_DATETIME,
- {"{2022-07-13: 2022-07-13 12:30:00, 2022-07-13
12:30:00: 2022-07-13 "
- "12:30:00, 2022-07-13 12:30:00.000: 2022-07-13
12:30:00.000, null: null, "
- "2022-07-13:'2022-07-13 12:30:00'}",
- // escaped char ':'
- "{2022-07-13 12\\:30\\:00: 2022-07-13, 2022-07-13
12\\:30\\:00.000: "
- "2022-07-13 12:30:00.000, 2022-07-13:\'2022-07-13
12:30:00\'}"},
- {"{\"2022-07-13\":\"2022-07-13 12:30:00\",
\"2022-07-13\":null, "
- "\"2022-07-13\":null, null:null,
\"2022-07-13\":\"2022-07-13 12:30:00\"}",
- "{\"2022-07-13\":\"2022-07-13 00:00:00\",
\"2022-07-13\":\"2022-07-13 "
- "12:30:00\", "
- "\"2022-07-13\":\"2022-07-13 12:30:00\"}"}),
+ {
+ "{2022-07-13: 2022-07-13 12:30:00, 2022-07-13
12:30:00: 2022-07-13 "
+ "12:30:00, 2022-07-13 12:30:00.000: 2022-07-13
12:30:00.000, null: "
+ "null, "
+ "2022-07-13:'2022-07-13 12:30:00'}",
+ // R"({2022-07-13 12\:30\:00: 2022-07-13,
2022-07-13 12\:30\:00.000: 2022-07-13 12:30:00.000, 2022-07-13:'2022-07-13
12:30:00'})"
+ },
+ {
+ R"({"2022-07-13":"2022-07-13 12:30:00",
"2022-07-13":null, "2022-07-13":null, null:null, "2022-07-13":"2022-07-13
12:30:00"})",
+ // R"({"2022-07-13":"2022-07-13 00:00:00",
"2022-07-13":"2022-07-13 12:30:00", "2022-07-13":"2022-07-13 12:30:00"})"
+ }),
FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_DATETIME,
FieldType::OLAP_FIELD_TYPE_DECIMAL,
- {"{2022-07-13 12:30:00: 12.45675432, 2022-07-13:
12.45675432, null: null}",
- "{\"2022-07-13 12:30:00\": \"12.45675432\"}",
- "{2022-07-13 12\\:30\\:00:12.45675432,
2022-07-13#12:30:00: 12.45675432}",
- "{2022-07-13 12\\:30\\:00.0000:12.45675432,
null:12.34}"},
- {"{\"2022-07-13 12:00:00\":null, \"2022-07-13 "
- "00:00:00\":12.456754320, "
- "null:null}",
- "{\"2022-07-13 12:30:00\":12.456754320}",
- "{\"2022-07-13 12:30:00\":12.456754320, \"2022-07-13 "
- "12:00:00\":null}",
- "{\"2022-07-13 12:30:00\":12.456754320,
null:12.340000000}"}),
+ {
+ "{2022-07-13 12:30:00: 12.45675432,
2022-07-13: 12.45675432, null: "
+ "null}",
+ R"({"2022-07-13 12:30:00": "12.45675432"})",
+ // "{2022-07-13 12\\:30\\:00:12.45675432,
2022-07-13#12:30:00: 12.45675432}",
+ // "{2022-07-13
12\\:30\\:00.0000:12.45675432, null:12.34}"
+ },
+ {
+ R"({"2022-07-13 12:00:00":null, "2022-07-13
00:00:00":12.456754320, null:null})",
+ "{\"2022-07-13 12:30:00\":12.456754320}",
+ // R"({"2022-07-13 12:30:00":12.456754320,
"2022-07-13 12:00:00":null})",
+ // "{\"2022-07-13 12:30:00\":12.456754320,
null:12.340000000}"
+ }),
};
for (auto type_pair : field_types) {
auto key_type = std::get<0>(type_pair);
@@ -785,22 +761,15 @@ TEST(TextSerde, ComplexTypeWithNestedSerdeTextTest) {
// array-array<string>
{
// nested type,test string, expect
string(option.converted_from_string=false), expect_from_string, expect
string(option.converted_from_string=true)
- typedef std::tuple<FieldType, std::vector<std::string>,
std::vector<std::string>,
- std::vector<std::string>, std::vector<std::string>>
- FieldType_RandStr;
+ using FieldType_RandStr =
+ std::tuple<FieldType, std::vector<std::string>,
std::vector<std::string>,
+ std::vector<std::string>, std::vector<std::string>>;
std::vector<FieldType_RandStr> nested_field_types = {FieldType_RandStr(
FieldType::OLAP_FIELD_TYPE_STRING,
- {"[[With, special, \"characters\"], [like, @, #, $, % \"^\",
&, *, (, ), "
- "-, _], [=, +, [, ], {, }, |, \\, ;, :, ', '\', <, >, ,, .,
/, ?, ~]]"},
- {"[[\"With\", \"special\", \"characters\"], [\"like\", \"@\",
\"#\", "
- "\"$\", \"% \"^\"\", \"&\", \"*\", \"(\", \")\", \"-\", "
- "\"_\"], [\"=\", \"+\", \"[, ]\", \"{, }\", \"|\", \"\\\",
\";\", "
- "\":\", \"', '', <, >, ,, ., /, ?, ~\"]]"},
+ {R"([[With, special, "characters"], [like, @, #, $, % "^", &,
*, (, ), -, _], [=, +, [, ], {, }, |, \, ;, :, ', '', <, >, ,, ., /, ?, ~]])"},
+ {R"lit([["With", "special", "characters"], ["like", "@", "#",
"$", "% "^"", "&", "*", "(", ")", "-", "_"], ["=", "+", "[, ]", "{, }", "|",
"\", ";", ":", "', '', <, >, ,, ., /, ?, ~"]])lit"},
{""},
- {"[[\"With\", \"special\", \"characters\"], [\"like\", \"@\",
\"#\", "
- "\"$\", \"% \"^\"\", \"&\", \"*\", \"(\", \")\", \"-\", "
- "\"_\"], [\"=\", \"+\", \"[, ]\", \"{, }\", \"|\", \"\\\",
\";\", "
- "\":\", \"', '', <, >, ,, ., /, ?, ~\"]]"})};
+ {R"lit([["With", "special", "characters"], ["like", "@", "#",
"$", "% "^"", "&", "*", "(", ")", "-", "_"], ["=", "+", "[, ]", "{, }", "|",
"\", ";", ":", "', '', <, >, ,, ., /, ?, ~"]])lit"})};
// array type
for (auto type_pair : nested_field_types) {
auto type = std::get<0>(type_pair);
@@ -903,56 +872,16 @@ TEST(TextSerde, ComplexTypeWithNestedSerdeTextTest) {
// array-map<string, double>
{
// nested type,test string, expect
string(option.converted_from_string=false), expect_from_string, expect
string(option.converted_from_string=true)
- typedef std::tuple<FieldType, FieldType, std::vector<std::string>,
std::vector<std::string>,
- std::vector<std::string>, std::vector<std::string>>
- FieldType_RandStr;
- std::vector<FieldType_RandStr> nested_field_types = {FieldType_RandStr(
- FieldType::OLAP_FIELD_TYPE_STRING,
FieldType::OLAP_FIELD_TYPE_DOUBLE,
-
{"[{\"2cKtIM-L1mOcEm-udR-HcB2\":0.23929040957798242,\"eof2UN-Is0EEuA-H5D-hE58\":0."
-
"42373055809540094,\"FwUSOB-R8rtK9W-BVG-8wYZ\":0.7680704548628841},{\"qDXU9D-"
-
"7orr51d-g80-6t5k\":0.6446245786874659,\"bkLjmx-uZ2Ez7F-536-PGqy\":0."
-
"8880791950937957,\"9Etq4o-FPm37O4-5fk-QWh7\":0.08630489716260481},{\"tu3OMw-"
-
"mzS0jAx-Dnj-Xm3G\":0.1184199213706042,\"XkhTn0-QFLo8Ks-JXR-k4zk\":0."
-
"5181239375482816,\"EYC8Dj-GTTp9iB-b4O-QBkO\":0.4491897722178303},{\"sHFGPg-"
-
"cfA8gya-kfw-IugT\":0.20842299487398452,\"BBQ6e5-OJYRJhC-zki-7rQj\":0."
-
"3050124830713523,\"mKH57V-YmwCNFq-vs8-vUIX\":0.36446683035480754},{\"HfhEMX-"
-
"oAMBJCC-YIC-hCqN\":0.8131454631693608,\"xrnTFd-ikONWik-T7J-sL8J\":0."
-
"37509722558990855,\"SVyEes-77mlzIr-N6c-DkYw\":0.4703053945053086,"
- "\"null\":0.1,\"null\":0.1,null:null}, {null:0.1, null:null,
\"null\":0}]"},
- {"[{\"2cKtIM-L1mOcEm-udR-HcB2\":0.2392904095779824, "
- "\"eof2UN-Is0EEuA-H5D-hE58\":0.4237305580954009, "
- "\"FwUSOB-R8rtK9W-BVG-8wYZ\":0.7680704548628841}, "
- "{\"qDXU9D-7orr51d-g80-6t5k\":0.6446245786874659, "
- "\"bkLjmx-uZ2Ez7F-536-PGqy\":0.8880791950937957, "
- "\"9Etq4o-FPm37O4-5fk-QWh7\":0.08630489716260481}, "
- "{\"tu3OMw-mzS0jAx-Dnj-Xm3G\":0.1184199213706042, "
- "\"XkhTn0-QFLo8Ks-JXR-k4zk\":0.5181239375482816, "
- "\"EYC8Dj-GTTp9iB-b4O-QBkO\":0.4491897722178303}, "
- "{\"sHFGPg-cfA8gya-kfw-IugT\":0.2084229948739845, "
- "\"BBQ6e5-OJYRJhC-zki-7rQj\":0.3050124830713523, "
- "\"mKH57V-YmwCNFq-vs8-vUIX\":0.3644668303548075}, "
- "{\"HfhEMX-oAMBJCC-YIC-hCqN\":0.8131454631693608, "
- "\"xrnTFd-ikONWik-T7J-sL8J\":0.3750972255899085, "
- "\"SVyEes-77mlzIr-N6c-DkYw\":0.4703053945053086,
\"null\":0.1, \"null\":0.1, "
- "null:null}, "
- "{null:0.1, null:null, \"null\":0}]"},
- {""},
- {"[{\"2cKtIM-L1mOcEm-udR-HcB2\":0.2392904095779824, "
- "\"eof2UN-Is0EEuA-H5D-hE58\":0.4237305580954009, "
- "\"FwUSOB-R8rtK9W-BVG-8wYZ\":0.7680704548628841}, "
- "{\"qDXU9D-7orr51d-g80-6t5k\":0.6446245786874659, "
- "\"bkLjmx-uZ2Ez7F-536-PGqy\":0.8880791950937957, "
- "\"9Etq4o-FPm37O4-5fk-QWh7\":0.08630489716260481}, "
- "{\"tu3OMw-mzS0jAx-Dnj-Xm3G\":0.1184199213706042, "
- "\"XkhTn0-QFLo8Ks-JXR-k4zk\":0.5181239375482816, "
- "\"EYC8Dj-GTTp9iB-b4O-QBkO\":0.4491897722178303}, "
- "{\"sHFGPg-cfA8gya-kfw-IugT\":0.2084229948739845, "
- "\"BBQ6e5-OJYRJhC-zki-7rQj\":0.3050124830713523, "
- "\"mKH57V-YmwCNFq-vs8-vUIX\":0.3644668303548075}, "
- "{\"HfhEMX-oAMBJCC-YIC-hCqN\":0.8131454631693608, "
- "\"xrnTFd-ikONWik-T7J-sL8J\":0.3750972255899085, "
- "\"SVyEes-77mlzIr-N6c-DkYw\":0.4703053945053086, "
- "\"null\":0.1, \"null\":0.1, null:null}, {null:0.1,
null:null, \"null\":0}]"})};
+ using FieldType_RandStr =
+ std::tuple<FieldType, FieldType, std::vector<std::string>,
std::vector<std::string>,
+ std::vector<std::string>, std::vector<std::string>>;
+ std::
+ vector<FieldType_RandStr>
+ nested_field_types =
+ {
+
FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_STRING,
+
FieldType::OLAP_FIELD_TYPE_DOUBLE,
+
{R"([{"2cKtIM-L1mOcEm-udR-HcB2":0.23929040957798242,"eof2UN-Is0EEuA-H5D-hE58":0.42373055809540094,"FwUSOB-R8rtK9W-BVG-8wYZ":0.7680704548628841},{"qDXU9D-7orr51d-g80-6t5k":0.6446245786874659,"bkLjmx-uZ2Ez7F-536-PGqy":0.8880791950937957,"9Etq4o-FPm37O4-5fk-QWh7":0.08630489716260481},{"tu3OMw-mzS0jAx-Dnj-Xm3G":0.1184199213706042,"XkhTn0-QFLo8Ks-JXR-k4zk":0.5181239375482816,"EYC8Dj-GTTp9iB-b4O-QBkO":0.4491897722178303},{"sHFGPg-cfA8gy
[...]
for (auto type_pair : nested_field_types) {
auto key_type = std::get<0>(type_pair);
DataTypePtr nested_key_data_type_ptr =
@@ -1058,116 +987,15 @@ TEST(TextSerde, ComplexTypeWithNestedSerdeTextTest) {
// map-scala-array (map<scala,array<scala>>)
{
// nested type,test string, expect
string(option.converted_from_string=false), expect_from_string, expect
string(option.converted_from_string=true)
- typedef std::tuple<FieldType, FieldType, std::vector<std::string>,
std::vector<std::string>,
- std::vector<std::string>, std::vector<std::string>>
- FieldType_RandStr;
+ using FieldType_RandStr =
+ std::tuple<FieldType, FieldType, std::vector<std::string>,
std::vector<std::string>,
+ std::vector<std::string>, std::vector<std::string>>;
std::vector<FieldType_RandStr> nested_field_types = {FieldType_RandStr(
// map<string,array<double>>
FieldType::OLAP_FIELD_TYPE_STRING,
FieldType::OLAP_FIELD_TYPE_DOUBLE,
-
{"{\"5Srn6n-SP9fOS3-khz-Ljwt\":[0.8537551959339321,0.13473869413865858,0."
-
"9806016478238296,0.23014415892941564,0.26853530959759686,0.05484935641143551,0."
-
"11181328816302816,0.26510985318905933,0.6350885463275475,0.18209889263574142],"
-
"\"vrQmBC-2WlpWML-V5S-OLgM\":[0.6982221340596457,0.9260447299229463,0."
-
"12488042737255534,0.8859407191137862,0.03201490973378984,0.8371916387557367,0."
-
"7894434066323907,0.29667576138232743,0.9837777568426148,0.7773721913552772],"
-
"\"3ZbiXK-VvmhFcg-09V-w3g3\":[0.20509046053951785,0.9175575704931109,0."
-
"305788438361256,0.9923240410251069,0.6612939841907548,0.5922056063112593,0."
-
"15750800821536715,0.6374743124669565,0.4158097731627699,0.00302193321816846],"
-
"\"gMswpS-Ele9wHM-Uxp-VxzC\":[0.14378032144751685,0.627919779177473,0."
-
"6188731271454715,0.8088384184584442,0.8169160298605824,0.9051151670055427,0."
-
"558001941204895,0.029409463113641787,0.9532987674717762,0.20833228278241533],"
-
"\"TT9P9f-PXjQnvN-RBx-xRiS\":[0.8276005878909756,0.470950932860423,0."
-
"2442851528127543,0.710599416715854,0.3353731152359334,0.622947602340124,0."
-
"30675353671676797,0.8190741661938367,0.633630372770242,0.9436322366112492],"
-
"\"gLAnZc-oF7PC9o-ryd-MOXr\":[0.9742716809818137,0.9114038616933997,0."
-
"47459239268645104,0.6054569900795078,0.5515590901916287,0.8833310208917589,0."
-
"96476090778518,0.8873874315592357,0.3577701257062156,0.6993447306713452],"
-
"\"zrq6BY-7FJg3hc-Dd1-bAJn\":[0.1038405592062176,0.6757819253774818,0."
-
"6386535502499314,0.23598674876945303,0.11046582465777044,0.6426056925348297,0."
-
"17289073092250662,0.37116009951425233,0.594677969672274,0.49351456402872274],"
-
"\"gCKqtW-bLaoxgZ-CuW-M2re\":[0.934169137905867,0.12015121444469123,0."
-
"5009923777544698,0.4689139716802634,0.7226298925299507,0.33486164698864984,0."
-
"32944768657449996,0.5051366150918063,0.03228636228382431,0.48211773870118435],"
-
"\"SWqhI2-XnF9jVR-dT1-Yrtt\":[0.8005897112110444,0.899180582368993,0."
-
"9232176819588501,0.8615673086606942,0.9248122266449379,0.5586489299212893,0."
-
"40494513773898455,0.4752644689010731,0.6668395567417462,0.9068738374244337],"
-
"\"Z85F6M-cy5K4GP-7I5-5KS9\":[0.34761241187833714,0.46467162849990507,0."
-
"009781307454025168,0.3174295126364216,0.6405423361175397,0.33838144910731327,0."
-
"328860321648657,0.032638966917555856,0.32782524002924884,0.7675689545937956],"
-
"\"rlcnbo-tFg1FfP-ra6-D9Z8\":[0.7450713997349928,0.792502852203968,0."
-
"9034039182796755,0.49131654565079996,0.25223293077647946,0.9827253462450637,0."
-
"1684868582627418,0.0417161505112974,0.8498128570850716,0.8948779001812955]}"},
- {"{\"5Srn6n-SP9fOS3-khz-Ljwt\":[0.8537551959339321,
0.1347386941386586, "
- "0.9806016478238296, 0.2301441589294156, 0.2685353095975969,
0.05484935641143551, "
- "0.1118132881630282, 0.2651098531890593, 0.6350885463275475,
0.1820988926357414], "
- "\"vrQmBC-2WlpWML-V5S-OLgM\":[0.6982221340596457,
0.9260447299229463, "
- "0.1248804273725553, 0.8859407191137862, 0.03201490973378984,
0.8371916387557367, "
- "0.7894434066323907, 0.2966757613823274, 0.9837777568426148,
0.7773721913552772], "
- "\"3ZbiXK-VvmhFcg-09V-w3g3\":[0.2050904605395178,
0.9175575704931109, "
- "0.305788438361256, 0.9923240410251069, 0.6612939841907548,
0.5922056063112593, "
- "0.1575080082153671, 0.6374743124669565, 0.4158097731627699, "
- "0.00302193321816846],
\"gMswpS-Ele9wHM-Uxp-VxzC\":[0.1437803214475168, "
- "0.627919779177473, 0.6188731271454715, 0.8088384184584442,
0.8169160298605824, "
- "0.9051151670055427, 0.558001941204895, 0.02940946311364179,
0.9532987674717762, "
- "0.2083322827824153],
\"TT9P9f-PXjQnvN-RBx-xRiS\":[0.8276005878909756, "
- "0.470950932860423, 0.2442851528127543, 0.710599416715854,
0.3353731152359334, "
- "0.622947602340124, 0.306753536716768, 0.8190741661938367,
0.633630372770242, "
- "0.9436322366112492],
\"gLAnZc-oF7PC9o-ryd-MOXr\":[0.9742716809818137, "
- "0.9114038616933997, 0.474592392686451, 0.6054569900795078,
0.5515590901916287, "
- "0.8833310208917589, 0.96476090778518, 0.8873874315592357,
0.3577701257062156, "
- "0.6993447306713452],
\"zrq6BY-7FJg3hc-Dd1-bAJn\":[0.1038405592062176, "
- "0.6757819253774818, 0.6386535502499314, 0.235986748769453,
0.1104658246577704, "
- "0.6426056925348297, 0.1728907309225066, 0.3711600995142523,
0.594677969672274, "
- "0.4935145640287227],
\"gCKqtW-bLaoxgZ-CuW-M2re\":[0.934169137905867, "
- "0.1201512144446912, 0.5009923777544698, 0.4689139716802634,
0.7226298925299507, "
- "0.3348616469886498, 0.3294476865745, 0.5051366150918063,
0.03228636228382431, "
- "0.4821177387011844],
\"SWqhI2-XnF9jVR-dT1-Yrtt\":[0.8005897112110444, "
- "0.899180582368993, 0.9232176819588501, 0.8615673086606942,
0.9248122266449379, "
- "0.5586489299212893, 0.4049451377389846, 0.4752644689010731,
0.6668395567417462, "
- "0.9068738374244337],
\"Z85F6M-cy5K4GP-7I5-5KS9\":[0.3476124118783371, "
- "0.4646716284999051, 0.009781307454025168,
0.3174295126364216, "
- "0.6405423361175397, 0.3383814491073133, 0.328860321648657,
0.03263896691755586, "
- "0.3278252400292488, 0.7675689545937956], "
- "\"rlcnbo-tFg1FfP-ra6-D9Z8\":[0.7450713997349928,
0.792502852203968, "
- "0.9034039182796755, 0.4913165456508, 0.2522329307764795,
0.9827253462450637, "
- "0.1684868582627418, 0.0417161505112974, 0.8498128570850716, "
- "0.8948779001812955]}"},
+
{R"({"5Srn6n-SP9fOS3-khz-Ljwt":[0.8537551959339321,0.13473869413865858,0.9806016478238296,0.23014415892941564,0.26853530959759686,0.05484935641143551,0.11181328816302816,0.26510985318905933,0.6350885463275475,0.18209889263574142],"vrQmBC-2WlpWML-V5S-OLgM":[0.6982221340596457,0.9260447299229463,0.12488042737255534,0.8859407191137862,0.03201490973378984,0.8371916387557367,0.7894434066323907,0.29667576138232743,0.9837777568426148,0.7773721913552772],"3ZbiXK-VvmhFcg-09V-w3g3"
[...]
{""},
- {"{\"5Srn6n-SP9fOS3-khz-Ljwt\":[0.8537551959339321,
0.1347386941386586, "
- "0.9806016478238296, 0.2301441589294156, 0.2685353095975969,
0.05484935641143551, "
- "0.1118132881630282, 0.2651098531890593, 0.6350885463275475,
0.1820988926357414], "
- "\"vrQmBC-2WlpWML-V5S-OLgM\":[0.6982221340596457,
0.9260447299229463, "
- "0.1248804273725553, 0.8859407191137862, 0.03201490973378984,
0.8371916387557367, "
- "0.7894434066323907, 0.2966757613823274, 0.9837777568426148,
0.7773721913552772], "
- "\"3ZbiXK-VvmhFcg-09V-w3g3\":[0.2050904605395178,
0.9175575704931109, "
- "0.305788438361256, 0.9923240410251069, 0.6612939841907548,
0.5922056063112593, "
- "0.1575080082153671, 0.6374743124669565, 0.4158097731627699, "
- "0.00302193321816846],
\"gMswpS-Ele9wHM-Uxp-VxzC\":[0.1437803214475168, "
- "0.627919779177473, 0.6188731271454715, 0.8088384184584442,
0.8169160298605824, "
- "0.9051151670055427, 0.558001941204895, 0.02940946311364179,
0.9532987674717762, "
- "0.2083322827824153],
\"TT9P9f-PXjQnvN-RBx-xRiS\":[0.8276005878909756, "
- "0.470950932860423, 0.2442851528127543, 0.710599416715854,
0.3353731152359334, "
- "0.622947602340124, 0.306753536716768, 0.8190741661938367,
0.633630372770242, "
- "0.9436322366112492],
\"gLAnZc-oF7PC9o-ryd-MOXr\":[0.9742716809818137, "
- "0.9114038616933997, 0.474592392686451, 0.6054569900795078,
0.5515590901916287, "
- "0.8833310208917589, 0.96476090778518, 0.8873874315592357,
0.3577701257062156, "
- "0.6993447306713452],
\"zrq6BY-7FJg3hc-Dd1-bAJn\":[0.1038405592062176, "
- "0.6757819253774818, 0.6386535502499314, 0.235986748769453,
0.1104658246577704, "
- "0.6426056925348297, 0.1728907309225066, 0.3711600995142523,
0.594677969672274, "
- "0.4935145640287227],
\"gCKqtW-bLaoxgZ-CuW-M2re\":[0.934169137905867, "
- "0.1201512144446912, 0.5009923777544698, 0.4689139716802634,
0.7226298925299507, "
- "0.3348616469886498, 0.3294476865745, 0.5051366150918063,
0.03228636228382431, "
- "0.4821177387011844],
\"SWqhI2-XnF9jVR-dT1-Yrtt\":[0.8005897112110444, "
- "0.899180582368993, 0.9232176819588501, 0.8615673086606942,
0.9248122266449379, "
- "0.5586489299212893, 0.4049451377389846, 0.4752644689010731,
0.6668395567417462, "
- "0.9068738374244337],
\"Z85F6M-cy5K4GP-7I5-5KS9\":[0.3476124118783371, "
- "0.4646716284999051, 0.009781307454025168,
0.3174295126364216, "
- "0.6405423361175397, 0.3383814491073133, 0.328860321648657,
0.03263896691755586, "
- "0.3278252400292488, 0.7675689545937956], "
- "\"rlcnbo-tFg1FfP-ra6-D9Z8\":[0.7450713997349928,
0.792502852203968, "
- "0.9034039182796755, 0.4913165456508, 0.2522329307764795,
0.9827253462450637, "
- "0.1684868582627418, 0.0417161505112974, 0.8498128570850716, "
- "0.8948779001812955]}"})};
+ {R"({"5Srn6n-SP9fOS3-khz-Ljwt":[0.8537551959339321,
0.1347386941386586, 0.9806016478238296, 0.2301441589294156, 0.2685353095975969,
0.05484935641143551, 0.1118132881630282, 0.2651098531890593,
0.6350885463275475, 0.1820988926357414],
"vrQmBC-2WlpWML-V5S-OLgM":[0.6982221340596457, 0.9260447299229463,
0.1248804273725553, 0.8859407191137862, 0.03201490973378984,
0.8371916387557367, 0.7894434066323907, 0.2966757613823274, 0.9837777568426148,
0.7773721913552772], "3ZbiXK-VvmhF [...]
for (auto type_pair : nested_field_types) {
auto key_type = std::get<0>(type_pair);
DataTypePtr nested_key_data_type_ptr =
@@ -1272,40 +1100,12 @@ TEST(TextSerde, ComplexTypeWithNestedSerdeTextTest) {
// map-scala-map (map<string,map<string,double>>)
{
// nested type,test string, expect
string(option.converted_from_string=false), expect_from_string, expect
string(option.converted_from_string=true)
- typedef std::tuple<FieldType, FieldType, std::vector<std::string>,
std::vector<std::string>,
- std::vector<std::string>, std::vector<std::string>>
- FieldType_RandStr;
- std::vector<FieldType_RandStr> nested_field_types = {FieldType_RandStr(
- FieldType::OLAP_FIELD_TYPE_STRING,
FieldType::OLAP_FIELD_TYPE_DOUBLE,
-
{"{\"5H6iPe-CRvVE5Q-QnG-8WQb\":{},\"stDa6g-GML89aZ-w5u-LBe0\":{\"Vlekcq-LDCMo6f-"
-
"J7U-6rwB\":0.15375824233866453,\"4ljyNE-JMK1bSp-c05-EajL\":0.36153399717116075},"
-
"\"URvXyY-SMttaG4-Zol-mPak\":{\"xVaeqR-cj8I6EM-3Nt-queD\":0.003968938824538082,"
-
"\"Vt2mSs-wacYDvl-qUi-B7kI\":0.6900852274982441,\"i3cJJh-oskdqti-KGU-U6gC\":0."
-
"40773692843073994},\"N3R9TI-jtBPGOQ-uRc-aWAD\":{\"xmGI09-FaCFrrR-O5J-29eu\":0."
-
"7166939407858642,\"fbxIwJ-HLvW94X-tPn-JgKT\":0.05904881148976504,\"ylE7y1-"
-
"wI3UhjR-ecQ-bNfo\":0.9293354174058581,\"zA0pEV-Lm8g4wq-NJc-TDou\":0."
- "4000067127237942}}"},
- {"{\"5H6iPe-CRvVE5Q-QnG-8WQb\":{}, "
-
"\"stDa6g-GML89aZ-w5u-LBe0\":{\"Vlekcq-LDCMo6f-J7U-6rwB\":0.1537582423386645, "
- "\"4ljyNE-JMK1bSp-c05-EajL\":0.3615339971711607}, "
-
"\"URvXyY-SMttaG4-Zol-mPak\":{\"xVaeqR-cj8I6EM-3Nt-queD\":0.003968938824538082,
"
- "\"Vt2mSs-wacYDvl-qUi-B7kI\":0.6900852274982441, "
- "\"i3cJJh-oskdqti-KGU-U6gC\":0.4077369284307399}, "
-
"\"N3R9TI-jtBPGOQ-uRc-aWAD\":{\"xmGI09-FaCFrrR-O5J-29eu\":0.7166939407858642, "
- "\"fbxIwJ-HLvW94X-tPn-JgKT\":0.05904881148976504, "
- "\"ylE7y1-wI3UhjR-ecQ-bNfo\":0.9293354174058581, "
- "\"zA0pEV-Lm8g4wq-NJc-TDou\":0.4000067127237942}}"},
- {""},
- {"{\"5H6iPe-CRvVE5Q-QnG-8WQb\":{}, "
-
"\"stDa6g-GML89aZ-w5u-LBe0\":{\"Vlekcq-LDCMo6f-J7U-6rwB\":0.1537582423386645, "
- "\"4ljyNE-JMK1bSp-c05-EajL\":0.3615339971711607}, "
-
"\"URvXyY-SMttaG4-Zol-mPak\":{\"xVaeqR-cj8I6EM-3Nt-queD\":0.003968938824538082,
"
- "\"Vt2mSs-wacYDvl-qUi-B7kI\":0.6900852274982441, "
- "\"i3cJJh-oskdqti-KGU-U6gC\":0.4077369284307399}, "
-
"\"N3R9TI-jtBPGOQ-uRc-aWAD\":{\"xmGI09-FaCFrrR-O5J-29eu\":0.7166939407858642, "
- "\"fbxIwJ-HLvW94X-tPn-JgKT\":0.05904881148976504, "
- "\"ylE7y1-wI3UhjR-ecQ-bNfo\":0.9293354174058581, "
- "\"zA0pEV-Lm8g4wq-NJc-TDou\":0.4000067127237942}}"})};
+ using FieldType_RandStr =
+ std::tuple<FieldType, FieldType, std::vector<std::string>,
std::vector<std::string>,
+ std::vector<std::string>, std::vector<std::string>>;
+ std::
+ vector<FieldType_RandStr>
+ nested_field_types =
{FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_STRING,
FieldType::OLAP_FIELD_TYPE_DOUBLE,
{R"({"5H6iPe-CRvVE5Q-QnG-8WQb":{},"stDa6g-GML89aZ-w5u-LBe0":{"Vlekcq-LDCMo6f-J7U-6rwB":0.15375824233866453,"4ljyNE-JMK1bSp-c05-EajL":0.36153399717116075},"URvXyY-SMttaG4-Zol-mPak":{"xVaeqR-cj8I6EM-3Nt-queD":0.003968938824538082,"Vt2mSs-wacYDvl-qUi-B7kI":0.6900852274982441,"i3cJJh-oskdqti-KGU-U6gC":0.40773692843073994},"N3R9TI-jtBPGOQ-uRc-aWAD":{"xmGI09-FaC
[...]
for (auto type_pair : nested_field_types) {
auto key_type = std::get<0>(type_pair);
DataTypePtr nested_key_data_type_ptr =
@@ -1412,7 +1212,7 @@ TEST(TextSerde, ComplexTypeWithNestedSerdeTextTest) {
}
TEST(TextSerde, test_slice) {
- Slice slice("[\"hello\", \"world\"]");
+ Slice slice(R"(["hello", "world"])");
slice.remove_prefix(1);
slice.remove_suffix(1);
std::vector<Slice> slices;
diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp
index 00a1daa851a..a855b64e5c1 100644
--- a/be/test/vec/exprs/vexpr_test.cpp
+++ b/be/test/vec/exprs/vexpr_test.cpp
@@ -534,7 +534,7 @@ TEST(TEST_VEXPR, LITERALTEST) {
uint32_t microsecond = 999999; // target scale is 4, so the
microsecond will be rounded up
DateV2Value<DateTimeV2ValueType> datetime_v2;
datetime_v2.unchecked_set_time(year, month, day, hour, minute, second,
microsecond);
- std::string date = datetime_v2.debug_string();
+ std::string date = datetime_v2.to_string();
VLiteral literal(create_literal<TYPE_DATETIMEV2, std::string>(date,
4));
Block block;
@@ -549,11 +549,12 @@ TEST(TEST_VEXPR, LITERALTEST) {
}
// date
{
- VecDateTimeValue data_time_value;
+ VecDateTimeValue date_time_value;
+ date_time_value.set_type(TIME_DATE);
const char* date = "20210407";
- data_time_value.from_date_str(date, strlen(date));
+ date_time_value.from_date_str(date, strlen(date));
__int64_t dt;
- memcpy(&dt, &data_time_value, sizeof(__int64_t));
+ memcpy(&dt, &date_time_value, sizeof(__int64_t));
VLiteral literal(create_literal<TYPE_DATE,
std::string>(std::string(date)));
Block block;
int ret = -1;
@@ -587,6 +588,7 @@ TEST(TEST_VEXPR, LITERALTEST) {
create_texpr_node_from((*ctn.column)[0], TYPE_DATEV2, 0, 0),
true);
EXPECT_EQ("2021-04-07", node->value());
}
+ config::allow_zero_date = true;
{
DateV2Value<DateV2ValueType> data_time_value;
const char* date = "00000000";
@@ -597,7 +599,7 @@ TEST(TEST_VEXPR, LITERALTEST) {
EXPECT_EQ(data_time_value1.from_date_str(date1, strlen(date1), -1,
true), true);
EXPECT_EQ(data_time_value.to_int64(), data_time_value1.to_int64());
- EXPECT_EQ(data_time_value.from_date_str(date, strlen(date)), false);
+ EXPECT_EQ(data_time_value.from_date_str(date, strlen(date)), true);
}
{
DateV2Value<DateTimeV2ValueType> data_time_value;
@@ -609,7 +611,7 @@ TEST(TEST_VEXPR, LITERALTEST) {
EXPECT_EQ(data_time_value1.from_date_str(date1, strlen(date1), -1,
true), true);
EXPECT_EQ(data_time_value.to_int64(), data_time_value1.to_int64());
- EXPECT_EQ(data_time_value.from_date_str(date, strlen(date)), false);
+ EXPECT_EQ(data_time_value.from_date_str(date, strlen(date)), true);
}
// jsonb
{
@@ -740,4 +742,5 @@ TEST(TEST_VEXPR, LITERALTEST) {
EXPECT_EQ("1.23456789", node->value());
}
}
+ config::allow_zero_date = false;
}
diff --git a/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
b/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
index 978ed7408a2..ef67c30fb06 100644
--- a/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
+++ b/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
@@ -183,9 +183,14 @@ void test_datetime(std::shared_ptr<ArrowType> type, const
std::vector<std::strin
DataTypePtr data_type = DataTypeFactory::instance().create_data_type(pt,
true);
MutableColumnPtr data_column = data_type->create_column();
ColumnWithTypeAndName column(std::move(data_column), data_type,
"test_datatime_column");
- for (auto& value : test_cases) {
+ for (const auto& value : test_cases) {
ArrowCppType arrow_datetime =
string_to_arrow_datetime<ArrowType>(type, value);
VecDateTimeValue tv;
+ if constexpr (std::is_same_v<ArrowType, arrow::Date32Type>) {
+ tv.set_type(TimeType::TIME_DATE);
+ } else {
+ tv.set_type(TimeType::TIME_DATETIME);
+ }
tv.from_date_str(value.c_str(), value.size());
test_arrow_to_datetime_column<ArrowType, ColumnType, is_nullable>(
type, column, num_elements, arrow_datetime, tv, counter);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
