This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new f06c13a [feature](vec)(function) support function `convert_tz()`
(#8060)
f06c13a is described below
commit f06c13a82870bf115866e0afeea2b327f9f336b8
Author: Pxl <[email protected]>
AuthorDate: Thu Feb 17 10:51:32 2022 +0800
[feature](vec)(function) support function `convert_tz()` (#8060)
---
be/src/vec/CMakeLists.txt | 1 +
be/src/vec/columns/columns_number.h | 3 +
.../function_convert_tz.cpp} | 27 +----
be/src/vec/functions/function_convert_tz.h | 111 +++++++++++++++++++++
be/src/vec/functions/simple_function_factory.h | 2 +
be/test/vec/function/function_test_util.h | 29 +++---
be/test/vec/function/function_time_test.cpp | 16 +++
7 files changed, 153 insertions(+), 36 deletions(-)
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index d9d9c43..0ad0ab1 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -145,6 +145,7 @@ set(VEC_FILES
functions/function_datetime_string_to_string.cpp
functions/function_grouping.cpp
functions/function_rpc.cpp
+ functions/function_convert_tz.cpp
olap/vgeneric_iterators.cpp
olap/vcollect_iterator.cpp
olap/block_reader.cpp
diff --git a/be/src/vec/columns/columns_number.h
b/be/src/vec/columns/columns_number.h
index da3c011..500bab4 100644
--- a/be/src/vec/columns/columns_number.h
+++ b/be/src/vec/columns/columns_number.h
@@ -39,6 +39,9 @@ using ColumnInt32 = ColumnVector<Int32>;
using ColumnInt64 = ColumnVector<Int64>;
using ColumnInt128 = ColumnVector<Int128>;
+using ColumnDate = ColumnVector<Int64>;
+using ColumnDateTime = ColumnVector<Int64>;
+
using ColumnFloat32 = ColumnVector<Float32>;
using ColumnFloat64 = ColumnVector<Float64>;
diff --git a/be/src/vec/columns/columns_number.h
b/be/src/vec/functions/function_convert_tz.cpp
similarity index 53%
copy from be/src/vec/columns/columns_number.h
copy to be/src/vec/functions/function_convert_tz.cpp
index da3c011..56461db 100644
--- a/be/src/vec/columns/columns_number.h
+++ b/be/src/vec/functions/function_convert_tz.cpp
@@ -14,32 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-// This file is copied from
-//
https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnsNumber.h
-// and modified by Doris
-#pragma once
-
-#include "vec/columns/column_vector.h"
-#include "vec/core/types.h"
+#include "vec/functions/function_convert_tz.h"
namespace doris::vectorized {
-/** Columns with numbers. */
-
-using ColumnUInt8 = ColumnVector<UInt8>;
-using ColumnUInt16 = ColumnVector<UInt16>;
-using ColumnUInt32 = ColumnVector<UInt32>;
-using ColumnUInt64 = ColumnVector<UInt64>;
-using ColumnUInt128 = ColumnVector<UInt128>;
-
-using ColumnInt8 = ColumnVector<Int8>;
-using ColumnInt16 = ColumnVector<Int16>;
-using ColumnInt32 = ColumnVector<Int32>;
-using ColumnInt64 = ColumnVector<Int64>;
-using ColumnInt128 = ColumnVector<Int128>;
-
-using ColumnFloat32 = ColumnVector<Float32>;
-using ColumnFloat64 = ColumnVector<Float64>;
+void register_function_convert_tz(SimpleFunctionFactory& factory) {
+ factory.register_function<FunctionConvertTZ>();
+}
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_convert_tz.h
b/be/src/vec/functions/function_convert_tz.h
new file mode 100644
index 0000000..739279d
--- /dev/null
+++ b/be/src/vec/functions/function_convert_tz.h
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+//
https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionStringOrArrayToT.h
+// and modified by Doris
+
+#include "vec/columns/columns_number.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/types.h"
+#include "vec/functions/simple_function_factory.h"
+#include "vec/utils/util.hpp"
+
+namespace doris::vectorized {
+
+class FunctionConvertTZ : public IFunction {
+public:
+ static constexpr auto name = "convert_tz";
+
+ static FunctionPtr create() { return
std::make_shared<FunctionConvertTZ>(); }
+
+ String get_name() const override { return name; }
+
+ size_t get_number_of_arguments() const override { return 3; }
+
+ DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
+ return make_nullable(std::make_shared<DataTypeDateTime>());
+ }
+
+ bool use_default_implementation_for_constants() const override { return
true; }
+ bool use_default_implementation_for_nulls() const override { return false;
}
+
+ Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
+ size_t result, size_t input_rows_count) override {
+ auto result_column = ColumnDateTime::create();
+ auto result_null_map_column = ColumnUInt8::create(input_rows_count, 0);
+
+ ColumnPtr argument_columns[3];
+
+ for (int i = 0; i < 3; ++i) {
+ argument_columns[i] =
+
block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
+ if (auto* nullable =
check_and_get_column<ColumnNullable>(*argument_columns[i])) {
+ argument_columns[i] = nullable->get_nested_column_ptr();
+
VectorizedUtils::update_null_map(result_null_map_column->get_data(),
+
nullable->get_null_map_data());
+ }
+ }
+
+ execute_straight(context, assert_cast<const
ColumnDateTime*>(argument_columns[0].get()),
+ assert_cast<const
ColumnString*>(argument_columns[1].get()),
+ assert_cast<const
ColumnString*>(argument_columns[2].get()),
+ assert_cast<ColumnDateTime*>(result_column.get()),
+
assert_cast<ColumnUInt8*>(result_null_map_column.get())->get_data(),
+ input_rows_count);
+
+ block.get_by_position(result).column =
+ ColumnNullable::create(std::move(result_column),
std::move(result_null_map_column));
+ return Status::OK();
+ }
+
+private:
+ void execute_straight(FunctionContext* context, const ColumnDateTime*
date_column,
+ const ColumnString* from_tz_column, const
ColumnString* to_tz_column,
+ ColumnDateTime* result_column, NullMap&
result_null_map,
+ size_t input_rows_count) {
+ for (size_t i = 0; i < input_rows_count; i++) {
+ if (result_null_map[i]) {
+ result_column->insert_default();
+ continue;
+ }
+
+ StringRef from_tz = from_tz_column->get_data_at(i);
+ StringRef to_tz = to_tz_column->get_data_at(i);
+
+ VecDateTimeValue ts_value =
+ binary_cast<Int64,
VecDateTimeValue>(date_column->get_element(i));
+ int64_t timestamp;
+
+ if (!ts_value.unix_timestamp(×tamp, from_tz.to_string())) {
+ result_null_map[i] = true;
+ result_column->insert_default();
+ continue;
+ }
+
+ VecDateTimeValue ts_value2;
+ if (!ts_value2.from_unixtime(timestamp, to_tz.to_string())) {
+ result_null_map[i] = true;
+ result_column->insert_default();
+ continue;
+ }
+
+ result_column->insert(binary_cast<VecDateTimeValue,
Int64>(ts_value2));
+ }
+ }
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/functions/simple_function_factory.h
b/be/src/vec/functions/simple_function_factory.h
index ab3968c..600f84e 100644
--- a/be/src/vec/functions/simple_function_factory.h
+++ b/be/src/vec/functions/simple_function_factory.h
@@ -67,6 +67,7 @@ void register_function_regexp(SimpleFunctionFactory& factory);
void register_function_random(SimpleFunctionFactory& factory);
void register_function_coalesce(SimpleFunctionFactory& factory);
void register_function_grouping(SimpleFunctionFactory& factory);
+void register_function_convert_tz(SimpleFunctionFactory& factory);
class SimpleFunctionFactory {
using Creator = std::function<FunctionBuilderPtr()>;
@@ -183,6 +184,7 @@ public:
register_function_random(instance);
register_function_coalesce(instance);
register_function_grouping(instance);
+ register_function_convert_tz(instance);
});
return instance;
}
diff --git a/be/test/vec/function/function_test_util.h
b/be/test/vec/function/function_test_util.h
index d543178..c1dc3d1 100644
--- a/be/test/vec/function/function_test_util.h
+++ b/be/test/vec/function/function_test_util.h
@@ -40,6 +40,17 @@ namespace doris::vectorized {
using DataSet = std::vector<std::pair<std::vector<std::any>, std::any>>;
using InputTypeSet = std::vector<std::any>;
+int64_t str_to_data_time(std::string datetime_str, bool data_time = true) {
+ VecDateTimeValue v;
+ v.from_date_str(datetime_str.c_str(), datetime_str.size());
+ if (data_time) { //bool data_time only to simplifly means data_time or
data to cast, just use in time-functions uint test
+ v.to_datetime();
+ } else {
+ v.cast_to_date();
+ }
+ return binary_cast<VecDateTimeValue, Int64>(v);
+}
+
namespace ut_type {
using TINYINT = int8_t;
using SMALLINT = int16_t;
@@ -53,19 +64,11 @@ using STRING = std::string;
using DOUBLE = double;
using FLOAT = float;
+
inline auto DECIMAL = Decimal<Int128>::double_to_decimal;
-} // namespace ut_type
-int64_t str_to_data_time(std::string datetime_str, bool data_time = true) {
- VecDateTimeValue v;
- v.from_date_str(datetime_str.c_str(), datetime_str.size());
- if (data_time) { //bool data_time only to simplifly means data_time or
data to cast, just use in time-functions uint test
- v.to_datetime();
- } else {
- v.cast_to_date();
- }
- return binary_cast<VecDateTimeValue, Int64>(v);
-}
+using DATETIME = std::string;
+} // namespace ut_type
template <typename ColumnType, typename Column, typename NullColumn>
void insert_column_to_block(std::list<ColumnPtr>& columns,
ColumnsWithTypeAndName& ctn,
@@ -250,8 +253,8 @@ void check_function(const std::string& func_name, const
std::vector<std::any>& i
col->insert_data(reinterpret_cast<char*>(&value), 0);
}
insert_column_to_block<DataTypeDecimal<Decimal128>>(columns, ctn,
std::move(col),
- std::move(null_map),
block, col_name, i,
- is_const, row_size);
+
std::move(null_map), block,
+ col_name, i,
is_const, row_size);
arg_type.type = doris_udf::FunctionContext::TYPE_DECIMALV2;
} else if (tp == TypeIndex::DateTime) {
static std::string date_time_format("%Y-%m-%d %H:%i:%s");
diff --git a/be/test/vec/function/function_time_test.cpp
b/be/test/vec/function/function_time_test.cpp
index 31b2350..7e45295 100644
--- a/be/test/vec/function/function_time_test.cpp
+++ b/be/test/vec/function/function_time_test.cpp
@@ -29,6 +29,7 @@
#include "vec/functions/simple_function_factory.h"
namespace doris::vectorized {
+using namespace ut_type;
TEST(TimestampFunctionsTest, day_of_week_test) {
std::string func_name = "dayofweek";
@@ -535,6 +536,21 @@ TEST(TimestampFunctionsTest, makedate_test) {
check_function<DataTypeDate, true>(func_name, input_types, data_set);
}
+
+TEST(TimestampFunctionsTest, convert_tz_test) {
+ std::string func_name = "convert_tz";
+
+ InputTypeSet input_types = {TypeIndex::DateTime, TypeIndex::String,
TypeIndex::String};
+
+ DataSet data_set = {
+ {{DATETIME("2019-08-01 13:21:03"), STRING("Asia/Shanghai"),
+ STRING("America/Los_Angeles")},
+ str_to_data_time("2019-07-31 22:21:03", true)},
+ {{DATETIME("2019-08-01 13:21:03"), STRING("+08:00"),
STRING("America/Los_Angeles")},
+ str_to_data_time("2019-07-31 22:21:03", true)}};
+
+ check_function<DataTypeDate, true>(func_name, input_types, data_set);
+}
} // namespace doris::vectorized
int main(int argc, char** argv) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
