This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 58f8cd61fec [fix](be) Preserve DATE type in min/max runtime filters
(#70222)
58f8cd61fec is described below
commit 58f8cd61fec797084489f270319ba2b88203e74a
Author: Jerry Hu <[email protected]>
AuthorDate: Thu Nov 5 17:15:48 2026 +0800
[fix](be) Preserve DATE type in min/max runtime filters (#70222)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #69963
Problem Summary: When a DATE min/max runtime filter is received through
the protobuf path, its bounds are decoded with the DATETIME target
because `TYPE_DATE` and `TYPE_DATETIME` share one branch. This changes
the DATE bounds' internal type tags to DATETIME. The consumer then
creates DATETIME literals for DATE scan predicates, which can abort the
BE on a Field type mismatch.
Decode `TYPE_DATE` bounds with the DATE target in the shared min/max
receive path. Keep the existing `TYPE_DATETIME` decoding unchanged. The
wire format and other runtime-filter types are unchanged.
### Release note
Fix a BE crash when receiving DATEV1 min/max runtime filters.
### Check List (For Author)
- Test:
- ASAN BE+FE build (`BUILD_TYPE=ASAN ./build.sh --be --fe -j 40`):
passed, including FE checkstyle.
- ASAN `RuntimeFilterWrapperTest.*`: 15 cases passed, including
DATE/DATETIME type, min/max mode, and merge/publish request coverage.
- `test_datev1_minmax_runtime_filter`: generated expected output and
passed a separate verification run; `test_datev1_common`: passed.
- C++ formatting, build hygiene, and changed-line clang-tidy checks:
passed.
- SQL regression ran on a single-BE cluster; cross-BE and mixed-version
execution were not exercised.
- Behavior changed: Yes. Received DATE min/max bounds retain the DATE
type instead of being converted to DATETIME.
- Does this need documentation: No
---
.../exec/runtime_filter/runtime_filter_wrapper.cpp | 16 ++-
.../runtime_filter/runtime_filter_wrapper_test.cpp | 128 +++++++++++++++++++++
.../datev1/test_datev1_minmax_runtime_filter.out | 8 ++
.../test_datev1_minmax_runtime_filter.groovy | 82 +++++++++++++
4 files changed, 232 insertions(+), 2 deletions(-)
diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
index 66fa692b65a..9125b250c32 100644
--- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
+++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
@@ -561,8 +561,7 @@ Status RuntimeFilterWrapper::_assign(const PMinMaxFilter&
minmax_filter, bool co
int64_t max_val = minmax_filter.max_val().longval();
return _minmax_func->assign(&min_val, &max_val);
}
- case TYPE_DATETIME:
- case TYPE_DATE: {
+ case TYPE_DATETIME: {
const auto& min_val_ref = minmax_filter.min_val().stringval();
const auto& max_val_ref = minmax_filter.max_val().stringval();
VecDateTimeValue min_val;
@@ -574,6 +573,19 @@ Status RuntimeFilterWrapper::_assign(const PMinMaxFilter&
minmax_filter, bool co
{max_val_ref.c_str(), max_val_ref.length()}, max_val, nullptr,
params);
return _minmax_func->assign(&min_val, &max_val);
}
+ case TYPE_DATE: {
+ const auto& min_val_ref = minmax_filter.min_val().stringval();
+ const auto& max_val_ref = minmax_filter.max_val().stringval();
+ VecDateTimeValue min_val;
+ VecDateTimeValue max_val;
+ CastParameters params;
+ // Preserve the DATE tag so the consumer creates DATE literals for
DATE scan predicates.
+
CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE>(
+ {min_val_ref.c_str(), min_val_ref.length()}, min_val, nullptr,
params);
+
CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE>(
+ {max_val_ref.c_str(), max_val_ref.length()}, max_val, nullptr,
params);
+ return _minmax_func->assign(&min_val, &max_val);
+ }
case TYPE_DECIMALV2: {
const auto& min_val_ref = minmax_filter.min_val().stringval();
const auto& max_val_ref = minmax_filter.max_val().stringval();
diff --git a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
index f31fcc68d67..fef8ed15ea9 100644
--- a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
@@ -20,12 +20,21 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
+#include <string>
+#include <utility>
+
#include "core/data_type/data_type_date.h"
+#include "core/data_type/data_type_date_time.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
+#include "core/field.h"
+#include "core/value/vdatetime_value.h"
+#include "exec/runtime_filter/utils.h"
#include "exprs/bloom_filter_func.h"
#include "exprs/hybrid_set.h"
#include "exprs/minmax_predicate.h"
+#include "exprs/vexpr.h"
+#include "exprs/vliteral.h"
#include "testutil/column_helper.h"
#include "util/raw_value.h"
@@ -361,6 +370,125 @@ TEST_F(RuntimeFilterWrapperTest, TestMinMaxAssign) {
#undef APPLY_FOR_PRIMITIVE_BASE_TYPE
}
+TEST_F(RuntimeFilterWrapperTest, DateMinMaxAssignPreservesDateType) {
+ RuntimeFilterParams params {.filter_id = 0,
+ .filter_type =
RuntimeFilterType::MINMAX_FILTER,
+ .column_return_type = TYPE_DATE,
+ .null_aware = false};
+ PMergeFilterRequest request;
+ request.set_filter_type(PFilterType::MINMAX_FILTER);
+
request.mutable_minmax_filter()->mutable_min_val()->set_stringval("2012-01-02");
+
request.mutable_minmax_filter()->mutable_max_val()->set_stringval("2012-03-02");
+
+ RuntimeFilterWrapper wrapper(¶ms);
+ ASSERT_TRUE(wrapper.assign(request, nullptr).ok());
+
EXPECT_EQ(static_cast<VecDateTimeValue*>(wrapper.minmax_func()->get_min())->type(),
TIME_DATE);
+
EXPECT_EQ(static_cast<VecDateTimeValue*>(wrapper.minmax_func()->get_max())->type(),
TIME_DATE);
+}
+
+static void check_date_like_minmax_bound(const void* bound, TimeType time_type,
+ PrimitiveType column_type, const
std::string& text,
+ const DataTypePtr& data_type) {
+ const auto* value = static_cast<const VecDateTimeValue*>(bound);
+ EXPECT_EQ(value->type(), time_type);
+ char buffer[30];
+ value->to_string(buffer);
+ EXPECT_EQ(std::string(buffer), text);
+
+ VExprSPtr literal;
+ ASSERT_TRUE(create_literal(data_type, value, literal).ok());
+ ASSERT_TRUE(literal->is_literal());
+ EXPECT_EQ(literal->result_type(), column_type);
+ Field field;
+ static_cast<VLiteral*>(literal.get())->get_column_ptr()->get(0, field);
+ EXPECT_EQ(field.get_type(), column_type);
+}
+
+template <typename Request>
+static void check_date_like_minmax_request(RuntimeFilterWrapper& producer,
+ const RuntimeFilterParams& params,
TimeType time_type,
+ const std::string& min_text, const
std::string& max_text,
+ const DataTypePtr& data_type) {
+ Request request;
+ request.set_filter_type(get_type(params.filter_type));
+ ASSERT_TRUE(producer.to_protobuf(request.mutable_minmax_filter()).ok());
+ RuntimeFilterWrapper receiver(¶ms);
+ ASSERT_TRUE(receiver.assign(request, nullptr).ok());
+ EXPECT_EQ(receiver.get_state(), RuntimeFilterWrapper::State::READY);
+ EXPECT_EQ(receiver.column_type(), params.column_return_type);
+ // MIN_FILTER and MAX_FILTER serialize only their owned bound, so the
other bound is absent.
+ if (params.filter_type != RuntimeFilterType::MAX_FILTER) {
+ check_date_like_minmax_bound(receiver.minmax_func()->get_min(),
time_type,
+ params.column_return_type, min_text,
data_type);
+ }
+ if (params.filter_type != RuntimeFilterType::MIN_FILTER) {
+ check_date_like_minmax_bound(receiver.minmax_func()->get_max(),
time_type,
+ params.column_return_type, max_text,
data_type);
+ }
+}
+
+static void check_date_like_minmax_round_trip(
+ RuntimeFilterType filter_type, PrimitiveType column_type, TimeType
time_type,
+ VecDateTimeValue min_value, VecDateTimeValue max_value, const
std::string& min_text,
+ const std::string& max_text, const DataTypePtr& data_type) {
+ RuntimeFilterParams params {.filter_id = 0,
+ .filter_type = filter_type,
+ .column_return_type = column_type,
+ .null_aware = false};
+ RuntimeFilterWrapper producer(¶ms);
+ ASSERT_TRUE(producer.minmax_func()->assign(&min_value, &max_value).ok());
+ check_date_like_minmax_request<PMergeFilterRequest>(producer, params,
time_type, min_text,
+ max_text, data_type);
+ check_date_like_minmax_request<PPublishFilterRequestV2>(producer, params,
time_type, min_text,
+ max_text,
data_type);
+}
+
+TEST_F(RuntimeFilterWrapperTest, DateLikeMinMaxRoundTripPreservesType) {
+ for (const auto filter_type : {RuntimeFilterType::MIN_FILTER,
RuntimeFilterType::MAX_FILTER,
+ RuntimeFilterType::MINMAX_FILTER}) {
+ check_date_like_minmax_round_trip(filter_type, TYPE_DATE, TIME_DATE,
+ VecDateTimeValue(0, TIME_DATE, 0, 0,
0, 2012, 1, 2),
+ VecDateTimeValue(0, TIME_DATE, 0, 0,
0, 2012, 3, 2),
+ "2012-01-02", "2012-03-02",
+ std::make_shared<DataTypeDate>());
+ check_date_like_minmax_round_trip(filter_type, TYPE_DATETIME,
TIME_DATETIME,
+ VecDateTimeValue(0, TIME_DATETIME,
1, 2, 3, 2012, 1, 2),
+ VecDateTimeValue(0, TIME_DATETIME,
4, 5, 6, 2012, 3, 2),
+ "2012-01-02 01:02:03", "2012-03-02
04:05:06",
+
std::make_shared<DataTypeDateTime>());
+ }
+}
+
+TEST_F(RuntimeFilterWrapperTest, DateMinMaxAssignAcceptsMidnightDateTimeText) {
+ RuntimeFilterParams params {.filter_id = 0,
+ .filter_type =
RuntimeFilterType::MINMAX_FILTER,
+ .column_return_type = TYPE_DATE,
+ .null_aware = false};
+ PMergeFilterRequest request;
+ request.set_filter_type(PFilterType::MINMAX_FILTER);
+
request.mutable_minmax_filter()->mutable_min_val()->set_stringval("2012-01-02
00:00:00");
+
request.mutable_minmax_filter()->mutable_max_val()->set_stringval("2012-03-02
00:00:00");
+
+ RuntimeFilterWrapper wrapper(¶ms);
+ ASSERT_TRUE(wrapper.assign(request, nullptr).ok());
+ auto data_type = std::make_shared<DataTypeDate>();
+ for (const auto& [bound, expected] :
+ {std::pair {wrapper.minmax_func()->get_min(), "2012-01-02"},
+ std::pair {wrapper.minmax_func()->get_max(), "2012-03-02"}}) {
+ const auto* value = static_cast<const VecDateTimeValue*>(bound);
+ EXPECT_EQ(value->type(), TIME_DATE);
+ char buffer[30];
+ value->to_string(buffer);
+ EXPECT_EQ(std::string(buffer), expected);
+ VExprSPtr literal;
+ ASSERT_TRUE(create_literal(data_type, value, literal).ok());
+ EXPECT_EQ(literal->result_type(), TYPE_DATE);
+ Field field;
+ static_cast<VLiteral*>(literal.get())->get_column_ptr()->get(0, field);
+ EXPECT_EQ(field.get_type(), TYPE_DATE);
+ }
+}
+
TEST_F(RuntimeFilterWrapperTest, TestBloom) {
std::vector<int> data_vector(10);
std::iota(data_vector.begin(), data_vector.end(), 0);
diff --git
a/regression-test/data/datatype_p0/datev1/test_datev1_minmax_runtime_filter.out
b/regression-test/data/datatype_p0/datev1/test_datev1_minmax_runtime_filter.out
new file mode 100644
index 00000000000..696ea959edf
--- /dev/null
+++
b/regression-test/data/datatype_p0/datev1/test_datev1_minmax_runtime_filter.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !with_minmax --
+2 2012-02-02 2 2012-02-02
+3 2012-03-02 3 2012-03-02
+
+-- !without_runtime_filter --
+2 2012-02-02 2 2012-02-02
+3 2012-03-02 3 2012-03-02
diff --git
a/regression-test/suites/datatype_p0/datev1/test_datev1_minmax_runtime_filter.groovy
b/regression-test/suites/datatype_p0/datev1/test_datev1_minmax_runtime_filter.groovy
new file mode 100644
index 00000000000..5ae4e0767ba
--- /dev/null
+++
b/regression-test/suites/datatype_p0/datev1/test_datev1_minmax_runtime_filter.groovy
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_datev1_minmax_runtime_filter", "nonConcurrent") {
+ def disableDatev1 = sql("ADMIN SHOW FRONTEND CONFIG LIKE
'disable_datev1'")[0][1]
+ def enableDateConversion = sql("ADMIN SHOW FRONTEND CONFIG LIKE
'enable_date_conversion'")[0][1]
+ def sessionVars = [
+ "runtime_filter_type", "runtime_filter_mode",
"runtime_filter_wait_infinitely",
+ "enable_runtime_filter_prune", "disable_join_reorder",
"parallel_pipeline_task_num"
+ ]
+ def previous = sessionVars.collectEntries { name -> [(name): sql("SELECT
@@${name}")[0][0]] }
+
+ try {
+ sql 'ADMIN SET FRONTEND CONFIG ("disable_datev1" = "false")'
+ sql 'ADMIN SET FRONTEND CONFIG ("enable_date_conversion" = "false")'
+
+ sql "DROP TABLE IF EXISTS test_datev1_minmax_runtime_filter"
+ sql """
+ CREATE TABLE test_datev1_minmax_runtime_filter (
+ id INT,
+ d DATEV1 NULL
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 2
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_datev1_minmax_runtime_filter VALUES
+ (1, '2012-01-02'), (2, '2012-02-02'), (3, '2012-03-02'),
+ (4, '2013-01-01'), (5, NULL)
+ """
+
+ sql "SET runtime_filter_type = 4"
+ sql "SET runtime_filter_mode = 'GLOBAL'"
+ sql "SET runtime_filter_wait_infinitely = true"
+ sql "SET enable_runtime_filter_prune = false"
+ sql "SET disable_join_reorder = true"
+ sql "SET parallel_pipeline_task_num = 2"
+
+ def query = """
+ SELECT a.id, a.d, b.id, b.d
+ FROM test_datev1_minmax_runtime_filter a
+ INNER JOIN [shuffle] test_datev1_minmax_runtime_filter b ON a.d =
b.d
+ WHERE b.id IN (2, 3)
+ ORDER BY a.id, a.d, b.id, b.d
+ """
+ explain {
+ sql query
+ contains "HASH JOIN"
+ contains "PARTITIONED"
+ contains "VEXCHANGE"
+ contains "[min_max] -> d"
+ }
+ qt_with_minmax query
+
+ sql "SET runtime_filter_type = 0"
+ qt_without_runtime_filter query
+ } finally {
+ previous.each { name, value ->
+ if (name == "runtime_filter_type" || name ==
"runtime_filter_mode") {
+ sql "SET ${name} = '${value}'"
+ } else {
+ sql "SET ${name} = ${value}"
+ }
+ }
+ sql "ADMIN SET FRONTEND CONFIG (\"enable_date_conversion\" =
\"${enableDateConversion}\")"
+ sql "ADMIN SET FRONTEND CONFIG (\"disable_datev1\" =
\"${disableDatev1}\")"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
