This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new a43275467ad branch-4.2: [fix](be) Preserve DATE type in min/max
runtime filters (#70333)
a43275467ad is described below
commit a43275467ad93edf931a0596b5e276c057b327a5
Author: Jerry Hu <[email protected]>
AuthorDate: Thu Nov 19 18:05:17 2026 +0800
branch-4.2: [fix](be) Preserve DATE type in min/max runtime filters (#70333)
### What problem does this PR solve?
Issue Number: None
Related PR: #70222
DATE min/max runtime-filter bounds were decoded as DATETIME when
received from protobuf. This could give DATE bounds a DATETIME type tag
and cause a type mismatch when building DATE scan predicates.
### What changes are included?
- Backport the DATE receive-path fix from #70222 to branch-4.2. DATE
bounds are now decoded with the DATE target type; DATETIME and other
types retain their existing behavior.
- Add focused BE unit coverage for DATE/DATETIME min/max filter round
trips and a DATEV1 SQL regression.
### Tests
- ASAN BE+FE build, including FE Checkstyle: passed.
- ASAN `RuntimeFilterWrapperTest.*`: 11 tests passed.
- `test_datev1_minmax_runtime_filter` regression: passed.
- `test_datev1_common` regression with DATEV1 enabled: passed.
- `clang-format.sh`, `check-format.sh`, and `git diff --check`: passed.
### Release note
Fix DATEV1 min/max runtime-filter bounds being treated as DATETIME on
receipt.
---
.../exec/runtime_filter/runtime_filter_wrapper.cpp | 16 ++-
.../runtime_filter/runtime_filter_wrapper_test.cpp | 130 +++++++++++++++++++++
.../datev1/test_datev1_minmax_runtime_filter.out | 8 ++
.../test_datev1_minmax_runtime_filter.groovy | 82 +++++++++++++
4 files changed, 234 insertions(+), 2 deletions(-)
diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
index a0049fe82f5..079d87ad487 100644
--- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
+++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
@@ -475,8 +475,7 @@ Status RuntimeFilterWrapper::_assign(const PMinMaxFilter&
minmax_filter, bool co
int64_t max_val = minmax_filter.max_val().longval();
return _minmax_func->assign(&min_val, &max_val);
}
- case TYPE_DATETIME:
- case TYPE_DATE: {
+ case TYPE_DATETIME: {
const auto& min_val_ref = minmax_filter.min_val().stringval();
const auto& max_val_ref = minmax_filter.max_val().stringval();
VecDateTimeValue min_val;
@@ -488,6 +487,19 @@ Status RuntimeFilterWrapper::_assign(const PMinMaxFilter&
minmax_filter, bool co
{max_val_ref.c_str(), max_val_ref.length()}, max_val, nullptr,
params);
return _minmax_func->assign(&min_val, &max_val);
}
+ case TYPE_DATE: {
+ const auto& min_val_ref = minmax_filter.min_val().stringval();
+ const auto& max_val_ref = minmax_filter.max_val().stringval();
+ VecDateTimeValue min_val;
+ VecDateTimeValue max_val;
+ CastParameters params;
+ // Preserve the DATE tag so the consumer creates DATE literals for
DATE scan predicates.
+
CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE>(
+ {min_val_ref.c_str(), min_val_ref.length()}, min_val, nullptr,
params);
+
CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE>(
+ {max_val_ref.c_str(), max_val_ref.length()}, max_val, nullptr,
params);
+ return _minmax_func->assign(&min_val, &max_val);
+ }
case TYPE_DECIMALV2: {
const auto& min_val_ref = minmax_filter.min_val().stringval();
const auto& max_val_ref = minmax_filter.max_val().stringval();
diff --git a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
index 27a9e6507f5..842bce360da 100644
--- a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
@@ -20,10 +20,21 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
+#include <string>
+#include <utility>
+
+#include "core/data_type/data_type_date.h"
+#include "core/data_type/data_type_date_time.h"
#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/field.h"
+#include "core/value/vdatetime_value.h"
+#include "exec/runtime_filter/utils.h"
#include "exprs/bloom_filter_func.h"
#include "exprs/hybrid_set.h"
#include "exprs/minmax_predicate.h"
+#include "exprs/vexpr.h"
+#include "exprs/vliteral.h"
#include "testutil/column_helper.h"
namespace doris {
@@ -329,6 +340,125 @@ TEST_F(RuntimeFilterWrapperTest, TestMinMaxAssign) {
#undef APPLY_FOR_PRIMITIVE_BASE_TYPE
}
+TEST_F(RuntimeFilterWrapperTest, DateMinMaxAssignPreservesDateType) {
+ RuntimeFilterParams params {.filter_id = 0,
+ .filter_type =
RuntimeFilterType::MINMAX_FILTER,
+ .column_return_type = TYPE_DATE,
+ .null_aware = false};
+ PMergeFilterRequest request;
+ request.set_filter_type(PFilterType::MINMAX_FILTER);
+
request.mutable_minmax_filter()->mutable_min_val()->set_stringval("2012-01-02");
+
request.mutable_minmax_filter()->mutable_max_val()->set_stringval("2012-03-02");
+
+ RuntimeFilterWrapper wrapper(¶ms);
+ ASSERT_TRUE(wrapper.assign(request, nullptr).ok());
+
EXPECT_EQ(static_cast<VecDateTimeValue*>(wrapper.minmax_func()->get_min())->type(),
TIME_DATE);
+
EXPECT_EQ(static_cast<VecDateTimeValue*>(wrapper.minmax_func()->get_max())->type(),
TIME_DATE);
+}
+
+static void check_date_like_minmax_bound(const void* bound, TimeType time_type,
+ PrimitiveType column_type, const
std::string& text,
+ const DataTypePtr& data_type) {
+ const auto* value = static_cast<const VecDateTimeValue*>(bound);
+ EXPECT_EQ(value->type(), time_type);
+ char buffer[30];
+ value->to_string(buffer);
+ EXPECT_EQ(std::string(buffer), text);
+
+ VExprSPtr literal;
+ ASSERT_TRUE(create_literal(data_type, value, literal).ok());
+ ASSERT_TRUE(literal->is_literal());
+ EXPECT_EQ(literal->result_type(), column_type);
+ Field field;
+ static_cast<VLiteral*>(literal.get())->get_column_ptr()->get(0, field);
+ EXPECT_EQ(field.get_type(), column_type);
+}
+
+template <typename Request>
+static void check_date_like_minmax_request(RuntimeFilterWrapper& producer,
+ const RuntimeFilterParams& params,
TimeType time_type,
+ const std::string& min_text, const
std::string& max_text,
+ const DataTypePtr& data_type) {
+ Request request;
+ request.set_filter_type(get_type(params.filter_type));
+ ASSERT_TRUE(producer.to_protobuf(request.mutable_minmax_filter()).ok());
+ RuntimeFilterWrapper receiver(¶ms);
+ ASSERT_TRUE(receiver.assign(request, nullptr).ok());
+ EXPECT_EQ(receiver.get_state(), RuntimeFilterWrapper::State::READY);
+ EXPECT_EQ(receiver.column_type(), params.column_return_type);
+ // MIN_FILTER and MAX_FILTER serialize only their owned bound, so the
other bound is absent.
+ if (params.filter_type != RuntimeFilterType::MAX_FILTER) {
+ check_date_like_minmax_bound(receiver.minmax_func()->get_min(),
time_type,
+ params.column_return_type, min_text,
data_type);
+ }
+ if (params.filter_type != RuntimeFilterType::MIN_FILTER) {
+ check_date_like_minmax_bound(receiver.minmax_func()->get_max(),
time_type,
+ params.column_return_type, max_text,
data_type);
+ }
+}
+
+static void check_date_like_minmax_round_trip(
+ RuntimeFilterType filter_type, PrimitiveType column_type, TimeType
time_type,
+ VecDateTimeValue min_value, VecDateTimeValue max_value, const
std::string& min_text,
+ const std::string& max_text, const DataTypePtr& data_type) {
+ RuntimeFilterParams params {.filter_id = 0,
+ .filter_type = filter_type,
+ .column_return_type = column_type,
+ .null_aware = false};
+ RuntimeFilterWrapper producer(¶ms);
+ ASSERT_TRUE(producer.minmax_func()->assign(&min_value, &max_value).ok());
+ check_date_like_minmax_request<PMergeFilterRequest>(producer, params,
time_type, min_text,
+ max_text, data_type);
+ check_date_like_minmax_request<PPublishFilterRequestV2>(producer, params,
time_type, min_text,
+ max_text,
data_type);
+}
+
+TEST_F(RuntimeFilterWrapperTest, DateLikeMinMaxRoundTripPreservesType) {
+ for (const auto filter_type : {RuntimeFilterType::MIN_FILTER,
RuntimeFilterType::MAX_FILTER,
+ RuntimeFilterType::MINMAX_FILTER}) {
+ check_date_like_minmax_round_trip(filter_type, TYPE_DATE, TIME_DATE,
+ VecDateTimeValue(0, TIME_DATE, 0, 0,
0, 2012, 1, 2),
+ VecDateTimeValue(0, TIME_DATE, 0, 0,
0, 2012, 3, 2),
+ "2012-01-02", "2012-03-02",
+ std::make_shared<DataTypeDate>());
+ check_date_like_minmax_round_trip(filter_type, TYPE_DATETIME,
TIME_DATETIME,
+ VecDateTimeValue(0, TIME_DATETIME,
1, 2, 3, 2012, 1, 2),
+ VecDateTimeValue(0, TIME_DATETIME,
4, 5, 6, 2012, 3, 2),
+ "2012-01-02 01:02:03", "2012-03-02
04:05:06",
+
std::make_shared<DataTypeDateTime>());
+ }
+}
+
+TEST_F(RuntimeFilterWrapperTest, DateMinMaxAssignAcceptsMidnightDateTimeText) {
+ RuntimeFilterParams params {.filter_id = 0,
+ .filter_type =
RuntimeFilterType::MINMAX_FILTER,
+ .column_return_type = TYPE_DATE,
+ .null_aware = false};
+ PMergeFilterRequest request;
+ request.set_filter_type(PFilterType::MINMAX_FILTER);
+
request.mutable_minmax_filter()->mutable_min_val()->set_stringval("2012-01-02
00:00:00");
+
request.mutable_minmax_filter()->mutable_max_val()->set_stringval("2012-03-02
00:00:00");
+
+ RuntimeFilterWrapper wrapper(¶ms);
+ ASSERT_TRUE(wrapper.assign(request, nullptr).ok());
+ auto data_type = std::make_shared<DataTypeDate>();
+ for (const auto& [bound, expected] :
+ {std::pair {wrapper.minmax_func()->get_min(), "2012-01-02"},
+ std::pair {wrapper.minmax_func()->get_max(), "2012-03-02"}}) {
+ const auto* value = static_cast<const VecDateTimeValue*>(bound);
+ EXPECT_EQ(value->type(), TIME_DATE);
+ char buffer[30];
+ value->to_string(buffer);
+ EXPECT_EQ(std::string(buffer), expected);
+ VExprSPtr literal;
+ ASSERT_TRUE(create_literal(data_type, value, literal).ok());
+ EXPECT_EQ(literal->result_type(), TYPE_DATE);
+ Field field;
+ static_cast<VLiteral*>(literal.get())->get_column_ptr()->get(0, field);
+ EXPECT_EQ(field.get_type(), TYPE_DATE);
+ }
+}
+
TEST_F(RuntimeFilterWrapperTest, TestBloom) {
std::vector<int> data_vector(10);
std::iota(data_vector.begin(), data_vector.end(), 0);
diff --git
a/regression-test/data/datatype_p0/datev1/test_datev1_minmax_runtime_filter.out
b/regression-test/data/datatype_p0/datev1/test_datev1_minmax_runtime_filter.out
new file mode 100644
index 00000000000..696ea959edf
--- /dev/null
+++
b/regression-test/data/datatype_p0/datev1/test_datev1_minmax_runtime_filter.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !with_minmax --
+2 2012-02-02 2 2012-02-02
+3 2012-03-02 3 2012-03-02
+
+-- !without_runtime_filter --
+2 2012-02-02 2 2012-02-02
+3 2012-03-02 3 2012-03-02
diff --git
a/regression-test/suites/datatype_p0/datev1/test_datev1_minmax_runtime_filter.groovy
b/regression-test/suites/datatype_p0/datev1/test_datev1_minmax_runtime_filter.groovy
new file mode 100644
index 00000000000..5ae4e0767ba
--- /dev/null
+++
b/regression-test/suites/datatype_p0/datev1/test_datev1_minmax_runtime_filter.groovy
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_datev1_minmax_runtime_filter", "nonConcurrent") {
+ def disableDatev1 = sql("ADMIN SHOW FRONTEND CONFIG LIKE
'disable_datev1'")[0][1]
+ def enableDateConversion = sql("ADMIN SHOW FRONTEND CONFIG LIKE
'enable_date_conversion'")[0][1]
+ def sessionVars = [
+ "runtime_filter_type", "runtime_filter_mode",
"runtime_filter_wait_infinitely",
+ "enable_runtime_filter_prune", "disable_join_reorder",
"parallel_pipeline_task_num"
+ ]
+ def previous = sessionVars.collectEntries { name -> [(name): sql("SELECT
@@${name}")[0][0]] }
+
+ try {
+ sql 'ADMIN SET FRONTEND CONFIG ("disable_datev1" = "false")'
+ sql 'ADMIN SET FRONTEND CONFIG ("enable_date_conversion" = "false")'
+
+ sql "DROP TABLE IF EXISTS test_datev1_minmax_runtime_filter"
+ sql """
+ CREATE TABLE test_datev1_minmax_runtime_filter (
+ id INT,
+ d DATEV1 NULL
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 2
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_datev1_minmax_runtime_filter VALUES
+ (1, '2012-01-02'), (2, '2012-02-02'), (3, '2012-03-02'),
+ (4, '2013-01-01'), (5, NULL)
+ """
+
+ sql "SET runtime_filter_type = 4"
+ sql "SET runtime_filter_mode = 'GLOBAL'"
+ sql "SET runtime_filter_wait_infinitely = true"
+ sql "SET enable_runtime_filter_prune = false"
+ sql "SET disable_join_reorder = true"
+ sql "SET parallel_pipeline_task_num = 2"
+
+ def query = """
+ SELECT a.id, a.d, b.id, b.d
+ FROM test_datev1_minmax_runtime_filter a
+ INNER JOIN [shuffle] test_datev1_minmax_runtime_filter b ON a.d =
b.d
+ WHERE b.id IN (2, 3)
+ ORDER BY a.id, a.d, b.id, b.d
+ """
+ explain {
+ sql query
+ contains "HASH JOIN"
+ contains "PARTITIONED"
+ contains "VEXCHANGE"
+ contains "[min_max] -> d"
+ }
+ qt_with_minmax query
+
+ sql "SET runtime_filter_type = 0"
+ qt_without_runtime_filter query
+ } finally {
+ previous.each { name, value ->
+ if (name == "runtime_filter_type" || name ==
"runtime_filter_mode") {
+ sql "SET ${name} = '${value}'"
+ } else {
+ sql "SET ${name} = ${value}"
+ }
+ }
+ sql "ADMIN SET FRONTEND CONFIG (\"enable_date_conversion\" =
\"${enableDateConversion}\")"
+ sql "ADMIN SET FRONTEND CONFIG (\"disable_datev1\" =
\"${disableDatev1}\")"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]