This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new aa68b2593e2 [fix](be) Reset null flag for mark join no-match sentinel
(#69762)
aa68b2593e2 is described below
commit aa68b2593e298b4db18800129ea2fa4635928de2
Author: Jerry Hu <[email protected]>
AuthorDate: Fri Oct 30 10:50:11 2026 +0800
[fix](be) Reset null flag for mark join no-match sentinel (#69762)
### What problem does this PR solve?
Issue Number: N/A
Related PR: N/A
Problem Summary: In a null-aware mark join with an additional join
predicate, real build candidates receive a null flag, but the no-match
sentinel did not. A reused probe buffer can retain a null flag from a
previous NULL-key probe, causing a correlated `IN` over an empty result
to return `NULL` instead of `FALSE`. Initialize the sentinel flag to
zero when it is emitted, while preserving the flags of real candidates.
Add a focused hash-table unit test and an ordered SQL regression
covering correlated `IN` and `NOT IN`.
### Release note
Fix incorrect NULL results for correlated IN/NOT IN when the correlated
subquery is empty after a nullable probe.
### Check List (For Author)
- Test
- [x] Regression test: `mark_join_correlated_empty` passed against an
isolated local ASAN FE/BE cluster, including a final rerun against
generated expected output.
- [x] Unit Test:
`JoinHashTableTest.NullAwareOtherConjunctSentinelResetsNullFlag` passed.
- [ ] Manual test
- [ ] No need to test or manual test
- Build and style: ASAN BE/FE build (including FE checkstyle), BE header
hygiene, clang-format, check-format, and diff whitespace check passed.
- Clang-tidy: attempted but did not pass locally.
- Behavior changed:
- [x] Yes. Empty correlated `IN` returns `FALSE` and `NOT IN` returns
`TRUE` for this join path, independent of the previous probe row.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label if needed
---
be/src/exec/common/hash_table/join_hash_table.h | 2 +
.../common/hash_table/join_hash_table_test.cpp | 60 +++++++++++++++++++
.../mark_join_correlated_empty.out | 11 ++++
.../mark_join_correlated_empty.groovy | 70 ++++++++++++++++++++++
4 files changed, 143 insertions(+)
diff --git a/be/src/exec/common/hash_table/join_hash_table.h
b/be/src/exec/common/hash_table/join_hash_table.h
index 1a622ba00aa..28a7ef14986 100644
--- a/be/src/exec/common/hash_table/join_hash_table.h
+++ b/be/src/exec/common/hash_table/join_hash_table.h
@@ -463,6 +463,8 @@ private:
if (!build_idx) {
probe_idxs[matched_cnt] = probe_idx;
build_idxs[matched_cnt] = 0;
+ // A previous NULL-key probe may leave 1 here; the no-match
sentinel is not NULL.
+ null_flags[matched_cnt] = 0;
picking_null_keys = false;
matched_cnt++;
}
diff --git a/be/test/exec/common/hash_table/join_hash_table_test.cpp
b/be/test/exec/common/hash_table/join_hash_table_test.cpp
new file mode 100644
index 00000000000..293c98067a5
--- /dev/null
+++ b/be/test/exec/common/hash_table/join_hash_table_test.cpp
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/common/hash_table/join_hash_table.h"
+
+#include <gtest/gtest.h>
+
+#include <array>
+#include <cstdint>
+#include <functional>
+#include <tuple>
+
+namespace doris {
+
+TEST(JoinHashTableTest, NullAwareOtherConjunctSentinelResetsNullFlag) {
+ JoinHashTable<uint32_t, std::hash<uint32_t>, false> table;
+ table.prepare_build<TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN>(2, 8, false, 0);
+
+ const std::array<uint32_t, 2> build_keys = {0, 7};
+ const std::array<uint32_t, 2> build_buckets = {
+ 0, table.hash(build_keys[1]) & (table.get_bucket_size() - 1)};
+ table.build(build_keys.data(), build_buckets.data(), build_keys.size(),
false);
+
+ const std::array<uint32_t, 1> probe_keys = {42};
+ const uint32_t probe_bucket = table.hash(probe_keys[0]) &
(table.get_bucket_size() - 1);
+ const std::array<uint32_t, 1> build_idx_map =
{table.get_first()[probe_bucket]};
+ std::array<uint32_t, 9> probe_idxs = {};
+ std::array<uint32_t, 9> build_idxs = {};
+ std::array<uint8_t, 9> null_flags = {};
+ null_flags[0] = 1; // A previous NULL-key probe left a NULL mark at this
output position.
+
+ const auto [probe_idx, build_idx, matched_cnt, picking_null_keys] =
+ table.find_null_aware_with_other_conjuncts(
+ probe_keys.data(), build_idx_map.data(), 0, 0,
probe_keys.size(),
+ probe_idxs.data(), build_idxs.data(), null_flags.data(),
false, nullptr);
+
+ EXPECT_EQ(1, probe_idx);
+ EXPECT_EQ(0, build_idx);
+ ASSERT_EQ(1, matched_cnt);
+ EXPECT_FALSE(picking_null_keys);
+ EXPECT_EQ(0, probe_idxs[0]);
+ EXPECT_EQ(0, build_idxs[0]);
+ EXPECT_EQ(0, null_flags[0]);
+}
+
+} // namespace doris
diff --git
a/regression-test/data/nereids_syntax_p0/mark_join_correlated_empty.out
b/regression-test/data/nereids_syntax_p0/mark_join_correlated_empty.out
new file mode 100644
index 00000000000..8b2156d2441
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/mark_join_correlated_empty.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !in_empty --
+1 false
+2 false
+
+-- !not_in_empty --
+1 true
+2 true
+
+-- !in_empty_single_row --
+2 false
diff --git
a/regression-test/suites/nereids_syntax_p0/mark_join_correlated_empty.groovy
b/regression-test/suites/nereids_syntax_p0/mark_join_correlated_empty.groovy
new file mode 100644
index 00000000000..bbdadcf46fd
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/mark_join_correlated_empty.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("mark_join_correlated_empty") {
+ sql "DROP TABLE IF EXISTS mark_join_correlated_empty_l"
+ sql "DROP TABLE IF EXISTS mark_join_correlated_empty_r"
+
+ sql """
+ CREATE TABLE mark_join_correlated_empty_l (
+ id INT NOT NULL, k INT NULL, v INT NOT NULL
+ ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ """
+ sql """
+ CREATE TABLE mark_join_correlated_empty_r (
+ id INT NOT NULL, k INT NOT NULL, v INT NOT NULL
+ ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ """
+
+ sql "INSERT INTO mark_join_correlated_empty_l VALUES (1, NULL, 0), (2, 42,
-1)"
+ sql "INSERT INTO mark_join_correlated_empty_r VALUES (1, 7, -2)"
+
+ explain {
+ sql """
+ SELECT l.id, l.k IN (SELECT r.k FROM mark_join_correlated_empty_r r
+ WHERE r.v > l.v) AS m
+ FROM (SELECT * FROM mark_join_correlated_empty_l ORDER BY id LIMIT
2) l
+ ORDER BY l.id
+ """
+ contains "NULL AWARE LEFT SEMI JOIN"
+ contains "mark join predicates"
+ contains "other join predicates"
+ }
+
+ order_qt_in_empty """
+ SELECT l.id, l.k IN (SELECT r.k FROM mark_join_correlated_empty_r r
+ WHERE r.v > l.v) AS m
+ FROM (SELECT * FROM mark_join_correlated_empty_l ORDER BY id LIMIT 2) l
+ ORDER BY l.id
+ """
+
+ order_qt_not_in_empty """
+ SELECT l.id, l.k NOT IN (SELECT r.k FROM mark_join_correlated_empty_r r
+ WHERE r.v > l.v) AS m
+ FROM (SELECT * FROM mark_join_correlated_empty_l ORDER BY id LIMIT 2) l
+ ORDER BY l.id
+ """
+
+ order_qt_in_empty_single_row """
+ SELECT l.id, l.k IN (SELECT r.k FROM mark_join_correlated_empty_r r
+ WHERE r.v > l.v) AS m
+ FROM (SELECT * FROM mark_join_correlated_empty_l WHERE id = 2 ORDER BY
id LIMIT 1) l
+ ORDER BY l.id
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
