This is an automated email from the ASF dual-hosted git repository.
Mryange pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1e47dc9b7c2 [fix](pipeline) prevent ColumnString64 across operator
boundaries (#65518)
1e47dc9b7c2 is described below
commit 1e47dc9b7c281da0dd9fe2545989231cac671683
Author: Mryange <[email protected]>
AuthorDate: Tue Jul 14 14:29:39 2026 +0800
[fix](pipeline) prevent ColumnString64 across operator boundaries (#65518)
### What problem does this PR solve?
`ColumnString64` is an internal representation used by specific
operators when string offsets overflow 32 bits. It must not be passed
between operators, but operator input and output blocks did not enforce
this constraint.
This change recursively detects `ColumnString64` in column trees and
validates blocks at the common sink and source boundaries. It also adds
a BE unit test covering regular strings, top-level `ColumnString64`, and
nested nullable `ColumnString64`.
### Release note
None
---
be/src/core/block/block.cpp | 14 ++++++++++++++
be/src/core/block/block.h | 2 ++
be/src/core/column/column.h | 15 +++++++++++++++
be/src/exec/operator/operator.h | 2 ++
be/test/core/data_type/block_check_type.cpp | 22 ++++++++++++++++++++++
5 files changed, 55 insertions(+)
diff --git a/be/src/core/block/block.cpp b/be/src/core/block/block.cpp
index 88006f7b2fd..8b92c2e3db9 100644
--- a/be/src/core/block/block.cpp
+++ b/be/src/core/block/block.cpp
@@ -354,6 +354,20 @@ Status Block::check_column_and_type_not_null() const {
return Status::OK();
}
+Status Block::check_no_column_string64() const {
+ for (size_t i = 0; i != data.size(); ++i) {
+ const auto& elem = data[i];
+ DCHECK(elem.column);
+ if (elem.column->contains_column_string64()) {
+ return Status::InternalError(
+ "ColumnString64 is not allowed at operator boundaries,
column index: {}, "
+ "name: {}, structure: {}",
+ i, elem.name, elem.column->dump_structure());
+ }
+ }
+ return Status::OK();
+}
+
size_t Block::rows() const {
for (const auto& elem : data) {
if (elem.column) {
diff --git a/be/src/core/block/block.h b/be/src/core/block/block.h
index 3dfef246bba..6031be051f9 100644
--- a/be/src/core/block/block.h
+++ b/be/src/core/block/block.h
@@ -185,6 +185,8 @@ public:
Status check_column_and_type_not_null() const;
+ Status check_no_column_string64() const;
+
/// Approximate number of bytes used by column data in memory.
/// This reflects the actual data footprint (e.g. string contents, numeric
arrays)
/// and is the metric used by adaptive batch size byte budgets.
diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h
index f27cbd1646e..d09263a9ae6 100644
--- a/be/src/core/column/column.h
+++ b/be/src/core/column/column.h
@@ -676,6 +676,21 @@ public:
virtual bool is_column_string64() const { return false; }
+ bool contains_column_string64() const {
+ if (is_column_string64()) {
+ return true;
+ }
+
+ bool contains = false;
+ ColumnCallback callback = [&](const IColumn& subcolumn) {
+ if (!contains) {
+ contains = subcolumn.contains_column_string64();
+ }
+ };
+ for_each_subcolumn(callback);
+ return contains;
+ }
+
virtual bool is_column_dictionary() const { return false; }
/// If the only value column can contain is NULL.
diff --git a/be/src/exec/operator/operator.h b/be/src/exec/operator/operator.h
index 49018fb57d8..4654ee7e542 100644
--- a/be/src/exec/operator/operator.h
+++ b/be/src/exec/operator/operator.h
@@ -616,6 +616,7 @@ public:
[[nodiscard]] Status sink(RuntimeState* state, Block* block, bool eos) {
RETURN_IF_ERROR(block->check_column_and_type_not_null());
+ RETURN_IF_ERROR(block->check_no_column_string64());
RETURN_IF_ERROR(block->check_type_and_column());
return sink_impl(state, block, eos);
}
@@ -878,6 +879,7 @@ public:
[[nodiscard]] Status get_block(RuntimeState* state, Block* block, bool*
eos) {
RETURN_IF_ERROR(get_block_impl(state, block, eos));
RETURN_IF_ERROR(block->check_column_and_type_not_null());
+ RETURN_IF_ERROR(block->check_no_column_string64());
RETURN_IF_ERROR(block->check_type_and_column());
return Status::OK();
}
diff --git a/be/test/core/data_type/block_check_type.cpp
b/be/test/core/data_type/block_check_type.cpp
index 6915aabb92f..af3afbe9ae7 100644
--- a/be/test/core/data_type/block_check_type.cpp
+++ b/be/test/core/data_type/block_check_type.cpp
@@ -19,7 +19,12 @@
#include <gtest/gtest.h>
#include "core/block/block.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
#include "core/data_type/primitive_type.h"
#include "testutil/column_helper.h"
@@ -57,4 +62,21 @@ TEST(BlockCheckType, CheckColumnAndTypeNotNull) {
st = block.check_column_and_type_not_null();
EXPECT_FALSE(st.ok());
}
+
+TEST(BlockCheckType, CheckNoColumnString64) {
+ Block block {{ColumnString::create(), std::make_shared<DataTypeString>(),
"string"}};
+ EXPECT_TRUE(block.check_no_column_string64());
+
+ block.get_by_position(0).column = ColumnString64::create();
+ auto st = block.check_no_column_string64();
+ EXPECT_FALSE(st.ok());
+ EXPECT_NE(st.msg().find("column index: 0, name: string"),
std::string::npos);
+
+ block.get_by_position(0).column =
+ ColumnNullable::create(ColumnString64::create(),
ColumnUInt8::create());
+ block.get_by_position(0).type =
+
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>());
+ st = block.check_no_column_string64();
+ EXPECT_FALSE(st.ok());
+}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
