This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4571c06e825 [fix](be) Reject unsafe ordered links for hidden columns
(#69395)
4571c06e825 is described below
commit 4571c06e825076dad3d70fbedb9e479a86a3c6e8
Author: Jerry Hu <[email protected]>
AuthorDate: Mon Oct 26 09:41:36 2026 +0800
[fix](be) Reject unsafe ordered links for hidden columns (#69395)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary: A non-empty single-version rowset can store physical
placeholders for publish-time hidden VERSION and TSO columns. Its reader
synthesizes the published values from rowset metadata, but ordered
file-link compaction can combine such rowsets into a multi-version
output whose reader uses the physical placeholders. This can return
incorrect hidden-column values. The same risk applies to the changelog
quick-merge link path.
Reject a link only when the rowsets actually selected for linking
include such a singleton, so the existing physical merge materializes
the values. A UNIQUE KEY merge-on-read table with
`light_schema_change=false` can retain the VERSION column while
`version_col_idx` is unset; look up VERSION by name to cover that
supported case. Safe materialized range rowsets and tidy prefixes remain
linkable.
Scope: This PR changes ordered-link admission, not row-store JSONB
generation, point/Row-ID reads, or historical rowsets already containing
incorrect physical values. It does not add a rowset metadata certificate
or change the protobuf/storage format.
### Release note
Prevent ordered file-link compaction from exposing publish-time hidden
VERSION/TSO placeholders.
### Check List (For Author)
- Test:
- ASAN BE+FE build: `BUILD_TYPE=ASAN ./build.sh --be --fe -j32` passed,
including FE Checkstyle.
- BE unit tests: `BUILD_TYPE=ASAN ./run-be-ut.sh --run
--filter='OrderedDataCompactionTest.*' -j32` passed (7/7).
- Worktree-isolated SQL regression:
`test_ordered_compaction_hidden_version_no_light_schema_change` and
`test_ordered_compaction_hidden_version_column` passed (1/1 each) using
the local regression runner on private FE/BE ports. The latter suite's
COMMIT_TSO/time-travel branch is feature-flag dependent and was not
independently confirmed in this run.
- `build-support/check-build-hygiene.sh`,
`build-support/check-format.sh`, and `git diff --check` passed.
- Changed-file `clang-tidy` was attempted but did not complete cleanly:
the local toolchain analysis cannot find `stddef.h` from
`/usr/include/wchar.h`. No clang-tidy pass is claimed.
- Behavior changed: Yes. Unsafe ordered links fall back to ordinary
physical merge; safe range rowsets and selected tidy prefixes can still
use file linking.
- Does this need documentation: No.
---
be/src/storage/compaction/compaction.cpp | 24 +++
.../compaction/ordered_data_compaction_test.cpp | 203 ++++++++++++++++++++-
...st_ordered_compaction_hidden_version_column.out | 22 +++
...ction_hidden_version_no_light_schema_change.out | 12 ++
...ordered_compaction_hidden_version_column.groovy | 174 ++++++++++++++++++
...on_hidden_version_no_light_schema_change.groovy | 104 +++++++++++
6 files changed, 536 insertions(+), 3 deletions(-)
diff --git a/be/src/storage/compaction/compaction.cpp
b/be/src/storage/compaction/compaction.cpp
index c7e9f3beb33..51c59e545f1 100644
--- a/be/src/storage/compaction/compaction.cpp
+++ b/be/src/storage/compaction/compaction.cpp
@@ -753,6 +753,24 @@ bool CompactionMixin::handle_ordered_data_compaction() {
}
}
+ // A non-empty singleton stores placeholders for publish-time VERSION and
TSO columns.
+ // Linking [2-2] and [3-3] into [2-3] would expose those placeholders
rather than the
+ // synthesized values. Check only the rowsets actually linked: the tidy
scan below can
+ // drop an unsafe tail [10-10] while retaining a safe range prefix [2-9].
+ auto links_unmaterialized_hidden_column = [this] {
+ return _input_rowsets.front()->start_version() !=
_input_rowsets.back()->end_version() &&
+ std::ranges::any_of(_input_rowsets, [](const RowsetSharedPtr&
rowset) {
+ const auto& schema = rowset->tablet_schema();
+ return rowset->num_rows() > 0 &&
+ rowset->start_version() == rowset->end_version() &&
+ // A non-light-schema-change tablet can retain
VERSION by name while
+ // version_col_idx is -1; linking its singleton
would expose stored 0.
+ (schema->field_index(VERSION_COL) >= 0 ||
+ schema->commit_tso_col_idx() >= 0 ||
+ schema->changelog_tso_col_idx() >= 0);
+ });
+ };
+
if (is_changelog_compaction) {
DCHECK(!_input_rowsets.empty()) << "tablet=" << _tablet->tablet_id();
auto compaction_level =
_input_rowsets.front()->rowset_meta()->compaction_level();
@@ -763,6 +781,9 @@ bool CompactionMixin::handle_ordered_data_compaction() {
if (!can_quick_merge_changelog) {
return false;
}
+ if (links_unmaterialized_hidden_column()) {
+ return false;
+ }
_trigger_quick_merge_by_changelog = true;
// Changelog quick merge at LMax is a special meta/link compaction
path selected by
@@ -791,6 +812,9 @@ bool CompactionMixin::handle_ordered_data_compaction() {
}
}
}
+ if (links_unmaterialized_hidden_column()) {
+ return false;
+ }
}
// most rowset of current compaction is nonoverlapping
diff --git a/be/test/storage/compaction/ordered_data_compaction_test.cpp
b/be/test/storage/compaction/ordered_data_compaction_test.cpp
index 39d7911244c..b91cd6be66a 100644
--- a/be/test/storage/compaction/ordered_data_compaction_test.cpp
+++ b/be/test/storage/compaction/ordered_data_compaction_test.cpp
@@ -130,14 +130,17 @@ protected:
int32_t _saved_ordered_data_compaction_min_segment_size = 0;
int32_t _saved_segments_key_bounds_truncation_threshold = 0;
- TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS) {
+ TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS, bool
with_version_column = false,
+ bool with_commit_tso_column = false,
+ bool unset_version_col_idx = false) {
TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
TabletSchemaPB tablet_schema_pb;
tablet_schema_pb.set_keys_type(keys_type);
tablet_schema_pb.set_num_short_key_columns(1);
tablet_schema_pb.set_num_rows_per_row_block(1024);
tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
- tablet_schema_pb.set_next_column_unique_id(4);
+ tablet_schema_pb.set_next_column_unique_id(4 + with_version_column +
+ with_commit_tso_column);
ColumnPB* column_1 = tablet_schema_pb.add_column();
column_1->set_unique_id(1);
@@ -174,6 +177,38 @@ protected:
column_3->set_is_bf_column(false);
}
+ if (with_version_column) {
+ ColumnPB* version_column = tablet_schema_pb.add_column();
+ version_column->set_unique_id(4);
+ version_column->set_name(VERSION_COL);
+ version_column->set_type("BIGINT");
+ version_column->set_is_key(false);
+ version_column->set_is_nullable(false);
+ version_column->set_visible(false);
+ version_column->set_default_value("0");
+ version_column->set_length(8);
+ version_column->set_index_length(8);
+ version_column->set_aggregation("REPLACE");
+ if (!unset_version_col_idx) {
+ tablet_schema_pb.set_version_col_idx(3);
+ }
+ }
+
+ if (with_commit_tso_column) {
+
tablet_schema_pb.set_commit_tso_col_idx(tablet_schema_pb.column_size());
+ ColumnPB* tso_column = tablet_schema_pb.add_column();
+ tso_column->set_unique_id(tablet_schema_pb.column_size());
+ tso_column->set_name(COMMIT_TSO_COL);
+ tso_column->set_type("BIGINT");
+ tso_column->set_is_key(false);
+ tso_column->set_is_nullable(false);
+ tso_column->set_visible(false);
+ tso_column->set_default_value("0");
+ tso_column->set_length(8);
+ tso_column->set_index_length(8);
+ tso_column->set_aggregation("NONE");
+ }
+
tablet_schema->init_from_pb(tablet_schema_pb);
return tablet_schema;
}
@@ -301,7 +336,8 @@ protected:
RowsetSharedPtr create_rowset(
TabletSchemaSPtr tablet_schema, TabletSharedPtr tablet,
const SegmentsOverlapPB& overlap,
- std::vector<std::vector<std::tuple<int64_t, int64_t>>>
rowset_data) {
+ std::vector<std::vector<std::tuple<int64_t, int64_t>>> rowset_data,
+ Version version = Version(-1, -1), int64_t stored_version = 0) {
RowsetWriterContext writer_context;
if (overlap == NONOVERLAPPING) {
for (auto i = 1; i < rowset_data.size(); i++) {
@@ -314,6 +350,9 @@ protected:
}
create_rowset_writer_context(tablet_schema, tablet->tablet_path(),
overlap, UINT32_MAX,
&writer_context);
+ if (version.first >= 0) {
+ writer_context.version = version;
+ }
auto res = RowsetFactory::create_rowset_writer(*engine_ref,
writer_context, true);
EXPECT_TRUE(res.has_value()) << res.error();
@@ -333,6 +372,15 @@ protected:
uint8_t num = 0;
columns[2]->insert_data((const char*)&num, sizeof(num));
}
+ if (auto version_idx =
tablet_schema->field_index(VERSION_COL); version_idx >= 0) {
+ columns[version_idx]->insert_data((const
char*)&stored_version,
+ sizeof(stored_version));
+ }
+ if (tablet_schema->commit_tso_col_idx() >= 0) {
+ int64_t tso_placeholder = 0;
+ columns[tablet_schema->commit_tso_col_idx()]->insert_data(
+ (const char*)&tso_placeholder,
sizeof(tso_placeholder));
+ }
num_rows++;
}
block.set_columns(std::move(columns));
@@ -349,6 +397,32 @@ protected:
return rowset;
}
+ std::vector<int64_t> read_hidden_values(Rowset* rowset, const
TabletSchemaSPtr& tablet_schema,
+ ColumnId hidden_column_id) {
+ auto read_schema =
std::make_shared<ReadSchema>(project_columns_by_ordinal(
+ tablet_schema->columns(), std::vector<ColumnId> {0,
hidden_column_id}));
+ EXPECT_TRUE(read_schema->init_from_tablet_schema(*tablet_schema,
false, false).ok());
+ RowsetReaderContext reader_context;
+ reader_context.read_schema = read_schema;
+ RowsetReaderSharedPtr reader;
+ create_and_init_rowset_reader(rowset, reader_context, &reader);
+ std::vector<int64_t> values;
+ Status status = Status::OK();
+ do {
+ Block block = read_schema->create_read_block();
+ status = reader->next_batch(&block);
+ for (size_t i = 0; i < block.rows(); ++i) {
+ values.push_back(block.get_by_position(1).column->get_int(i));
+ }
+ } while (status.ok());
+ EXPECT_EQ(status, Status::Error<END_OF_FILE>(""));
+ return values;
+ }
+
+ std::vector<int64_t> read_versions(Rowset* rowset, const TabletSchemaSPtr&
tablet_schema) {
+ return read_hidden_values(rowset, tablet_schema,
tablet_schema->field_index(VERSION_COL));
+ }
+
void init_rs_meta(RowsetMetaSharedPtr& pb1, int64_t start, int64_t end) {
std::string json_rowset_meta = R"({
"rowset_id": 540085,
@@ -559,6 +633,129 @@ TEST_F(OrderedDataCompactionTest, test_01) {
}
}
+TEST_F(OrderedDataCompactionTest,
test_version_col_singleton_rowset_not_linked) {
+ auto tablet_schema = create_schema(UNIQUE_KEYS, true);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(2, 1, 2, input_data);
+ std::vector<RowsetSharedPtr> input_rowsets;
+ for (const auto& rowset_data : input_data) {
+ auto rowset = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
rowset_data);
+ ASSERT_EQ(rowset->start_version(), rowset->end_version());
+ ASSERT_EQ(rowset->num_rows(), 2);
+ EXPECT_EQ(read_versions(rowset.get(), tablet_schema),
+ (std::vector<int64_t> {rowset->end_version(),
rowset->end_version()}));
+ input_rowsets.push_back(std::move(rowset));
+ }
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = std::move(input_rowsets);
+ ASSERT_FALSE(compaction.handle_ordered_data_compaction());
+}
+
+TEST_F(OrderedDataCompactionTest,
test_version_col_with_unset_index_singleton_not_linked) {
+ // A schema created without light schema change has the VERSION column but
no index in PB.
+ auto tablet_schema = create_schema(UNIQUE_KEYS, true, false, true);
+ ASSERT_EQ(tablet_schema->version_col_idx(), -1);
+ ASSERT_EQ(tablet_schema->field_index(VERSION_COL), 3);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(2, 1, 2, input_data);
+ std::vector<RowsetSharedPtr> input_rowsets;
+ for (const auto& rowset_data : input_data) {
+ auto rowset = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
rowset_data);
+ ASSERT_EQ(rowset->start_version(), rowset->end_version());
+ ASSERT_EQ(rowset->num_rows(), 2);
+ EXPECT_EQ(read_versions(rowset.get(), tablet_schema),
+ (std::vector<int64_t> {rowset->end_version(),
rowset->end_version()}));
+ input_rowsets.push_back(std::move(rowset));
+ }
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = std::move(input_rowsets);
+ ASSERT_FALSE(compaction.handle_ordered_data_compaction());
+}
+
+TEST_F(OrderedDataCompactionTest,
test_version_col_materialized_range_rowsets_can_link) {
+ auto tablet_schema = create_schema(UNIQUE_KEYS, true);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(2, 1, 2, input_data);
+ auto first = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
input_data[0],
+ Version(1000, 1001), 1001);
+ auto second = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
input_data[1],
+ Version(1002, 1003), 1003);
+ EXPECT_EQ(read_versions(first.get(), tablet_schema), (std::vector<int64_t>
{1001, 1001}));
+ EXPECT_EQ(read_versions(second.get(), tablet_schema),
(std::vector<int64_t> {1003, 1003}));
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = {first, second};
+ ASSERT_TRUE(compaction.handle_ordered_data_compaction());
+ EXPECT_EQ(compaction._output_rowset->start_version(), 1000);
+ EXPECT_EQ(compaction._output_rowset->end_version(), 1003);
+ EXPECT_EQ(read_versions(compaction._output_rowset.get(), tablet_schema),
+ (std::vector<int64_t> {1001, 1001, 1003, 1003}));
+}
+
+TEST_F(OrderedDataCompactionTest,
test_commit_tso_singleton_rowsets_not_linked) {
+ auto tablet_schema = create_schema(DUP_KEYS, false, true);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(2, 1, 2, input_data);
+ std::vector<RowsetSharedPtr> input_rowsets;
+ for (size_t i = 0; i < input_data.size(); ++i) {
+ auto rowset = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
input_data[i]);
+ rowset->rowset_meta()->set_commit_tso(100 + i);
+ EXPECT_EQ(read_hidden_values(rowset.get(), tablet_schema,
+ tablet_schema->commit_tso_col_idx()),
+ (std::vector<int64_t> {100 + static_cast<int64_t>(i),
+ 100 + static_cast<int64_t>(i)}));
+ input_rowsets.push_back(std::move(rowset));
+ }
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = std::move(input_rowsets);
+ ASSERT_FALSE(compaction.handle_ordered_data_compaction());
+}
+
+TEST_F(OrderedDataCompactionTest, test_unsafe_tail_does_not_block_safe_prefix)
{
+ auto tablet_schema = create_schema(UNIQUE_KEYS, true);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(5, 1, 2, input_data);
+ std::vector<RowsetSharedPtr> input_rowsets;
+ for (size_t i = 0; i < input_data.size() - 1; ++i) {
+ input_rowsets.push_back(create_rowset(tablet_schema, tablet,
NONOVERLAPPING, input_data[i],
+ Version(1000 + 2 * i, 1001 + 2 *
i), 1001 + 2 * i));
+ }
+ // Mark a two-segment singleton tail untidy; it must not veto the tidy
range prefix.
+ auto tail_data = std::vector<std::vector<std::tuple<int64_t, int64_t>>> {
+ {input_data.back()[0][0]}, {input_data.back()[0][0]}};
+ auto tail = create_rowset(tablet_schema, tablet, OVERLAPPING, tail_data,
Version(1008, 1008));
+ ASSERT_EQ(tail->num_segments(), 2);
+ ASSERT_TRUE(tail->is_segments_overlapping());
+ input_rowsets.push_back(tail);
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = std::move(input_rowsets);
+ ASSERT_TRUE(compaction.handle_ordered_data_compaction());
+ EXPECT_EQ(compaction._input_rowsets.size(), 4);
+ EXPECT_EQ(compaction._output_rowset->start_version(), 1000);
+ EXPECT_EQ(compaction._output_rowset->end_version(), 1007);
+ EXPECT_EQ(read_versions(compaction._output_rowset.get(), tablet_schema),
+ (std::vector<int64_t> {1001, 1001, 1003, 1003, 1005, 1005, 1007,
1007}));
+}
+
TEST_F(OrderedDataCompactionTest, test_index_disk_size) {
auto num_input_rowset = 3;
auto num_segments = 2;
diff --git
a/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_column.out
b/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_column.out
new file mode 100644
index 00000000000..cd4ecbbf743
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_column.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !versions --
+10 true
+11 true
+20 true
+21 true
+
+-- !min_max --
+true true
+
+-- !version_filters --
+2 2
+
+-- !commit_tso --
+10 true
+11 true
+20 true
+21 true
+
+-- !time_travel --
+10 10
+11 11
diff --git
a/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.out
b/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.out
new file mode 100644
index 00000000000..db1ee8131fa
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !versions --
+10 true
+11 true
+20 true
+21 true
+
+-- !min_max --
+true true
+
+-- !version_filters --
+2 2
diff --git
a/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_column.groovy
b/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_column.groovy
new file mode 100644
index 00000000000..ced7e4b2240
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_column.groovy
@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_ordered_compaction_hidden_version_column", "nonConcurrent") {
+ if (isCloudMode()) {
+ return
+ }
+
+ setBeConfigTemporary([
+ enable_ordered_data_compaction: true,
+ ordered_data_compaction_min_segment_size: 1,
+ segments_key_bounds_truncation_threshold: -1
+ ]) {
+ sql "DROP TABLE IF EXISTS
test_ordered_compaction_hidden_version_column"
+ sql """
+ CREATE TABLE test_ordered_compaction_hidden_version_column (
+ k INT NOT NULL,
+ v INT NOT NULL
+ ) UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true")
+ """
+
+ sql "INSERT INTO test_ordered_compaction_hidden_version_column VALUES
(10, 10), (11, 11)"
+ sql "INSERT INTO test_ordered_compaction_hidden_version_column VALUES
(20, 20), (21, 21)"
+ sql "SET show_hidden_columns = true"
+ def firstVersions = sql """SELECT __DORIS_VERSION_COL__
+ FROM test_ordered_compaction_hidden_version_column WHERE k < 20
ORDER BY k"""
+ def secondVersions = sql """SELECT __DORIS_VERSION_COL__
+ FROM test_ordered_compaction_hidden_version_column WHERE k >= 20
ORDER BY k"""
+ assert firstVersions.size() == 2 && secondVersions.size() == 2
+ long v1 = firstVersions[0][0].toString().toLong()
+ long v2 = secondVersions[0][0].toString().toLong()
+ assert firstVersions.every { it[0].toString().toLong() == v1 }
+ assert secondVersions.every { it[0].toString().toLong() == v2 }
+ assert v1 > 0 && v2 == v1 + 1
+
+ def tabletInfo = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_version_column").get(0)
+ def tabletId = tabletInfo.TabletId
+ def (inputCode, inputBody, inputError) = curl("GET",
tabletInfo.MetaUrl)
+ assert inputCode == 0 : inputError
+ def inputRowsets = parseJson(inputBody.trim()).rs_metas
+ for (def version : [v1, v2]) {
+ assert inputRowsets.count {
+ it.start_version.toString().toLong() == version &&
+ it.end_version.toString().toLong() == version &&
+ it.num_rows.toString().toLong() == 2 &&
+ it.num_segments.toString().toLong() == 1
+ } == 1
+ }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets",
+ [tablet_id: "${tabletId}", start_version: "${v1}",
end_version: "${v2}"])
+
trigger_and_wait_compaction("test_ordered_compaction_hidden_version_column",
"cumulative")
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets")
+ }
+
+ def metaUrl = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_version_column").get(0).MetaUrl
+ def (code, body, error) = curl("GET", metaUrl)
+ assert code == 0 : error
+ def rowsets = parseJson(body.trim()).rs_metas
+ assert rowsets.count {
+ it.start_version.toString().toLong() == v1 &&
+ it.end_version.toString().toLong() == v2 &&
+ it.num_rows.toString().toLong() == 4
+ } == 1
+
+ order_qt_versions """SELECT k,
+ (__DORIS_VERSION_COL__ = CASE WHEN k < 20 THEN ${v1} ELSE ${v2}
END) AS correct_version
+ FROM test_ordered_compaction_hidden_version_column ORDER BY k"""
+ qt_min_max """SELECT MIN(__DORIS_VERSION_COL__) = ${v1},
+ MAX(__DORIS_VERSION_COL__) = ${v2}
+ FROM test_ordered_compaction_hidden_version_column"""
+ qt_version_filters """SELECT
+ (SELECT COUNT(*) FROM test_ordered_compaction_hidden_version_column
+ WHERE __DORIS_VERSION_COL__ = ${v1}),
+ (SELECT COUNT(*) FROM test_ordered_compaction_hidden_version_column
+ WHERE __DORIS_VERSION_COL__ = ${v2})"""
+
+ def featureEnabled = { String name ->
+ def row = sql("SHOW FRONTEND CONFIG LIKE '%${name}'").find {
+ it[0] == name || it[0] == "experimental_${name}"
+ }
+ return row != null && row[1] == "true"
+ }
+ if (featureEnabled("enable_feature_changelog") &&
featureEnabled("enable_feature_binlog")) {
+ sql "DROP TABLE IF EXISTS
test_ordered_compaction_hidden_tso_column"
+ sql """
+ CREATE TABLE test_ordered_compaction_hidden_tso_column (
+ k INT NOT NULL,
+ v INT NOT NULL
+ ) DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "changelog.enable" = "true",
+ "disable_auto_compaction" = "true")
+ """
+ sql "INSERT INTO test_ordered_compaction_hidden_tso_column VALUES
(10, 10), (11, 11)"
+ sql "INSERT INTO test_ordered_compaction_hidden_tso_column VALUES
(20, 20), (21, 21)"
+ def firstTsos = sql """SELECT __DORIS_COMMIT_TSO_COL__
+ FROM test_ordered_compaction_hidden_tso_column WHERE k < 20
ORDER BY k"""
+ def secondTsos = sql """SELECT __DORIS_COMMIT_TSO_COL__
+ FROM test_ordered_compaction_hidden_tso_column WHERE k >= 20
ORDER BY k"""
+ assert firstTsos.size() == 2 && secondTsos.size() == 2
+ long t1 = firstTsos[0][0].toString().toLong()
+ long t2 = secondTsos[0][0].toString().toLong()
+ assert t1 > 0 && t2 > t1
+ assert firstTsos.every { it[0].toString().toLong() == t1 }
+ assert secondTsos.every { it[0].toString().toLong() == t2 }
+
+ def tsoTabletInfo = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_tso_column").get(0)
+ def (tsoInputCode, tsoInputBody, tsoInputError) = curl("GET",
tsoTabletInfo.MetaUrl)
+ assert tsoInputCode == 0 : tsoInputError
+ def tsoInputRowsets =
parseJson(tsoInputBody.trim()).rs_metas.findAll {
+ it.start_version.toString().toLong() ==
it.end_version.toString().toLong() &&
+ it.num_rows.toString().toLong() == 2 &&
+ it.num_segments.toString().toLong() == 1
+ }.sort { it.start_version.toString().toLong() }
+ assert tsoInputRowsets.size() == 2
+ long firstVersion =
tsoInputRowsets[0].start_version.toString().toLong()
+ long secondVersion =
tsoInputRowsets[1].end_version.toString().toLong()
+ assert secondVersion == firstVersion + 1
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets",
+ [tablet_id: "${tsoTabletInfo.TabletId}",
start_version: "${firstVersion}",
+ end_version: "${secondVersion}"])
+
trigger_and_wait_compaction("test_ordered_compaction_hidden_tso_column",
"cumulative")
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets")
+ }
+
+ def tsoMetaUrl = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_tso_column").get(0).MetaUrl
+ def (tsoCode, tsoBody, tsoError) = curl("GET", tsoMetaUrl)
+ assert tsoCode == 0 : tsoError
+ assert parseJson(tsoBody.trim()).rs_metas.count {
+ it.start_version.toString().toLong() == firstVersion &&
+ it.end_version.toString().toLong() == secondVersion &&
+ it.num_rows.toString().toLong() == 4
+ } == 1
+ order_qt_commit_tso """SELECT k,
+ (__DORIS_COMMIT_TSO_COL__ = CASE WHEN k < 20 THEN ${t1} ELSE
${t2} END) AS correct_tso
+ FROM test_ordered_compaction_hidden_tso_column ORDER BY k"""
+ order_qt_time_travel """SELECT k, v FROM
test_ordered_compaction_hidden_tso_column
+ FOR VERSION AS OF ${t1} ORDER BY k"""
+ }
+ }
+}
diff --git
a/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.groovy
b/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.groovy
new file mode 100644
index 00000000000..5b245cd885c
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.groovy
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_ordered_compaction_hidden_version_no_light_schema_change",
"nonConcurrent") {
+ if (isCloudMode()) {
+ return
+ }
+
+ setBeConfigTemporary([
+ enable_ordered_data_compaction: true,
+ ordered_data_compaction_min_segment_size: 1,
+ segments_key_bounds_truncation_threshold: -1
+ ]) {
+ sql "DROP TABLE IF EXISTS
test_ordered_compaction_hidden_version_no_light_schema_change"
+ sql """
+ CREATE TABLE
test_ordered_compaction_hidden_version_no_light_schema_change (
+ k INT NOT NULL,
+ v INT NOT NULL
+ ) UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false",
+ "light_schema_change" = "false",
+ "disable_auto_compaction" = "true")
+ """
+
+ sql "INSERT INTO
test_ordered_compaction_hidden_version_no_light_schema_change VALUES (10, 10),
(11, 11)"
+ sql "INSERT INTO
test_ordered_compaction_hidden_version_no_light_schema_change VALUES (20, 20),
(21, 21)"
+ sql "SET show_hidden_columns = true"
+ def firstVersions = sql """SELECT __DORIS_VERSION_COL__
+ FROM test_ordered_compaction_hidden_version_no_light_schema_change
WHERE k < 20 ORDER BY k"""
+ def secondVersions = sql """SELECT __DORIS_VERSION_COL__
+ FROM test_ordered_compaction_hidden_version_no_light_schema_change
WHERE k >= 20 ORDER BY k"""
+ assert firstVersions.size() == 2 && secondVersions.size() == 2
+ long v1 = firstVersions[0][0].toString().toLong()
+ long v2 = secondVersions[0][0].toString().toLong()
+ assert firstVersions.every { it[0].toString().toLong() == v1 }
+ assert secondVersions.every { it[0].toString().toLong() == v2 }
+ assert v1 > 0 && v2 == v1 + 1
+
+ def tabletInfo = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_version_no_light_schema_change").get(0)
+ def tabletId = tabletInfo.TabletId
+ def (inputCode, inputBody, inputError) = curl("GET",
tabletInfo.MetaUrl)
+ assert inputCode == 0 : inputError
+ def inputRowsets = parseJson(inputBody.trim()).rs_metas
+ for (def version : [v1, v2]) {
+ assert inputRowsets.count {
+ it.start_version.toString().toLong() == version &&
+ it.end_version.toString().toLong() == version &&
+ it.num_rows.toString().toLong() == 2 &&
+ it.num_segments.toString().toLong() == 1
+ } == 1
+ }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets",
+ [tablet_id: "${tabletId}", start_version: "${v1}",
end_version: "${v2}"])
+
trigger_and_wait_compaction("test_ordered_compaction_hidden_version_no_light_schema_change",
"cumulative")
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets")
+ }
+
+ def metaUrl = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_version_no_light_schema_change").get(0).MetaUrl
+ def (code, body, error) = curl("GET", metaUrl)
+ assert code == 0 : error
+ def rowsets = parseJson(body.trim()).rs_metas
+ assert rowsets.count {
+ it.start_version.toString().toLong() == v1 &&
+ it.end_version.toString().toLong() == v2 &&
+ it.num_rows.toString().toLong() == 4
+ } == 1
+
+ order_qt_versions """SELECT k,
+ (__DORIS_VERSION_COL__ = CASE WHEN k < 20 THEN ${v1} ELSE ${v2}
END) AS correct_version
+ FROM test_ordered_compaction_hidden_version_no_light_schema_change
ORDER BY k"""
+ qt_min_max """SELECT MIN(__DORIS_VERSION_COL__) = ${v1},
+ MAX(__DORIS_VERSION_COL__) = ${v2}
+ FROM
test_ordered_compaction_hidden_version_no_light_schema_change"""
+ qt_version_filters """SELECT
+ (SELECT COUNT(*) FROM
test_ordered_compaction_hidden_version_no_light_schema_change
+ WHERE __DORIS_VERSION_COL__ = ${v1}),
+ (SELECT COUNT(*) FROM
test_ordered_compaction_hidden_version_no_light_schema_change
+ WHERE __DORIS_VERSION_COL__ = ${v2})"""
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
