This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 211a79a0115 [fix](be) Count variant data in
information_schema.column_data_sizes (#68444)
211a79a0115 is described below
commit 211a79a011554bb561192e83d28cbd4fb5e25021
Author: lihangyu <[email protected]>
AuthorDate: Mon Oct 19 10:52:21 2026 +0800
[fix](be) Count variant data in information_schema.column_data_sizes
(#68444)
---
.../schema_column_data_sizes_scanner.cpp | 53 +++++----
be/src/storage/merger.cpp | 5 +-
be/src/storage/segment/column_writer.cpp | 13 +++
be/src/storage/segment/column_writer.h | 22 ++--
.../segment/variant/v2/variant_column_writer.cpp | 13 +++
.../segment/variant/v2/variant_column_writer.h | 3 +
.../segment/variant/variant_column_writer_impl.cpp | 28 +++++
.../segment/variant/variant_column_writer_impl.h | 20 +++-
.../segment_0.dat | Bin 5586 -> 5588 bytes
.../segment_1.dat | Bin 5588 -> 5590 bytes
.../segment_0.dat | Bin 5402 -> 5404 bytes
.../segment_1.dat | Bin 5404 -> 5406 bytes
.../segment_0.dat | Bin 5446 -> 5448 bytes
.../segment_1.dat | Bin 5258 -> 5260 bytes
.../segment_0.dat | Bin 5262 -> 5264 bytes
.../segment_1.dat | Bin 5074 -> 5076 bytes
.../segment_0.dat | Bin 5582 -> 5584 bytes
.../segment_1.dat | Bin 5582 -> 5584 bytes
.../segment_0.dat | Bin 5398 -> 5400 bytes
.../segment_1.dat | Bin 5398 -> 5400 bytes
.../segment_0.dat | Bin 5384 -> 5386 bytes
.../segment_1.dat | Bin 5161 -> 5163 bytes
.../segment_0.dat | Bin 5200 -> 5202 bytes
.../segment_1.dat | Bin 4977 -> 4979 bytes
.../segment_0.dat | Bin 12635 -> 12639 bytes
.../segment_1.dat | Bin 12645 -> 12649 bytes
.../variant_row_store_0_horizontal/segment_0.dat | Bin 3513 -> 3515 bytes
.../variant_row_store_0_horizontal/segment_1.dat | Bin 3525 -> 3527 bytes
.../variant_row_store_1_vertical/segment_0.dat | Bin 3515 -> 3517 bytes
.../variant_row_store_1_vertical/segment_1.dat | Bin 3527 -> 3529 bytes
.../variant_row_store_2_horizontal/segment_0.dat | Bin 3513 -> 3515 bytes
.../variant_row_store_2_horizontal/segment_1.dat | Bin 3525 -> 3527 bytes
.../variant_row_store_3_vertical/segment_0.dat | Bin 3513 -> 3515 bytes
.../variant_row_store_3_vertical/segment_1.dat | Bin 3525 -> 3527 bytes
.../segment_0.dat | Bin 2854 -> 2856 bytes
.../segment_1.dat | Bin 3056 -> 3058 bytes
.../segment_2.dat | Bin 2448 -> 2450 bytes
.../variant/variant_column_writer_reader_test.cpp | 14 +++
.../test_query_sys_column_data_sizes_variant.out | 33 ++++++
...test_query_sys_column_data_sizes_variant.groovy | 120 +++++++++++++++++++++
40 files changed, 287 insertions(+), 37 deletions(-)
diff --git a/be/src/information_schema/schema_column_data_sizes_scanner.cpp
b/be/src/information_schema/schema_column_data_sizes_scanner.cpp
index 52df991c710..2ffba6a1563 100644
--- a/be/src/information_schema/schema_column_data_sizes_scanner.cpp
+++ b/be/src/information_schema/schema_column_data_sizes_scanner.cpp
@@ -111,40 +111,57 @@ void
SchemaColumnDataSizesScanner::_collect_column_data_sizes_from_rowsets(
// Get column data page stats from each segment footer and aggregate
by column_unique_id
for (const auto& segment : segments) {
- auto collector = [&](const segment_v2::ColumnMetaPB& column_meta) {
+ auto collector = [&](const segment_v2::ColumnMetaPB& column_meta,
+ uint32_t column_unique_id) {
if (column_meta.has_compressed_data_bytes() &&
column_meta.has_uncompressed_data_bytes()) {
- auto cid = schema->field_index(column_meta.unique_id());
+ auto cid = schema->field_index(column_unique_id);
if (cid == -1) {
return;
}
- // Aggregate stats by column_unique_id
- if (aggregated_stats.contains(column_meta.unique_id())) {
- auto& existing_stats =
aggregated_stats[column_meta.unique_id()];
- existing_stats.compressed_data_bytes +=
column_meta.compressed_data_bytes();
- existing_stats.uncompressed_data_bytes +=
- column_meta.uncompressed_data_bytes();
- existing_stats.raw_data_bytes +=
column_meta.raw_data_bytes();
- } else {
- aggregated_stats[column_meta.unique_id()] =
ColumnDataSizeInfo {
+ auto [it, inserted] =
aggregated_stats.try_emplace(column_unique_id);
+ auto& stats = it->second;
+ if (inserted) {
+ stats = ColumnDataSizeInfo {
.backend_id = backend_id_,
.table_id = table_id,
.index_id = index_id,
.partition_id = partition_id,
.tablet_id = tablet_id,
.rowset_id = rowset_id,
- .column_unique_id = column_meta.unique_id(),
+ .column_unique_id = column_unique_id,
.column_name = schema->column(cid).name(),
.column_type =
TabletColumn::get_string_by_field_type(
-
static_cast<FieldType>(column_meta.type())),
- .compressed_data_bytes =
column_meta.compressed_data_bytes(),
- .uncompressed_data_bytes =
column_meta.uncompressed_data_bytes(),
- .raw_data_bytes = column_meta.raw_data_bytes(),
+ schema->column(cid).type()),
+ .compressed_data_bytes = 0,
+ .uncompressed_data_bytes = 0,
+ .raw_data_bytes = 0,
};
}
+ stats.compressed_data_bytes +=
column_meta.compressed_data_bytes();
+ stats.uncompressed_data_bytes +=
column_meta.uncompressed_data_bytes();
+ stats.raw_data_bytes += column_meta.raw_data_bytes();
}
};
- st = segment->traverse_column_meta_pbs(collector);
+ auto footer_collector = [&](const segment_v2::ColumnMetaPB&
column_meta) {
+ // A variant meta counts the data of all its inner writers,
whose own metas carry no
+ // data bytes. Compaction writes variant subcolumns as columns
of their own, with
+ // data bytes but no unique id; count them to their root
variant column.
+ uint32_t column_unique_id = column_meta.unique_id();
+ if (column_meta.unique_id() == -1 &&
column_meta.has_compressed_data_bytes()) {
+ DORIS_CHECK(column_meta.has_column_path_info());
+ column_unique_id =
column_meta.column_path_info().parrent_column_unique_id();
+ }
+ collector(column_meta, column_unique_id);
+ if (column_meta.type() ==
static_cast<int>(FieldType::OLAP_FIELD_TYPE_VARIANT)) {
+ // External column meta (V3) embeds the sparse and doc
value columns of a
+ // variant into its root column meta.
+ for (const auto& child : column_meta.children_columns()) {
+ collector(child, column_unique_id);
+ }
+ }
+ };
+ st = segment->traverse_column_meta_pbs(footer_collector);
if (!st.ok()) {
continue;
}
@@ -354,4 +371,4 @@ Status
SchemaColumnDataSizesScanner::_fill_block_impl(Block* block) {
return Status::OK();
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp
index 566adea31ac..a743e90467e 100644
--- a/be/src/storage/merger.cpp
+++ b/be/src/storage/merger.cpp
@@ -706,8 +706,9 @@ Status Merger::vertical_merge_rowsets(
const auto& col = tablet_schema.column(col_ordinal);
int32_t uid = col.unique_id();
- // Variant columns (root or subcolumn): raw_data_bytes is 0 (TODO
in writer),
- // cannot estimate from footer, fallback to default for the entire
group.
+ // Variant columns (root or subcolumn): subcolumns written as
columns of their own
+ // (e.g. by compaction) are not counted in the root's
raw_data_bytes, cannot estimate
+ // from footer, fallback to default for the entire group.
if (uid < 0 || col.is_variant_type()) {
need_fallback = true;
break;
diff --git a/be/src/storage/segment/column_writer.cpp
b/be/src/storage/segment/column_writer.cpp
index 685076a9402..9b94884c8be 100644
--- a/be/src/storage/segment/column_writer.cpp
+++ b/be/src/storage/segment/column_writer.cpp
@@ -1342,6 +1342,19 @@ uint64_t VariantColumnWriter::estimate_buffer_size() {
return _impl->estimate_buffer_size();
}
+uint64_t VariantColumnWriter::get_raw_data_bytes() const {
+ return
_impl->get_total_data_pages_bytes(&ColumnWriter::get_raw_data_bytes);
+}
+
+uint64_t VariantColumnWriter::get_total_uncompressed_data_pages_bytes() const {
+ return _impl->get_total_data_pages_bytes(
+ &ColumnWriter::get_total_uncompressed_data_pages_bytes);
+}
+
+uint64_t VariantColumnWriter::get_total_compressed_data_pages_bytes() const {
+ return
_impl->get_total_data_pages_bytes(&ColumnWriter::get_total_compressed_data_pages_bytes);
+}
+
Status VariantColumnWriter::finish() {
return _impl->finish();
}
diff --git a/be/src/storage/segment/column_writer.h
b/be/src/storage/segment/column_writer.h
index fc8c7db59ba..d79b35d1647 100644
--- a/be/src/storage/segment/column_writer.h
+++ b/be/src/storage/segment/column_writer.h
@@ -210,6 +210,8 @@ public:
virtual uint64_t get_raw_data_bytes() const = 0;
virtual uint64_t get_total_uncompressed_data_pages_bytes() const = 0;
virtual uint64_t get_total_compressed_data_pages_bytes() const = 0;
+ // Points to one of the three data bytes getters above.
+ using DataBytesGetter = uint64_t (ColumnWriter::*)() const;
// used for append not null data.
virtual Status append_data(const uint8_t** ptr, size_t num_rows) = 0;
@@ -620,16 +622,14 @@ public:
Status write_bloom_filter_index() override;
ordinal_t get_next_rowid() const override { return _next_rowid; }
- uint64_t get_raw_data_bytes() const override {
- return 0; // TODO
- }
+ uint64_t get_raw_data_bytes() const override { return
_writer->get_raw_data_bytes(); }
uint64_t get_total_uncompressed_data_pages_bytes() const override {
- return 0; // TODO
+ return _writer->get_total_uncompressed_data_pages_bytes();
}
uint64_t get_total_compressed_data_pages_bytes() const override {
- return 0; // TODO
+ return _writer->get_total_compressed_data_pages_bytes();
}
Status append_nulls(size_t num_rows) override {
@@ -688,17 +688,11 @@ public:
Status write_bloom_filter_index() override;
ordinal_t get_next_rowid() const override { return _next_rowid; }
- uint64_t get_raw_data_bytes() const override {
- return 0; // TODO
- }
+ uint64_t get_raw_data_bytes() const override;
- uint64_t get_total_uncompressed_data_pages_bytes() const override {
- return 0; // TODO
- }
+ uint64_t get_total_uncompressed_data_pages_bytes() const override;
- uint64_t get_total_compressed_data_pages_bytes() const override {
- return 0; // TODO
- }
+ uint64_t get_total_compressed_data_pages_bytes() const override;
Status append_nulls(size_t num_rows) override {
return Status::NotSupported("variant writer can not append_nulls");
diff --git a/be/src/storage/segment/variant/v2/variant_column_writer.cpp
b/be/src/storage/segment/variant/v2/variant_column_writer.cpp
index 12a06a84ab2..0790f1c8db0 100644
--- a/be/src/storage/segment/variant/v2/variant_column_writer.cpp
+++ b/be/src/storage/segment/variant/v2/variant_column_writer.cpp
@@ -359,4 +359,17 @@ uint64_t VariantV2ColumnWriter::estimate_buffer_size() {
return size;
}
+uint64_t VariantV2ColumnWriter::get_total_data_pages_bytes(
+ ColumnWriter::DataBytesGetter getter) const {
+ DORIS_CHECK(_is_finalized);
+ uint64_t size = std::invoke(getter, _root_writer.get());
+ for (const auto& writer : _subcolumn_writers) {
+ size += std::invoke(getter, writer.get());
+ }
+ if (_binary_writer != nullptr) {
+ size += _binary_writer->get_total_data_pages_bytes(getter);
+ }
+ return size;
+}
+
} // namespace doris::segment_v2
diff --git a/be/src/storage/segment/variant/v2/variant_column_writer.h
b/be/src/storage/segment/variant/v2/variant_column_writer.h
index 6a41bf6e821..ea982f58a87 100644
--- a/be/src/storage/segment/variant/v2/variant_column_writer.h
+++ b/be/src/storage/segment/variant/v2/variant_column_writer.h
@@ -59,6 +59,9 @@ public:
Status write_inverted_index();
Status write_bloom_filter_index();
uint64_t estimate_buffer_size();
+ // Sums `getter`, one of the ColumnWriter data bytes getters, over every
column writer of this
+ // variant: the root, the materialized subcolumns and the sparse or doc
value columns.
+ uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter)
const;
private:
Status _write_root(const IColumn* root_jsonb, int& column_id);
diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp
b/be/src/storage/segment/variant/variant_column_writer_impl.cpp
index bb4559884d8..a443482a9e6 100644
--- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp
+++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp
@@ -211,6 +211,18 @@ uint64_t UnifiedSparseColumnWriter::estimate_buffer_size()
const {
return size;
}
+uint64_t UnifiedSparseColumnWriter::get_total_data_pages_bytes(
+ ColumnWriter::DataBytesGetter getter) const {
+ uint64_t size = 0;
+ if (_single_writer) {
+ size += std::invoke(getter, _single_writer.get());
+ }
+ for (const auto& w : _bucket_writers) {
+ size += std::invoke(getter, w.get());
+ }
+ return size;
+}
+
Status UnifiedSparseColumnWriter::finish() {
if (_single_writer) {
RETURN_IF_ERROR(_single_writer->finish());
@@ -350,6 +362,17 @@ uint64_t VariantDocWriter::estimate_buffer_size() const {
return size;
}
+uint64_t
VariantDocWriter::get_total_data_pages_bytes(ColumnWriter::DataBytesGetter
getter) const {
+ uint64_t size = 0;
+ for (const auto& writer : _subcolumn_writers) {
+ size += std::invoke(getter, writer.get());
+ }
+ for (const auto& writer : _doc_value_column_writers) {
+ size += std::invoke(getter, writer.get());
+ }
+ return size;
+}
+
Status VariantDocWriter::finish() {
for (auto& writer : _subcolumn_writers) {
RETURN_IF_ERROR(writer->finish());
@@ -520,6 +543,11 @@ uint64_t VariantColumnWriterImpl::estimate_buffer_size() {
return _v2_writer ? _v2_writer->estimate_buffer_size() : 0;
}
+uint64_t VariantColumnWriterImpl::get_total_data_pages_bytes(
+ ColumnWriter::DataBytesGetter getter) const {
+ return _v2_writer->get_total_data_pages_bytes(getter);
+}
+
VariantSubcolumnWriter::VariantSubcolumnWriter(const ColumnWriterOptions& opts,
TabletColumnPtr column)
: ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta)
{
diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.h
b/be/src/storage/segment/variant/variant_column_writer_impl.h
index 6ba064c3927..4cd882eaed2 100644
--- a/be/src/storage/segment/variant/variant_column_writer_impl.h
+++ b/be/src/storage/segment/variant/variant_column_writer_impl.h
@@ -62,6 +62,8 @@ public:
virtual Status write_inverted_index() = 0;
virtual Status write_bloom_filter_index() = 0;
virtual uint64_t estimate_buffer_size() const = 0;
+ // Sums `getter`, one of the ColumnWriter data bytes getters, over the
owned column writers.
+ virtual uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter
getter) const = 0;
virtual void merge_stats_to(VariantStatistics* stats) const = 0;
};
@@ -80,6 +82,7 @@ public:
Status write_inverted_index() override;
Status write_bloom_filter_index() override;
uint64_t estimate_buffer_size() const override;
+ uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter)
const override;
void merge_stats_to(VariantStatistics* stats) const override;
private:
@@ -127,6 +130,7 @@ public:
Status write_zone_map() override;
Status write_inverted_index() override;
Status write_bloom_filter_index() override;
+ uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter)
const override;
void merge_stats_to(VariantStatistics* stats) const override;
private:
@@ -171,6 +175,7 @@ public:
Status write_inverted_index();
Status write_bloom_filter_index();
uint64_t estimate_buffer_size();
+ uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter)
const;
private:
Status _ensure_writer();
@@ -205,15 +210,15 @@ public:
ordinal_t get_next_rowid() const override { return _next_rowid; }
uint64_t get_raw_data_bytes() const override {
- return 0; // TODO
+ return _get_total_data_pages_bytes(&ColumnWriter::get_raw_data_bytes);
}
uint64_t get_total_uncompressed_data_pages_bytes() const override {
- return 0; // TODO
+ return
_get_total_data_pages_bytes(&ColumnWriter::get_total_uncompressed_data_pages_bytes);
}
uint64_t get_total_compressed_data_pages_bytes() const override {
- return 0; // TODO
+ return
_get_total_data_pages_bytes(&ColumnWriter::get_total_compressed_data_pages_bytes);
}
Status append_nulls(size_t num_rows) override {
@@ -228,6 +233,15 @@ public:
Status finalize();
private:
+ template <typename Func>
+ uint64_t _get_total_data_pages_bytes(Func func) const {
+ uint64_t size = std::invoke(func, _doc_value_column_writer.get());
+ for (const auto& writer : _subcolumn_writers) {
+ size += std::invoke(func, writer.get());
+ }
+ return size;
+ }
+
Status _append(const uint8_t* null_map, const uint8_t** ptr, size_t
num_rows);
Status _ensure_input_format(const VariantColumnData& column);
Status _initialize_v2_shredder();
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
index 7e5fce17d26..f4c19d63ff3 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat
index b6d9603d749..dcdd42cbf18 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_0.dat
index dc385012095..c18164da395 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat
index d5bfc49f8e7..027959a37a1 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_0.dat
index 2aac77babc1..776cac5ba8f 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_1.dat
index 4ca8478218c..83ec1ff3d36 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_0.dat
index a2bcbf9c6f6..6a6a987fd58 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_1.dat
index 60a3d754fda..c525473cbd2 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat
index cf870d36a45..274293b55a6 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat
index 0c70bedef55..50b2ccd9208 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat
index c92c2d9d7cb..1605671feda 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_1.dat
index 45607d468cc..98df5f82029 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
index 38b7998ace2..30ac1ee33ed 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat
index bd262d787f1..1b055d7e1fd 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_0.dat
index 148bfe5a0fb..e6abeb09547 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat
index ffe02c1030f..156d248d7ad 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_0.dat
index 2194947a023..8cff17ddb9c 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_1.dat
index a37cdfa9235..ed73f8873c9 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_0.dat
index fe23b80c28d..5273fd05748 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_1.dat
index 04ce67e4f77..ddd81eb8f4d 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_0.dat
index fa7e17a77bc..0d221a14cf9 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_1.dat
index 8edad123858..4eb8de195a4 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_0.dat
index fe23b80c28d..5273fd05748 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_1.dat
index 04ce67e4f77..ddd81eb8f4d 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_0.dat
index 10a3e9baea0..0d08884ec60 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_1.dat
index 3ca53da8412..19744cf5b27 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_0.dat
index 9a119a5b17d..f9625d6766c 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_1.dat
index b4c7f6a3705..74a3a39f727 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_2.dat
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_2.dat
index 540713ba363..af4d09fc4be 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_2.dat
and
b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_2.dat
differ
diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp
b/be/test/storage/variant/variant_column_writer_reader_test.cpp
index 423152955ed..0e907d08924 100644
--- a/be/test/storage/variant/variant_column_writer_reader_test.cpp
+++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp
@@ -1704,6 +1704,8 @@ protected:
RETURN_IF_ERROR(writer->finish());
RETURN_IF_ERROR(writer->write_data());
+ // The file holds only the data pages of this variant so far.
+ EXPECT_EQ(writer->get_total_compressed_data_pages_bytes(),
file_writer->bytes_appended());
RETURN_IF_ERROR(writer->write_ordinal_index());
RETURN_IF_ERROR(writer->write_zone_map());
if (index_write_policy == VariantIndexWritePolicy::BLOOM_AND_INVERTED)
{
@@ -1775,6 +1777,8 @@ protected:
RETURN_IF_ERROR(writer->finish());
RETURN_IF_ERROR(writer->write_data());
+ // The file holds only the data pages of this subcolumn so far.
+ EXPECT_EQ(writer->get_total_compressed_data_pages_bytes(),
file_writer->bytes_appended());
RETURN_IF_ERROR(writer->write_ordinal_index());
RETURN_IF_ERROR(writer->write_zone_map());
RETURN_IF_ERROR(file_writer->close());
@@ -3951,6 +3955,10 @@ TEST_F(VariantColumnWriterReaderTest,
test_write_data_normal) {
EXPECT_TRUE(st.ok()) << st.msg();
st = writer->write_data();
EXPECT_TRUE(st.ok()) << st.msg();
+ // The root, the subcolumns and the sparse columns all count to the
variant.
+ EXPECT_EQ(writer->get_total_compressed_data_pages_bytes(),
file_writer->bytes_appended());
+ EXPECT_GT(writer->get_total_uncompressed_data_pages_bytes(), 0);
+ EXPECT_GT(writer->get_raw_data_bytes(), 0);
st = writer->write_ordinal_index();
EXPECT_TRUE(st.ok()) << st.msg();
st = writer->write_zone_map();
@@ -5241,6 +5249,12 @@ TEST_P(VariantSpecializedWriterCompatibilityTest,
doc_compact_writer_round_trip)
EXPECT_TRUE(doc_compact_writer->finish().ok());
EXPECT_TRUE(root_writer->write_data().ok());
EXPECT_TRUE(doc_compact_writer->write_data().ok());
+ // The file holds only data pages so far: the root of the root-only
writer, plus the doc value
+ // column and the materialized subcolumns of the doc compact writer.
+ EXPECT_GT(doc_compact_writer->get_total_compressed_data_pages_bytes(), 0);
+ EXPECT_EQ(root_writer->get_total_compressed_data_pages_bytes() +
+
doc_compact_writer->get_total_compressed_data_pages_bytes(),
+ file_writer->bytes_appended());
EXPECT_TRUE(root_writer->write_ordinal_index().ok());
EXPECT_TRUE(doc_compact_writer->write_ordinal_index().ok());
EXPECT_TRUE(file_writer->close().ok());
diff --git
a/regression-test/data/query_p0/system/test_query_sys_column_data_sizes_variant.out
b/regression-test/data/query_p0/system/test_query_sys_column_data_sizes_variant.out
new file mode 100644
index 00000000000..9233a27e436
--- /dev/null
+++
b/regression-test/data/query_p0/system/test_query_sys_column_data_sizes_variant.out
@@ -0,0 +1,33 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sizes --
+id BIGINT true true true
+v VARIANT true true true
+
+-- !sizes --
+id BIGINT true true true
+v VARIANT true true true
+
+-- !sizes --
+id BIGINT true true true
+v VARIANT true true true
+
+-- !sizes --
+id BIGINT true true true
+v VARIANT true true true
+
+-- !sizes --
+id BIGINT true true true
+v VARIANT true true true
+
+-- !sizes --
+id BIGINT true true true
+v VARIANT true true true
+
+-- !sizes --
+id BIGINT true true true
+v VARIANT true true true
+
+-- !sizes --
+id BIGINT true true true
+v VARIANT true true true
+
diff --git
a/regression-test/suites/query_p0/system/test_query_sys_column_data_sizes_variant.groovy
b/regression-test/suites/query_p0/system/test_query_sys_column_data_sizes_variant.groovy
new file mode 100644
index 00000000000..71463b2c933
--- /dev/null
+++
b/regression-test/suites/query_p0/system/test_query_sys_column_data_sizes_variant.groovy
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_query_sys_column_data_sizes_variant", "query,p0") {
+ // A variant column keeps its data in the root, the materialized
subcolumns and the sparse or
+ // doc value columns. Compaction writes them as columns of their own, and
external column meta
+ // (V3) embeds the sparse and doc value columns into the root meta. All of
them count to the
+ // variant column.
+ def checkVariantDataSizes = { String tableName ->
+ for (int i = 0; i < 3; i++) {
+ sql """
+ INSERT INTO ${tableName}
+ SELECT number, parse_to_variant(concat('{"a":', number,
',"b":"', repeat('x', number % 32),
+ '","c":{"d":', number * 2, '},"k', number % 16, '":"',
repeat('y', number % 64), '"}'))
+ FROM numbers("number" = "10000")
+ """
+ }
+ def tabletId = sql("SHOW TABLETS FROM ${tableName}")[0][0]
+ def check = {
+ qt_sizes """
+ SELECT COLUMN_NAME, COLUMN_TYPE, SUM(COMPRESSED_DATA_BYTES) >
0,
+ SUM(UNCOMPRESSED_DATA_BYTES) > 0, SUM(RAW_DATA_BYTES) > 0
+ FROM information_schema.column_data_sizes
+ WHERE TABLET_ID = ${tabletId}
+ GROUP BY COLUMN_NAME, COLUMN_TYPE
+ ORDER BY COLUMN_NAME
+ """
+ // The data pages of all columns lie in the segment files and make
up most of them, so
+ // a part of the variant that is not counted, or counted twice,
breaks the bounds.
+ def bytes = sql """
+ SELECT SUM(c.data_page_bytes), SUM(r.DATA_DISK_SIZE)
+ FROM (
+ SELECT BACKEND_ID, ROWSET_ID, SUM(COMPRESSED_DATA_BYTES)
AS data_page_bytes
+ FROM information_schema.column_data_sizes
+ WHERE TABLET_ID = ${tabletId}
+ GROUP BY BACKEND_ID, ROWSET_ID
+ ) c
+ JOIN information_schema.rowsets r
+ ON r.BACKEND_ID = c.BACKEND_ID AND r.ROWSET_ID =
c.ROWSET_ID
+ """
+ long dataPageBytes = bytes[0][0] as long
+ long segmentBytes = bytes[0][1] as long
+ logger.info("${tableName}: ${dataPageBytes} data page bytes in
${segmentBytes} segment bytes")
+ assertTrue(dataPageBytes <= segmentBytes && dataPageBytes * 2 >
segmentBytes,
+ "${tableName}: ${dataPageBytes} data page bytes in
${segmentBytes} segment bytes")
+ }
+
+ check()
+ trigger_and_wait_compaction(tableName, "full")
+ check()
+ }
+
+ // Materialized subcolumns and bucketized sparse columns, with the column
metas in the footer
+ // (V2) and in the external column meta region (V3).
+ sql "set default_variant_enable_doc_mode = false"
+ sql "DROP TABLE IF EXISTS test_column_data_sizes_variant_v2"
+ sql """
+ CREATE TABLE test_column_data_sizes_variant_v2 (
+ id BIGINT,
+ v VARIANT<PROPERTIES("variant_max_subcolumns_count" = "1",
"variant_sparse_hash_shard_count" = "2")>
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1", "disable_auto_compaction" =
"true", "storage_format" = "V2")
+ """
+ checkVariantDataSizes("test_column_data_sizes_variant_v2")
+
+ sql "DROP TABLE IF EXISTS test_column_data_sizes_variant_v3"
+ sql """
+ CREATE TABLE test_column_data_sizes_variant_v3 (
+ id BIGINT,
+ v VARIANT<PROPERTIES("variant_max_subcolumns_count" = "1",
"variant_sparse_hash_shard_count" = "2")>
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1", "disable_auto_compaction" =
"true", "storage_format" = "V3")
+ """
+ checkVariantDataSizes("test_column_data_sizes_variant_v3")
+
+ // Doc value columns and materialized subcolumns in doc mode.
+ sql "DROP TABLE IF EXISTS test_column_data_sizes_variant_doc_v2"
+ sql """
+ CREATE TABLE test_column_data_sizes_variant_doc_v2 (
+ id BIGINT,
+ v VARIANT<PROPERTIES("variant_enable_doc_mode" = "true",
"variant_doc_materialization_min_rows" = "0",
+ "variant_doc_hash_shard_count" = "2")>
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1", "disable_auto_compaction" =
"true", "storage_format" = "V2")
+ """
+ checkVariantDataSizes("test_column_data_sizes_variant_doc_v2")
+
+ sql "DROP TABLE IF EXISTS test_column_data_sizes_variant_doc_v3"
+ sql """
+ CREATE TABLE test_column_data_sizes_variant_doc_v3 (
+ id BIGINT,
+ v VARIANT<PROPERTIES("variant_enable_doc_mode" = "true",
"variant_doc_materialization_min_rows" = "0",
+ "variant_doc_hash_shard_count" = "2")>
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1", "disable_auto_compaction" =
"true", "storage_format" = "V3")
+ """
+ checkVariantDataSizes("test_column_data_sizes_variant_doc_v3")
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
