This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a37fc36a0ff [opt](variant) Walk only non-null materialized slots per
row in VariantAssembler (#69914)
a37fc36a0ff is described below
commit a37fc36a0ffdc6bc666d7069eb865038aa302857
Author: lihangyu <[email protected]>
AuthorDate: Tue Nov 3 12:31:52 2026 +0800
[opt](variant) Walk only non-null materialized slots per row in
VariantAssembler (#69914)
---
be/benchmark/benchmark_main.cpp | 1 +
be/benchmark/benchmark_variant_assembler.hpp | 90 +++++++
.../segment/variant/v2/variant_assembler.cpp | 107 ++++++---
.../segment/variant_assembler_legacy_test.cpp | 258 +++++++++++++++++++++
4 files changed, 426 insertions(+), 30 deletions(-)
diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp
index 77ba90b09b2..00166122c8b 100644
--- a/be/benchmark/benchmark_main.cpp
+++ b/be/benchmark/benchmark_main.cpp
@@ -40,6 +40,7 @@
#include "benchmark_hybrid_set.hpp"
#include "benchmark_json_extract.hpp"
#include "benchmark_relative_day.hpp"
+#include "benchmark_variant_assembler.hpp"
#include "benchmark_zone_map_index.hpp"
#include "binary_cast_benchmark.hpp"
#include "parquet/benchmark_file_scanner_expr.hpp"
diff --git a/be/benchmark/benchmark_variant_assembler.hpp
b/be/benchmark/benchmark_variant_assembler.hpp
new file mode 100644
index 00000000000..4dfe6e20162
--- /dev/null
+++ b/be/benchmark/benchmark_variant_assembler.hpp
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <benchmark/benchmark.h>
+
+#include <algorithm>
+#include <memory>
+#include <numeric>
+#include <random>
+#include <string>
+#include <utility>
+
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
+#include "core/custom_allocator.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "storage/segment/variant/v2/variant_assembler.h"
+
+namespace doris::variant_assembler_benchmark {
+
+static void BM_VariantAssembler(benchmark::State& state, size_t slots, size_t
visible) {
+ constexpr size_t ROWS = 4096;
+ auto type = make_nullable(std::make_shared<DataTypeInt64>());
+ segment_v2::variant_v2::VariantAssemblerOptions options;
+ DorisVector<ColumnNullable::MutablePtr> columns;
+ DorisVector<const IColumn*> materialized;
+ for (size_t slot = 0; slot < slots; ++slot) {
+ options.materialized_paths.push_back(
+ {.path = PathInData("k" + std::to_string(slot)), .type =
type});
+ columns.push_back(ColumnNullable::create(
+ ColumnInt64::create(ROWS, 1), ColumnUInt8::create(ROWS,
visible == slots ? 0 : 1)));
+ materialized.push_back(columns.back().get());
+ }
+ if (visible != slots) {
+ std::mt19937 random(20261031);
+ DorisVector<size_t> slot_order(slots);
+ std::iota(slot_order.begin(), slot_order.end(), 0);
+ for (size_t row = 0; row < ROWS; ++row) {
+ std::shuffle(slot_order.begin(), slot_order.end(), random);
+ for (size_t index = 0; index < visible; ++index) {
+ columns[slot_order[index]]->get_null_map_data()[row] = 0;
+ }
+ }
+ }
+ auto result =
segment_v2::variant_v2::VariantAssembler::create(std::move(options));
+ if (!result.has_value()) {
+ state.SkipWithError(result.error().to_string().c_str());
+ return;
+ }
+ auto assembler = std::move(result).value();
+ segment_v2::variant_v2::VariantAssemblerBatchView batch;
+ batch.num_rows = ROWS;
+ batch.materialized_columns = materialized;
+ for (auto _ : state) {
+ ColumnNullable::MutablePtr output;
+ const Status status = assembler->assemble(batch, &output);
+ if (!status.ok()) {
+ state.SkipWithError(status.to_string().c_str());
+ return;
+ }
+ benchmark::DoNotOptimize(output);
+ }
+ state.SetItemsProcessed(state.iterations() * ROWS);
+ state.counters["rows/s"] =
benchmark::Counter(static_cast<double>(state.iterations()) * ROWS,
+ benchmark::Counter::kIsRate);
+}
+
+BENCHMARK_CAPTURE(BM_VariantAssembler, WideSparse, 2048, 20);
+BENCHMARK_CAPTURE(BM_VariantAssembler, Dense, 200, 200);
+BENCHMARK_CAPTURE(BM_VariantAssembler, Narrow, 10, 10);
+BENCHMARK_CAPTURE(BM_VariantAssembler, MediumSparse, 200, 75);
+
+} // namespace doris::variant_assembler_benchmark
diff --git a/be/src/storage/segment/variant/v2/variant_assembler.cpp
b/be/src/storage/segment/variant/v2/variant_assembler.cpp
index 20e5c2d2889..44833aa0e78 100644
--- a/be/src/storage/segment/variant/v2/variant_assembler.cpp
+++ b/be/src/storage/segment/variant/v2/variant_assembler.cpp
@@ -19,6 +19,8 @@
#include <algorithm>
#include <cstring>
+#include <limits>
+#include <numeric>
#include <utility>
#include <vector>
@@ -34,6 +36,7 @@
#include "exprs/function/parse/variant_jsonb_parse.h"
#include "storage/segment/variant/v2/variant_assembler_internal.h"
#include "storage/segment/variant/v2/variant_storage_cell.h"
+#include "util/simd/bits.h"
namespace doris::segment_v2::variant_v2 {
namespace {
@@ -296,22 +299,17 @@ struct PreparedHierarchicalBatch {
const ColumnString* root_values = nullptr;
const uint8_t* root_nulls = nullptr;
DorisVector<variant_assembler_detail::PreparedMaterializedColumn>
materialized;
-};
+ DorisVector<size_t> candidate_offsets;
+ DorisVector<uint32_t> candidate_slots;
-bool has_materialized_value(
- std::span<const variant_assembler_detail::PreparedMaterializedColumn>
materialized,
- std::span<const MaterializedSlot> materialized_slots, size_t row,
- bool reads_whole_variant) {
- DORIS_CHECK_EQ(materialized.size(), materialized_slots.size());
- for (size_t index = 0; index < materialized.size(); ++index) {
- if (variant_assembler_detail::is_materialized_value_visible(
- materialized[index], row,
- reads_whole_variant &&
materialized_slots[index].relative_path.empty())) {
- return true;
- }
+ std::span<const uint32_t> candidate_materialized_slots(size_t row) const {
+ const std::span<const uint32_t> slots(candidate_slots);
+ return candidate_offsets.empty()
+ ? slots
+ : slots.subspan(candidate_offsets[row],
+ candidate_offsets[row + 1] -
candidate_offsets[row]);
}
- return false;
-}
+};
PreparedHierarchicalBatch prepare_hierarchical_batch(
StorageMapKind storage_map_kind, bool has_root,
@@ -336,9 +334,49 @@ PreparedHierarchicalBatch prepare_hierarchical_batch(
}
output.materialized.reserve(materialized_slots.size());
+ size_t candidate_count = 0;
for (const MaterializedSlot& slot : materialized_slots) {
-
output.materialized.push_back(variant_assembler_detail::prepare_materialized_column(
- slot.type, batch.materialized_columns[slot.batch_index],
batch.num_rows));
+ auto column = variant_assembler_detail::prepare_materialized_column(
+ slot.type, batch.materialized_columns[slot.batch_index],
batch.num_rows);
+ candidate_count +=
+ column.nulls == nullptr
+ ? batch.num_rows
+ : simd::count_zero_num(reinterpret_cast<const
int8_t*>(column.nulls),
+ batch.num_rows);
+ output.materialized.push_back(std::move(column));
+ }
+ DORIS_CHECK_LE(materialized_slots.size(),
std::numeric_limits<uint32_t>::max());
+ // At 37.5% candidates, CSR was about 5% faster than identity in the
4096-row benchmark.
+ if (candidate_count >= batch.num_rows * materialized_slots.size() / 2) {
+ output.candidate_slots.resize(materialized_slots.size());
+ std::iota(output.candidate_slots.begin(),
output.candidate_slots.end(), uint32_t {0});
+ } else {
+ auto for_each_candidate = [&](auto&& visit) {
+ for (size_t index = 0; index < output.materialized.size();
++index) {
+ const auto* nulls = output.materialized[index].nulls;
+ for (size_t row = 0; row < batch.num_rows; ++row) {
+ if (nulls != nullptr) {
+ row = simd::find_byte<uint8_t>(nulls, row,
batch.num_rows, 0);
+ if (row == batch.num_rows) {
+ break;
+ }
+ }
+ visit(row, static_cast<uint32_t>(index));
+ }
+ }
+ };
+ output.candidate_offsets.resize(batch.num_rows + 1, 0);
+ for_each_candidate([&](size_t row, uint32_t) {
++output.candidate_offsets[row + 1]; });
+ for (size_t row = 0; row < batch.num_rows; ++row) {
+ output.candidate_offsets[row + 1] += output.candidate_offsets[row];
+ }
+ output.candidate_slots.resize(candidate_count);
+ DorisVector<size_t> write_positions(output.candidate_offsets.begin(),
+ output.candidate_offsets.end() -
1);
+ // Filling in sorted slot order keeps each row ordered, e.g. [a, a-,
a.b].
+ for_each_candidate([&](size_t row, uint32_t index) {
+ output.candidate_slots[write_positions[row]++] = index;
+ });
}
if (storage_map_kind != StorageMapKind::NONE) {
map_cursor->bind(batch.storage_map, batch.num_rows);
@@ -446,25 +484,30 @@ Status emit_doc_row(size_t row, StringRef requested_raw,
bool reads_whole_varian
Status emit_merged_row(
std::span<const variant_assembler_detail::PreparedMaterializedColumn>
materialized,
- bool has_sparse, std::span<const MaterializedSlot> materialized_slots,
size_t row,
- StringRef requested_raw, bool reads_whole_variant,
StorageMapRowCursor* sparse_cursor,
+ bool has_sparse, std::span<const MaterializedSlot> materialized_slots,
+ std::span<const uint32_t> candidate_slots, size_t row, StringRef
requested_raw,
+ bool reads_whole_variant, StorageMapRowCursor* sparse_cursor,
DorisVector<MergeValue>* pending, ObjectEmitter* emitter) {
if (has_sparse) {
RETURN_IF_ERROR(sparse_cursor->start_row(row, requested_raw,
reads_whole_variant));
}
pending->clear();
+ size_t candidate_index = 0;
size_t materialized_index = 0;
- while (true) {
- while (materialized_index < materialized.size()) {
+ auto advance_materialized = [&] {
+ while (candidate_index < candidate_slots.size()) {
+ materialized_index = candidate_slots[candidate_index++];
if (variant_assembler_detail::is_materialized_value_visible(
materialized[materialized_index], row,
reads_whole_variant &&
materialized_slots[materialized_index].relative_path.empty())) {
- break;
+ return true;
}
- ++materialized_index;
}
- const bool materialized_available = materialized_index <
materialized_slots.size();
+ return false;
+ };
+ bool materialized_available = advance_materialized();
+ while (true) {
StringRef materialized_raw_path;
if (materialized_available) {
const std::string& path =
@@ -490,7 +533,7 @@ Status emit_merged_row(
if (!use_sparse) {
current.set_materialized(materialized_slots[materialized_index].relative_path,
materialized_raw_path,
&materialized[materialized_index]);
- ++materialized_index;
+ materialized_available = advance_materialized();
} else {
current.set_cell(sparse_cursor->sort_key, sparse_cursor->cell,
sparse_cursor->is_direct_subtree_value,
sparse_cursor->depth);
@@ -513,6 +556,7 @@ Status assemble_hierarchical_row(StorageMapKind
storage_map_kind, bool has_root,
batch.root_values->get_data_at(row_index).size != 0;
const bool has_doc_row = has_doc && !map_cursor->row_empty(row_index);
const bool reads_whole_variant = requested.empty();
+ const auto candidate_slots = batch.candidate_materialized_slots(row_index);
// Physical row precedence:
// 1. A non-empty doc row exclusively defines the row.
@@ -521,10 +565,13 @@ Status assemble_hierarchical_row(StorageMapKind
storage_map_kind, bool has_root,
// 4. An empty whole-Variant row is {}; an absent subtree is SQL NULL.
if (has_root_sidecar_value && !has_doc_row &&
(!has_sparse || map_cursor->row_empty(row_index))) {
- // Only the root fast-path candidate needs this O(materialized-paths)
scan. Ordinary object
- // and doc rows are scanned once later by
emit_merged_row()/emit_doc_row().
- if (!has_materialized_value(batch.materialized, materialized_slots,
row_index,
- reads_whole_variant)) {
+ const bool has_materialized_value =
+ std::ranges::any_of(candidate_slots, [&](uint32_t index) {
+ return
variant_assembler_detail::is_materialized_value_visible(
+ batch.materialized[index], row_index,
+ reads_whole_variant &&
materialized_slots[index].relative_path.empty());
+ });
+ if (!has_materialized_value) {
jsonb_to_variant(batch.root_values->get_data_at(row_index), *row,
0, nullptr);
return Status::OK();
}
@@ -538,8 +585,8 @@ Status assemble_hierarchical_row(StorageMapKind
storage_map_kind, bool has_root,
pending, emitter));
} else {
RETURN_IF_ERROR(emit_merged_row(batch.materialized, has_sparse,
materialized_slots,
- row_index, requested_raw,
reads_whole_variant, map_cursor,
- pending, emitter));
+ candidate_slots, row_index,
requested_raw,
+ reads_whole_variant, map_cursor,
pending, emitter));
}
if (!emitter->emitted) {
if (reads_whole_variant && has_root && storage_map_kind ==
StorageMapKind::NONE &&
diff --git a/be/test/storage/segment/variant_assembler_legacy_test.cpp
b/be/test/storage/segment/variant_assembler_legacy_test.cpp
index 2be5ba56f86..f65f2a404d6 100644
--- a/be/test/storage/segment/variant_assembler_legacy_test.cpp
+++ b/be/test/storage/segment/variant_assembler_legacy_test.cpp
@@ -160,6 +160,19 @@ std::unique_ptr<VariantAssembler>
create_assembler(VariantAssemblerOptions optio
return std::move(result).value();
}
+void expect_assembled_rows(VariantAssembler& assembler, const
VariantAssemblerBatchView& batch,
+ const std::vector<std::string_view>& expected,
+ const std::vector<uint8_t>& outer_nulls = {}) {
+ ColumnNullable::MutablePtr output;
+ const Status status = assembler.assemble(batch, &output);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_EQ(output->size(), expected.size());
+ for (size_t row = 0; row < expected.size(); ++row) {
+ EXPECT_EQ(json_at(assembled_values(output), row), expected[row]) <<
"row=" << row;
+ EXPECT_EQ(output->get_null_map_data()[row], outer_nulls.empty() ? 0 :
outer_nulls[row]);
+ }
+}
+
Status assemble_binary_cells(std::span<const StringRef> cells,
ColumnNullable::MutablePtr* output,
std::span<const uint8_t> outer_nulls = {},
std::span<const uint8_t> missing = {}) {
@@ -819,6 +832,251 @@ TEST(VariantAssemblerLegacyTest,
MaterializedArraysPreserveExplicitValues) {
EXPECT_EQ(subtree_output->get_null_map_data(), (PaddedPODArray<uint8_t>
{1, 0, 0, 0, 0, 0, 1}));
}
+TEST(VariantAssemblerLegacyTest, MaterializedVisibilityMatrix) {
+ auto check = [](const DataTypePtr& type, const IColumn* column, const
PathInData& path,
+ const PathInData& requested, const
std::vector<std::string_view>& expected,
+ const std::vector<uint8_t>& outer_nulls = {}) {
+ auto absent_type = make_nullable(std::make_shared<DataTypeInt32>());
+ auto absent =
ColumnNullable::create(ColumnInt32::create(column->size(), 0),
+
ColumnUInt8::create(column->size(), 255));
+ for (size_t padding : {0, 8}) {
+ VariantAssemblerOptions options;
+ options.requested_path = requested;
+ options.materialized_paths = {{.path = path, .type = type}};
+ DorisVector<const IColumn*> materialized {column};
+ for (size_t index = 0; index < padding; ++index) {
+ const std::string prefix = requested.empty() ? "" :
requested.get_path() + ".";
+ options.materialized_paths.push_back(
+ {.path = PathInData(prefix + "absent" +
std::to_string(index)),
+ .type = absent_type});
+ materialized.push_back(absent.get());
+ }
+ auto assembler = create_assembler(std::move(options));
+ VariantAssemblerBatchView batch;
+ batch.num_rows = column->size();
+ batch.materialized_columns = materialized;
+ expect_assembled_rows(*assembler, batch, expected, outer_nulls);
+ }
+ };
+
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto defaults = ColumnInt32::create(3, 0);
+ check(int_type, defaults.get(), PathInData("a"), {},
+ {R"({"a":0})", R"({"a":0})", R"({"a":0})"});
+ auto nulls = ColumnUInt8::create();
+ for (uint8_t value : {0, 1, 255}) {
+ nulls->insert_value(value);
+ }
+ auto nullable = ColumnNullable::create(std::move(defaults),
std::move(nulls));
+ for (bool typed : {false, true}) {
+ check(make_nullable(int_type), nullable.get(), PathInData("a", typed),
{},
+ {R"({"a":0})", "{}", "{}"});
+ }
+
+ auto jsonb = ColumnString::create();
+ for (std::string_view text : {"null", "{}", "[]"}) {
+ auto value = jsonb_column(text);
+ jsonb->insert_range_from(*value, 0, 1);
+ }
+ check(std::make_shared<DataTypeJsonb>(), jsonb.get(), PathInData("a"), {},
+ {R"({"a":null})", R"({"a":{}})", R"({"a":[]})"});
+
+ auto elements = ColumnString::create();
+ elements->insert_default();
+ auto empty_array = jsonb_column("[]");
+ elements->insert_range_from(*empty_array, 0, 1);
+ auto element_nulls = ColumnUInt8::create();
+ element_nulls->insert_value(1);
+ element_nulls->insert_value(0);
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ auto array_nulls = ColumnUInt8::create();
+ for (uint8_t value : {1, 0, 0, 0, 255}) {
+ array_nulls->insert_value(value);
+ }
+ for (size_t offset : {0, 0, 1, 2, 2}) {
+ offsets->insert_value(offset);
+ }
+ auto arrays = ColumnNullable::create(
+ ColumnArray::create(
+ ColumnNullable::create(std::move(elements),
std::move(element_nulls)),
+ std::move(offsets)),
+ std::move(array_nulls));
+ auto array_type = make_nullable(
+
std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeJsonb>())));
+ check(array_type, arrays.get(), PathInData("a"), {},
+ {"{}", "{}", R"({"a":[null]})", R"({"a":[[]]})", "{}"});
+ check(array_type, arrays.get(), {}, {}, {"{}", "[]", "[null]", "[[]]",
"{}"});
+ check(array_type, arrays.get(), PathInData("a"), PathInData("a"),
+ {"null", "null", "[null]", "[[]]", "null"}, {1, 1, 0, 0, 1});
+}
+
+TEST(VariantAssemblerLegacyTest,
MaterializedSparseMergeKeepsTiesAndPrefixOrdering) {
+ struct Case {
+ std::string_view path;
+ std::vector<std::pair<std::string, int32_t>> sparse;
+ std::string_view expected;
+ };
+ const std::vector<Case> cases {
+ {.path = "a", .sparse = {{"a", 20}}, .expected = ""},
+ {.path = "a",
+ .sparse = {{"a-", 20}, {"a.b", 30}},
+ .expected = R"({"a":{"b":30},"a-":20})"},
+ {.path = "a.b",
+ .sparse = {{"a", 20}, {"a-", 30}},
+ .expected = R"({"a":{"b":10},"a-":30})"},
+ {.path = "a-",
+ .sparse = {{"a", 20}, {"a.b", 30}},
+ .expected = R"({"a":{"b":30},"a-":10})"},
+ {.path = "a",
+ .sparse = {{"a", 20}, {"a-", 30}, {"a.b", 40}},
+ .expected = R"({"a":{"b":40},"a-":30})"},
+ };
+ auto check = [](const Status& status, const ColumnNullable::MutablePtr&
output,
+ std::string_view expected) {
+ if (expected.empty()) {
+ EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status;
+ EXPECT_NE(status.to_string().find("Variant object value must be
preceded by add_key"),
+ std::string::npos);
+ EXPECT_EQ(output.get(), nullptr);
+ return;
+ }
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(json_at(assembled_values(output), 0), expected);
+ };
+ for (const auto& test : cases) {
+ SCOPED_TRACE(test.path);
+ MapEntries entries;
+ for (const auto& [path, value] : test.sparse) {
+ entries.emplace_back(
+ path,
fixed_storage_cell<int32_t>(FieldType::OLAP_FIELD_TYPE_INT, value));
+ }
+ auto sparse = map_column(entries);
+ for (size_t padding : {0, 8}) {
+ VariantAssemblerOptions options;
+ options.materialized_paths = {
+ {.path = PathInData(test.path), .type =
std::make_shared<DataTypeInt32>()}};
+ options.storage_map_kind = StorageMapKind::SPARSE;
+ auto values = ColumnInt32::create(1, 10);
+ auto absent =
+ ColumnNullable::create(ColumnInt32::create(1, 0),
ColumnUInt8::create(1, 255));
+ DorisVector<const IColumn*> materialized {values.get()};
+ for (size_t index = 0; index < padding; ++index) {
+ options.materialized_paths.push_back(
+ {.path = PathInData("absent" + std::to_string(index)),
+ .type =
make_nullable(std::make_shared<DataTypeInt32>())});
+ materialized.push_back(absent.get());
+ }
+ auto assembler = create_assembler(std::move(options));
+ VariantAssemblerBatchView batch;
+ batch.num_rows = 1;
+ batch.materialized_columns = materialized;
+ batch.storage_map = sparse.get();
+ ColumnNullable::MutablePtr output;
+ const Status status = assembler->assemble(batch, &output);
+ check(status, output, test.expected);
+ }
+ }
+}
+
+TEST(VariantAssemblerLegacyTest,
MixedDocRowsPreserveMaterializedAndRootPrecedence) {
+ auto root_value = jsonb_column(R"({"root":1})");
+ auto roots = ColumnString::create();
+ for (size_t row = 0; row < 4; ++row) {
+ roots->insert_range_from(*root_value, 0, 1);
+ }
+ auto root_nulls = ColumnUInt8::create(4, 0);
+ root_nulls->get_data()[3] = 1;
+ auto root = ColumnNullable::create(std::move(roots),
std::move(root_nulls));
+ auto nulls = ColumnUInt8::create(4, 0);
+ nulls->get_data()[2] = 255;
+ auto values = ColumnNullable::create(ColumnInt32::create(4, 10),
std::move(nulls));
+ auto array_type =
std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>());
+ auto empty_arrays = array_type->create_column();
+ empty_arrays->insert_many_defaults(4);
+ auto absent = ColumnNullable::create(ColumnInt32::create(4, 0),
ColumnUInt8::create(4, 255));
+ const auto cell =
fixed_storage_cell<int32_t>(FieldType::OLAP_FIELD_TYPE_INT, 20);
+ auto doc = map_column_rows({{{"d", cell}}, {}, {}, {{"d", cell}}});
+ for (size_t padding : {0, 8}) {
+ for (bool subtree : {false, true}) {
+ VariantAssemblerOptions options;
+ options.storage_map_kind = StorageMapKind::DOC;
+ options.has_root = !subtree;
+ options.requested_path = subtree ? PathInData("a") : PathInData();
+ options.materialized_paths = {
+ {.path = PathInData("a"),
+ .type = make_nullable(std::make_shared<DataTypeInt32>())},
+ {.path = PathInData("a.empty"), .type = array_type}};
+ DorisVector<const IColumn*> materialized {values.get(),
empty_arrays.get()};
+ for (size_t index = 0; index < padding; ++index) {
+ options.materialized_paths.push_back(
+ {.path = PathInData("a.absent" +
std::to_string(index)),
+ .type =
make_nullable(std::make_shared<DataTypeInt32>())});
+ materialized.push_back(absent.get());
+ }
+ auto assembler = create_assembler(std::move(options));
+ VariantAssemblerBatchView batch;
+ batch.num_rows = 4;
+ batch.root_jsonb = subtree ? nullptr : root.get();
+ batch.materialized_columns = materialized;
+ batch.storage_map = doc.get();
+ if (subtree) {
+ expect_assembled_rows(*assembler, batch, {"null", "10",
"null", "null"},
+ {1, 0, 1, 1});
+ } else {
+ expect_assembled_rows(*assembler, batch,
+ {R"({"d":20})", R"({"a":10})",
R"({"root":1})", "null"},
+ {0, 0, 0, 1});
+ }
+ }
+ }
+}
+
+TEST(VariantAssemblerLegacyTest, VisibleSlotsResetAcrossBatchBoundaries) {
+ auto type = make_nullable(std::make_shared<DataTypeInt32>());
+ VariantAssemblerOptions options;
+ options.materialized_paths = {{.path = PathInData("z"), .type = type},
+ {.path = PathInData("a"), .type = type}};
+ auto assembler = create_assembler(std::move(options));
+ VariantAssemblerOptions empty_options;
+ empty_options.storage_map_kind = StorageMapKind::SPARSE;
+ auto empty_assembler = create_assembler(std::move(empty_options));
+ for (size_t rows : {15, 16, 17, 33, 0}) {
+ for (size_t pattern = 0; pattern < 3; ++pattern) {
+ std::array<ColumnNullable::MutablePtr, 2> columns;
+ std::array<const IColumn*, 2> materialized;
+ for (size_t slot = 0; slot < columns.size(); ++slot) {
+ auto nulls = ColumnUInt8::create();
+ for (size_t row = 0; row < rows; ++row) {
+ constexpr std::array<uint8_t, 8> NULL_BYTES {0, 1, 255, 1,
255, 1, 255, 1};
+ nulls->insert_value(
+ NULL_BYTES[pattern == 1 ? (row + slot) %
NULL_BYTES.size() : pattern]);
+ }
+ columns[slot] =
+ ColumnNullable::create(ColumnInt32::create(rows, 0),
std::move(nulls));
+ materialized[slot] = columns[slot].get();
+ }
+ VariantAssemblerBatchView batch;
+ batch.num_rows = rows;
+ batch.materialized_columns = materialized;
+ constexpr std::array<std::string_view, 4> EXPECTED_JSON {
+ "{}", R"({"a":0})", R"({"z":0})", R"({"a":0,"z":0})"};
+ std::vector<std::string_view> expected;
+ for (size_t row = 0; row < rows; ++row) {
+ const bool a = columns[1]->get_null_map_data()[row] == 0;
+ const bool z = columns[0]->get_null_map_data()[row] == 0;
+ expected.push_back(EXPECTED_JSON[a + 2 * z]);
+ }
+ expect_assembled_rows(*assembler, batch, expected);
+ }
+ auto empty_map = map_column_rows(std::vector<MapEntries>(rows));
+ VariantAssemblerBatchView empty_batch;
+ empty_batch.num_rows = rows;
+ empty_batch.storage_map = empty_map.get();
+ expect_assembled_rows(*empty_assembler, empty_batch,
+ std::vector<std::string_view>(rows, "{}"));
+ }
+}
+
TEST(VariantAssemblerLegacyTest,
HierarchicalMaterializedLegacyDateTimeAndDecimalV2) {
LegacyCells source;
VariantAssemblerOptions options;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
