Gabriel39 commented on code in PR #66302:
URL: https://github.com/apache/doris/pull/66302#discussion_r3689499170
##########
be/src/format_v2/parquet/native_schema_desc.cpp:
##########
@@ -517,6 +726,21 @@ Status NativeFieldDescriptor::parse_group_field(
const std::vector<tparquet::SchemaElement>& t_schemas, size_t curr_pos,
NativeFieldSchema* group_field) {
auto& group_schema = t_schemas[curr_pos];
+ group_field->parquet_schema = group_schema;
+ if (is_variant_node(group_schema)) {
Review Comment:
Fixed. Required and optional Variant groups preserve their
repetition/definition levels; repeated Variant groups are rejected explicitly
because the current Doris Variant column is scalar. Schema/read tests cover all
three forms.
##########
be/src/format_v2/parquet/parquet_reader.cpp:
##########
@@ -57,6 +58,99 @@ struct ParquetReaderScanState {
bool enable_strict_mode = false;
};
+const ParquetColumnSchema* projected_schema_child(const ParquetColumnSchema&
schema,
+ int32_t local_id) {
+ const auto child_it = std::ranges::find_if(
+ schema.children, [local_id](const auto& child) { return
child->local_id == local_id; });
+ return child_it == schema.children.end() ? nullptr : child_it->get();
+}
+
+const ParquetColumnSchema* schema_child_by_name(const ParquetColumnSchema&
schema,
+ std::string_view name) {
+ const auto child_it = std::ranges::find_if(
+ schema.children, [name](const auto& child) { return child->name ==
name; });
+ return child_it == schema.children.end() ? nullptr : child_it->get();
+}
+
+bool collect_variant_residual_leaf_ids(const ParquetColumnSchema& schema,
+ const format::LocalColumnIndex&
projection,
+ std::vector<int>* residual_leaf_ids) {
+ DORIS_CHECK(residual_leaf_ids != nullptr);
+ const auto* value = schema_child_by_name(schema, "value");
+ const auto* typed_value = schema_child_by_name(schema, "typed_value");
+ if (value != nullptr && typed_value != nullptr) {
+ if (value->kind != ParquetColumnSchemaKind::PRIMITIVE ||
value->leaf_column_id < 0) {
+ return false;
+ }
+ residual_leaf_ids->push_back(value->leaf_column_id);
+ }
+ for (const auto& child_projection : projection.children) {
+ const auto* child = projected_schema_child(schema,
child_projection.local_id());
+ if (child == nullptr ||
+ !collect_variant_residual_leaf_ids(*child, child_projection,
residual_leaf_ids)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool detail::variant_projection_is_fully_shredded(const
tparquet::FileMetaData& metadata,
+ const ParquetColumnSchema&
schema,
+ const
format::LocalColumnIndex& projection) {
+ if (schema.kind != ParquetColumnSchemaKind::VARIANT ||
+ !format::is_partial_projection(&projection)) {
+ return false;
+ }
+ std::vector<int> residual_leaf_ids;
+ if (!collect_variant_residual_leaf_ids(schema, projection,
&residual_leaf_ids)) {
+ return false;
+ }
+ std::ranges::sort(residual_leaf_ids);
+ residual_leaf_ids.erase(std::unique(residual_leaf_ids.begin(),
residual_leaf_ids.end()),
+ residual_leaf_ids.end());
+ for (const auto& row_group : metadata.row_groups) {
+ for (const int leaf_id : residual_leaf_ids) {
+ if (leaf_id < 0 || leaf_id >=
static_cast<int>(row_group.columns.size())) {
+ return false;
+ }
+ const auto& chunk = row_group.columns[leaf_id];
+ if (!chunk.__isset.meta_data ||
!chunk.meta_data.__isset.statistics ||
+ !chunk.meta_data.statistics.__isset.null_count ||
+ chunk.meta_data.statistics.null_count != row_group.num_rows) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+size_t finalize_variant_leaf_projections(
+ const NativeParquetMetadata& metadata,
+ const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
+ std::vector<format::LocalColumnIndex>* projections) {
+ DORIS_CHECK(projections != nullptr);
+ size_t retained = 0;
+ for (auto& projection : *projections) {
+ const int32_t local_id = projection.local_id();
+ if (local_id < 0 || local_id >=
static_cast<int32_t>(file_schema.size()) ||
+ file_schema[local_id]->kind != ParquetColumnSchemaKind::VARIANT ||
Review Comment:
Fixed. Variant projection finalization now walks nested STRUCT/LIST/MAP
mappings recursively, expands unsafe residual paths, and validates fallback
absence in the repeated-instance domain. Tests cover nested typed-leaf
retention and fallback expansion.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
