Gabriel39 commented on code in PR #66206:
URL: https://github.com/apache/doris/pull/66206#discussion_r3674829091
##########
be/src/format_v2/parquet/parquet_statistics.cpp:
##########
@@ -614,12 +927,17 @@ void collect_filtered_leaf_ids(const ParquetColumnSchema&
column_schema,
return;
}
for (const auto& child_schema : column_schema.children) {
- if (!format::is_child_projected(projection, child_schema->local_id)) {
+ if (column_schema.kind != ParquetColumnSchemaKind::VARIANT &&
Review Comment:
Fixed. Filtered-byte leaf collection now follows the actual Variant
projection; a leaf-only projection accounts only for retained leaf chunks,
while a full Variant projection traverses the complete subtree. The statistics
regression test checks the exact retained-byte total.
##########
be/src/format_v2/parquet/native_schema_desc.cpp:
##########
@@ -55,6 +55,84 @@ static bool is_map_node(const tparquet::SchemaElement&
schema) {
(schema.__isset.logicalType && schema.logicalType.__isset.MAP);
}
+static bool is_variant_node(const tparquet::SchemaElement& schema) {
+ return schema.__isset.logicalType && schema.logicalType.__isset.VARIANT;
+}
+
+class ScopedBoolOverride {
+public:
+ ScopedBoolOverride(bool& target, bool value) : _target(target),
_original(target) {
+ _target = value;
+ }
+ ~ScopedBoolOverride() { _target = _original; }
+
+private:
+ bool& _target;
+ bool _original;
+};
+
+static Status validate_variant_layout(const tparquet::SchemaElement&
group_schema,
+ const NativeFieldSchema& group_field) {
+ const auto& annotation = group_schema.logicalType.VARIANT;
+ if (annotation.__isset.specification_version &&
annotation.specification_version != 1) {
+ return Status::NotSupported("Parquet Variant specification version {}
is not supported",
+ annotation.specification_version);
+ }
+ if (group_field.children.size() < 2 || group_field.children.size() > 3) {
+ return Status::Corruption(
+ "Parquet Variant {} must contain metadata, value, and optional
typed_value",
+ group_schema.name);
+ }
+
+ const NativeFieldSchema* metadata = nullptr;
+ const NativeFieldSchema* value = nullptr;
+ const NativeFieldSchema* typed_value = nullptr;
+ for (const auto& child : group_field.children) {
+ const NativeFieldSchema** target = nullptr;
+ if (child.name == "metadata") {
+ target = &metadata;
+ } else if (child.name == "value") {
+ target = &value;
+ } else if (child.name == "typed_value") {
+ target = &typed_value;
+ } else {
+ return Status::Corruption("Parquet Variant {} has unexpected child
{}",
+ group_schema.name, child.name);
+ }
+ if (*target != nullptr) {
+ return Status::Corruption("Parquet Variant {} has duplicate child
{}",
+ group_schema.name, child.name);
+ }
+ *target = &child;
+ }
+ if (metadata == nullptr || value == nullptr) {
+ return Status::Corruption("Parquet Variant {} requires metadata and
value children",
+ group_schema.name);
+ }
+ if (!metadata->children.empty() || metadata->physical_type !=
tparquet::Type::BYTE_ARRAY ||
+ metadata->parquet_schema.repetition_type !=
tparquet::FieldRepetitionType::REQUIRED) {
+ return Status::Corruption("Parquet Variant {} metadata must be a
required BYTE_ARRAY",
+ group_schema.name);
+ }
+ const auto expected_value_repetition = typed_value == nullptr
+ ?
tparquet::FieldRepetitionType::REQUIRED
+ :
tparquet::FieldRepetitionType::OPTIONAL;
+ // SQL nullability belongs to the outer Variant group. Only shredding
makes value optional,
+ // because typed_value may carry all or part of the logical value instead.
+ if (!value->children.empty() || value->physical_type !=
tparquet::Type::BYTE_ARRAY ||
+ value->parquet_schema.repetition_type != expected_value_repetition) {
+ return Status::Corruption("Parquet Variant {} value must be a {}
BYTE_ARRAY",
+ group_schema.name,
+ typed_value == nullptr ? "required" :
"optional");
+ }
+ if (typed_value != nullptr &&
Review Comment:
Fixed. Schema open now recursively validates required shredded wrappers,
optional value/typed_value children, complex shape, signed integers, local
TIME(MICROS), and rejects adjusted TIME, TIME(MILLIS), unsigned types, and
TIMESTAMP(NANOS). Added negative schema tests.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java:
##########
@@ -713,7 +713,11 @@ public static Type
icebergTypeToDorisType(org.apache.iceberg.types.Type type, bo
.collect(Collectors.toCollection(ArrayList::new));
return new StructType(nestedTypes);
case VARIANT:
- return Type.UNSUPPORTED;
+ // Iceberg Variant uses the Parquet Variant encoding directly.
Mark it compute-only
+ // so BE scanners materialize ColumnVariantV2 without changing
persisted Doris
+ // table metadata semantics.
+ return new org.apache.doris.catalog.VariantType(
Review Comment:
Fixed. Iceberg sink binding now rejects tables containing root or nested
Variant during FE analysis, including inserts that omit the Variant column,
because this feature remains read-only. Added root and nested DML validation
tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
