JingsongLi commented on code in PR #9389: URL: https://github.com/apache/paimon/pull/9389#discussion_r3886660173
########## paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java: ########## @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.parquet; + +import org.apache.paimon.data.variant.PaimonShreddingUtils; +import org.apache.paimon.data.variant.VariantMetadataUtils; +import org.apache.paimon.data.variant.VariantPathSegment; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; + +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.Type; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType; +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** + * Prunes a shredded Variant Parquet type according to a logical Variant projection. + * + * <p>It builds a trie from the requested object/array paths and recursively removes unneeded fields + * from the {@code typed_value} (and list element) groups while preserving {@code value} fallbacks + * when a requested path cannot be satisfied from typed columns. + * + * <p>Variant object keys are matched case-sensitively, independently of Parquet column-name + * resolution, because Variant path extraction downstream resolves keys exactly through {@code + * objectSchemaMap}. + */ +public class VariantShreddingTypePruner { + private static final String LIST_WRAPPER_NAME = "list"; + private static final String LIST_ELEMENT_NAME = "element"; + + @Nullable private final PathNode root; + + VariantShreddingTypePruner(RowType variantRowType) { + this.root = buildPathTree(variantRowType); + } + + /** + * Clips the given Parquet Variant type to only include fields needed for {@code + * variantRowType}. + * + * @param variantRowType the logical Variant projection row type + * @param parquetType the physical Parquet Variant type + * @return a clipped Parquet type + */ + public static Type clip(RowType variantRowType, GroupType parquetType) { + return new VariantShreddingTypePruner(variantRowType).clip(parquetType); + } + + private Type clip(GroupType parquetType) { + return clipShreddingRow(parquetType, root); + } + + /** A projection trie for Variant object paths and array element paths. */ + private static class PathNode { + private final Map<String, PathNode> children = new HashMap<>(); + private PathNode arrayElement; + private boolean keepAll; + + private PathNode getOrCreateChild(String key) { + return children.computeIfAbsent(key, k -> new PathNode()); + } + } + + @Nullable + private PathNode buildPathTree(RowType variantRowType) { + PathNode root = new PathNode(); + for (DataField field : variantRowType.getFields()) { + String path = VariantMetadataUtils.path(field.description()); + VariantPathSegment[] segments = VariantPathSegment.parse(path); + if (segments.length == 0) { + return null; + } + + PathNode node = root; + for (VariantPathSegment segment : segments) { + if (segment instanceof VariantPathSegment.ArrayExtraction) { + // Array indices cannot prune individual elements at the Parquet level, + // but we can still prune nested fields inside each array element. + if (node.arrayElement == null) { + node.arrayElement = new PathNode(); + } + node = node.arrayElement; + } else if (segment instanceof VariantPathSegment.ObjectExtraction) { + String key = ((VariantPathSegment.ObjectExtraction) segment).getKey(); + node = node.getOrCreateChild(key); + } else { + return null; + } + } + node.keepAll = true; + } + return root; + } + + private Type clipShreddingRow(Type type, PathNode node) { + if (type.isPrimitive() || node == null) { + return type; + } + + GroupType group = type.asGroupType(); + if (node.keepAll || !group.containsField(PaimonShreddingUtils.TYPED_VALUE_FIELD_NAME)) { + return group; + } + + List<Type> newFields = new ArrayList<>(); + if (group.containsField(PaimonShreddingUtils.METADATA_FIELD_NAME)) { + newFields.add(group.getType(PaimonShreddingUtils.METADATA_FIELD_NAME)); + } + + Type typedValue = group.getType(PaimonShreddingUtils.TYPED_VALUE_FIELD_NAME); Review Comment: [P1] Preserve the fallback for mixed array/object projections This branch selects list pruning whenever an array projection is present, even if the same Variant column also has object projections in `node.children`. For a list-typed shredding schema, querying `$[0].x` and `$.y` together therefore removes the parent `value` field; an object-shaped row such as `{"y": 2}` is stored only in that fallback and `$.y` silently returns null. I reproduced this on this head, and adding `node.children.isEmpty()` to this condition makes the regression test pass. Please either skip list pruning when object children coexist or retain `value` in that case. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
