This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new aacb745227 [format] Follow Parquet list backward-compatibility rules
in schema clipping (#9550)
aacb745227 is described below
commit aacb74522769d5e4ae42af7d6ebda631a301f242
Author: Juntao Zhang <[email protected]>
AuthorDate: Sun Sep 20 11:43:34 2026 +0800
[format] Follow Parquet list backward-compatibility rules in schema
clipping (#9550)
---
.../format/parquet/ParquetListLayoutResolver.java | 226 ++++++++
.../format/parquet/ParquetReaderFactory.java | 64 ++-
.../format/parquet/ParquetSchemaConverter.java | 346 +++++++-----
.../parquet/VariantShreddingReadPlanFactory.java | 6 +-
.../format/parquet/VariantShreddingTypePruner.java | 51 +-
.../format/parquet/reader/ParquetReaderUtil.java | 112 ++--
.../format/parquet/ParquetLegacyListReadTest.java | 595 +++++++++++++++++++++
.../parquet/ParquetListLayoutResolverTest.java | 191 +++++++
.../format/parquet/ParquetSchemaConverterTest.java | 244 +++++++++
9 files changed, 1556 insertions(+), 279 deletions(-)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetListLayoutResolver.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetListLayoutResolver.java
new file mode 100644
index 0000000000..2ae53e83aa
--- /dev/null
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetListLayoutResolver.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.format.parquet;
+
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.Type;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.LIST_ELEMENT_NAME;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/**
+ * Resolves Parquet list layouts, following the backward-compatibility rules
in the Parquet spec: <a
+ *
href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules";>LogicalTypes#Backward-compatibility-rules</a>.
+ *
+ * <p>All list layout decisions should be made through this class so that
schema inference,
+ * requested-schema clipping and reader construction share a single
interpretation.
+ */
+public final class ParquetListLayoutResolver {
+
+ private static final String LIST_WRAPPER_NAME = "list";
+ private static final String LEGACY_LIST_ARRAY_NAME = "array";
+
+ private ParquetListLayoutResolver() {}
+
+ /** Returns true if the given group is annotated as a Parquet LIST logical
type. */
+ public static boolean isList(GroupType listType) {
+ return listType.getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation;
+ }
+
+ /**
+ * Whether the given group has the legacy nested-list shape: an
unannotated {@code REPEATED}
+ * group whose single child is also {@code REPEATED}.
+ *
+ * <p>This is the Parquet spec's backward-compatibility <b>Rule 3</b> and
appears as the element
+ * of an annotated list, e.g. {@code repeated group element { repeated
int32 array; }}. The
+ * repeated child is the element of the nested list.
+ */
+ public static boolean isLegacyNestedList(GroupType groupType) {
+ return groupType.isRepetition(Type.Repetition.REPEATED)
+ && groupType.getLogicalTypeAnnotation() == null
+ && groupType.getFieldCount() == 1
+ && groupType.getType(0).getRepetition() ==
Type.Repetition.REPEATED;
+ }
+
+ /**
+ * Returns true if the given group is a three-level Parquet list.
+ *
+ * <p>In a three-level list the immediate repeated child is a wrapper
group whose single
+ * non-repeated child is the actual element type. This covers the
canonical layout ({@code list
+ * -> element}) as well as legacy wrappers such as Hive's {@code bag}
layout.
+ *
+ * <p>This corresponds to the Parquet spec's backward-compatibility
<b>Rule 5</b>: a repeated
+ * group that contains exactly one non-repeated child is a wrapper, unless
it matches one of
+ * Rules 1-4.
+ *
+ * <p>The compatibility encodings that are <em>not</em> three-level are:
+ *
+ * <ul>
+ * <li><b>Rule 1</b>: the repeated field is a primitive and is itself
the element type.
+ * <li><b>Rule 2</b>: the repeated field is a group with multiple fields
and is itself the
+ * element type.
+ * <li><b>Rule 3</b>: the repeated field is a group whose single child
is also repeated; the
+ * group itself is the element type.
+ * <li><b>Rule 4</b>: the repeated field is a group named {@code
"array"} or {@code
+ * "<list>_tuple"} with a single child; the group itself is the
element type.
+ * </ul>
+ */
+ public static boolean isThreeLevelList(Type type) {
+ if (type.isPrimitive()) {
+ return false;
+ }
+
+ GroupType listType = type.asGroupType();
+ if (!isList(listType)) {
+ return false;
+ }
+
+ // A list must have exactly one repeated child (the middle level).
+ if (listType.getFieldCount() != 1) {
+ return false;
+ }
+ Type middle = listType.getType(0);
+ if (middle.isPrimitive() || middle.getRepetition() !=
Type.Repetition.REPEATED) {
+ return false;
+ }
+ GroupType repeatedGroup = middle.asGroupType();
+
+ // Rule 5: the repeated group is a wrapper containing exactly one
non-repeated child.
+ if (repeatedGroup.getFieldCount() != 1
+ || repeatedGroup.getType(0).getRepetition() ==
Type.Repetition.REPEATED) {
+ return false;
+ }
+
+ // Rule 4: legacy "array" and "<list>_tuple" encodings are not
wrappers; the repeated
+ // group itself is the element type.
+ return !LEGACY_LIST_ARRAY_NAME.equals(repeatedGroup.getName())
+ && !(listType.getName() +
"_tuple").equals(repeatedGroup.getName());
+ }
+
+ /**
+ * Returns true if the given group follows the canonical three-level
Parquet list layout ({@code
+ * list -> element}).
+ *
+ * <p>The canonical layout is described in the Parquet spec: <a
+ *
href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists";>LogicalTypes#Lists</a>
+ */
+ public static boolean isCanonicalList(Type type) {
+ if (!isThreeLevelList(type)) {
+ return false;
+ }
+
+ Type middle = type.asGroupType().getType(0);
+ Type element = middle.asGroupType().getType(0);
+ return LIST_WRAPPER_NAME.equals(middle.getName())
+ && LIST_ELEMENT_NAME.equals(element.getName());
+ }
+
+ /**
+ * Resolves the element type of the given LIST-annotated group according
to the Parquet spec's
+ * backward-compatibility rules for lists.
+ *
+ * <p>For a three-level list (Rule 5) the returned type is the single
child of the repeated
+ * wrapper. For Rules 1-4 the repeated field itself is returned because it
is the element type.
+ */
+ public static Type resolveElementType(GroupType listType) {
+ checkArgument(
+ isList(listType) || isLegacyNestedList(listType),
+ "Expected LIST-annotated group but got: %s",
+ listType);
+
+ if (isThreeLevelList(listType)) {
+ return listType.getType(0).asGroupType().getType(0);
+ }
+
+ return listType.getType(0);
+ }
+
+ /**
+ * A schema-level manifest of list layouts, modeled after parquet-cpp's
{@code SchemaManifest}.
+ *
+ * <p>Built once from the file schema, it resolves the three-level verdict
of every
+ * LIST-annotated node in the file and records it by field path, so that
reader construction
+ * interprets list layouts against the file schema rather than the
(possibly reshaped) requested
+ * schema. Field paths are the identifier shared by the file schema, the
requested schema and
+ * the ColumnIO tree: clipping may rebuild nodes, but it preserves names,
so paths stay stable
+ * where node identity would not.
+ *
+ * <p>Paths absent from this context belong to nodes that are not
LIST-annotated in the file
+ * schema (or synthetic fill fields, whose requested shape is
Paimon-canonical); the verdict
+ * falls back to interpreting the requested node itself.
+ */
+ public static final class LayoutContext {
+ private static final LayoutContext EMPTY = new LayoutContext();
+
+ private final Map<List<String>, Boolean> threeLevelMapping = new
HashMap<>();
+
+ /**
+ * Returns a context carrying no file layout information. Lookups
always fall back to
+ * interpreting the requested node itself, which is the shape-based
behavior.
+ */
+ public static LayoutContext empty() {
+ return EMPTY;
+ }
+
+ /** Builds the manifest by resolving every LIST-annotated node of the
file schema. */
+ public static LayoutContext fromFileSchema(GroupType fileSchema) {
+ LayoutContext context = new LayoutContext();
+ for (Type field : fileSchema.getFields()) {
+ collect(field, Collections.singletonList(field.getName()),
context);
+ }
+ return context;
+ }
+
+ private static void collect(Type node, List<String> path,
LayoutContext context) {
+ if (node.isPrimitive()) {
+ return;
+ }
+ GroupType group = node.asGroupType();
+ if (isList(group)) {
+ context.threeLevelMapping.put(
+ path,
ParquetListLayoutResolver.isThreeLevelList(group));
+ }
+ for (Type child : group.getFields()) {
+ List<String> childPath = new ArrayList<>(path);
+ childPath.add(child.getName());
+ collect(child, childPath, context);
+ }
+ }
+
+ /**
+ * Returns whether the list at {@code path} is a three-level list, as
resolved against the
+ * file schema; falls back to interpreting {@code requestedGroup}
itself when the path is
+ * not an annotated list in the file schema.
+ */
+ public boolean isThreeLevelList(GroupType requestedGroup, List<String>
path) {
+ Boolean threeLevel = threeLevelMapping.get(path);
+ return threeLevel != null
+ ? threeLevel
+ :
ParquetListLayoutResolver.isThreeLevelList(requestedGroup);
+ }
+ }
+}
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java
index e66edb6f20..557002831d 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java
@@ -26,6 +26,7 @@ import
org.apache.paimon.data.shredding.MapSharedShreddingReadPlanFactory;
import org.apache.paimon.data.shredding.ShreddingReadPlan;
import org.apache.paimon.format.FormatMetadataUtils;
import org.apache.paimon.format.FormatReaderFactory;
+import
org.apache.paimon.format.parquet.ParquetListLayoutResolver.LayoutContext;
import org.apache.paimon.format.parquet.reader.VectorizedParquetRecordReader;
import org.apache.paimon.format.parquet.type.ParquetField;
import org.apache.paimon.format.shredding.ShreddingFormatReader;
@@ -58,7 +59,6 @@ import org.apache.parquet.schema.ConversionPatterns;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Types;
@@ -81,8 +81,11 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.function.IntFunction;
import static
org.apache.paimon.data.columnar.ColumnVectorUtils.createParquetWritableColumnVector;
+import static
org.apache.paimon.format.parquet.ParquetListLayoutResolver.isLegacyNestedList;
+import static
org.apache.paimon.format.parquet.ParquetListLayoutResolver.isList;
+import static
org.apache.paimon.format.parquet.ParquetListLayoutResolver.isThreeLevelList;
+import static
org.apache.paimon.format.parquet.ParquetListLayoutResolver.resolveElementType;
import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.PAIMON_SCHEMA;
-import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType;
import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetMapKeyValueType;
import static
org.apache.paimon.format.parquet.reader.ParquetReaderUtil.buildFieldsList;
@@ -258,9 +261,10 @@ public class ParquetReaderFactory implements
FormatReaderFactory {
}
private RequestedSchema createRequestedSchema(MessageType fileSchema,
DataField[] readFields) {
+ LayoutContext listLayout = LayoutContext.fromFileSchema(fileSchema);
MessageType rs = clipParquetSchema(fileSchema, readFields);
MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(rs);
- List<ParquetField> f = buildFieldsList(readFields, columnIO, rs);
+ List<ParquetField> f = buildFieldsList(readFields, columnIO, rs,
listLayout);
return new RequestedSchema(rs, f);
}
@@ -362,31 +366,37 @@ public class ParquetReaderFactory implements
FormatReaderFactory {
Preconditions.checkArgument(
listSubFields == 1,
"Parquet list group type should only have one middle
level REPEATED field.");
- // There are two representations for array type in parquet.
- // See link:
- //
https://impala.apache.org/docs/build/html/topics/impala_parquet_array_resolution.html.
- int level = arrayGroup.getType(0) instanceof GroupType ? 3 : 2;
- Type elementType =
- clipParquetType(elementReadType,
parquetListElementType(arrayGroup));
-
- if (level == 3) {
- // In case that the name in middle level is not "list".
- Type groupMiddle =
- new GroupType(
- Type.Repetition.REPEATED,
- arrayGroup.getType(0).getName(),
- elementType);
- return new GroupType(
- arrayGroup.getRepetition(),
- arrayGroup.getName(),
- OriginalType.LIST,
- groupMiddle);
+ if (isList(arrayGroup)) {
+ boolean threeLevel = isThreeLevelList(arrayGroup);
+ Type originalElement = resolveElementType(arrayGroup);
+ Type elementType = clipParquetType(elementReadType,
originalElement);
+ if (threeLevel) {
+ Type clippedMiddle =
+ arrayGroup
+ .getType(0)
+ .asGroupType()
+
.withNewFields(Collections.singletonList(elementType));
+ return
arrayGroup.withNewFields(Collections.singletonList(clippedMiddle));
+ } else {
+ // Rules 1-4: the repeated field itself is the
element. Keep it (clipped)
+ // in place so that reader construction unwraps by
path against the file
+ // schema instead of guessing from this reshaped
requested shape.
+ return
arrayGroup.withNewFields(Collections.singletonList(elementType));
+ }
+ } else if (isLegacyNestedList(arrayGroup)) {
+ // Rule 3: an unannotated repeated group is itself the
element of the annotated
+ // outer list, and its single repeated child is the
element of the nested inner
+ // list.
+ Type originalElement = arrayGroup.getType(0);
+ Type elementType = clipParquetType(elementReadType,
originalElement);
+ return
arrayGroup.withNewFields(Collections.singletonList(elementType));
} else {
- return new GroupType(
- arrayGroup.getRepetition(),
- arrayGroup.getName(),
- OriginalType.LIST,
- elementType);
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot read Parquet group '%s' as an
ARRAY: it is neither "
+ + "LIST-annotated nor a legacy
nested list. Parquet type: %s, "
+ + "read type: %s",
+ arrayGroup.getName(), arrayGroup,
readType));
}
default:
return parquetType;
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
index 912baec072..1b6a177e9d 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
@@ -19,6 +19,7 @@
package org.apache.paimon.format.parquet;
import org.apache.paimon.data.variant.Variant;
+import
org.apache.paimon.format.parquet.ParquetListLayoutResolver.LayoutContext;
import org.apache.paimon.table.SpecialFields;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
@@ -47,6 +48,7 @@ import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Types;
+import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
@@ -350,159 +352,233 @@ public class ParquetSchemaConverter {
/** Convert parquet {@link Type} to paimon {@link DataField} to. */
public static DataField convertToPaimonField(Type parquetType) {
- LogicalTypeAnnotation logicalType =
parquetType.getLogicalTypeAnnotation();
- DataType paimonDataType;
-
- if (parquetType.isPrimitive()) {
- switch (parquetType.asPrimitiveType().getPrimitiveTypeName()) {
- case BINARY:
- if (logicalType instanceof
LogicalTypeAnnotation.StringLogicalTypeAnnotation) {
- paimonDataType = DataTypes.STRING();
- } else if (logicalType
- instanceof
LogicalTypeAnnotation.GeometryLogicalTypeAnnotation) {
- paimonDataType =
- DataTypes.GEOMETRY(
-
((LogicalTypeAnnotation.GeometryLogicalTypeAnnotation)
- logicalType)
- .getCrs());
- } else if (logicalType
- instanceof
LogicalTypeAnnotation.GeographyLogicalTypeAnnotation) {
- LogicalTypeAnnotation.GeographyLogicalTypeAnnotation
geography =
-
(LogicalTypeAnnotation.GeographyLogicalTypeAnnotation) logicalType;
- EdgeAlgorithm algorithm =
- geography.getAlgorithm() == null
- ? null
- :
EdgeAlgorithm.valueOf(geography.getAlgorithm().name());
- paimonDataType =
DataTypes.GEOGRAPHY(geography.getCrs(), algorithm);
- } else {
- paimonDataType = DataTypes.BYTES();
- }
- break;
- case BOOLEAN:
- paimonDataType = DataTypes.BOOLEAN();
- break;
- case FLOAT:
- paimonDataType = DataTypes.FLOAT();
- break;
- case DOUBLE:
- paimonDataType = DataTypes.DOUBLE();
- break;
- case INT32:
- if (logicalType instanceof
LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) {
- LogicalTypeAnnotation.DecimalLogicalTypeAnnotation
decimalType =
-
(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalType;
- paimonDataType =
- new DecimalType(decimalType.getPrecision(),
decimalType.getScale());
- } else if (logicalType
- instanceof
LogicalTypeAnnotation.IntLogicalTypeAnnotation) {
- LogicalTypeAnnotation.IntLogicalTypeAnnotation intType
=
-
(LogicalTypeAnnotation.IntLogicalTypeAnnotation) logicalType;
- int bitWidth = intType.getBitWidth();
- if (bitWidth == 8) {
- paimonDataType = DataTypes.TINYINT();
- } else if (bitWidth == 16) {
- paimonDataType = DataTypes.SMALLINT();
+ return new FieldConverter(LayoutContext.empty()).convert(parquetType);
+ }
+
+ public static DataField convertToPaimonField(
+ Type parquetType, LayoutContext listLayout, String[] parentPath) {
+ return new FieldConverter(listLayout).convertAt(parquetType,
parentPath);
+ }
+
+ private static final class FieldConverter {
+
+ private final LayoutContext listLayout;
+ private final LinkedList<String> path = new LinkedList<>();
+
+ FieldConverter(LayoutContext listLayout) {
+ this.listLayout = listLayout == null ? LayoutContext.empty() :
listLayout;
+ }
+
+ /**
+ * Converts {@code parquetType} whose parent in the file schema sits
at {@code parentPath}.
+ * The converter instance is discarded after this call, so the seeded
path stack does not
+ * need to be unwound.
+ */
+ DataField convertAt(Type parquetType, String[] parentPath) {
+ for (String segment : parentPath) {
+ path.addLast(segment);
+ }
+ path.addLast(parquetType.getName());
+ return convert(parquetType);
+ }
+
+ DataField convert(Type parquetType) {
+ LogicalTypeAnnotation logicalType =
parquetType.getLogicalTypeAnnotation();
+ DataType paimonDataType;
+
+ if (parquetType.isPrimitive()) {
+ switch (parquetType.asPrimitiveType().getPrimitiveTypeName()) {
+ case BINARY:
+ if (logicalType
+ instanceof
LogicalTypeAnnotation.StringLogicalTypeAnnotation) {
+ paimonDataType = DataTypes.STRING();
+ } else if (logicalType
+ instanceof
LogicalTypeAnnotation.GeometryLogicalTypeAnnotation) {
+ paimonDataType =
+ DataTypes.GEOMETRY(
+
((LogicalTypeAnnotation.GeometryLogicalTypeAnnotation)
+ logicalType)
+ .getCrs());
+ } else if (logicalType
+ instanceof
LogicalTypeAnnotation.GeographyLogicalTypeAnnotation) {
+
LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geography =
+
(LogicalTypeAnnotation.GeographyLogicalTypeAnnotation)
+ logicalType;
+ EdgeAlgorithm algorithm =
+ geography.getAlgorithm() == null
+ ? null
+ : EdgeAlgorithm.valueOf(
+
geography.getAlgorithm().name());
+ paimonDataType =
DataTypes.GEOGRAPHY(geography.getCrs(), algorithm);
+ } else {
+ paimonDataType = DataTypes.BYTES();
+ }
+ break;
+ case BOOLEAN:
+ paimonDataType = DataTypes.BOOLEAN();
+ break;
+ case FLOAT:
+ paimonDataType = DataTypes.FLOAT();
+ break;
+ case DOUBLE:
+ paimonDataType = DataTypes.DOUBLE();
+ break;
+ case INT32:
+ if (logicalType
+ instanceof
LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) {
+ LogicalTypeAnnotation.DecimalLogicalTypeAnnotation
decimalType =
+
(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation)
+ logicalType;
+ paimonDataType =
+ new DecimalType(
+ decimalType.getPrecision(),
decimalType.getScale());
+ } else if (logicalType
+ instanceof
LogicalTypeAnnotation.IntLogicalTypeAnnotation) {
+ LogicalTypeAnnotation.IntLogicalTypeAnnotation
intType =
+
(LogicalTypeAnnotation.IntLogicalTypeAnnotation) logicalType;
+ int bitWidth = intType.getBitWidth();
+ if (bitWidth == 8) {
+ paimonDataType = DataTypes.TINYINT();
+ } else if (bitWidth == 16) {
+ paimonDataType = DataTypes.SMALLINT();
+ } else {
+ paimonDataType = DataTypes.INT();
+ }
+ } else if (logicalType
+ instanceof
LogicalTypeAnnotation.DateLogicalTypeAnnotation) {
+ paimonDataType = DataTypes.DATE();
+ } else if (logicalType
+ instanceof
LogicalTypeAnnotation.TimeLogicalTypeAnnotation) {
+ paimonDataType = DataTypes.TIME();
} else {
paimonDataType = DataTypes.INT();
}
- } else if (logicalType
- instanceof
LogicalTypeAnnotation.DateLogicalTypeAnnotation) {
- paimonDataType = DataTypes.DATE();
- } else if (logicalType
- instanceof
LogicalTypeAnnotation.TimeLogicalTypeAnnotation) {
- paimonDataType = DataTypes.TIME();
- } else {
- paimonDataType = DataTypes.INT();
- }
- break;
- case INT64:
- if (logicalType instanceof
LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) {
+ break;
+ case INT64:
+ if (logicalType
+ instanceof
LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) {
+ LogicalTypeAnnotation.DecimalLogicalTypeAnnotation
decimalType =
+
(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation)
+ logicalType;
+ paimonDataType =
+ new DecimalType(
+ decimalType.getPrecision(),
decimalType.getScale());
+ } else if (logicalType
+ instanceof
LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) {
+
LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampType =
+
(LogicalTypeAnnotation.TimestampLogicalTypeAnnotation)
+ logicalType;
+ int precision;
+ if (timestampType
+ .getUnit()
+
.equals(LogicalTypeAnnotation.TimeUnit.MILLIS)) {
+ precision = 3;
+ } else if (timestampType
+ .getUnit()
+
.equals(LogicalTypeAnnotation.TimeUnit.MICROS)) {
+ precision = 6;
+ } else {
+ precision = 9;
+ }
+ paimonDataType =
+ timestampType.isAdjustedToUTC()
+ ? new
LocalZonedTimestampType(precision)
+ : new TimestampType(precision);
+ } else {
+ paimonDataType = DataTypes.BIGINT();
+ }
+ break;
+ case INT96:
+ paimonDataType = new TimestampType(9);
+ break;
+ case FIXED_LEN_BYTE_ARRAY:
LogicalTypeAnnotation.DecimalLogicalTypeAnnotation
decimalType =
(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalType;
paimonDataType =
new DecimalType(decimalType.getPrecision(),
decimalType.getScale());
- } else if (logicalType
- instanceof
LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) {
- LogicalTypeAnnotation.TimestampLogicalTypeAnnotation
timestampType =
-
(LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType;
- int precision;
- if
(timestampType.getUnit().equals(LogicalTypeAnnotation.TimeUnit.MILLIS)) {
- precision = 3;
- } else if (timestampType
- .getUnit()
-
.equals(LogicalTypeAnnotation.TimeUnit.MICROS)) {
- precision = 6;
- } else {
- precision = 9;
- }
- paimonDataType =
- timestampType.isAdjustedToUTC()
- ? new
LocalZonedTimestampType(precision)
- : new TimestampType(precision);
- } else {
- paimonDataType = DataTypes.BIGINT();
- }
- break;
- case INT96:
- paimonDataType = new TimestampType(9);
- break;
- case FIXED_LEN_BYTE_ARRAY:
- LogicalTypeAnnotation.DecimalLogicalTypeAnnotation
decimalType =
-
(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalType;
- paimonDataType =
- new DecimalType(decimalType.getPrecision(),
decimalType.getScale());
- break;
- default:
- throw new UnsupportedOperationException("Unsupported type:
" + parquetType);
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported
type: " + parquetType);
+ }
+ if
(parquetType.getRepetition().equals(Type.Repetition.REQUIRED)) {
+ paimonDataType = paimonDataType.notNull();
+ }
+ return new DataField(
+ parquetType.getId().intValue(), parquetType.getName(),
paimonDataType);
+ } else {
+ GroupType groupType = parquetType.asGroupType();
+ if (ParquetListLayoutResolver.isList(groupType)) {
+ paimonDataType = convertList(groupType);
+ } else if (logicalType instanceof
LogicalTypeAnnotation.MapLogicalTypeAnnotation) {
+ paimonDataType = convertMap(groupType);
+ } else if
(ParquetListLayoutResolver.isLegacyNestedList(groupType)) {
+ paimonDataType = convertTwoLevelList(groupType);
+ } else {
+ paimonDataType = convertRow(groupType);
+ }
}
+
if (parquetType.getRepetition().equals(Type.Repetition.REQUIRED)) {
paimonDataType = paimonDataType.notNull();
}
+
return new DataField(
parquetType.getId().intValue(), parquetType.getName(),
paimonDataType);
- } else {
- GroupType groupType = parquetType.asGroupType();
- if (logicalType instanceof
LogicalTypeAnnotation.ListLogicalTypeAnnotation) {
- paimonDataType =
- new ArrayType(
-
convertToPaimonField(parquetListElementType(groupType)).type());
- } else if (logicalType instanceof
LogicalTypeAnnotation.MapLogicalTypeAnnotation) {
- Pair<Type, Type> keyValueType =
parquetMapKeyValueType(groupType);
- paimonDataType =
- new MapType(
- // Since parquet does not support nullable
key, when converting
- // back to Paimon, set as nullable by default.
-
convertToPaimonField(keyValueType.getLeft()).type().nullable(),
-
convertToPaimonField(keyValueType.getRight()).type());
- } else {
- paimonDataType =
- new RowType(
- groupType.getFields().stream()
-
.map(ParquetSchemaConverter::convertToPaimonField)
- .collect(Collectors.toList()));
- }
}
- if (parquetType.getRepetition().equals(Type.Repetition.REQUIRED)) {
- paimonDataType = paimonDataType.notNull();
+ private DataType convertList(GroupType groupType) {
+ return listLayout.isThreeLevelList(groupType, path)
+ ? convertThreeLevelList(groupType)
+ : convertTwoLevelList(groupType);
}
- return new DataField(parquetType.getId().intValue(),
parquetType.getName(), paimonDataType);
- }
+ private DataType convertThreeLevelList(GroupType groupType) {
+ GroupType repeated = groupType.getType(0).asGroupType();
+ DataType elementType = convertChild(repeated.getType(0),
repeated.getName()).type();
+ return new ArrayType(elementType);
+ }
- public static Type parquetListElementType(GroupType listType) {
- int level = listType.getType(0) instanceof GroupType ? 3 : 2;
- if (level == 3) {
- // Level 3 representation of list type.
- // List type should only have one middle group type, which is
repeated, and one element
- // type, which is optional.
- return listType.getType(0).asGroupType().getType(0);
- } else if (level == 2) {
- // Level 2 representation of list type
- return listType.getType(0);
- } else {
- throw new UnsupportedOperationException(
- "Parquet list type only have two level representation and
three level representation.");
+ /**
+ * Rules 1-4: the repeated node itself is the element. A REPEATED node
is never null, so the
+ * element type is not nullable.
+ */
+ private DataType convertTwoLevelList(GroupType groupType) {
+ return new
ArrayType(convertChild(groupType.getType(0)).type().notNull());
+ }
+
+ private DataType convertRow(GroupType groupType) {
+ return new RowType(
+ groupType.getFields().stream()
+ .map(this::convertChild)
+ .collect(Collectors.toList()));
+ }
+
+ private DataType convertMap(GroupType groupType) {
+ Pair<Type, Type> keyValueType = parquetMapKeyValueType(groupType);
+ String keyValueName = groupType.getType(0).getName();
+ // Since parquet does not support nullable key, when converting
back to Paimon, set as
+ // nullable by default.
+ return new MapType(
+ convertChild(keyValueType.getLeft(),
keyValueName).type().nullable(),
+ convertChild(keyValueType.getRight(),
keyValueName).type());
+ }
+
+ private DataField convertChild(Type child) {
+ path.addLast(child.getName());
+ try {
+ return convert(child);
+ } finally {
+ path.removeLast();
+ }
+ }
+
+ private DataField convertChild(Type child, String parentName) {
+ path.addLast(parentName);
+ try {
+ return convertChild(child);
+ } finally {
+ path.removeLast();
+ }
}
}
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingReadPlanFactory.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingReadPlanFactory.java
index 0e902c56e3..3d33f6c7d3 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingReadPlanFactory.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingReadPlanFactory.java
@@ -56,7 +56,7 @@ import java.util.Map;
import static org.apache.paimon.data.variant.Variant.METADATA;
import static org.apache.paimon.data.variant.Variant.VALUE;
-import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType;
+import static
org.apache.paimon.format.parquet.ParquetListLayoutResolver.resolveElementType;
import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetMapKeyValueType;
/**
@@ -194,7 +194,7 @@ public class VariantShreddingReadPlanFactory implements
ShreddingReadPlanFactory
arrayType.isNullable(),
buildPhysicalType(
arrayType.getElementType(),
- parquetListElementType(fileType.asGroupType()),
+ resolveElementType(fileType.asGroupType()),
caseSensitive));
case VECTOR:
VectorType vectorType = (VectorType) logicalType;
@@ -203,7 +203,7 @@ public class VariantShreddingReadPlanFactory implements
ShreddingReadPlanFactory
vectorType.getLength(),
buildPhysicalType(
vectorType.getElementType(),
- parquetListElementType(fileType.asGroupType()),
+ resolveElementType(fileType.asGroupType()),
caseSensitive));
case MAP:
MapType mapType = (MapType) logicalType;
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java
index f9e5075342..860c5df9d9 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java
@@ -25,7 +25,6 @@ import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
import org.apache.parquet.schema.GroupType;
-import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.Type;
import javax.annotation.Nullable;
@@ -38,7 +37,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType;
+import static
org.apache.paimon.format.parquet.ParquetListLayoutResolver.isCanonicalList;
+import static
org.apache.paimon.format.parquet.ParquetListLayoutResolver.resolveElementType;
import static org.apache.paimon.utils.Preconditions.checkArgument;
/**
@@ -53,9 +53,6 @@ import static
org.apache.paimon.utils.Preconditions.checkArgument;
* objectSchemaMap}.
*/
public class VariantShreddingTypePruner {
- private static final String LIST_WRAPPER_NAME = "list";
- private static final String LIST_ELEMENT_NAME = "element";
-
@Nullable private final PathNode root;
VariantShreddingTypePruner(RowType variantRowType) {
@@ -197,7 +194,7 @@ public class VariantShreddingTypePruner {
newFields.add(group.getType(PaimonShreddingUtils.VARIANT_VALUE_FIELD_NAME));
}
- Type elementType = parquetListElementType(listGroup);
+ Type elementType = resolveElementType(listGroup);
Type clippedElement = clipShreddingRow(elementType.asGroupType(),
node.arrayElement);
GroupType repeated = listGroup.getType(0).asGroupType();
GroupType clippedRepeated =
@@ -206,48 +203,6 @@ public class VariantShreddingTypePruner {
return group.withNewFields(newFields);
}
- /**
- * Returns true if the given group follows the canonical three-level
Parquet list layout.
- *
- * <p>The canonical layout is described in the Parquet spec: <a
- *
href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists";>LogicalTypes#Lists</a>
- */
- private static boolean isCanonicalList(Type type) {
- if (type.isPrimitive()) {
- return false;
- }
-
- GroupType listGroup = type.asGroupType();
- // 1. Must be a LIST logical type.
- if (!(listGroup.getLogicalTypeAnnotation()
- instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation)) {
- return false;
- }
-
- // 2. LIST group must have exactly one child named "list".
- if (listGroup.getFieldCount() != 1) {
- return false;
- }
- Type middle = listGroup.getType(0);
- if (!LIST_WRAPPER_NAME.equals(middle.getName())) {
- return false;
- }
-
- // 3. The child must be a repeated group.
- if (middle.isPrimitive() || middle.getRepetition() !=
Type.Repetition.REPEATED) {
- return false;
- }
- GroupType repeatedWrapper = middle.asGroupType();
-
- // 4. The repeated wrapper must contain exactly one child named
"element".
- if (repeatedWrapper.getFieldCount() != 1) {
- return false;
- }
-
- Type element = repeatedWrapper.getType(0);
- return LIST_ELEMENT_NAME.equals(element.getName());
- }
-
/** Returns true if the given group is a plain struct (not a Parquet list
or map). */
private static boolean isObjectGroup(Type type) {
if (type.isPrimitive()) {
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java
index 48f98b785b..d2ea25e455 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java
@@ -18,6 +18,7 @@
package org.apache.paimon.format.parquet.reader;
+import org.apache.paimon.format.parquet.ParquetListLayoutResolver;
import org.apache.paimon.format.parquet.type.ParquetField;
import org.apache.paimon.format.parquet.type.ParquetGroupField;
import org.apache.paimon.format.parquet.type.ParquetPrimitiveField;
@@ -30,7 +31,6 @@ import org.apache.paimon.types.MultisetType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VectorType;
import org.apache.paimon.utils.Pair;
-import org.apache.paimon.utils.StringUtils;
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList;
@@ -43,38 +43,43 @@ import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.Type;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.convertToPaimonField;
-import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType;
import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetMapKeyValueType;
-import static org.apache.parquet.schema.Type.Repetition.REPEATED;
import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
/** Util for generating parquet readers. */
public class ParquetReaderUtil {
public static List<ParquetField> buildFieldsList(
- DataField[] readFields, MessageColumnIO columnIO, MessageType
requestedFileSchema) {
+ DataField[] readFields,
+ MessageColumnIO columnIO,
+ MessageType requestedSchema,
+ ParquetListLayoutResolver.LayoutContext listLayout) {
List<ParquetField> list = new ArrayList<>();
for (int i = 0; i < readFields.length; i++) {
list.add(
constructField(
readFields[i],
lookupColumnByName(columnIO, readFields[i].name()),
- requestedFileSchema.getType(i)));
+ requestedSchema.getType(i),
+ listLayout));
}
return list;
}
private static ParquetField constructField(
- DataField dataField, ColumnIO columnIO, Type parquetType) {
+ DataField dataField,
+ ColumnIO columnIO,
+ Type parquetType,
+ ParquetListLayoutResolver.LayoutContext listLayout) {
boolean required = columnIO.getType().getRepetition() == REQUIRED;
int repetitionLevel = columnIO.getRepetitionLevel();
int definitionLevel = columnIO.getDefinitionLevel();
DataType type = dataField.type();
- String fieldName = dataField.name();
if (type instanceof RowType) {
GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
RowType rowType = (RowType) type;
@@ -88,17 +93,23 @@ public class ParquetReaderUtil {
constructField(
children.get(i),
lookupColumnByName(groupColumnIO, childName),
- getTypeIgnoreCase(parquetType.asGroupType(),
childName)));
+ getTypeIgnoreCase(parquetType.asGroupType(),
childName),
+ listLayout));
}
GroupType parquetGroup = parquetType.asGroupType();
for (int i = children.size(); i < parquetGroup.getFieldCount();
i++) {
Type extraType = parquetGroup.getType(i);
- DataField extraField =
convertToPaimonField(addFallbackFieldIds(extraType));
+ DataField extraField =
+ convertToPaimonField(
+ addFallbackFieldIds(extraType),
+ listLayout,
+ groupColumnIO.getFieldPath());
fieldsBuilder.add(
constructField(
extraField,
lookupColumnByName(groupColumnIO,
extraType.getName()),
- extraType));
+ extraType,
+ listLayout));
}
return new ParquetGroupField(
@@ -119,12 +130,14 @@ public class ParquetReaderUtil {
constructField(
new DataField(0, "", mapType.getKeyType()),
keyValueColumnIO.getChild(0),
- keyValueType.getKey());
+ keyValueType.getKey(),
+ listLayout);
ParquetField valueField =
constructField(
new DataField(0, "", mapType.getValueType()),
keyValueColumnIO.getChild(1),
- keyValueType.getValue());
+ keyValueType.getValue(),
+ listLayout);
return new ParquetGroupField(
type,
repetitionLevel,
@@ -143,12 +156,14 @@ public class ParquetReaderUtil {
constructField(
new DataField(0, "",
multisetType.getElementType()),
keyValueColumnIO.getChild(0),
- keyValueType.getKey());
+ keyValueType.getKey(),
+ listLayout);
ParquetField valueField =
constructField(
new DataField(0, "", new IntType()),
keyValueColumnIO.getChild(1),
- keyValueType.getValue());
+ keyValueType.getValue(),
+ listLayout);
return new ParquetGroupField(
type,
repetitionLevel,
@@ -163,34 +178,27 @@ public class ParquetReaderUtil {
type instanceof ArrayType
? ((ArrayType) type).getElementType()
: ((VectorType) type).getElementType();
- ColumnIO elementTypeColumnIO;
- if (columnIO instanceof GroupColumnIO) {
- GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
- if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) {
- // Column lookup is case-insensitive; the wrapper-group
names can
- // therefore differ in case from the requested field name.
- while
(!groupColumnIO.getName().equalsIgnoreCase(fieldName)) {
- groupColumnIO = (GroupColumnIO)
groupColumnIO.getChild(0);
- }
- elementTypeColumnIO = groupColumnIO;
- } else {
- if (elementType instanceof RowType) {
- elementTypeColumnIO = groupColumnIO;
- } else {
- elementTypeColumnIO = groupColumnIO.getChild(0);
- }
- }
- } else if (columnIO instanceof PrimitiveColumnIO) {
- elementTypeColumnIO = columnIO;
- } else {
- throw new RuntimeException(String.format("Unknown ColumnIO,
%s", columnIO));
- }
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ GroupType requestedGroup = parquetType.asGroupType();
+
+ boolean threeLevel =
+ listLayout.isThreeLevelList(
+ requestedGroup,
Arrays.asList(groupColumnIO.getFieldPath()));
+ Type requestedElementType =
+ threeLevel
+ ?
requestedGroup.getType(0).asGroupType().getType(0)
+ : requestedGroup.getType(0);
+
+ ColumnIO middleColumnIO = groupColumnIO.getChild(0);
+ ColumnIO elementColumnIO =
+ threeLevel ? ((GroupColumnIO) middleColumnIO).getChild(0)
: middleColumnIO;
ParquetField field =
constructField(
new DataField(0, "", elementType),
- getArrayElementColumn(elementTypeColumnIO),
- parquetListElementType(parquetType.asGroupType()));
+ elementColumnIO,
+ requestedElementType,
+ listLayout);
if (repetitionLevel == field.getRepetitionLevel()) {
repetitionLevel = columnIO.getParent().getRepetitionLevel();
}
@@ -270,32 +278,4 @@ public class ParquetReaderUtil {
}
return groupColumnIO;
}
-
- public static ColumnIO getArrayElementColumn(ColumnIO columnIO) {
- while (columnIO instanceof GroupColumnIO &&
!columnIO.getType().isRepetition(REPEATED)) {
- columnIO = ((GroupColumnIO) columnIO).getChild(0);
- }
-
- /* Compatible with array has a standard 3-level structure:
- * optional group my_list (LIST) {
- * repeated group element {
- * required binary str (UTF8);
- * };
- * }
- */
- if (columnIO instanceof GroupColumnIO
- && columnIO.getType().getLogicalTypeAnnotation() == null
- && ((GroupColumnIO) columnIO).getChildrenCount() == 1
- && !columnIO.getName().equals("array")
- && !columnIO.getName().equals(columnIO.getParent().getName() +
"_tuple")) {
- return ((GroupColumnIO) columnIO).getChild(0);
- }
-
- /* Compatible with array for 2-level arrays where a repeated field is
not a group:
- * optional group my_list (LIST) {
- * repeated int32 element;
- * }
- */
- return columnIO;
- }
}
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetLegacyListReadTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetLegacyListReadTest.java
new file mode 100644
index 0000000000..af3a309bd8
--- /dev/null
+++
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetLegacyListReadTest.java
@@ -0,0 +1,595 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.format.parquet;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.format.FormatReaderContext;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.VarCharType;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.hadoop.ParquetFileWriter;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.util.HadoopOutputFile;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * End-to-end tests for reading Parquet files that use legacy list encodings,
as described by the
+ * backward-compatibility rules in the Parquet spec.
+ */
+public class ParquetLegacyListReadTest {
+
+ private static final int ROW_COUNT = 10;
+
+ @TempDir public File folder;
+
+ /** Backward-compatibility Rule 1: a repeated primitive field is itself
the element type. */
+ @Test
+ public void testReadRepeatedPrimitiveElementList() throws Exception {
+ // optional group my_list (LIST) { repeated int32 element; }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .repeated(INT32)
+ .named("element")
+ .named("my_list"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Group row = factory.newGroup();
+ Group array = row.addGroup("my_list");
+ int size = i % 3 + 1;
+ for (int j = 0; j < size; j++) {
+ array.append("element", i * 10 + j);
+ }
+ writer.write(row);
+ }
+ }
+
+ AtomicInteger i = new AtomicInteger();
+ try (RecordReader<InternalRow> reader =
+ createReader(
+ path,
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 0, "my_list", new
ArrayType(new IntType())))))) {
+ reader.forEachRemaining(
+ row -> {
+ int index = i.getAndIncrement();
+ InternalArray array = row.getArray(0);
+ int size = index % 3 + 1;
+ assertThat(array.size()).isEqualTo(size);
+ for (int j = 0; j < size; j++) {
+ assertThat(array.getInt(j)).isEqualTo(index * 10 +
j);
+ }
+ });
+ }
+ assertThat(i.get()).isEqualTo(ROW_COUNT);
+ }
+
+ /**
+ * Backward-compatibility Rule 2: a repeated group with multiple fields is
itself the element
+ * type.
+ */
+ @Test
+ public void testReadRepeatedStructElementList() throws Exception {
+ // optional group my_list (LIST) { repeated group element { optional
int32 x; optional int32
+ // y; } }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .optional(INT32)
+ .named("x")
+ .optional(INT32)
+ .named("y")
+ .named("element"))
+ .named("my_list"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Group row = factory.newGroup();
+ Group array = row.addGroup("my_list");
+ int size = i % 3 + 1;
+ for (int j = 0; j < size; j++) {
+ array.addGroup("element").append("x", i + j).append("y", i
* j);
+ }
+ writer.write(row);
+ }
+ }
+
+ RowType elementType =
+ new RowType(
+ Arrays.asList(
+ new DataField(0, "x", new IntType()),
+ new DataField(1, "y", new IntType())));
+ AtomicInteger i = new AtomicInteger();
+ try (RecordReader<InternalRow> reader =
+ createReader(
+ path,
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 0, "my_list", new
ArrayType(elementType)))))) {
+ reader.forEachRemaining(
+ row -> {
+ int index = i.getAndIncrement();
+ InternalArray array = row.getArray(0);
+ int size = index % 3 + 1;
+ assertThat(array.size()).isEqualTo(size);
+ for (int j = 0; j < size; j++) {
+ InternalRow element = array.getRow(j, 2);
+ assertThat(element.getInt(0)).isEqualTo(index + j);
+ assertThat(element.getInt(1)).isEqualTo(index * j);
+ }
+ });
+ }
+ assertThat(i.get()).isEqualTo(ROW_COUNT);
+ }
+
+ /**
+ * Backward-compatibility Rule 2 with projection: a repeated group with
multiple fields is the
+ * element type, and projecting it to a single field must not reclassify
the clipped element
+ * group as a Rule 5 wrapper.
+ */
+ @Test
+ public void testReadStructElementWithProjection() throws Exception {
+ // optional group my_list (LIST) { repeated group element { optional
int32 x; optional int32
+ // y; } }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .optional(INT32)
+ .named("x")
+ .optional(INT32)
+ .named("y")
+ .named("element"))
+ .named("my_list"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Group row = factory.newGroup();
+ Group array = row.addGroup("my_list");
+ int size = i % 3 + 1;
+ for (int j = 0; j < size; j++) {
+ array.addGroup("element").append("x", i + j).append("y", i
* j);
+ }
+ writer.write(row);
+ }
+ }
+
+ RowType elementType =
+ new RowType(Collections.singletonList(new DataField(0, "x",
new IntType())));
+ AtomicInteger i = new AtomicInteger();
+ try (RecordReader<InternalRow> reader =
+ createReader(
+ path,
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 0, "my_list", new
ArrayType(elementType)))))) {
+ reader.forEachRemaining(
+ row -> {
+ int index = i.getAndIncrement();
+ InternalArray array = row.getArray(0);
+ int size = index % 3 + 1;
+ assertThat(array.size()).isEqualTo(size);
+ for (int j = 0; j < size; j++) {
+ assertThat(array.getRow(j,
1).getInt(0)).isEqualTo(index + j);
+ }
+ });
+ }
+ assertThat(i.get()).isEqualTo(ROW_COUNT);
+ }
+
+ /**
+ * Backward-compatibility Rule 3: a repeated group whose single child is
also repeated is the
+ * element type, and the repeated child is a nested list. Reading {@code
[[8, 9], ...]} must
+ * return every inner value, not just the first one.
+ */
+ @Test
+ public void testReadNestedLegacyList() throws Exception {
+ // optional group my_list (LIST) { repeated group element { repeated
int32 array; } }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .repeated(INT32)
+ .named("array")
+ .named("array"))
+ .named("my_list"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Group row = factory.newGroup();
+ Group array = row.addGroup("my_list");
+ int size = i % 3 + 1;
+ for (int j = 0; j < size; j++) {
+ Group element = array.addGroup("array");
+ element.append("array", i * 10 + j);
+ element.append("array", i * 10 + j + 100);
+ }
+ writer.write(row);
+ }
+ }
+
+ AtomicInteger i = new AtomicInteger();
+ try (RecordReader<InternalRow> reader =
+ createReader(
+ path,
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 0,
+ "my_list",
+ new ArrayType(new
ArrayType(new IntType()))))))) {
+ reader.forEachRemaining(
+ row -> {
+ int index = i.getAndIncrement();
+ InternalArray array = row.getArray(0);
+ int size = index % 3 + 1;
+ assertThat(array.size()).isEqualTo(size);
+ for (int j = 0; j < size; j++) {
+ InternalArray nested = array.getArray(j);
+ assertThat(nested.size()).isEqualTo(2);
+ assertThat(nested.getInt(0)).isEqualTo(index * 10
+ j);
+ assertThat(nested.getInt(1)).isEqualTo(index * 10
+ j + 100);
+ }
+ });
+ }
+ assertThat(i.get()).isEqualTo(ROW_COUNT);
+ }
+
+ /**
+ * Backward-compatibility Rule 4: a repeated group named {@code "array"}
with one field is the
+ * element type.
+ */
+ @Test
+ public void testReadArrayNamedGroupElementList() throws Exception {
+ // optional group my_list (LIST) { repeated group array { optional
int32 foo; } }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .optional(INT32)
+ .named("foo")
+ .named("array"))
+ .named("my_list"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Group row = factory.newGroup();
+ Group array = row.addGroup("my_list");
+ int size = i % 3 + 1;
+ for (int j = 0; j < size; j++) {
+ array.addGroup("array").append("foo", i * 10 + j);
+ }
+ writer.write(row);
+ }
+ }
+
+ RowType elementType =
+ new RowType(Collections.singletonList(new DataField(0, "foo",
new IntType())));
+ AtomicInteger i = new AtomicInteger();
+ try (RecordReader<InternalRow> reader =
+ createReader(
+ path,
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 0, "my_list", new
ArrayType(elementType)))))) {
+ reader.forEachRemaining(
+ row -> {
+ int index = i.getAndIncrement();
+ InternalArray array = row.getArray(0);
+ int size = index % 3 + 1;
+ assertThat(array.size()).isEqualTo(size);
+ for (int j = 0; j < size; j++) {
+ assertThat(array.getRow(j,
1).getInt(0)).isEqualTo(index * 10 + j);
+ }
+ });
+ }
+ assertThat(i.get()).isEqualTo(ROW_COUNT);
+ }
+
+ /**
+ * Backward-compatibility Rule 4: a repeated group named {@code
"<list>_tuple"} with one field
+ * is the element type.
+ */
+ @Test
+ public void testReadListTupleNamedGroupElementList() throws Exception {
+ // optional group my_list (LIST) { repeated group my_list_tuple {
required binary str
+ // (STRING); } }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .required(BINARY)
+
.as(LogicalTypeAnnotation.stringType())
+ .named("str")
+ .named("my_list_tuple"))
+ .named("my_list"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Group row = factory.newGroup();
+ Group array = row.addGroup("my_list");
+ int size = i % 3 + 1;
+ for (int j = 0; j < size; j++) {
+ array.addGroup("my_list_tuple")
+ .append("str", Binary.fromString("v" + (i * 10 +
j)));
+ }
+ writer.write(row);
+ }
+ }
+
+ RowType elementType =
+ new RowType(
+ Collections.singletonList(
+ new DataField(0, "str", new
VarCharType(VarCharType.MAX_LENGTH))));
+ AtomicInteger i = new AtomicInteger();
+ try (RecordReader<InternalRow> reader =
+ createReader(
+ path,
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 0, "my_list", new
ArrayType(elementType)))))) {
+ reader.forEachRemaining(
+ row -> {
+ int index = i.getAndIncrement();
+ InternalArray array = row.getArray(0);
+ int size = index % 3 + 1;
+ assertThat(array.size()).isEqualTo(size);
+ for (int j = 0; j < size; j++) {
+ assertThat(array.getRow(j, 1).getString(0))
+ .isEqualTo(BinaryString.fromString("v" +
(index * 10 + j)));
+ }
+ });
+ }
+ assertThat(i.get()).isEqualTo(ROW_COUNT);
+ }
+
+ /**
+ * Backward-compatibility Rule 5: a repeated group with a single
non-repeated child is a
+ * wrapper, so the child is the element (Hive bag style).
+ */
+ @Test
+ public void testReadElementWrappedList() throws Exception {
+ // optional group my_list (LIST) { repeated group element { optional
int32 num; } }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .optional(INT32)
+ .named("num")
+ .named("element"))
+ .named("my_list"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Group row = factory.newGroup();
+ Group array = row.addGroup("my_list");
+ int size = i % 3 + 1;
+ for (int j = 0; j < size; j++) {
+ array.addGroup("element").append("num", i * 10 + j);
+ }
+ writer.write(row);
+ }
+ }
+
+ AtomicInteger i = new AtomicInteger();
+ try (RecordReader<InternalRow> reader =
+ createReader(
+ path,
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 0, "my_list", new
ArrayType(new IntType())))))) {
+ reader.forEachRemaining(
+ row -> {
+ int index = i.getAndIncrement();
+ InternalArray array = row.getArray(0);
+ int size = index % 3 + 1;
+ assertThat(array.size()).isEqualTo(size);
+ for (int j = 0; j < size; j++) {
+ assertThat(array.getInt(j)).isEqualTo(index * 10 +
j);
+ }
+ });
+ }
+ assertThat(i.get()).isEqualTo(ROW_COUNT);
+ }
+
+ /**
+ * A plain struct group is neither LIST-annotated nor a legacy nested
list, so reading it as an
+ * ARRAY must fail fast in schema clipping instead of silently producing
undefined data.
+ */
+ @Test
+ public void testReadNonListGroupAsArrayFails() throws Exception {
+ // optional group arr { optional int32 x; optional int32 y; }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .addField(
+
Types.buildGroup(Type.Repetition.OPTIONAL)
+ .optional(INT32)
+ .named("x")
+ .optional(INT32)
+ .named("y")
+ .named("tuple"))
+ .named("arr"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Group row = factory.newGroup();
+ row.addGroup("arr").addGroup("tuple").append("x",
i).append("y", i * 2);
+ writer.write(row);
+ }
+ }
+
+ RowType readType =
+ new RowType(
+ Collections.singletonList(
+ new DataField(0, "arr", new ArrayType(new
IntType()))));
+ assertThatThrownBy(() -> createReader(path, readType))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Cannot read Parquet group 'arr' as an
ARRAY");
+ }
+
+ @Test
+ public void testReadMissingNestedFieldWithListFallback() throws Exception {
+ // optional group s { optional group a (LIST) { repeated group element
{ optional int32 x;
+ // optional int32 y; } } }
+ MessageType schema =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .addField(
+
Types.buildGroup(Type.Repetition.OPTIONAL)
+
.as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+
.optional(INT32)
+ .named("x")
+
.optional(INT32)
+ .named("y")
+
.named("element"))
+ .named("a"))
+ .named("s"));
+ Path path = new Path(folder.getPath(), UUID.randomUUID().toString());
+ try (ParquetWriter<Group> writer = createWriter(path, schema)) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ // row0: a null parent struct
+ writer.write(factory.newGroup());
+ // row1: a present struct with an empty list
+ Group emptyListRow = factory.newGroup();
+ emptyListRow.addGroup("s").addGroup("a");
+ writer.write(emptyListRow);
+ // row2: a present struct with one element
+ Group row = factory.newGroup();
+ row.addGroup("s").addGroup("a").addGroup("element").append("x",
1).append("y", 2);
+ writer.write(row);
+ }
+
+ RowType readType =
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 0,
+ "s",
+ new RowType(
+ Collections.singletonList(
+ new DataField(
+ 1, "newField",
new IntType()))))));
+
+ List<Boolean> sNulls = new ArrayList<>();
+ List<Boolean> newFieldNulls = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = createReader(path, readType)) {
+ reader.forEachRemaining(
+ row -> {
+ sNulls.add(row.isNullAt(0));
+ newFieldNulls.add(row.isNullAt(0) || row.getRow(0,
1).isNullAt(0));
+ });
+ }
+ assertThat(sNulls).containsExactly(true, false, false);
+ assertThat(newFieldNulls).containsExactly(true, true, true);
+ }
+
+ private ParquetWriter<Group> createWriter(Path path, MessageType schema)
throws IOException {
+ Configuration conf = new Configuration();
+ return ExampleParquetWriter.builder(
+ HadoopOutputFile.fromPath(
+ new
org.apache.hadoop.fs.Path(path.toString()), conf))
+ .withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
+ .withConf(conf)
+ .withType(schema)
+ .build();
+ }
+
+ private RecordReader<InternalRow> createReader(Path path, RowType
readType) throws IOException {
+ ParquetReaderFactory factory =
+ new ParquetReaderFactory(new Options(), readType, 1024, null);
+ LocalFileIO fileIO = new LocalFileIO();
+ return factory.createReader(
+ new FormatReaderContext(fileIO, path,
fileIO.getFileSize(path), null, null));
+ }
+}
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetListLayoutResolverTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetListLayoutResolverTest.java
new file mode 100644
index 0000000000..e190cda801
--- /dev/null
+++
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetListLayoutResolverTest.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.format.parquet;
+
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+import org.junit.jupiter.api.Test;
+
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests for {@link ParquetListLayoutResolver}.
+ *
+ * <p>All list layout decisions should be made through {@link
ParquetListLayoutResolver} so that
+ * schema inference, requested-schema clipping and reader construction share a
single
+ * interpretation; the layouts below follow the backward-compatibility rules
in the Parquet spec.
+ */
+public class ParquetListLayoutResolverTest {
+
+ @Test
+ public void testResolveThreeLevelElementType() {
+ // Rule 5: canonical three-level list (list -> element) with a
primitive element.
+ GroupType threeLevelPrimitiveList =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+ Types.buildGroup(Type.Repetition.REPEATED)
+ .optional(INT32)
+ .named("element")
+ .named("list"))
+ .named("my_list");
+ Type threeLevelPrimitiveElement =
+
ParquetListLayoutResolver.resolveElementType(threeLevelPrimitiveList);
+ assertThat(threeLevelPrimitiveElement.isPrimitive()).isEqualTo(true);
+ assertThat(threeLevelPrimitiveElement.getName()).isEqualTo("element");
+
+ // Rule 5: canonical three-level list (list -> element) with a group
element.
+ GroupType threeLevelGroupElement =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .optional(INT32)
+ .named("x")
+ .optional(INT32)
+ .named("y")
+ .named("element");
+ GroupType threeLevelGroupList =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+ Types.buildGroup(Type.Repetition.REPEATED)
+ .addField(threeLevelGroupElement)
+ .named("list"))
+ .named("my_list");
+ Type threeLevelGroupElementType =
+
ParquetListLayoutResolver.resolveElementType(threeLevelGroupList);
+ assertThat(threeLevelGroupElementType.isPrimitive()).isEqualTo(false);
+ assertThat(threeLevelGroupElementType.getName()).isEqualTo("element");
+
assertThat(threeLevelGroupElementType.asGroupType().getFieldCount()).isEqualTo(2);
+ }
+
+ @Test
+ public void testResolveTwoLevelElementType() {
+ // Rule 1: a repeated primitive field is itself the element type.
+ GroupType twoLevelPrimitiveList =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .repeated(INT32)
+ .named("element")
+ .named("my_list");
+ Type twoLevelPrimitiveElement =
+
ParquetListLayoutResolver.resolveElementType(twoLevelPrimitiveList);
+ assertThat(twoLevelPrimitiveElement.isPrimitive()).isEqualTo(true);
+ assertThat(twoLevelPrimitiveElement.getName()).isEqualTo("element");
+
+ // Rule 2: a repeated group with multiple fields is itself the element
type.
+ GroupType twoLevelGroupElement =
+ Types.buildGroup(Type.Repetition.REPEATED)
+ .optional(INT32)
+ .named("x")
+ .optional(INT32)
+ .named("y")
+ .named("element");
+ GroupType twoLevelGroupList =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(twoLevelGroupElement)
+ .named("my_list");
+ Type twoLevelGroupElementType =
+
ParquetListLayoutResolver.resolveElementType(twoLevelGroupList);
+ assertThat(twoLevelGroupElementType.isPrimitive()).isEqualTo(false);
+ assertThat(twoLevelGroupElementType.getName()).isEqualTo("element");
+
assertThat(twoLevelGroupElementType.asGroupType().getFieldCount()).isEqualTo(2);
+ }
+
+ @Test
+ public void testResolveLegacyNestedElementType() {
+ // Rule 3: a repeated group with a single repeated field is the
element type.
+ GroupType nestedRepeatedWrapper =
+ Types.buildGroup(Type.Repetition.REPEATED)
+ .repeated(INT32)
+ .named("array")
+ .named("array");
+ GroupType nestedRepeatedList =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(nestedRepeatedWrapper)
+ .named("my_list");
+ Type nestedRepeatedElement =
+
ParquetListLayoutResolver.resolveElementType(nestedRepeatedList);
+ assertThat(nestedRepeatedElement.isPrimitive()).isEqualTo(false);
+ assertThat(nestedRepeatedElement.getName()).isEqualTo("array");
+
assertThat(nestedRepeatedElement.asGroupType().getFieldCount()).isEqualTo(1);
+
assertThat(nestedRepeatedElement.asGroupType().getType(0).getName()).isEqualTo("array");
+ }
+
+ @Test
+ public void testResolveLegacyNamedElementType() {
+ // Rule 4: a repeated group named "array" with one field is the
element type.
+ GroupType arrayWrapper =
+ Types.buildGroup(Type.Repetition.REPEATED)
+ .optional(INT32)
+ .named("foo")
+ .named("array");
+ GroupType arrayWrapperList =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(arrayWrapper)
+ .named("my_list");
+ Type arrayWrapperElement =
ParquetListLayoutResolver.resolveElementType(arrayWrapperList);
+ assertThat(arrayWrapperElement.isPrimitive()).isEqualTo(false);
+ assertThat(arrayWrapperElement.getName()).isEqualTo("array");
+
+ // Rule 4: a repeated group named "<list>_tuple" with one field is the
element type.
+ GroupType tupleWrapper =
+ Types.buildGroup(Type.Repetition.REPEATED)
+ .required(BINARY)
+ .as(LogicalTypeAnnotation.stringType())
+ .named("str")
+ .named("my_list_tuple");
+ GroupType tupleWrapperList =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(tupleWrapper)
+ .named("my_list");
+ Type tupleWrapperElement =
ParquetListLayoutResolver.resolveElementType(tupleWrapperList);
+ assertThat(tupleWrapperElement.isPrimitive()).isEqualTo(false);
+ assertThat(tupleWrapperElement.getName()).isEqualTo("my_list_tuple");
+
assertThat(tupleWrapperElement.asGroupType().getFieldCount()).isEqualTo(1);
+
assertThat(tupleWrapperElement.asGroupType().getType(0).getName()).isEqualTo("str");
+ }
+
+ @Test
+ public void testResolveLegacyBagElementType() {
+ // Rule 5: a repeated group with a single non-repeated field that is
neither "array" nor
+ // "<list>_tuple" unwraps to the single child (e.g. Hive's
bag/array_element encoding).
+ GroupType bagWrapper =
+ Types.buildGroup(Type.Repetition.REPEATED)
+ .optional(INT32)
+ .named("array_element")
+ .named("bag");
+ GroupType bagWrapperList =
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(bagWrapper)
+ .named("my_list");
+
assertThat(ParquetListLayoutResolver.isThreeLevelList(bagWrapperList)).isTrue();
+
assertThat(ParquetListLayoutResolver.isCanonicalList(bagWrapperList)).isFalse();
+ Type bagWrapperElement =
ParquetListLayoutResolver.resolveElementType(bagWrapperList);
+ assertThat(bagWrapperElement.isPrimitive()).isEqualTo(true);
+ assertThat(bagWrapperElement.getName()).isEqualTo("array_element");
+ }
+}
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java
index f40550831d..e93cf01b46 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java
@@ -39,6 +39,7 @@ import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.convertToP
import static
org.apache.paimon.format.parquet.ParquetSchemaConverter.convertToParquetMessageType;
import static org.apache.paimon.types.DataTypesTest.assertThat;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
/** Test for {@link ParquetSchemaConverter}. */
@@ -206,4 +207,247 @@ public class ParquetSchemaConverterTest {
assertThat(expected).isEqualTo(convertToPaimonRowType(messageType));
}
+
+ /**
+ * Backward-compatibility Rules 1, 2 and 4: two-level lists (a repeated
primitive, a repeated
+ * struct, and a legacy {@code array} wrapper) infer a non-nullable
element, because a {@code
+ * REPEATED} node is never null. This matches parquet-cpp's {@code
SchemaManifest} contract and
+ * keeps the inferred schema symmetric with what Paimon's own writer
produces.
+ *
+ * <p>Rule 5 (three-level wrapper) is the contrast case: the wrapper
child's own nullability is
+ * preserved, so an {@code OPTIONAL} element stays nullable.
+ */
+ @Test
+ public void testInferTwoLevelListElementNotNull() {
+ // Rule 1: optional group my_list (LIST) { repeated int32 element; }
+ MessageType rule1 =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+ Types.primitive(INT32,
Type.Repetition.REPEATED)
+ .named("element")
+ .withId(1))
+ .named("my_list")
+ .withId(0));
+ assertThat(
+ new RowType(
+ Arrays.asList(
+ new DataField(
+ 0,
+ "my_list",
+ new
ArrayType(DataTypes.INT().notNull())))))
+ .isEqualTo(convertToPaimonRowType(rule1));
+
+ // Rule 2: optional group my_list (LIST) { repeated group element {
optional int32 x;
+ // optional int32 y; } }
+ MessageType rule2 =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .addField(
+ Types.primitive(
+ INT32,
+
Type.Repetition.OPTIONAL)
+ .named("x")
+ .withId(2))
+ .addField(
+ Types.primitive(
+ INT32,
+
Type.Repetition.OPTIONAL)
+ .named("y")
+ .withId(3))
+ .named("element")
+ .withId(1))
+ .named("my_list")
+ .withId(0));
+ RowType rule2Element =
+ new RowType(
+ Arrays.asList(
+ new DataField(2, "x", DataTypes.INT()),
+ new DataField(3, "y", DataTypes.INT())));
+ assertThat(
+ new RowType(
+ Arrays.asList(
+ new DataField(
+ 0,
+ "my_list",
+ new
ArrayType(rule2Element.notNull())))))
+ .isEqualTo(convertToPaimonRowType(rule2));
+
+ // Rule 4: optional group my_list (LIST) { repeated group array {
optional int32 foo; } }
+ MessageType rule4 =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .addField(
+ Types.primitive(
+ INT32,
+
Type.Repetition.OPTIONAL)
+ .named("foo")
+ .withId(2))
+ .named("array")
+ .withId(1))
+ .named("my_list")
+ .withId(0));
+ RowType rule4Element = new RowType(Arrays.asList(new DataField(2,
"foo", DataTypes.INT())));
+ assertThat(
+ new RowType(
+ Arrays.asList(
+ new DataField(
+ 0,
+ "my_list",
+ new
ArrayType(rule4Element.notNull())))))
+ .isEqualTo(convertToPaimonRowType(rule4));
+
+ // Rule 5 (contrast): optional group my_list (LIST) { repeated group
bag { optional int32
+ // array_element; } } keeps the OPTIONAL element nullable.
+ MessageType rule5 =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .addField(
+ Types.primitive(
+ INT32,
+
Type.Repetition.OPTIONAL)
+
.named("array_element")
+ .withId(2))
+ .named("bag")
+ .withId(1))
+ .named("my_list")
+ .withId(0));
+ assertThat(
+ new RowType(
+ Arrays.asList(
+ new DataField(
+ 0, "my_list", new
ArrayType(DataTypes.INT())))))
+ .isEqualTo(convertToPaimonRowType(rule5));
+ }
+
+ /**
+ * Backward-compatibility Rule 3: an annotated list whose element is a
nested legacy list infers
+ * {@code ARRAY<ARRAY<INT NOT NULL> NOT NULL>}.
+ */
+ @Test
+ public void testInferNestedLegacyList() {
+ // Rule 3: optional group my_list (LIST) { repeated group element {
repeated int32 array; }
+ // }
+ MessageType annotated =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .addField(
+ Types.primitive(
+ INT32,
+
Type.Repetition.REPEATED)
+ .named("array")
+ .withId(2))
+ .named("element")
+ .withId(1))
+ .named("my_list")
+ .withId(0));
+ assertThat(
+ new RowType(
+ Arrays.asList(
+ new DataField(
+ 0,
+ "my_list",
+ new ArrayType(
+ new
ArrayType(DataTypes.INT().notNull())
+ .notNull())))))
+ .isEqualTo(convertToPaimonRowType(annotated));
+
+ // Without the annotation: repeated group my_list { repeated group
array { optional int32
+ // x; } } infers ARRAY<ROW<x INT> NOT NULL>.
+ MessageType unannotated =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.REPEATED)
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .addField(
+ Types.primitive(
+ INT32,
+
Type.Repetition.OPTIONAL)
+ .named("x")
+ .withId(2))
+ .named("array")
+ .withId(1))
+ .named("my_list")
+ .withId(0));
+ RowType unannotatedElement =
+ new RowType(Arrays.asList(new DataField(2, "x",
DataTypes.INT())));
+ assertThat(
+ new RowType(
+ Arrays.asList(
+ new DataField(
+ 0,
+ "my_list",
+ new
ArrayType(unannotatedElement.notNull())))))
+ .isEqualTo(convertToPaimonRowType(unannotated));
+ }
+
+ @Test
+ public void testInferArrayOfMapsFromOldAvroList() {
+ MessageType oldAvroListOfMaps =
+ new MessageType(
+ "origin-parquet",
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.listType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+
.as(LogicalTypeAnnotation.mapType())
+ .addField(
+
Types.buildGroup(Type.Repetition.REPEATED)
+ .addField(
+
Types.primitive(
+
BINARY,
+
Type
+
.Repetition
+
.REQUIRED)
+
.as(
+
LogicalTypeAnnotation
+
.stringType())
+
.named("key")
+
.withId(3))
+ .addField(
+
Types.primitive(
+
INT32,
+
Type
+
.Repetition
+
.OPTIONAL)
+
.named("value")
+
.withId(4))
+
.named("key_value")
+ .withId(2))
+ .named("array")
+ .withId(1))
+ .named("my_list")
+ .withId(0));
+ assertThat(
+ new RowType(
+ Arrays.asList(
+ new DataField(
+ 0,
+ "my_list",
+ new ArrayType(
+ new MapType(
+
DataTypes.STRING(),
+
DataTypes.INT())
+ .notNull())))))
+ .isEqualTo(convertToPaimonRowType(oldAvroListOfMaps));
+ }
}
