pvary commented on code in PR #17320:
URL: https://github.com/apache/iceberg/pull/17320#discussion_r3844674271
##########
parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueReaders.java:
##########
@@ -231,6 +251,105 @@ public static ParquetValueReader<?>
replaceWithMetadataReader(
return reader;
}
+ /**
+ * Builds readers for a struct's expected fields, in field order. A field
present in the file uses
+ * its column reader; a field missing from the file uses a metadata or
partition constant, or its
+ * initial default. When no expected field reads a file column, one default
reader is given a
+ * probe column so its definition level tracks the struct's null-ness.
+ */
+ public static List<ParquetValueReader<?>> structFieldReaders(
+ MessageType fileSchema,
+ String[] structPath,
+ List<Types.NestedField> expectedFields,
+ Map<Integer, ParquetValueReader<?>> readersById,
+ Map<Integer, ?> idToConstant,
+ BiFunction<org.apache.iceberg.types.Type, Object, Object>
convertConstant) {
+ int constantDefinitionLevel = fileSchema.getMaxDefinitionLevel(structPath);
+ ColumnDescriptor probe =
+ definitionLevelProbe(
+ fileSchema, structPath, constantDefinitionLevel, expectedFields,
readersById);
+ Integer probeHostId = probe == null ? null :
firstInitialDefaultFieldId(expectedFields);
+
+ List<ParquetValueReader<?>> readers =
Lists.newArrayListWithExpectedSize(expectedFields.size());
+ for (Types.NestedField field : expectedFields) {
+ int id = field.fieldId();
+ ParquetValueReader<?> reader =
+ replaceWithMetadataReader(id, readersById.get(id), idToConstant,
constantDefinitionLevel);
+ ColumnDescriptor fieldProbe = probeHostId != null && id == probeHostId ?
probe : null;
+ readers.add(
+ defaultReader(field, reader, constantDefinitionLevel, fieldProbe,
convertConstant));
+ }
+
+ return readers;
+ }
+
+ private static ParquetValueReader<?> defaultReader(
+ Types.NestedField field,
+ ParquetValueReader<?> reader,
+ int constantDefinitionLevel,
+ ColumnDescriptor probe,
+ BiFunction<org.apache.iceberg.types.Type, Object, Object>
convertConstant) {
+ if (reader != null) {
+ return reader;
+ } else if (field.initialDefault() != null) {
+ Object value = convertConstant.apply(field.type(),
field.initialDefault());
+ return probe != null ? constant(value, probe) : constant(value,
constantDefinitionLevel);
+ } else if (field.isOptional()) {
+ return nulls();
+ }
+
+ throw new IllegalArgumentException(String.format("Missing required field:
%s", field.name()));
+ }
+
+ /**
+ * Returns the first leaf column under the struct, or null if an expected
field already reads a
+ * file column or the struct has no leaf columns.
+ */
+ private static ColumnDescriptor definitionLevelProbe(
Review Comment:
We should make this align with `firstNonNullColumn` too
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
