shangxinli commented on code in PR #20332:
URL: https://github.com/apache/hudi/pull/20332#discussion_r4304761388
##########
hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieVariantReconstruction.java:
##########
@@ -110,90 +124,153 @@ static HoodieVariantReconstruction create(HoodieSchema
fileSchema, HoodieSchema
+ HoodieStorageConfig.PARQUET_VARIANT_SHREDDING_PROVIDER_CLASS.key()
+ " or add a provider implementation (e.g. the Spark variant module)
to the classpath.");
}
+ for (VariantColumnRebuilder variantColumn : planner.variantColumns) {
+ variantColumn.prepare(provider);
+ }
return new HoodieVariantReconstruction(
- VariantSchemaUtils.toShreddedReadSchema(requestedSchema, fileSchema),
provider, rootRebuilder);
+ VariantSchemaUtils.toShreddedReadSchema(requestedSchema, fileSchema),
rootRebuilder);
}
/**
* Rebuilds shredded variant columns of {@code in} (read in the intermediate
shredded shape) into
* a record conforming to the unshredded output schema.
*/
IndexedRecord reconstruct(IndexedRecord in) {
- return (IndexedRecord) rootRebuilder.rebuild(in, provider);
+ return (IndexedRecord) rootRebuilder.rebuild(in);
}
- /**
- * Plans the rebuild for one schema position, or returns {@code null} when
nothing below it is a
- * shredded variant and the value can be passed through untouched. The walk
is driven by the
- * output (requested) side because that is the shape the record read from
parquet has; the file
- * side is matched into it by field name. Detection is anchored by the
requested side because the
- * file schema usually comes from converting the parquet footer MessageType,
which loses the
- * variant logical type; see VariantSchemaUtils.isShreddedVariantTarget
(#19567). Records, array
- * elements and map values are all descended into, matching what the writers
can emit; see
- * {@code VariantSchemaUtils.swapShreddedVariantFields} for what actually
produces a nested
- * shredded file today (both the row and the AVRO write path shred variant
record members at any
- * depth off the forced-shredding property; a variant that is directly an
array element or a map
- * value shreds on neither path unless the write schema itself declares
typed_value there).
- */
- private static Rebuilder buildRebuilder(HoodieSchema outputSchema,
HoodieSchema fileSchema) {
- if (VariantSchemaUtils.isShreddedVariantTarget(fileSchema, outputSchema)) {
- return new VariantRebuilder(fileSchema.getNonNullType().getAvroSchema(),
- outputSchema.getNonNullType().getAvroSchema());
- }
- HoodieSchema output = outputSchema.getNonNullType();
- HoodieSchema file = fileSchema.getNonNullType();
- if (output.getType() != file.getType()) {
- return null;
+ /** Plans the rebuild of a file, collecting its shredded variant columns for
preparation. */
+ private static final class Planner {
+ // Mirrors the file schema conversion's ADD_LIST_ELEMENT_RECORDS, which
decides the list element.
Review Comment:
nit: this and Spark4VariantShreddingProvider now carry separate list-element
rules for the same parquet group; they only disagree when the element is itself
a repeated group, and then the provider throws invalidShreddingLayout at open,
but it would be easy to change one and not the other.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
