voonhous commented on code in PR #18062:
URL: https://github.com/apache/hudi/pull/18062#discussion_r3069655808
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java:
##########
@@ -189,28 +233,26 @@ private StructType generateShreddedSchema(StructType
structType, HoodieSchema ho
.map(HoodieSchemaField::schema)
.orElse(null);
- // Check if this is a Variant field that should be shredded
- if (SparkAdapterSupport$.MODULE$.sparkAdapter().isVariantType(dataType))
{
- if (fieldHoodieSchema != null && fieldHoodieSchema.getType() ==
HoodieSchemaType.VARIANT) {
- HoodieSchema.Variant variantSchema = (HoodieSchema.Variant)
fieldHoodieSchema;
- // If typed_value field exists, the variant is shredded
- if (variantSchema.getTypedValueField().isPresent()) {
- // Use plain types for SparkShreddingUtils (unwraps nested {value,
typed_value} structs if present)
- HoodieSchema typedValueSchema =
variantSchema.getPlainTypedValueSchema().get();
- DataType typedValueDataType =
HoodieSchemaConversionUtils.convertHoodieSchemaToDataType(typedValueSchema);
-
- // Generate the shredding schema with write metadata using
SparkAdapter
- StructType markedShreddedStruct =
SparkAdapterSupport$.MODULE$.sparkAdapter()
- .generateVariantWriteShreddingSchema(typedValueDataType, true,
false);
-
- shreddedFields[i] = new StructField(field.name(),
markedShreddedStruct, field.nullable(), field.metadata());
- hasShredding = true;
- continue;
- }
- } else {
- LOG.warn("Variant field '{}' has no corresponding HoodieSchema; "
- + "shredding will be skipped. This may indicate a schema
mismatch.", field.name());
+ if (SparkAdapterSupport$.MODULE$.sparkAdapter().isVariantType(dataType)
+ && fieldHoodieSchema != null && fieldHoodieSchema.getType() ==
HoodieSchemaType.VARIANT) {
+ HoodieSchema.Variant variantSchema = (HoodieSchema.Variant)
fieldHoodieSchema;
Review Comment:
Addressed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
