voonhous commented on code in PR #18065:
URL: https://github.com/apache/hudi/pull/18065#discussion_r2841140804
##########
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieInternalRowUtils.scala:
##########
@@ -398,11 +399,47 @@ object HoodieInternalRowUtils {
(fieldUpdater, ordinal, value) =>
fieldUpdater.set(ordinal,
CatalystTypeConverters.convertToCatalyst(java.sql.Date.valueOf(value.toString)))
+ // Handle conversion from VariantType to variant struct representation
+ case (newStructType: StructType, _) if
sparkAdapter.isVariantType(prevDataType) &&
looksLikeVariantStruct(newStructType) =>
+ (fieldUpdater, ordinal, value) => {
+ if (value == null) {
+ fieldUpdater.setNullAt(ordinal)
+ } else {
+ val row = sparkAdapter.convertVariantToStruct(value, newStructType)
+ fieldUpdater.set(ordinal, row)
+ }
+ }
+
+ // Handle conversion from variant struct representation to VariantType
+ case (_, prevStructType: StructType) if
sparkAdapter.isVariantType(newDataType) &&
looksLikeVariantStruct(prevStructType) =>
+ (fieldUpdater, ordinal, value) => {
+ if (value == null) {
+ fieldUpdater.setNullAt(ordinal)
+ } else {
+ val row = value.asInstanceOf[InternalRow]
+ val variant = sparkAdapter.convertStructToVariant(row,
prevStructType)
+ fieldUpdater.set(ordinal, variant)
+ }
+ }
+
case (_, _) =>
throw new IllegalArgumentException(s"$prevDataType and $newDataType
are incompatible")
}
}
+ /**
+ * Checks if a StructType looks like a variant representation (has value and
metadata binary fields).
+ * This is a structural check that doesn't rely on metadata, useful during
schema reconciliation
Review Comment:
Will add this, else fall back to the behaviour we have as of now.
```
// Check if it has the expected metadata marker (fast path for Spark 4.x)
if (sparkAdapter.isVariantShreddingStruct(structType)) {
return true
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
