voonhous commented on code in PR #19403:
URL: https://github.com/apache/hudi/pull/19403#discussion_r3689100894
##########
hudi-spark-datasource/hudi-spark4-common/src/test/java/org/apache/hudi/variant/TestSpark4VariantShreddingProvider.java:
##########
@@ -100,15 +244,111 @@ void objectRoundTrips() throws Exception {
assertRoundTrips("{\"a\":\"x\",\"b\":5}",
HoodieSchema.createVariantShreddedObject(shreddedFields));
}
+ @Test
+ void partialObjectShreddingKeepsExtraFieldsInResidual() throws Exception {
+ // Shredded schema declares {a, b} but the variant provides {a, c}: "a"
shreds into typed_value,
+ // "b" is absent (null value + null typed_value), and the extra "c" lands
in the residual value.
+ Map<String, HoodieSchema> shreddedFields = new LinkedHashMap<>();
+ shreddedFields.put("a", HoodieSchema.create(HoodieSchemaType.STRING));
+ shreddedFields.put("b", HoodieSchema.create(HoodieSchemaType.LONG));
+ HoodieSchema.Variant shredded =
HoodieSchema.createVariantShreddedObject(shreddedFields);
+
+ Variant variant = VariantBuilder.parseJson("{\"a\":\"x\",\"c\":99}",
false);
+ GenericRecord shreddedRecord = shred(variant, shredded);
+
+ // The unmatched field forces a non-null residual value at the top level.
+ assertNotNull(shreddedRecord.get(VARIANT_VALUE_FIELD), "extra field must
be captured in residual value");
Review Comment:
Addressed: added the `a.typed_value == "x"` / `a.value == null` asserts,
plus a new `typeMismatchedObjectFieldFallsToFieldResidual` test pinning the
field-level residual with a null top-level `value`.
##########
hudi-spark-datasource/hudi-spark4-common/src/test/java/org/apache/hudi/variant/TestSpark4VariantShreddingProvider.java:
##########
@@ -87,11 +174,68 @@ void booleanRoundTrips() throws Exception {
assertScalarRoundTrips("true",
HoodieSchema.create(HoodieSchemaType.BOOLEAN));
}
+ @Test
+ void binaryShredsToByteBuffer() {
+ byte[] payload = "not-utf8-�ÿ".getBytes(StandardCharsets.ISO_8859_1);
+ assertScalarShredsTo(scalar(b -> b.appendBinary(payload)),
+ HoodieSchema.create(HoodieSchemaType.BYTES), ByteBuffer.wrap(payload));
+ }
+
+ @Test
+ void uuidShredsToString() {
+ UUID uuid = UUID.fromString("12345678-1234-1234-1234-123456789abc");
+ assertScalarShredsTo(scalar(b -> b.appendUuid(uuid)),
HoodieSchema.createUUID(), uuid.toString());
+ }
+
+ @Test
+ void dateShredsToDaysSinceEpoch() {
+ assertScalarShredsTo(scalar(b -> b.appendDate(19000)),
HoodieSchema.createDate(), 19000);
+ }
+
+ @Test
+ void timestampMicrosShredsToMicros() {
+ long micros = 1_700_000_000_000_000L;
+ assertScalarShredsTo(scalar(b -> b.appendTimestamp(micros)),
HoodieSchema.createTimestampMicros(), micros);
+ }
+
+ @Test
+ void localTimestampMicrosShredsToMicros() {
+ long micros = 1_700_000_000_000_000L;
+ assertScalarShredsTo(scalar(b -> b.appendTimestampNtz(micros)),
HoodieSchema.createLocalTimestampMicros(), micros);
+ }
+
@Test
void decimalRoundTrips() throws Exception {
assertScalarRoundTrips("123.45", HoodieSchema.createDecimal(10, 2));
}
+ //
---------------------------------------------------------------------------
+ // "Decline to shred" fallbacks: value stays in the residual binary.
+ //
---------------------------------------------------------------------------
+
+ @Test
+ void millisTimestampIsNotShreddedIntoMicrosLeaf() {
+ // A millisecond-precision typed_value cannot represent a micros variant
timestamp, so
+ // avroTypeToScalarType returns null and the value is left unshredded in
the residual.
+ assertStaysInResidual(scalar(b ->
b.appendTimestamp(1_700_000_000_000_000L)),
+ HoodieSchema.createTimestampMillis());
+ assertStaysInResidual(scalar(b ->
b.appendTimestampNtz(1_700_000_000_000_000L)),
+ HoodieSchema.createLocalTimestampMillis());
+ }
+
+ @Test
+ void fixedLeafShredsBinaryToByteBuffer() {
Review Comment:
Addressed: dropped `fixedLeafShredsBinaryToByteBuffer` rather than pin the
broken behavior (inverting it would fail until the provider is fixed). FIXED
handling tracked in #19442.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
