voonhous commented on code in PR #18961:
URL: https://github.com/apache/hudi/pull/18961#discussion_r3843794636
##########
hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchema.java:
##########
@@ -3087,4 +3087,120 @@ public void testGetPlainTypedValueSchemaEmpty() {
HoodieSchema.Variant unshreddedVariant = HoodieSchema.createVariant();
assertFalse(unshreddedVariant.getPlainTypedValueSchema().isPresent());
}
+
+ @Test
+ public void testGetPlainTypedValueSchemaNestedObjectRecursion() {
+ // Depth-2 spec form: typed_value { a: wrapper{value, typed_value: { b:
wrapper{value, typed_value: long} }} }.
+ // Both record levels are named "typed_value", as the schema converters
produce them.
+ HoodieSchema innerObject = HoodieSchema.createRecord("typed_value",
"inner.ns", null,
+ Collections.singletonList(HoodieSchemaField.of("b",
+
HoodieSchema.createNullable(HoodieSchema.createShreddedFieldStruct("b_wrapper",
HoodieSchema.create(HoodieSchemaType.LONG))))));
+ HoodieSchema topTypedValue = HoodieSchema.createRecord("typed_value",
"outer.ns", null,
+ Collections.singletonList(HoodieSchemaField.of("a",
+
HoodieSchema.createNullable(HoodieSchema.createShreddedFieldStruct("a_wrapper",
innerObject)))));
+ // Nullable typed_value, as produced by the inferred-shredding splice.
+ HoodieSchema.Variant variant =
HoodieSchema.createVariantShredded(HoodieSchema.createNullable(topTypedValue));
+
+ Option<HoodieSchema> plainOpt = variant.getPlainTypedValueSchema();
+ assertTrue(plainOpt.isPresent());
+ HoodieSchema plain = plainOpt.get();
+ assertEquals(HoodieSchemaType.RECORD, plain.getType());
+ assertEquals(1, plain.getFields().size());
+
+ HoodieSchema aPlain = plain.getFields().get(0).schema();
+ aPlain = aPlain.isNullable() ? aPlain.getNonNullType() : aPlain;
+ assertEquals(HoodieSchemaType.RECORD, aPlain.getType());
+ assertEquals(1, aPlain.getFields().size());
+
+ HoodieSchema bPlain = aPlain.getFields().get(0).schema();
+ bPlain = bPlain.isNullable() ? bPlain.getNonNullType() : bPlain;
+ assertEquals(HoodieSchemaType.LONG, bPlain.getType());
+
+ // Generated plain record names must be unique per nesting level: a nested
record carrying
+ // its ancestor's fullname is an Avro self-reference, which Spark rejects
as recursion.
+ assertNotEquals(plain.getFullName(), aPlain.getFullName());
+ }
+
+ @Test
+ public void testGetPlainTypedValueSchemaNamesDistinguishConcatenatingPaths()
{
+ // Two object paths whose segments concatenate to the same string ("x_y" >
"z" and "x" > "y_z")
+ // must still yield distinct plain record names for the objects at their
leaves; the path goes
+ // into the namespace, where the '.' separator keeps them apart (a flat
"<path>_plain" name
+ // gave both leaves "typed_value_x_y_z_plain").
+ HoodieSchema leafObject = HoodieSchema.createRecord("typed_value",
"leaf.ns", null,
+ Collections.singletonList(HoodieSchemaField.of("c",
+
HoodieSchema.createNullable(HoodieSchema.createShreddedFieldStruct("c_wrapper",
HoodieSchema.create(HoodieSchemaType.LONG))))));
+ HoodieSchema underXy = HoodieSchema.createRecord("typed_value", "a.ns",
null,
+ Collections.singletonList(HoodieSchemaField.of("z",
+
HoodieSchema.createNullable(HoodieSchema.createShreddedFieldStruct("z_wrapper",
leafObject)))));
+ HoodieSchema underX = HoodieSchema.createRecord("typed_value", "b.ns",
null,
+ Collections.singletonList(HoodieSchemaField.of("y_z",
+
HoodieSchema.createNullable(HoodieSchema.createShreddedFieldStruct("y_z_wrapper",
leafObject)))));
+ HoodieSchema topTypedValue = HoodieSchema.createRecord("typed_value",
"outer.ns", null, Arrays.asList(
+ HoodieSchemaField.of("x_y",
HoodieSchema.createNullable(HoodieSchema.createShreddedFieldStruct("x_y_wrapper",
underXy))),
+ HoodieSchemaField.of("x",
HoodieSchema.createNullable(HoodieSchema.createShreddedFieldStruct("x_wrapper",
underX)))));
+
+ HoodieSchema plain =
HoodieSchema.createVariantShredded(topTypedValue).getPlainTypedValueSchema().get();
+ HoodieSchema zLeaf = plain.getField("x_y").get().schema().getNonNullType()
+ .getField("z").get().schema().getNonNullType();
+ HoodieSchema yzLeaf = plain.getField("x").get().schema().getNonNullType()
+ .getField("y_z").get().schema().getNonNullType();
+ assertEquals(HoodieSchemaType.RECORD, zLeaf.getType());
+ assertEquals(HoodieSchemaType.RECORD, yzLeaf.getType());
+ assertNotEquals(zLeaf.getFullName(), yzLeaf.getFullName());
+ // Serializing the whole tree (as the config-splice path does) must not
alias the two leaves.
+ assertNotNull(plain.getAvroSchema().toString());
Review Comment:
Replaced with a re-parse of the serialized tree and an assertion that the
two leaf full names still differ.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
