yihua commented on code in PR #18007:
URL: https://github.com/apache/hudi/pull/18007#discussion_r2772492754
##########
hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java:
##########
@@ -184,12 +191,54 @@ public void testAddMetadataFields() {
assertEquals(schemaWithMetaNoOp.getFields().size(),
baseSchema.getFields().size() + 5);
}
+ @Test
+ public void testAddMetadataFieldsWithProps() {
+ HoodieSchema baseSchema = HoodieSchema.parse(EXAMPLE_SCHEMA_WITH_PROPS);
+ HoodieSchema schemaWithMetadata =
HoodieSchemaUtils.addMetadataFields(baseSchema);
+ List<HoodieSchemaField> updatedFields = schemaWithMetadata.getFields();
+ // assert fields added in expected order
+ assertEquals(HoodieRecord.COMMIT_TIME_METADATA_FIELD,
updatedFields.get(0).name());
+ assertEquals(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD,
updatedFields.get(1).name());
+ assertEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD,
updatedFields.get(2).name());
+ assertEquals(HoodieRecord.PARTITION_PATH_METADATA_FIELD,
updatedFields.get(3).name());
+ assertEquals(HoodieRecord.FILENAME_METADATA_FIELD,
updatedFields.get(4).name());
+ // assert original fields are copied over
+ List<HoodieSchemaField> originalFieldsInUpdatedSchema =
updatedFields.subList(5, updatedFields.size());
+ assertEquals(baseSchema.getFields(), originalFieldsInUpdatedSchema);
+ // validate properties are properly copied over
+ assertEquals("custom_schema_property_value",
schemaWithMetadata.getProp("custom_schema_property"));
+ assertEquals("value",
originalFieldsInUpdatedSchema.get(0).getProp("custom_field_property"));
+ }
+
@Test
public void testAddMetadataFieldsValidation() {
// Should throw on null schema
assertThrows(IllegalArgumentException.class, () ->
HoodieSchemaUtils.addMetadataFields(null, true));
}
+ @Test
+ public void testPropsPresent() {
+ HoodieSchema schema =
HoodieSchemaUtils.addMetadataFields(HoodieSchema.parse(EXAMPLE_SCHEMA));
+ boolean piiPresent = false;
+ for (HoodieSchemaField field : schema.getFields()) {
+ if (HoodieSchemaUtils.isMetadataField(field.name())) {
+ continue;
+ }
+
+ assertNotNull(field.name(), "field name is null");
+ Map<String, Object> props = field.getObjectProps();
+ assertNotNull(props, "The property is null");
+
+ if (field.name().equals("pii_col")) {
+ piiPresent = true;
+ assertTrue(props.containsKey("column_category"), "sensitivity_level is
removed in field 'pii_col'");
Review Comment:
nit: the error message does not match the actual validation
##########
hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java:
##########
@@ -336,17 +136,4 @@ public static Schema asNullable(Schema schema) {
(change, field) -> change.updateColumnNullability(field, true));
return convert(SchemaChangeUtils.applyTableChanges2Schema(internalSchema,
schemaChange), schema.getFullName()).toAvroSchema();
}
-
- /**
- * Helper to copy properties and logical types from source schema to target
schema.
- */
- private static Schema copyProperties(Schema source, Schema target) {
- for (Map.Entry<String, Object> prop : source.getObjectProps().entrySet()) {
- target.addProp(prop.getKey(), prop.getValue());
- }
- if (source.getLogicalType() != null) {
- source.getLogicalType().addToSchema(target);
- }
Review Comment:
Is this already accounted for when creating `HoodieSchema` of logical types
so this logic is no longer needed?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
