yihua commented on code in PR #12829:
URL: https://github.com/apache/hudi/pull/12829#discussion_r1970640794
##########
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java:
##########
@@ -190,6 +196,187 @@ private org.apache.hudi.avro.model.HoodieWriteStat
createWriteStat(String fileId
writeStat.setFileId(fileId);
writeStat.setBaseFile(baseFile);
writeStat.setLogFiles(logFiles);
- return writeStat;
+ return writeStat;
+ }
+
+ @Test
+ public void testSchemaEqualityForHoodieCommitMetaData() {
+ // Step 1: Get the schema from the Avro auto-generated class
+ Schema avroSchema =
org.apache.hudi.avro.model.HoodieCommitMetadata.SCHEMA$;
+
+ // Step 2: Convert the POJO class to an Avro schema
+ Schema pojoSchema =
ReflectData.get().getSchema(org.apache.hudi.common.model.HoodieCommitMetadata.class);
+
+ // Step 3: Validate schemas
+ ValidationResult result = validateSchemas(pojoSchema, avroSchema, "root");
+
+ // Print validation results
+ printValidationResults(result);
Review Comment:
Is printing necessary?
##########
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java:
##########
@@ -190,6 +196,187 @@ private org.apache.hudi.avro.model.HoodieWriteStat
createWriteStat(String fileId
writeStat.setFileId(fileId);
writeStat.setBaseFile(baseFile);
writeStat.setLogFiles(logFiles);
- return writeStat;
+ return writeStat;
+ }
+
+ @Test
+ public void testSchemaEqualityForHoodieCommitMetaData() {
+ // Step 1: Get the schema from the Avro auto-generated class
+ Schema avroSchema =
org.apache.hudi.avro.model.HoodieCommitMetadata.SCHEMA$;
+
+ // Step 2: Convert the POJO class to an Avro schema
+ Schema pojoSchema =
ReflectData.get().getSchema(org.apache.hudi.common.model.HoodieCommitMetadata.class);
+
+ // Step 3: Validate schemas
+ ValidationResult result = validateSchemas(pojoSchema, avroSchema, "root");
+
+ // Print validation results
+ printValidationResults(result);
+
+ // Fail if there are any critical errors (field missing or type mismatch
at current layer)
+ assertFalse(result.hasCriticalErrors(), "Critical validation errors
found");
+ }
+
+ @Test
+ public void testSchemaEqualityForHoodieReplaceCommitMetaData() {
+ // Step 1: Get the schema from the Avro auto-generated class
+ Schema avroSchema =
org.apache.hudi.avro.model.HoodieReplaceCommitMetadata.SCHEMA$;
+
+ // Step 2: Convert the POJO class to an Avro schema
+ Schema pojoSchema =
ReflectData.get().getSchema(org.apache.hudi.common.model.HoodieReplaceCommitMetadata.class);
+
+ // Step 3: Validate schemas
+ ValidationResult result = validateSchemas(pojoSchema, avroSchema, "root");
+
+ // Print validation results
+ printValidationResults(result);
+
+ // Fail if there are any critical errors (field missing or type mismatch
at current layer)
+ assertFalse(result.hasCriticalErrors(), "Critical validation errors
found");
+ }
+
+ private static class ValidationResult {
+ boolean hasCriticalErrors = false;
+ List<String> errors = new ArrayList<>();
+ List<ValidationResult> nestedResults = new ArrayList<>();
+
+ void addError(String error) {
+ errors.add(error);
+ }
+
+ void addNestedResult(ValidationResult result) {
+ nestedResults.add(result);
+ }
+
+ void markCritical() {
+ hasCriticalErrors = true;
+ }
+
+ boolean hasCriticalErrors() {
+ return hasCriticalErrors;
+ }
+ }
+
+ private void printValidationResults(ValidationResult result) {
+ printValidationResults(result, 0);
+ }
+
+ private void printValidationResults(ValidationResult result, int indent) {
+ // We don't have repeat method in java 8
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < indent; i++) {
+ sb.append(" ");
+ }
+ for (String error : result.errors) {
+ LOGGER.error(sb + error);
+ }
+ for (ValidationResult nested : result.nestedResults) {
+ printValidationResults(nested, indent + 1);
+ }
+ }
Review Comment:
Seems redundant
##########
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java:
##########
@@ -190,6 +196,187 @@ private org.apache.hudi.avro.model.HoodieWriteStat
createWriteStat(String fileId
writeStat.setFileId(fileId);
writeStat.setBaseFile(baseFile);
writeStat.setLogFiles(logFiles);
- return writeStat;
+ return writeStat;
+ }
+
+ @Test
+ public void testSchemaEqualityForHoodieCommitMetaData() {
+ // Step 1: Get the schema from the Avro auto-generated class
+ Schema avroSchema =
org.apache.hudi.avro.model.HoodieCommitMetadata.SCHEMA$;
+
+ // Step 2: Convert the POJO class to an Avro schema
+ Schema pojoSchema =
ReflectData.get().getSchema(org.apache.hudi.common.model.HoodieCommitMetadata.class);
+
+ // Step 3: Validate schemas
+ ValidationResult result = validateSchemas(pojoSchema, avroSchema, "root");
+
+ // Print validation results
+ printValidationResults(result);
+
+ // Fail if there are any critical errors (field missing or type mismatch
at current layer)
+ assertFalse(result.hasCriticalErrors(), "Critical validation errors
found");
+ }
+
+ @Test
+ public void testSchemaEqualityForHoodieReplaceCommitMetaData() {
+ // Step 1: Get the schema from the Avro auto-generated class
+ Schema avroSchema =
org.apache.hudi.avro.model.HoodieReplaceCommitMetadata.SCHEMA$;
+
+ // Step 2: Convert the POJO class to an Avro schema
+ Schema pojoSchema =
ReflectData.get().getSchema(org.apache.hudi.common.model.HoodieReplaceCommitMetadata.class);
+
+ // Step 3: Validate schemas
+ ValidationResult result = validateSchemas(pojoSchema, avroSchema, "root");
+
+ // Print validation results
+ printValidationResults(result);
+
+ // Fail if there are any critical errors (field missing or type mismatch
at current layer)
+ assertFalse(result.hasCriticalErrors(), "Critical validation errors
found");
+ }
+
+ private static class ValidationResult {
+ boolean hasCriticalErrors = false;
+ List<String> errors = new ArrayList<>();
+ List<ValidationResult> nestedResults = new ArrayList<>();
+
+ void addError(String error) {
+ errors.add(error);
+ }
+
+ void addNestedResult(ValidationResult result) {
+ nestedResults.add(result);
+ }
+
+ void markCritical() {
+ hasCriticalErrors = true;
+ }
+
+ boolean hasCriticalErrors() {
+ return hasCriticalErrors;
+ }
+ }
+
+ private void printValidationResults(ValidationResult result) {
+ printValidationResults(result, 0);
+ }
+
+ private void printValidationResults(ValidationResult result, int indent) {
+ // We don't have repeat method in java 8
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < indent; i++) {
+ sb.append(" ");
+ }
+ for (String error : result.errors) {
+ LOGGER.error(sb + error);
+ }
+ for (ValidationResult nested : result.nestedResults) {
+ printValidationResults(nested, indent + 1);
+ }
+ }
+
+ private ValidationResult validateSchemas(Schema pojoSchema, Schema
avroSchema, String path) {
Review Comment:
Could you avoid these schema validation logic and code duplication by
reusing utils in `AvroSchemaUtils`, e.g., by running
`isSchemaCompatible(pojoSchema, avroSchema)` and
`isSchemaCompatible(avroSchema, pojoSchema)`
##########
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java:
##########
@@ -190,6 +196,187 @@ private org.apache.hudi.avro.model.HoodieWriteStat
createWriteStat(String fileId
writeStat.setFileId(fileId);
writeStat.setBaseFile(baseFile);
writeStat.setLogFiles(logFiles);
- return writeStat;
+ return writeStat;
+ }
+
+ @Test
+ public void testSchemaEqualityForHoodieCommitMetaData() {
+ // Step 1: Get the schema from the Avro auto-generated class
+ Schema avroSchema =
org.apache.hudi.avro.model.HoodieCommitMetadata.SCHEMA$;
+
+ // Step 2: Convert the POJO class to an Avro schema
+ Schema pojoSchema =
ReflectData.get().getSchema(org.apache.hudi.common.model.HoodieCommitMetadata.class);
+
+ // Step 3: Validate schemas
+ ValidationResult result = validateSchemas(pojoSchema, avroSchema, "root");
+
+ // Print validation results
+ printValidationResults(result);
+
+ // Fail if there are any critical errors (field missing or type mismatch
at current layer)
+ assertFalse(result.hasCriticalErrors(), "Critical validation errors
found");
+ }
+
+ @Test
+ public void testSchemaEqualityForHoodieReplaceCommitMetaData() {
+ // Step 1: Get the schema from the Avro auto-generated class
+ Schema avroSchema =
org.apache.hudi.avro.model.HoodieReplaceCommitMetadata.SCHEMA$;
+
+ // Step 2: Convert the POJO class to an Avro schema
+ Schema pojoSchema =
ReflectData.get().getSchema(org.apache.hudi.common.model.HoodieReplaceCommitMetadata.class);
+
+ // Step 3: Validate schemas
+ ValidationResult result = validateSchemas(pojoSchema, avroSchema, "root");
+
+ // Print validation results
+ printValidationResults(result);
Review Comment:
Same here on removing the printing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
