[hudi] branch master updated: [HUDI-1375] Fix bug in HoodieAvroUtils.removeMetadataFields() method (#2232)

[uditme]

This is an automated email from the ASF dual-hosted git repository.

uditme pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 0364498  [HUDI-1375] Fix bug in HoodieAvroUtils.removeMetadataFields() 
method (#2232)
0364498 is described below

commit 0364498ae3be553f7c9fab78469283a6958da2ff
Author: wenningd <wenningdin...@gmail.com>
AuthorDate: Thu Nov 5 17:30:17 2020 -0800

    [HUDI-1375] Fix bug in HoodieAvroUtils.removeMetadataFields() method (#2232)
    
    Co-authored-by: Wenning Ding <wenni...@amazon.com>
---
 .../src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java    |  1 +
 .../test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java    | 10 ++++++++++
 2 files changed, 11 insertions(+)

diff --git 
a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java 
b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
index a7517a4..3b356a7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
@@ -204,6 +204,7 @@ public class HoodieAvroUtils {
     List<Schema.Field> filteredFields = schema.getFields()
                                               .stream()
                                               .filter(field -> 
!HoodieRecord.HOODIE_META_COLUMNS.contains(field.name()))
+                                              .map(field -> new 
Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()))
                                               .collect(Collectors.toList());
     Schema filteredSchema = Schema.createRecord(schema.getName(), 
schema.getDoc(), schema.getNamespace(), false);
     filteredSchema.setFields(filteredFields);
diff --git 
a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java 
b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
index f6854ca..40db67b 100644
--- a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
@@ -55,6 +55,8 @@ public class TestHoodieAvroUtils {
       + "{\"name\": \"non_pii_col\", \"type\": \"string\"},"
       + "{\"name\": \"pii_col\", \"type\": \"string\", \"column_category\": 
\"user_profile\"}]}";
 
+  private static int NUM_FIELDS_IN_EXAMPLE_SCHEMA = 4;
+
   private static String SCHEMA_WITH_METADATA_FIELD = "{\"type\": 
\"record\",\"name\": \"testrec2\",\"fields\": [ "
       + "{\"name\": \"timestamp\",\"type\": \"double\"},{\"name\": 
\"_row_key\", \"type\": \"string\"},"
       + "{\"name\": \"non_pii_col\", \"type\": \"string\"},"
@@ -197,4 +199,12 @@ public class TestHoodieAvroUtils {
     //evolvedField5.defaultVal() returns null.
     assertNull(rec1.get("evolved_field1"));
   }
+
+  @Test
+  public void testAddingAndRemovingMetadataFields() {
+    Schema schemaWithMetaCols = HoodieAvroUtils.addMetadataFields(new 
Schema.Parser().parse(EXAMPLE_SCHEMA));
+    assertEquals(schemaWithMetaCols.getFields().size(), 
NUM_FIELDS_IN_EXAMPLE_SCHEMA + HoodieRecord.HOODIE_META_COLUMNS.size());
+    Schema schemaWithoutMetaCols = 
HoodieAvroUtils.removeMetadataFields(schemaWithMetaCols);
+    assertEquals(schemaWithoutMetaCols.getFields().size(), 
NUM_FIELDS_IN_EXAMPLE_SCHEMA);
+  }
 }