This is an automated email from the ASF dual-hosted git repository.
hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 928e018 [GOBBLIN-896] Clone schema and field props in
AvroSchemaFieldRemover
928e018 is described below
commit 928e0180c471fc4b7a6caee041b001b5b34e1cc6
Author: zhchen <[email protected]>
AuthorDate: Thu Oct 10 14:45:42 2019 -0700
[GOBBLIN-896] Clone schema and field props in AvroSchemaFieldRemover
Closes #2750 from zxcware/props
---
.../apache/gobblin/converter/filter/AvroSchemaFieldRemover.java | 7 +++++++
.../src/test/resources/converter/recursive_schema_1.avsc | 2 +-
.../src/test/resources/converter/recursive_schema_1_converted.avsc | 2 +-
.../src/test/resources/converter/recursive_schema_2_converted.avsc | 2 +-
.../test/resources/converter/recursive_schema_2_not_converted.avsc | 2 +-
5 files changed, 11 insertions(+), 4 deletions(-)
diff --git
a/gobblin-core-base/src/main/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemover.java
b/gobblin-core-base/src/main/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemover.java
index d681619..22f4d80 100644
---
a/gobblin-core-base/src/main/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemover.java
+++
b/gobblin-core-base/src/main/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemover.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
+import org.codehaus.jackson.JsonNode;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
@@ -105,6 +106,9 @@ public class AvroSchemaFieldRemover {
private Schema removeFieldsFromRecords(Schema schema, Map<String, Schema>
schemaMap) {
Schema newRecord = Schema.createRecord(schema.getName(), schema.getDoc(),
schema.getNamespace(), schema.isError());
+ for (Map.Entry<String, JsonNode> stringJsonNodeEntry :
schema.getJsonProps().entrySet()) {
+ newRecord.addProp(stringJsonNodeEntry.getKey(),
stringJsonNodeEntry.getValue());
+ }
// Put an incomplete schema into schemaMap to avoid re-processing a
recursive field.
// The fields in the incomplete schema will be populated once the current
schema is completely processed.
@@ -121,6 +125,9 @@ public class AvroSchemaFieldRemover {
newField = new Field(field.name(),
DO_NOTHING_INSTANCE.removeFields(field.schema(), schemaMap), field.doc(),
field.defaultValue());
}
+ for (Map.Entry<String, JsonNode> stringJsonNodeEntry :
field.getJsonProps().entrySet()) {
+ newField.addProp(stringJsonNodeEntry.getKey(),
stringJsonNodeEntry.getValue());
+ }
newFields.add(newField);
}
}
diff --git
a/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
b/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
index 1cece39..4b41e2d 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
@@ -1 +1 @@
-{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj
NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw
tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll
lIVyeB"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj NXTFcrls gIn
WS oWm Z [...]
+{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj
NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw
tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll
lIVyeB","meta":"field_meta"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj
N [...]
diff --git
a/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
b/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
index ca6ac1e..66a8fae 100644
---
a/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
+++
b/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj
NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw
tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll
lIVyeB"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj NXTFcrls gIn
WS oWm Z [...]
+{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj
NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw
tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll
lIVyeB","meta":"field_meta"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj
N [...]
diff --git
a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
index 1e63079..a733019 100644
---
a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
+++
b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu
fS jty qGGt PpaPkENfDJ jty
HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty
FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty
kdYkvr"},{"name":"jfqWhtm","type":"string","doc":"BLD name fS jty
jfqWhtm"},{"name":"ORVhoRguhhZ","type":["nu [...]
+{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu
fS jty qGGt PpaPkENfDJ jty
HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty
FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty
kdYkvr"},{"name":"jfqWhtm","type":"string","doc":"BLD name fS jty
jfqWhtm"},{"name":"ORVhoRguhhZ","type":["nu [...]
\ No newline at end of file
diff --git
a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
index 428bec3..c3130f9 100644
---
a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
+++
b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"TzuYZajb","type":"int","doc":"BLD
isOtza xj fS jty qGGt PpaPkENfDJ jty
HTvfxa"},{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt
PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD
kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty
kdYkvr"},{"name":"jf [...]
\ No newline at end of file
+{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"TzuYZajb","type":"int","doc":"BLD
isOtza xj fS jty qGGt PpaPkENfDJ jty
HTvfxa"},{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt
PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD
kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty
kdYkvr"},{"name":"jf [...]
\ No newline at end of file
