Repository: incubator-gobblin Updated Branches: refs/heads/master d769b2144 -> 396fc40de
[GOBBLIN-228] Add config property to ignore fields Closes #2082 from jack-moseley/master Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/396fc40d Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/396fc40d Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/396fc40d Branch: refs/heads/master Commit: 396fc40dee6d4f7dac6029acdba3043305740ee8 Parents: d769b21 Author: Jack Moseley <[email protected]> Authored: Mon Aug 28 13:09:20 2017 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Mon Aug 28 13:09:20 2017 -0700 ---------------------------------------------------------------------- .../avro/JsonRecordAvroSchemaToAvroConverter.java | 13 ++++++++++++- .../avro/JsonRecordAvroSchemaToAvroConverterTest.java | 4 +++- .../src/test/resources/converter/jsonToAvroSchema.avsc | 4 ++++ 3 files changed, 19 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/396fc40d/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java ---------------------------------------------------------------------- diff --git a/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java b/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java index c3edd25..6f5e7b3 100644 --- a/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java +++ b/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java @@ -17,7 +17,6 @@ package org.apache.gobblin.converter.avro; -import com.google.common.base.Preconditions; import java.util.List; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; @@ -29,6 +28,8 @@ import org.apache.gobblin.converter.SchemaConversionException; import org.apache.gobblin.converter.SingleRecordIterable; import org.apache.gobblin.converter.ToAvroConverterBase; import com.google.gson.JsonObject; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; /** @@ -37,14 +38,19 @@ import com.google.gson.JsonObject; */ public class JsonRecordAvroSchemaToAvroConverter<SI> extends ToAvroConverterBase<SI, JsonObject> { + private static final Splitter SPLITTER_ON_COMMA = Splitter.on(',').trimResults().omitEmptyStrings(); + public static final String AVRO_SCHEMA_KEY = "gobblin.converter.avroSchema"; + public static final String IGNORE_FIELDS = "gobblin.converter.ignoreFields"; private Schema schema; + private List<String> ignoreFields; public ToAvroConverterBase<SI, JsonObject> init(WorkUnitState workUnit) { super.init(workUnit); Preconditions.checkArgument(workUnit.contains(AVRO_SCHEMA_KEY)); this.schema = new Schema.Parser().parse(workUnit.getProp(AVRO_SCHEMA_KEY)); + this.ignoreFields = SPLITTER_ON_COMMA.splitToList(workUnit.getProp(IGNORE_FIELDS, "")); return this; } @@ -70,6 +76,11 @@ public class JsonRecordAvroSchemaToAvroConverter<SI> extends ToAvroConverterBase GenericRecord avroRecord = new GenericData.Record(outputSchema); JsonElementConversionWithAvroSchemaFactory.JsonElementConverter converter; for (Schema.Field field : outputSchema.getFields()) { + + if (this.ignoreFields.contains(field.name())) { + continue; + } + if (inputRecord.get(field.name()) == null) { throw new DataConversionException("Field missing from record: " + field.name()); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/396fc40d/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java ---------------------------------------------------------------------- diff --git a/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java b/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java index 7da37b7..9971d83 100644 --- a/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java +++ b/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java @@ -53,12 +53,13 @@ public class JsonRecordAvroSchemaToAvroConverterTest { this.state = new WorkUnitState( source.createWorkUnit(source.createExtract(TableType.SNAPSHOT_ONLY, "test_table", "test_namespace"))); this.state.setProp(JsonRecordAvroSchemaToAvroConverter.AVRO_SCHEMA_KEY, avroSchemaString); + this.state.setProp(JsonRecordAvroSchemaToAvroConverter.IGNORE_FIELDS, "fieldToIgnore"); } @Test public void testConverter() throws Exception { - JsonRecordAvroSchemaToAvroConverter<String> converter = new JsonRecordAvroSchemaToAvroConverter(); + JsonRecordAvroSchemaToAvroConverter<String> converter = new JsonRecordAvroSchemaToAvroConverter<>(); converter.init(this.state); @@ -66,6 +67,7 @@ public class JsonRecordAvroSchemaToAvroConverterTest { GenericRecord record = converter.convertRecord(avroSchema, this.jsonRecord, this.state).iterator().next(); + Assert.assertEquals(record.get("fieldToIgnore"), null); Assert.assertEquals(record.get("nullableField"), null); Assert.assertEquals(record.get("longField"), 1234L); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/396fc40d/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc ---------------------------------------------------------------------- diff --git a/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc b/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc index 22fec73..effc91c 100644 --- a/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc +++ b/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc @@ -4,6 +4,10 @@ "namespace": "org.apache.gobblin.test", "fields": [ { + "name": "fieldToIgnore", + "type": "string" + }, + { "name": "nullableField", "type": ["string", "null"] },
