Repository: incubator-gobblin Updated Branches: refs/heads/master 01302a6db -> c47f43dc3
[GOBBLIN-496] Support nullable union when getting field schema Closes #2367 from jack-moseley/nullable-union Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/c47f43dc Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/c47f43dc Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/c47f43dc Branch: refs/heads/master Commit: c47f43dc3c0ce9cc9b1f11c07e0876df27afdd81 Parents: 01302a6 Author: Jack Moseley <[email protected]> Authored: Fri May 18 09:24:12 2018 -0700 Committer: Hung Tran <[email protected]> Committed: Fri May 18 09:24:12 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/gobblin/util/AvroUtils.java | 4 ++++ .../org/apache/gobblin/util/AvroUtilsTest.java | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c47f43dc/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java index 4226497..c1d65d8 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java @@ -57,6 +57,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; import org.codehaus.jackson.JsonNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,6 +127,9 @@ public class AvroUtils { } switch (schema.getType()) { case UNION: + if (AvroSerdeUtils.isNullableType(schema)) { + return AvroUtils.getFieldSchemaHelper(AvroSerdeUtils.getOtherTypeFromNullableType(schema), pathList, field); + } throw new AvroRuntimeException("Union of complex types cannot be handled : " + schema); case MAP: if ((field + 1) == pathList.size()) { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c47f43dc/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroUtilsTest.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroUtilsTest.java b/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroUtilsTest.java index e89b8b9..2188a46 100644 --- a/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroUtilsTest.java +++ b/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroUtilsTest.java @@ -279,4 +279,24 @@ public class AvroUtilsTest { AvroUtils.getFieldSchema(record.getSchema(), TEST_LOCATION); } + + @Test + public void testUnionWithNull() { + Schema nestedRecord = SchemaBuilder.record("nested").fields().requiredDouble("double") + .requiredString("string").endRecord(); + Schema union = SchemaBuilder.unionOf().nullType().and().type(nestedRecord).endUnion(); + Schema schema = SchemaBuilder.record("record").fields().name("union").type(union).noDefault().endRecord(); + + Schema doubleSchema = AvroUtils.getFieldSchema(schema, "union.double").get(); + Assert.assertEquals(doubleSchema.getType(), Schema.Type.DOUBLE); + + GenericRecord nested = new GenericData.Record(nestedRecord); + nested.put("double", 10); + nested.put("string", "testString"); + GenericRecord record = new GenericData.Record(schema); + record.put("union", nested); + + String stringValue = AvroUtils.getFieldValue(record, "union.string").get().toString(); + Assert.assertEquals(stringValue, "testString"); + } }
