Repository: incubator-gobblin Updated Branches: refs/heads/master b9a774a82 -> 28e3aece7
[GOBBLIN-556] Gobblin AvroUtils reads and writes UTF rather than chars[] Closes #2417 from erwa/gobblin-556-read-write-utf Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/28e3aece Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/28e3aece Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/28e3aece Branch: refs/heads/master Commit: 28e3aece7bda77d1e16906894e3974fa6908df9e Parents: b9a774a Author: Anthony Hsu <[email protected]> Authored: Fri Aug 3 12:09:24 2018 -0700 Committer: Hung Tran <[email protected]> Committed: Fri Aug 3 12:09:24 2018 -0700 ---------------------------------------------------------------------- .../hive/avro/AvroSchemaManagerTest.java | 17 ----------------- .../java/org/apache/gobblin/util/AvroUtils.java | 4 ++-- 2 files changed, 2 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/28e3aece/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/avro/AvroSchemaManagerTest.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/avro/AvroSchemaManagerTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/avro/AvroSchemaManagerTest.java index eeba93e..fa2a296 100644 --- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/avro/AvroSchemaManagerTest.java +++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/avro/AvroSchemaManagerTest.java @@ -61,23 +61,6 @@ public class AvroSchemaManagerTest { Assert.assertEquals(actualSchema.toString(), expectedSchema); } - @Test(expectedExceptions = SchemaParseException.class) - public void testExceptionWhenReadingSchemaUsingParser() - throws IOException, HiveException { - FileSystem fs = FileSystem.getLocal(new Configuration()); - - String jobId = "123"; - State state = new State(); - state.setProp(ConfigurationKeys.JOB_ID_KEY, jobId); - - AvroSchemaManager asm = new AvroSchemaManager(fs, state); - Partition partition = getTestPartition(new Table("testDb", "testTable")); - Path schemaPath = asm.getSchemaUrl(partition); - // parse operation tries to read using UTF-8 encoding and fails - // because schema is written using modified UTF-8 encoding - new Schema.Parser().parse(fs.open(schemaPath)); - } - private Partition getTestPartition(Table table) throws HiveException { Partition partition = new Partition(table, ImmutableMap.of("partition_key", "1"), null); StorageDescriptor sd = new StorageDescriptor(); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/28e3aece/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java index 7d7e497..52de135 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroUtils.java @@ -381,7 +381,7 @@ public class AvroUtils { Preconditions.checkArgument(fs.exists(filePath), filePath + " does not exist"); try (FSDataInputStream in = fs.open(filePath)) { - return new Schema.Parser().parse(in.readUTF()); + return new Schema.Parser().parse(in); } } @@ -399,7 +399,7 @@ public class AvroUtils { } try (DataOutputStream dos = fs.create(filePath)) { - dos.writeUTF(schema.toString()); + dos.writeChars(schema.toString()); } fs.setPermission(filePath, perm); }
