Repository: incubator-gobblin Updated Branches: refs/heads/master 0b1c52cd1 -> dc2d9e891
[GOBBLIN-660] Fix OracleExtractor datatype mapping Closes #2531 from FrancisLfg/master Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/dc2d9e89 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/dc2d9e89 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/dc2d9e89 Branch: refs/heads/master Commit: dc2d9e891b095e3b18ebf7efc1cd57d65d06013e Parents: 0b1c52c Author: Francis Laforge <[email protected]> Authored: Wed Jan 16 09:31:56 2019 -0800 Committer: Hung Tran <[email protected]> Committed: Wed Jan 16 09:31:56 2019 -0800 ---------------------------------------------------------------------- .../parquet/JsonElementConversionFactory.java | 4 ++++ .../gobblin/converter/parquet/JsonSchema.java | 2 +- .../JsonIntermediateToParquetConverter.json | 20 +++++++++++++++++--- .../gobblin/source/jdbc/OracleExtractor.java | 2 +- .../source/jdbc/OracleExtractorTest.java | 9 ++++----- 5 files changed, 27 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java b/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java index 1d3636a..46b264a 100644 --- a/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java +++ b/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java @@ -105,6 +105,10 @@ public class JsonElementConversionFactory { case MAP: return new MapConverter(schema); + case DATE: + case TIMESTAMP: + return new StringConverter(schema, repeated); + default: throw new UnsupportedOperationException(fieldType + " is unsupported"); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java b/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java index b7e001b..dc650a7 100644 --- a/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java +++ b/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java @@ -55,7 +55,7 @@ public class JsonSchema extends Schema { private final InputType type; public enum InputType { - STRING, INT, LONG, FLOAT, DOUBLE, BOOLEAN, ARRAY, ENUM, RECORD, MAP + STRING, INT, LONG, FLOAT, DOUBLE, BOOLEAN, ARRAY, ENUM, RECORD, MAP, DATE, TIMESTAMP } public JsonSchema(JsonArray jsonArray) { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json b/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json index e12325d..a588ea9 100644 --- a/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json +++ b/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json @@ -5,7 +5,9 @@ "b": 5.0, "c": 8.0, "d": true, - "e": "somestring" + "e": "somestring", + "f": "2018-01-01", + "g": 1545083047 }, "schema": [ { @@ -37,10 +39,22 @@ "dataType": { "type": "string" } + }, + { + "columnName": "f", + "dataType": { + "type": "date" + } + }, + { + "columnName": "g", + "dataType": { + "type": "timestamp" + } } ], - "expectedRecord": "a: 5 ; b: 5.0 ; c: 8.0 ; d: true ; e: somestring ; ", - "expectedSchema": "message test_table{ ; required int32 a ; ; required float b ; ; required double c ; ; required boolean d ; ; required binary e (UTF8) ; ; } ; " + "expectedRecord": "a: 5 ; b: 5.0 ; c: 8.0 ; d: true ; e: somestring ; f: 2018-01-01 ; g: 1545083047 ;", + "expectedSchema": "message test_table{ ; required int32 a ; ; required float b ; ; required double c ; ; required boolean d ; ; required binary e (UTF8) ; ; required binary f (UTF8) ; ; required binary g (UTF8) ; ; } ; " }, "array": { "record": { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java index e893ef4..01001eb 100644 --- a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java +++ b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java @@ -67,7 +67,7 @@ public class OracleExtractor extends JdbcExtractor { private static final String METADATA_SCHEMA_PSTMT_FORMAT = "SELECT " + "column_name, " + - "UPPER(data_type), " + + "LOWER(data_type), " + "NVL(data_length, 0) as length, " + "NVL(data_precision, 0) as precesion, " + "NVL(data_scale, 0) as scale, " + http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java b/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java index ab82a07..6aa914a 100644 --- a/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java +++ b/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java @@ -69,18 +69,17 @@ public class OracleExtractorTest { } @Test - public void testConstructSampleClause() throws Exception { + public void testConstructSampleClause() { String sClause = oracleExtractor.constructSampleClause(); assertEquals(sClause.trim(), (" rownum <= " + oracleExtractor.getSampleRecordCount()).trim()); } @Test - public void testRemoveSampleClauseFromQuery() throws Exception { + public void testRemoveSampleClauseFromQuery() { String q1Expected = "SELECT * FROM x WHERE 1=1"; String q2Expected = "SELECT * FROM x WHERE 1=1 AND x.a < 10"; String q3Expected = "SELECT * FROM x WHERE x.a < 10 AND 1=1"; String q4Expected = "SELECT * FROM x WHERE x.a < 10 AND 1=1 AND x.b = 20"; - String qEmptyClean = ""; String q1Parsed = oracleExtractor.removeSampleClauseFromQuery(QUERY_1); String q2Parsed = oracleExtractor.removeSampleClauseFromQuery(QUERY_2); @@ -94,7 +93,7 @@ public class OracleExtractorTest { } @Test - public void testExractSampleRecordCountFromQuery() throws Exception { + public void testExractSampleRecordCountFromQuery() { long res1 = oracleExtractor.exractSampleRecordCountFromQuery(QUERY_1); long res2 = oracleExtractor.exractSampleRecordCountFromQuery(QUERY_2); long res3 = oracleExtractor.exractSampleRecordCountFromQuery(QUERY_3); @@ -113,7 +112,7 @@ public class OracleExtractorTest { /** * Build a mock implementation of Result using Mockito */ - private ResultSet buildMockResultSet() throws Exception { + private ResultSet buildMockResultSet() { MockResultSet mrs = new MockResultSet(StringUtils.EMPTY); for (MockJdbcColumn column : COLUMNS) {
