Repository: sqoop Updated Branches: refs/heads/sqoop2 9febdf3ae -> c19f9c946
SQOOP-1765: Sqoop2: Time/Timestamp format support for CSV IDF (Veena Basavaraj via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c19f9c94 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c19f9c94 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c19f9c94 Branch: refs/heads/sqoop2 Commit: c19f9c9460b4f10466b15f20b0c2493e9ef09387 Parents: 9febdf3 Author: Abraham Elmahrek <[email protected]> Authored: Thu Dec 4 13:21:10 2014 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Thu Dec 4 13:21:50 2014 -0800 ---------------------------------------------------------------------- .../idf/CSVIntermediateDataFormat.java | 21 ++++-- .../idf/TestCSVIntermediateDataFormat.java | 72 +++++++++++++++++--- 2 files changed, 78 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/c19f9c94/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java index a075d3f..d481cce 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java @@ -29,6 +29,7 @@ import org.apache.sqoop.schema.type.FixedPoint; import org.apache.sqoop.schema.type.FloatingPoint; import org.joda.time.DateTime; import org.joda.time.LocalDate; +import org.joda.time.LocalTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.json.simple.JSONArray; @@ -83,12 +84,12 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { new String(new char[] { ESCAPE_CHARACTER, '\''}) }; - // ISO-8859-1 is an 8-bit codec that is supported in every java - // implementation. + // ISO-8859-1 is an 8-bit codec that is supported in every java implementation. static final String BYTE_FIELD_CHARSET = "ISO-8859-1"; - //http://www.joda.org/joda-time/key_format.html provides details on the formatter token - static final DateTimeFormatter dtf = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS'Z'"); + // http://www.joda.org/joda-time/key_format.html provides details on the formatter token + static final DateTimeFormatter dtf = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSSSSZ"); static final DateTimeFormatter df = DateTimeFormat.forPattern("yyyy-MM-dd"); + static final DateTimeFormatter tf = DateTimeFormat.forPattern("HH:mm:ss.SSSSSSZ"); private final List<Integer> stringTypeColumnIndices = new ArrayList<Integer>(); private final List<Integer> byteTypeColumnIndices = new ArrayList<Integer>(); @@ -96,6 +97,7 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { private final List<Integer> mapTypeColumnIndices = new ArrayList<Integer>(); private final List<Integer> dateTimeTypeColumnIndices = new ArrayList<Integer>(); private final List<Integer> dateTypeColumnIndices = new ArrayList<Integer>(); + private final List<Integer> timeColumnIndices = new ArrayList<Integer>(); private Schema schema; @@ -138,6 +140,8 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { stringTypeColumnIndices.add(i); } else if (col.getType() == ColumnType.DATE) { dateTypeColumnIndices.add(i); + } else if (col.getType() == ColumnType.TIME) { + timeColumnIndices.add(i); } else if (col.getType() == ColumnType.DATE_TIME) { dateTimeTypeColumnIndices.add(i); } else if (col.getType() == ColumnType.BINARY) { @@ -273,6 +277,9 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { case DATE: returnValue = LocalDate.parse(removeQuotes(fieldString)); break; + case TIME: + returnValue = LocalTime.parse(removeQuotes(fieldString)); + break; case DATE_TIME: // A datetime string with a space as date-time separator will not be // parsed expectedly. The expected separator is "T". See also: @@ -436,6 +443,10 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { org.joda.time.LocalDate date = (org.joda.time.LocalDate) stringArray[i]; stringArray[i] = encloseWithQuote(df.print(date)); } + for (int i : timeColumnIndices) { + org.joda.time.LocalTime date = (org.joda.time.LocalTime) stringArray[i]; + stringArray[i] = encloseWithQuote(tf.print(date)); + } for (int i : byteTypeColumnIndices) { stringArray[i] = escapeByteArrays((byte[]) stringArray[i]); } @@ -560,4 +571,4 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { public String toString() { return data; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/c19f9c94/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java index bf15c69..b348ed8 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java @@ -41,6 +41,7 @@ import org.apache.sqoop.schema.type.Date; import org.apache.sqoop.schema.type.DateTime; import org.apache.sqoop.schema.type.FixedPoint; import org.apache.sqoop.schema.type.Text; +import org.apache.sqoop.schema.type.Time; import org.junit.Before; import org.junit.Test; @@ -272,6 +273,49 @@ public class TestCSVIntermediateDataFormat { assertTrue(Arrays.deepEquals(inCopy, dataFormat.getObjectData())); } + // **************test cases for time******************* + + @Test + public void testTimeWithCSVTextInCSVTextOut() { + Schema schema = new Schema("test"); + schema.addColumn(new Time("1")); + dataFormat.setSchema(schema); + dataFormat.setTextData("'12:00:00'"); + assertEquals("'12:00:00'", dataFormat.getTextData()); + } + + @Test + public void testTimeWithCSVTextInObjectArrayOut() { + Schema schema = new Schema("test"); + schema.addColumn(new Time("1")); + dataFormat.setSchema(schema); + dataFormat.setTextData("'12:59:59'"); + org.joda.time.LocalTime time = new org.joda.time.LocalTime(12, 59, 59); + assertEquals(time.toString(), dataFormat.getObjectData()[0].toString()); + } + + @Test + public void testTimeWithObjectArrayInCSVTextOut() { + Schema schema = new Schema("test"); + schema.addColumn(new Time("1")).addColumn(new Text("2")); + dataFormat.setSchema(schema); + org.joda.time.LocalTime time = new org.joda.time.LocalTime(15, 0, 0); + Object[] in = { time, "test" }; + dataFormat.setObjectData(in); + assertEquals("'15:00:00.000000','test'", dataFormat.getTextData()); + } + + @Test + public void testTimeWithObjectArrayInObjectArrayOut() { + Schema schema = new Schema("test"); + schema.addColumn(new Time("1")); + dataFormat.setSchema(schema); + org.joda.time.LocalTime time = new org.joda.time.LocalTime(2, 23, 33); + Object[] in = { time }; + dataFormat.setObjectData(in); + assertEquals(time.toString(), dataFormat.getObjectData()[0].toString()); + } + // **************test cases for date******************* @Test @@ -328,6 +372,16 @@ public class TestCSVIntermediateDataFormat { } @Test + public void testDateTimeWithMilliSecsWithCSVTextInCSVTextOut() { + Schema schema = new Schema("test"); + schema.addColumn(new DateTime("1")); + dataFormat.setSchema(schema); + + dataFormat.setTextData("'2014-10-01 12:00:00.000'"); + assertEquals("'2014-10-01 12:00:00.000'", dataFormat.getTextData()); + } + + @Test public void testDateTimeWithCSVTextInObjectArrayOut() { Schema schema = new Schema("test"); schema.addColumn(new DateTime("1")); @@ -342,10 +396,11 @@ public class TestCSVIntermediateDataFormat { Schema schema = new Schema("test"); schema.addColumn(new DateTime("1")); dataFormat.setSchema(schema); - org.joda.time.DateTime dateTime = new org.joda.time.DateTime(2014, 10, 01, 12, 0, 0, 0); + org.joda.time.DateTime dateTime = new org.joda.time.DateTime(2014, 10, 01, 12, 0, 0, 1); Object[] in = { dateTime }; dataFormat.setObjectData(in); - assertEquals("'2014-10-01 12:00:00.000Z'", dataFormat.getTextData()); + // Note: DateTime has the timezone info + assertEquals("'2014-10-01 12:00:00.001000-0700'", dataFormat.getTextData()); } @Test @@ -353,11 +408,11 @@ public class TestCSVIntermediateDataFormat { Schema schema = new Schema("test"); schema.addColumn(new DateTime("1")); dataFormat.setSchema(schema); - org.joda.time.LocalDateTime dateTime = new org.joda.time.LocalDateTime(2014, 10, 01, 12, 0, 0, - 0); + org.joda.time.LocalDateTime dateTime = new org.joda.time.LocalDateTime(2014, 10, 01, 12, 0, 0, 2); Object[] in = { dateTime }; dataFormat.setObjectData(in); - assertEquals("'2014-10-01 12:00:00.000Z'", dataFormat.getTextData()); + // Note: LocalDateTime is missing the timezone info + assertEquals("'2014-10-01 12:00:00.002000'", dataFormat.getTextData()); } @Test @@ -365,11 +420,8 @@ public class TestCSVIntermediateDataFormat { Schema schema = new Schema("test"); schema.addColumn(new DateTime("1")); dataFormat.setSchema(schema); - - for (String dateTime : new String[] { "'2014-10-01 12:00:00.000'" }) { - dataFormat.setTextData(dateTime); - assertEquals("2014-10-01T12:00:00.000-07:00", dataFormat.getObjectData()[0].toString()); - } + dataFormat.setTextData("'2014-10-01 12:00:00.000'"); + assertEquals("2014-10-01T12:00:00.000-07:00", dataFormat.getObjectData()[0].toString()); } /**
