SQOOP-1561: Sqoop2: Date and DateTime handling in CSV IDF (Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/1e9db011 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/1e9db011 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/1e9db011 Branch: refs/heads/SQOOP-1367 Commit: 1e9db0113fe6b83f843faabc6a58eb07b19b0fee Parents: 8362c73 Author: Jarek Jarcec Cecho <[email protected]> Authored: Sat Oct 4 14:02:41 2014 -0700 Committer: Abraham Elmahrek <[email protected]> Committed: Thu Oct 9 18:22:51 2014 -0700 ---------------------------------------------------------------------- connector/connector-sdk/pom.xml | 4 +++ .../idf/CSVIntermediateDataFormat.java | 6 ++++ .../idf/TestCSVIntermediateDataFormat.java | 27 ++++++++++++++++ pom.xml | 33 ++++++++++++-------- 4 files changed, 57 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e9db011/connector/connector-sdk/pom.xml ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/pom.xml b/connector/connector-sdk/pom.xml index f54837d..6720027 100644 --- a/connector/connector-sdk/pom.xml +++ b/connector/connector-sdk/pom.xml @@ -34,6 +34,10 @@ limitations under the License. <dependencies> <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e9db011/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java index 02d1a51..3c3543c 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java @@ -27,6 +27,8 @@ import org.apache.sqoop.schema.type.Column; import org.apache.sqoop.schema.type.FixedPoint; import org.apache.sqoop.schema.type.FloatingPoint; import org.apache.sqoop.schema.type.Type; +import org.joda.time.LocalDate; +import org.joda.time.LocalDateTime; import java.io.DataInput; import java.io.DataOutput; @@ -215,7 +217,11 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { out[i] = new BigDecimal(fields[i]); break; case DATE: + out[i] = LocalDate.parse(fields[i]); + break; case DATE_TIME: + out[i] = LocalDateTime.parse(fields[i]); + break; case BIT: out[i] = fields[i]; break; http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e9db011/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java index f5fbab7..3159883 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java @@ -28,6 +28,8 @@ import java.util.Arrays; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.schema.Schema; import org.apache.sqoop.schema.type.Binary; +import org.apache.sqoop.schema.type.Date; +import org.apache.sqoop.schema.type.DateTime; import org.apache.sqoop.schema.type.FixedPoint; import org.apache.sqoop.schema.type.Text; import org.junit.Before; @@ -222,6 +224,31 @@ public class TestCSVIntermediateDataFormat { assertTrue(Arrays.deepEquals(inCopy, data.getObjectData())); } + @Test + public void testDate() { + Schema schema = new Schema("test"); + schema.addColumn(new Date("1")); + data.setSchema(schema); + + data.setTextData("2014-10-01"); + assertEquals("2014-10-01", data.getObjectData()[0].toString()); + } + + @Test + public void testDateTime() { + Schema schema = new Schema("test"); + schema.addColumn(new DateTime("1")); + data.setSchema(schema); + + for (String dateTime : new String[]{ + "2014-10-01T12:00:00", + "2014-10-01T12:00:00.000" + }) { + data.setTextData(dateTime); + assertEquals("2014-10-01T12:00:00.000", data.getObjectData()[0].toString()); + } + } + @Test(expected=SqoopException.class) public void testEmptySchema() { String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54}) http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e9db011/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f2cd070..f25a29f 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,8 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ---><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache</groupId> @@ -111,6 +112,7 @@ limitations under the License. <jdbc.sqlserver.version>4.0</jdbc.sqlserver.version> <jdbc.teradata.version>14.00.00.21</jdbc.teradata.version> <jdbc.netezza.version>6.0</jdbc.netezza.version> + <joda.version>2.4</joda.version> </properties> <dependencies> @@ -440,18 +442,23 @@ limitations under the License. <artifactId>nzjdbc3</artifactId> <version>${jdbc.netezza.version}</version> </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <version>${mockito.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.tomcat</groupId> - <artifactId>catalina</artifactId> - <version>${tomcat.version}</version> - <scope>provided</scope> - </dependency> + <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + <version>${joda.version}</version> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>${mockito.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.tomcat</groupId> + <artifactId>catalina</artifactId> + <version>${tomcat.version}</version> + <scope>provided</scope> + </dependency> </dependencies> </dependencyManagement>
