Repository: sqoop Updated Branches: refs/heads/SQOOP-1367 f63c080dd -> 3cb981d48
SQOOP-1561: Sqoop2: Date and DateTime handling in CSV IDF (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3cb981d4 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3cb981d4 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3cb981d4 Branch: refs/heads/SQOOP-1367 Commit: 3cb981d482b605fda2d61b45e7a7c7e2d3cb0ee8 Parents: f63c080 Author: Jarek Jarcec Cecho <[email protected]> Authored: Sat Oct 4 14:02:41 2014 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Sat Oct 4 14:02:41 2014 -0700 ---------------------------------------------------------------------- connector/connector-sdk/pom.xml | 4 +++ .../idf/CSVIntermediateDataFormat.java | 6 ++++ .../idf/TestCSVIntermediateDataFormat.java | 27 ++++++++++++++++ pom.xml | 33 ++++++++++++-------- 4 files changed, 57 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/3cb981d4/connector/connector-sdk/pom.xml ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/pom.xml b/connector/connector-sdk/pom.xml index f54837d..6720027 100644 --- a/connector/connector-sdk/pom.xml +++ b/connector/connector-sdk/pom.xml @@ -34,6 +34,10 @@ limitations under the License. <dependencies> <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/sqoop/blob/3cb981d4/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java index 02d1a51..3c3543c 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java @@ -27,6 +27,8 @@ import org.apache.sqoop.schema.type.Column; import org.apache.sqoop.schema.type.FixedPoint; import org.apache.sqoop.schema.type.FloatingPoint; import org.apache.sqoop.schema.type.Type; +import org.joda.time.LocalDate; +import org.joda.time.LocalDateTime; import java.io.DataInput; import java.io.DataOutput; @@ -215,7 +217,11 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { out[i] = new BigDecimal(fields[i]); break; case DATE: + out[i] = LocalDate.parse(fields[i]); + break; case DATE_TIME: + out[i] = LocalDateTime.parse(fields[i]); + break; case BIT: out[i] = fields[i]; break; http://git-wip-us.apache.org/repos/asf/sqoop/blob/3cb981d4/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java index f5fbab7..3159883 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java @@ -28,6 +28,8 @@ import java.util.Arrays; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.schema.Schema; import org.apache.sqoop.schema.type.Binary; +import org.apache.sqoop.schema.type.Date; +import org.apache.sqoop.schema.type.DateTime; import org.apache.sqoop.schema.type.FixedPoint; import org.apache.sqoop.schema.type.Text; import org.junit.Before; @@ -222,6 +224,31 @@ public class TestCSVIntermediateDataFormat { assertTrue(Arrays.deepEquals(inCopy, data.getObjectData())); } + @Test + public void testDate() { + Schema schema = new Schema("test"); + schema.addColumn(new Date("1")); + data.setSchema(schema); + + data.setTextData("2014-10-01"); + assertEquals("2014-10-01", data.getObjectData()[0].toString()); + } + + @Test + public void testDateTime() { + Schema schema = new Schema("test"); + schema.addColumn(new DateTime("1")); + data.setSchema(schema); + + for (String dateTime : new String[]{ + "2014-10-01T12:00:00", + "2014-10-01T12:00:00.000" + }) { + data.setTextData(dateTime); + assertEquals("2014-10-01T12:00:00.000", data.getObjectData()[0].toString()); + } + } + @Test(expected=SqoopException.class) public void testEmptySchema() { String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54}) http://git-wip-us.apache.org/repos/asf/sqoop/blob/3cb981d4/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9d9b508..9c5800f 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,8 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ---><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache</groupId> @@ -110,6 +111,7 @@ limitations under the License. <jdbc.sqlserver.version>4.0</jdbc.sqlserver.version> <jdbc.teradata.version>14.00.00.21</jdbc.teradata.version> <jdbc.netezza.version>6.0</jdbc.netezza.version> + <joda.version>2.4</joda.version> </properties> <dependencies> @@ -429,18 +431,23 @@ limitations under the License. <artifactId>nzjdbc3</artifactId> <version>${jdbc.netezza.version}</version> </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <version>${mockito.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.tomcat</groupId> - <artifactId>catalina</artifactId> - <version>${tomcat.version}</version> - <scope>provided</scope> - </dependency> + <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + <version>${joda.version}</version> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>${mockito.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.tomcat</groupId> + <artifactId>catalina</artifactId> + <version>${tomcat.version}</version> + <scope>provided</scope> + </dependency> </dependencies> </dependencyManagement>
