Updated Branches: refs/heads/sqoop2 66a296ead -> 05a73ef16
SQOOP-1080: Sqoop2: Investigate Jenkins test failures (Venkat Ranganathan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/05a73ef1 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/05a73ef1 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/05a73ef1 Branch: refs/heads/sqoop2 Commit: 05a73ef168f65fea155befb9dbbac01b61febed5 Parents: 66a296e Author: Jarek Jarcec Cecho <[email protected]> Authored: Sun Jun 16 06:40:13 2013 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Sun Jun 16 06:40:35 2013 -0700 ---------------------------------------------------------------------- .../jdbc/GenericJdbcImportPartitioner.java | 31 ++++++++++++++++---- .../connector/jdbc/TestImportPartitioner.java | 6 ++-- 2 files changed, 28 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/05a73ef1/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java index eef18f2..0bb979e 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java @@ -22,8 +22,11 @@ import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.LinkedList; import java.util.List; +import java.util.TimeZone; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration; @@ -106,27 +109,39 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur long minDateValue = 0; long maxDateValue = 0; - + SimpleDateFormat sdf = null; switch(partitionColumnType) { case Types.DATE: + sdf = new SimpleDateFormat("yyyy-MM-dd"); minDateValue = Date.valueOf(partitionMinValue).getTime(); maxDateValue = Date.valueOf(partitionMaxValue).getTime(); break; case Types.TIME: + sdf = new SimpleDateFormat("HH:mm:ss"); minDateValue = Time.valueOf(partitionMinValue).getTime(); maxDateValue = Time.valueOf(partitionMaxValue).getTime(); break; case Types.TIMESTAMP: + sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); minDateValue = Timestamp.valueOf(partitionMinValue).getTime(); maxDateValue = Timestamp.valueOf(partitionMaxValue).getTime(); break; } + + long tzOffset = TimeZone.getDefault().getRawOffset(); + + minDateValue += tzOffset; + maxDateValue += tzOffset; + + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + long interval = (maxDateValue - minDateValue) / numberPartitions; long remainder = (maxDateValue - minDateValue) % numberPartitions; if (interval == 0) { numberPartitions = (int)remainder; } + long lowerBound; long upperBound = minDateValue; @@ -146,6 +161,7 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur case Types.TIME: objLB = new Time(lowerBound); objUB = new Time(upperBound); + break; case Types.TIMESTAMP: objLB = new Timestamp(lowerBound); @@ -155,9 +171,10 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur GenericJdbcImportPartition partition = new GenericJdbcImportPartition(); partition.setConditions( - constructDateConditions(objLB, objUB, false)); + constructDateConditions(sdf, objLB, objUB, false)); partitions.add(partition); } + switch(partitionColumnType) { case Types.DATE: objLB = new Date(upperBound); @@ -172,9 +189,11 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur objUB = new Timestamp(maxDateValue); break; } + + GenericJdbcImportPartition partition = new GenericJdbcImportPartition(); partition.setConditions( - constructDateConditions(objLB, objUB, true)); + constructDateConditions(sdf, objLB, objUB, true)); partitions.add(partition); return partitions; } @@ -461,16 +480,16 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur ; } - protected String constructDateConditions( + protected String constructDateConditions(SimpleDateFormat sdf, Object lowerBound, Object upperBound, boolean lastOne) { StringBuilder conditions = new StringBuilder(); - conditions.append('\'').append(lowerBound.toString()).append('\''); + conditions.append('\'').append(sdf.format((java.util.Date)lowerBound)).append('\''); conditions.append(" <= "); conditions.append(partitionColumnName); conditions.append(" AND "); conditions.append(partitionColumnName); conditions.append(lastOne ? " <= " : " < "); - conditions.append('\'').append(upperBound.toString()).append('\''); + conditions.append('\'').append(sdf.format((java.util.Date)upperBound)).append('\''); return conditions.toString(); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/05a73ef1/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java index 522a515..1a3601a 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java @@ -333,9 +333,9 @@ public class TestImportPartitioner extends TestCase { 3); List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf); verifyResult(partitions, new String[]{ - "'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 13:14:17.634'", - "'2013-05-02 13:14:17.634' <= TSCOL AND TSCOL < '2013-09-01 00:27:34.144'", - "'2013-09-01 00:27:34.144' <= TSCOL AND TSCOL <= '2013-12-31 10:40:50.654'", + "'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 12:14:17.634'", + "'2013-05-02 12:14:17.634' <= TSCOL AND TSCOL < '2013-08-31 23:27:34.144'", + "'2013-08-31 23:27:34.144' <= TSCOL AND TSCOL <= '2013-12-31 10:40:50.654'", }); }
