Updated Branches: refs/heads/sqoop2 81e71dc45 -> 9c7adb4e1
SQOOP-1154: Sqoop2: Text partitioner might miss or include edge values (Jarek Jarcec Cecho via Venkat Ranganathan) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9c7adb4e Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9c7adb4e Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9c7adb4e Branch: refs/heads/sqoop2 Commit: 9c7adb4e112e57e5d2c4720be71b8a5278ef7638 Parents: 81e71dc Author: Venkat Ranganathan <[email protected]> Authored: Tue Jul 30 22:47:44 2013 -0700 Committer: Venkat Ranganathan <[email protected]> Committed: Tue Jul 30 22:47:44 2013 -0700 ---------------------------------------------------------------------- .../jdbc/GenericJdbcImportPartitioner.java | 25 +++++++------------- .../connector/jdbc/TestImportPartitioner.java | 5 ++++ 2 files changed, 13 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/9c7adb4e/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java index 8d0c4ab..d103223 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java @@ -250,7 +250,8 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur // Having one single value means that we can create only one single split if(minStringBD.equals(maxStringBD)) { GenericJdbcImportPartition partition = new GenericJdbcImportPartition(); - partition.setConditions(constructTextConditions(prefix, maxStringBD)); + partition.setConditions(constructTextConditions(prefix, 0, 0, + partitionMinValue, partitionMaxValue, true, true)); partitions.add(partition); return partitions; } @@ -294,8 +295,8 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur BigDecimal end = splitPoints.get(i); GenericJdbcImportPartition partition = new GenericJdbcImportPartition(); - partition.setConditions(constructTextConditions(prefix, start, - end, i == splitPoints.size() - 1)); + partition.setConditions(constructTextConditions(prefix, start, end, + partitionMinValue, partitionMaxValue, i == 1, i == splitPoints.size() - 1)); partitions.add(partition); start = end; @@ -521,31 +522,21 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur return conditions.toString(); } - protected String constructTextConditions(String prefix, - Object lowerBound, Object upperBound, boolean lastOne) { + protected String constructTextConditions(String prefix, Object lowerBound, Object upperBound, + String lowerStringBound, String upperStringBound, boolean firstOne, boolean lastOne) { StringBuilder conditions = new StringBuilder(); String lbString = prefix + bigDecimalToText((BigDecimal)lowerBound); String ubString = prefix + bigDecimalToText((BigDecimal)upperBound); - conditions.append('\'').append(lbString).append('\''); + conditions.append('\'').append(firstOne ? lowerStringBound : lbString).append('\''); conditions.append(" <= "); conditions.append(partitionColumnName); conditions.append(" AND "); conditions.append(partitionColumnName); conditions.append(lastOne ? " <= " : " < "); - conditions.append('\'').append(ubString).append('\''); + conditions.append('\'').append(lastOne ? upperStringBound : ubString).append('\''); return conditions.toString(); } - protected String constructTextConditions(String prefix, Object value) { - return new StringBuilder() - .append(partitionColumnName) - .append(" = ").append('\'') - .append(prefix + bigDecimalToText((BigDecimal)value)) - .append('\'').toString() - ; - } - - /** * Converts a string to a BigDecimal representation in Base 2^21 format. * The maximum Unicode code point value defined is 10FFFF. Although http://git-wip-us.apache.org/repos/asf/sqoop/blob/9c7adb4e/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java index a33dd6c..679accf 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java @@ -409,6 +409,7 @@ public class TestImportPartitioner extends TestCase { "'Y' <= VCCOL AND VCCOL <= 'Z'", }); } + public void testVarcharPartition2() throws Exception { MutableContext context = new MutableMapContext(); context.setString(GenericJdbcConnectorConstants @@ -426,6 +427,10 @@ public class TestImportPartitioner extends TestCase { PartitionerContext partitionerContext = new PartitionerContext(context, 5, null); List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf); assertEquals(partitions.size(), 5); + // First partition needs to contain entire upper bound + assertTrue(partitions.get(0).toString().contains("Breezy Badger")); + // Last partition needs to contain entire lower bound + assertTrue(partitions.get(4).toString().contains("Warty Warthog")); } public void testVarcharPartitionWithCommonPrefix() throws Exception {
