Repository: sqoop Updated Branches: refs/heads/trunk 7e14e380f -> 820a10fb5
SQOOP-1539: Empty partition keys and values in multiple partition key hcatalog usage should be validated (Venkat Ranganathan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/820a10fb Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/820a10fb Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/820a10fb Branch: refs/heads/trunk Commit: 820a10fb576513867651124985891d8672b25a0d Parents: 7e14e38 Author: Jarek Jarcec Cecho <[email protected]> Authored: Wed Sep 24 16:44:27 2014 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Wed Sep 24 16:44:27 2014 -0700 ---------------------------------------------------------------------- .../mapreduce/hcat/SqoopHCatUtilities.java | 25 ++++++++++++++++---- .../org/apache/sqoop/tool/BaseSqoopTool.java | 15 ++++++++++++ .../apache/sqoop/hcat/TestHCatalogBasic.java | 22 ++++++++++++++++- 3 files changed, 56 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/820a10fb/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java index b69b83e..599548d 100644 --- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java +++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java @@ -288,11 +288,18 @@ public final class SqoopHCatUtilities { hCatStaticPartitionValues = new ArrayList<String>(); String partKeysString = options.getHCatalogPartitionKeys(); String partKeysVals = options.getHCatalogPartitionValues(); + // Already validated if (partKeysString != null) { String[] keys = partKeysString.split(","); - hCatStaticPartitionKeys.addAll(Arrays.asList(keys)); + for (int i = 0; i < keys.length; ++i) { + String k = keys[i].trim(); + hCatStaticPartitionKeys.add(k); + } String[] vals = partKeysVals.split(","); - hCatStaticPartitionValues.addAll(Arrays.asList(vals)); + for (int i = 0; i < vals.length; ++i) { + String v = vals[i].trim(); + hCatStaticPartitionValues.add(v); + } } else { partKeysString = options.getHivePartitionKey(); if (partKeysString != null) { @@ -370,8 +377,16 @@ public final class SqoopHCatUtilities { throw new IOException("Database column " + col + " not found in " + " hcatalog table."); } - if (hCatStaticPartitionKeys != null - && hCatStaticPartitionKeys.equals(col)) { + boolean skip=false; + if (hCatStaticPartitionKeys != null) { + for (String key : hCatStaticPartitionKeys) { + if (col.equals(key)) { + skip=true; + break; + } + } + } + if (skip) { continue; } outputFieldList.add(hCatFullTableSchema.get(col)); @@ -670,7 +685,7 @@ public final class SqoopHCatUtilities { } IntWritable[] positions = new IntWritable[hCatFieldPositions.length]; - for (int i : hCatFieldPositions) { + for (int i = 0; i < hCatFieldPositions.length; ++i) { positions[i] = new IntWritable(hCatFieldPositions[i]); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/820a10fb/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 3104454..498ad79 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -1554,6 +1554,21 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { throw new InvalidOptionsException("Number of static partition keys " + "provided dpes match the number of partition values"); } + + for (int i = 0; i < keys.length; ++i) { + String k = keys[i].trim(); + if (k.isEmpty()) { + throw new InvalidOptionsException( + "Invalid HCatalog static partition key at position " + i); + } + } + for (int i = 0; i < vals.length; ++i) { + String v = vals[i].trim(); + if (v.isEmpty()) { + throw new InvalidOptionsException( + "Invalid HCatalog static partition key at position " + v); + } + } } else { if (options.getHivePartitionKey() != null && options.getHivePartitionValue() == null) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/820a10fb/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java index 388cce5..ec60c9f 100644 --- a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java +++ b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java @@ -284,7 +284,27 @@ public class TestHCatalogBasic extends TestCase { // expected. } } - + public void testHCatImportWithEmptyKeysAndVals() throws Exception { + String[] args = { + "--connect", + "jdbc:db:url", + "--table", + "dbtable", + "--hcatalog-table", + "table", + "--hcatalog-partition-keys", + "k1,", + "--hcatalog-partition-values", + ",v1", + }; + try { + SqoopOptions opts = parseImportArgs(args); + importTool.validateOptions(opts); + fail("Expected InvalidOptionsException"); + } catch (SqoopOptions.InvalidOptionsException ioe) { + // expected. + } + } public void testHCatImportWithBothHCatAndHivePartOptions() throws Exception { String[] args = { "--connect",
