Repository: sqoop Updated Branches: refs/heads/trunk 6d8a670d3 -> 803edc279
SQOOP-3152: Fix SqoopOptions#parseColumnMapping, thus Sqoop will support mappings with precisison (e.g. DECIMAL(10,5)) in the future properly (Eric Lin via Attila Szabo) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/803edc27 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/803edc27 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/803edc27 Branch: refs/heads/trunk Commit: 803edc27925ec6ee950b2e7fa2ffea5727248fa6 Parents: 6d8a670 Author: Attila Szabo <[email protected]> Authored: Mon Mar 20 16:37:49 2017 +0100 Committer: Attila Szabo <[email protected]> Committed: Mon Mar 20 16:37:49 2017 +0100 ---------------------------------------------------------------------- src/docs/web/images/Thumbs.db | Bin 0 -> 4608 bytes src/java/org/apache/sqoop/SqoopOptions.java | 9 +++-- .../com/cloudera/sqoop/hive/TestHiveImport.java | 17 ++++++++ .../cloudera/sqoop/hive/TestTableDefWriter.java | 39 ++++++++++++++++++- testdata/hive/scripts/decimalMapImport.q | 17 ++++++++ 5 files changed, 78 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/docs/web/images/Thumbs.db ---------------------------------------------------------------------- diff --git a/src/docs/web/images/Thumbs.db b/src/docs/web/images/Thumbs.db new file mode 100644 index 0000000..bc9428c Binary files /dev/null and b/src/docs/web/images/Thumbs.db differ http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/java/org/apache/sqoop/SqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index b33b54b..caf95f6 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -1322,11 +1322,14 @@ public class SqoopOptions implements Cloneable { public void setPasswordAlias(String alias) { this.passwordAlias = alias; } + protected void parseColumnMapping(String mapping, Properties output) { output.clear(); - String[] maps = mapping.split(","); + // replace (xx,xx) with (xx#xx), so that we can just split by "," afterwards + String[] maps = mapping.replaceAll("\\(([0-9]+),([0-9]+)\\)", "($1#$2)").split(","); + for(String map : maps) { String[] details = map.split("="); if (details.length != 2) { @@ -1336,8 +1339,8 @@ public class SqoopOptions implements Cloneable { try { output.put( - URLDecoder.decode(details[0], "UTF-8"), - URLDecoder.decode(details[1], "UTF-8")); + URLDecoder.decode(details[0].replace("#", ","), "UTF-8"), + URLDecoder.decode(details[1].replace("#", ","), "UTF-8")); } catch (UnsupportedEncodingException e) { throw new IllegalArgumentException("Encoding not supported. " + "Column mapping should be UTF-8 encoding."); http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/test/com/cloudera/sqoop/hive/TestHiveImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index 6f13fe2..33e0cc4 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -499,6 +499,23 @@ public class TestHiveImport extends ImportJobTestCase { getArgv(false, null), new ImportTool()); } + /** Test that DECIMALS using --map-column-hive option maps can run without issues. */ + @Test + public void testDecimalMapColumnHive() throws IOException { + final String TABLE_NAME = "DECIMAL_MAP_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(2); + String [] types = { "NUMERIC", "CHAR(64)" }; + String [] vals = { "12343.14159", "'foo'" }; + + ArrayList<String> args = new ArrayList<String>(); + args.add("--map-column-hive"); + args.add(BASE_COL_NAME + "0=DECIMAL(10,10)"); + + runImportTest(TABLE_NAME, types, vals, "decimalMapImport.q", + getArgv(false, args.toArray(new String[args.size()])), new ImportTool()); + } + /** If bin/hive returns an error exit status, we should get an IOException. */ @Test public void testHiveExitFails() throws IOException { http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java index 6af12da..dbf0dde 100644 --- a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java +++ b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java @@ -180,7 +180,7 @@ public class TestTableDefWriter { } @Test - public void testUserMapping() throws Exception { + public void testUserMappingNoDecimal() throws Exception { String[] args = { "--map-column-hive", "id=STRING,value=INTEGER", }; @@ -207,6 +207,43 @@ public class TestTableDefWriter { } @Test + public void testUserMappingWithDecimal() throws Exception { + String[] args = { + "--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," + + "value3=DECIMAL(4,5),value4=VARCHAR(255)", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + colTypes.put("id", Types.INTEGER); + colTypes.put("value1", Types.VARCHAR); + colTypes.put("value2", Types.DOUBLE); + colTypes.put("value3", Types.FLOAT); + colTypes.put("value4", Types.CHAR); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + + assertNotNull(createTable); + + assertTrue(createTable.contains("`id` STRING")); + assertTrue(createTable.contains("`value1` INTEGER")); + assertTrue(createTable.contains("`value2` DECIMAL(13,5)")); + assertTrue(createTable.contains("`value3` DECIMAL(4,5)")); + assertTrue(createTable.contains("`value4` VARCHAR(255)")); + + assertFalse(createTable.contains("`id` INTEGER")); + assertFalse(createTable.contains("`value1` STRING")); + assertFalse(createTable.contains("`value2` DOUBLE")); + assertFalse(createTable.contains("`value3` FLOAT")); + assertFalse(createTable.contains("`value4` CHAR")); + } + + @Test public void testUserMappingFailWhenCantBeApplied() throws Exception { String[] args = { "--map-column-hive", "id=STRING,value=INTEGER", http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/testdata/hive/scripts/decimalMapImport.q ---------------------------------------------------------------------- diff --git a/testdata/hive/scripts/decimalMapImport.q b/testdata/hive/scripts/decimalMapImport.q new file mode 100644 index 0000000..8f05d58 --- /dev/null +++ b/testdata/hive/scripts/decimalMapImport.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `DECIMAL_MAP_HIVE_IMPORT` ( `DATA_COL0` DECIMAL(10, 10), `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DECIMAL_MAP_HIVE_IMPORT' INTO TABLE `DECIMAL_MAP_HIVE_IMPORT`;
