Repository: sqoop Updated Branches: refs/heads/trunk a50869437 -> 855c56fef
SQOOP-3233: SqoopHCatImportHelper.convertNumberTypes check for Varchar instead of Char (Ferenc Szabo via Szabolcs Vasas) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/855c56fe Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/855c56fe Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/855c56fe Branch: refs/heads/trunk Commit: 855c56fef8ab833e63587f17f70030e2c7764f4a Parents: a508694 Author: Szabolcs Vasas <[email protected]> Authored: Fri Dec 8 11:39:24 2017 +0100 Committer: Szabolcs Vasas <[email protected]> Committed: Fri Dec 8 11:39:24 2017 +0100 ---------------------------------------------------------------------- .../mapreduce/hcat/SqoopHCatImportHelper.java | 78 +++++++++++--------- .../apache/sqoop/hcat/HCatalogImportTest.java | 12 +++ 2 files changed, 56 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/855c56fe/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java index 1c1ed1e..efaef17 100644 --- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java +++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java @@ -335,9 +335,7 @@ public class SqoopHCatImportHelper { private Object convertStringTypes(Object val, HCatFieldSchema hfs) { HCatFieldSchema.Type hfsType = hfs.getType(); - if (hfsType == HCatFieldSchema.Type.STRING - || hfsType == HCatFieldSchema.Type.VARCHAR - || hfsType == HCatFieldSchema.Type.CHAR) { + if (isStringType(hfsType)) { String str = val.toString(); if (doHiveDelimsReplacement) { str = FieldFormatter.hiveStringReplaceDelims(str, @@ -400,59 +398,71 @@ public class SqoopHCatImportHelper { return null; } if (val instanceof BigDecimal - && hfsType == HCatFieldSchema.Type.STRING - || hfsType == HCatFieldSchema.Type.VARCHAR - || hfsType == HCatFieldSchema.Type.CHAR) { + && isStringType(hfsType)) { BigDecimal bd = (BigDecimal) val; - String bdStr = null; - if (bigDecimalFormatString) { - bdStr = bd.toPlainString(); - } else { - bdStr = bd.toString(); - } - if (hfsType == HCatFieldSchema.Type.VARCHAR) { - VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo(); - HiveVarchar hvc = new HiveVarchar(bdStr, vti.getLength()); - return hvc; - } else if (hfsType == HCatFieldSchema.Type.VARCHAR) { - CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo(); - HiveChar hChar = new HiveChar(bdStr, cti.getLength()); - return hChar; - } else { - return bdStr; - } + return convertBigDecimalToTextTypes(hfs, hfsType, bd); } Number n = (Number) val; + return convertNumberToAnyType(hfs, hfsType, n); + } + + private boolean isStringType(HCatFieldSchema.Type hfsType) { + return hfsType == HCatFieldSchema.Type.STRING + || hfsType == HCatFieldSchema.Type.VARCHAR + || hfsType == HCatFieldSchema.Type.CHAR; + } + + private Object convertNumberToAnyType(HCatFieldSchema hfs, HCatFieldSchema.Type hfsType, Number number) { if (hfsType == HCatFieldSchema.Type.TINYINT) { - return n.byteValue(); + return number.byteValue(); } else if (hfsType == HCatFieldSchema.Type.SMALLINT) { - return n.shortValue(); + return number.shortValue(); } else if (hfsType == HCatFieldSchema.Type.INT) { - return n.intValue(); + return number.intValue(); } else if (hfsType == HCatFieldSchema.Type.BIGINT) { - return n.longValue(); + return number.longValue(); } else if (hfsType == HCatFieldSchema.Type.FLOAT) { - return n.floatValue(); + return number.floatValue(); } else if (hfsType == HCatFieldSchema.Type.DOUBLE) { - return n.doubleValue(); + return number.doubleValue(); } else if (hfsType == HCatFieldSchema.Type.BOOLEAN) { - return n.byteValue() == 0 ? Boolean.FALSE : Boolean.TRUE; + return number.byteValue() == 0 ? Boolean.FALSE : Boolean.TRUE; } else if (hfsType == HCatFieldSchema.Type.STRING) { - return n.toString(); + return number.toString(); } else if (hfsType == HCatFieldSchema.Type.VARCHAR) { VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo(); - HiveVarchar hvc = new HiveVarchar(val.toString(), vti.getLength()); + HiveVarchar hvc = new HiveVarchar(number.toString(), vti.getLength()); return hvc; } else if (hfsType == HCatFieldSchema.Type.CHAR) { CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo(); - HiveChar hChar = new HiveChar(val.toString(), cti.getLength()); + HiveChar hChar = new HiveChar(number.toString(), cti.getLength()); return hChar; } else if (hfsType == HCatFieldSchema.Type.DECIMAL) { - return convertNumberIntoHiveDecimal(n); + return convertNumberIntoHiveDecimal(number); } return null; } + private Object convertBigDecimalToTextTypes(HCatFieldSchema hfs, HCatFieldSchema.Type hfsType, BigDecimal bigDecimal) { + String bdStr = null; + if (bigDecimalFormatString) { + bdStr = bigDecimal.toPlainString(); + } else { + bdStr = bigDecimal.toString(); + } + if (hfsType == HCatFieldSchema.Type.VARCHAR) { + VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo(); + HiveVarchar hvc = new HiveVarchar(bdStr, vti.getLength()); + return hvc; + } else if (hfsType == HCatFieldSchema.Type.CHAR) { + CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo(); + HiveChar hChar = new HiveChar(bdStr, cti.getLength()); + return hChar; + } else { + return bdStr; + } + } + HiveDecimal convertNumberIntoHiveDecimal(Number number) { BigDecimal bigDecimal = null; http://git-wip-us.apache.org/repos/asf/sqoop/blob/855c56fe/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java index 4686493..b6741f4 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java @@ -446,6 +446,18 @@ public class HCatalogImportTest extends ImportJobTestCase { "decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.DECIMAL, 18, 2, HiveDecimal.create(new BigDecimal("2000")), new BigDecimal("2000"), KeyType.NOT_A_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3), + "decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.VARCHAR, 20, 0, + new HiveVarchar("1999", 20), new BigDecimal("1999"), KeyType.NOT_A_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(4), + "decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.CHAR, 20, 0, + new HiveChar("1998", 20), new BigDecimal("1998"), KeyType.NOT_A_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(5), + "int", Types.INTEGER, HCatFieldSchema.Type.VARCHAR, 20, 0, + new HiveVarchar("2001", 20 ), new BigDecimal("2001"), KeyType.NOT_A_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(6), + "int", Types.INTEGER, HCatFieldSchema.Type.CHAR, 20, 0, + new HiveChar("2002", 20), new BigDecimal("2002"), KeyType.NOT_A_KEY), }; List<String> addlArgsArray = new ArrayList<String>(); setExtraArgs(addlArgsArray);
