Repository: hive Updated Branches: refs/heads/master c23841e55 -> c0c19d073
HIVE-11835: Type decimal(1,1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0c19d07 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0c19d07 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0c19d07 Branch: refs/heads/master Commit: c0c19d0733a3a9fe785e120f618414212e4976be Parents: c23841e Author: Xuefu Zhang <[email protected]> Authored: Mon Oct 5 05:42:20 2015 -0700 Committer: Xuefu Zhang <[email protected]> Committed: Mon Oct 5 05:42:20 2015 -0700 ---------------------------------------------------------------------- .../hive/common/type/TestHiveDecimal.java | 12 +-- .../test/resources/testconfiguration.properties | 1 + .../test/queries/clientpositive/decimal_1_1.q | 9 ++ .../results/clientpositive/decimal_1_1.q.out | 104 +++++++++++++++++++ .../clientpositive/spark/decimal_1_1.q.out | 104 +++++++++++++++++++ .../hadoop/hive/common/type/HiveDecimal.java | 10 +- 6 files changed, 233 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java index ba5ef71..f68842c 100644 --- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java +++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java @@ -42,15 +42,15 @@ public class TestHiveDecimal { Assert.assertTrue("Decimal scale should not go above maximum", dec.scale() <= HiveDecimal.MAX_SCALE); decStr = "57847525803324040144343378.09799306448796128931113691624"; - BigDecimal bd = new BigDecimal(decStr); - BigDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5); + HiveDecimal bd = HiveDecimal.create(decStr); + HiveDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5); Assert.assertNull(bd1); bd1 = HiveDecimal.enforcePrecisionScale(bd, 35, 5); Assert.assertEquals("57847525803324040144343378.09799", bd1.toString()); bd1 = HiveDecimal.enforcePrecisionScale(bd, 45, 20); Assert.assertNull(bd1); - dec = HiveDecimal.create(bd, false); + dec = HiveDecimal.create(new BigDecimal(decStr), false); Assert.assertNull(dec); dec = HiveDecimal.create("-1786135888657847525803324040144343378.09799306448796128931113691624"); @@ -65,10 +65,10 @@ public class TestHiveDecimal { // Rounding numbers that increase int digits Assert.assertEquals("10", - HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 2, 0).toString()); - Assert.assertNull(HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 1, 0)); + HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 2, 0).toString()); + Assert.assertNull(HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 1, 0)); Assert.assertEquals("9", - HiveDecimal.enforcePrecisionScale(new BigDecimal("9.4"), 1, 0).toString()); + HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.4"), 1, 0).toString()); } @Test http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 700ea63..9c9f4cc 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -667,6 +667,7 @@ spark.query.files=add_part_multiple.q, \ custom_input_output_format.q, \ date_join1.q, \ date_udf.q, \ + decimal_1_1.q, \ decimal_join.q, \ disable_merge_for_bucketing.q, \ dynamic_rdd_cache.q, \ http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/ql/src/test/queries/clientpositive/decimal_1_1.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/decimal_1_1.q b/ql/src/test/queries/clientpositive/decimal_1_1.q new file mode 100644 index 0000000..83ce521 --- /dev/null +++ b/ql/src/test/queries/clientpositive/decimal_1_1.q @@ -0,0 +1,9 @@ +drop table if exists decimal_1_1; + +create table decimal_1_1 (d decimal(1,1)); +load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1; +select * from decimal_1_1; + +select d from decimal_1_1 order by d desc; + +drop table decimal_1_1; http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/ql/src/test/results/clientpositive/decimal_1_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/decimal_1_1.q.out b/ql/src/test/results/clientpositive/decimal_1_1.q.out new file mode 100644 index 0000000..b2704c6 --- /dev/null +++ b/ql/src/test/results/clientpositive/decimal_1_1.q.out @@ -0,0 +1,104 @@ +PREHOOK: query: drop table if exists decimal_1_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists decimal_1_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table decimal_1_1 (d decimal(1,1)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@decimal_1_1 +POSTHOOK: query: create table decimal_1_1 (d decimal(1,1)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@decimal_1_1 +PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@decimal_1_1 +POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@decimal_1_1 +PREHOOK: query: select * from decimal_1_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from decimal_1_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1_1 +#### A masked pattern was here #### +0 +0 +0 +0.1 +0.2 +0.9 +0.9 +NULL +0.3 +NULL +NULL +0 +0 +NULL +0 +0 +0 +0 +-0.1 +-0.2 +-0.9 +-0.9 +NULL +-0.3 +NULL +NULL +0 +0 +NULL +0 +PREHOOK: query: select d from decimal_1_1 order by d desc +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1_1 +#### A masked pattern was here #### +POSTHOOK: query: select d from decimal_1_1 order by d desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1_1 +#### A masked pattern was here #### +0.9 +0.9 +0.3 +0.2 +0.1 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +-0.1 +-0.2 +-0.3 +-0.9 +-0.9 +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +PREHOOK: query: drop table decimal_1_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_1_1 +PREHOOK: Output: default@decimal_1_1 +POSTHOOK: query: drop table decimal_1_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_1_1 +POSTHOOK: Output: default@decimal_1_1 http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out new file mode 100644 index 0000000..b2704c6 --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out @@ -0,0 +1,104 @@ +PREHOOK: query: drop table if exists decimal_1_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists decimal_1_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table decimal_1_1 (d decimal(1,1)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@decimal_1_1 +POSTHOOK: query: create table decimal_1_1 (d decimal(1,1)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@decimal_1_1 +PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@decimal_1_1 +POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@decimal_1_1 +PREHOOK: query: select * from decimal_1_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from decimal_1_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1_1 +#### A masked pattern was here #### +0 +0 +0 +0.1 +0.2 +0.9 +0.9 +NULL +0.3 +NULL +NULL +0 +0 +NULL +0 +0 +0 +0 +-0.1 +-0.2 +-0.9 +-0.9 +NULL +-0.3 +NULL +NULL +0 +0 +NULL +0 +PREHOOK: query: select d from decimal_1_1 order by d desc +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1_1 +#### A masked pattern was here #### +POSTHOOK: query: select d from decimal_1_1 order by d desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1_1 +#### A masked pattern was here #### +0.9 +0.9 +0.3 +0.2 +0.1 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +-0.1 +-0.2 +-0.3 +-0.9 +-0.9 +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +PREHOOK: query: drop table decimal_1_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_1_1 +PREHOOK: Output: default@decimal_1_1 +POSTHOOK: query: drop table decimal_1_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_1_1 +POSTHOOK: Output: default@decimal_1_1 http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java index 12a3936..4ed17a2 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java +++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java @@ -272,11 +272,19 @@ public class HiveDecimal implements Comparable<HiveDecimal> { return bd; } - public static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) { + private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) { if (bd == null) { return null; } + /** + * Specially handling the case that bd=0, and we are converting it to a type where precision=scale, + * such as decimal(1, 1). + */ + if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) { + return bd.setScale(maxScale); + } + bd = trim(bd); if (bd.scale() > maxScale) {
