spark git commit: [SPARK-10245] [SQL] Fix decimal literals with precision scale

2015-08-25 Thread yhuai 
Repository: spark
Updated Branches:
  refs/heads/master 00ae4be97 - ec89bd840


[SPARK-10245] [SQL] Fix decimal literals with precision  scale

In BigDecimal or java.math.BigDecimal, the precision could be smaller than 
scale, for example, BigDecimal(0.001) has precision = 1 and scale = 3. But 
DecimalType require that the precision should be larger than scale, so we 
should use the maximum of precision and scale when inferring the schema from 
decimal literal.

Author: Davies Liu dav...@databricks.com

Closes #8428 from davies/smaller_decimal.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ec89bd84
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ec89bd84
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ec89bd84

Branch: refs/heads/master
Commit: ec89bd840a6862751999d612f586a962cae63f6d
Parents: 00ae4be
Author: Davies Liu dav...@databricks.com
Authored: Tue Aug 25 14:55:34 2015 -0700
Committer: Yin Huai yh...@databricks.com
Committed: Tue Aug 25 14:55:34 2015 -0700

--
 .../apache/spark/sql/catalyst/expressions/literals.scala  |  7 ---
 .../sql/catalyst/expressions/LiteralExpressionSuite.scala |  8 +---
 .../test/scala/org/apache/spark/sql/SQLQuerySuite.scala   | 10 ++
 3 files changed, 19 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/ec89bd84/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 34bad23..8c0c5d5 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -36,9 +36,10 @@ object Literal {
 case s: Short = Literal(s, ShortType)
 case s: String = Literal(UTF8String.fromString(s), StringType)
 case b: Boolean = Literal(b, BooleanType)
-case d: BigDecimal = Literal(Decimal(d), DecimalType(d.precision, 
d.scale))
-case d: java.math.BigDecimal = Literal(Decimal(d), 
DecimalType(d.precision(), d.scale()))
-case d: Decimal = Literal(d, DecimalType(d.precision, d.scale))
+case d: BigDecimal = Literal(Decimal(d), 
DecimalType(Math.max(d.precision, d.scale), d.scale))
+case d: java.math.BigDecimal =
+  Literal(Decimal(d), DecimalType(Math.max(d.precision, d.scale), 
d.scale()))
+case d: Decimal = Literal(d, DecimalType(Math.max(d.precision, d.scale), 
d.scale))
 case t: Timestamp = Literal(DateTimeUtils.fromJavaTimestamp(t), 
TimestampType)
 case d: Date = Literal(DateTimeUtils.fromJavaDate(d), DateType)
 case a: Array[Byte] = Literal(a, BinaryType)

http://git-wip-us.apache.org/repos/asf/spark/blob/ec89bd84/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index f6404d2..015eb18 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -83,12 +83,14 @@ class LiteralExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
   }
 
   test(decimal) {
-List(0.0, 1.2, 1., 5).foreach { d =
+List(-0.0001, 0.0, 0.001, 1.2, 1., 5).foreach { d =
   checkEvaluation(Literal(Decimal(d)), Decimal(d))
   checkEvaluation(Literal(Decimal(d.toInt)), Decimal(d.toInt))
   checkEvaluation(Literal(Decimal(d.toLong)), Decimal(d.toLong))
-  checkEvaluation(Literal(Decimal((d * 1000L).toLong, 10, 1)),
-Decimal((d * 1000L).toLong, 10, 1))
+  checkEvaluation(Literal(Decimal((d * 1000L).toLong, 10, 3)),
+Decimal((d * 1000L).toLong, 10, 3))
+  checkEvaluation(Literal(BigDecimal(d.toString)), Decimal(d))
+  checkEvaluation(Literal(new java.math.BigDecimal(d.toString)), 
Decimal(d))
 }
   }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/ec89bd84/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
--
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index dcb4e83..aa07665 100644
---

spark git commit: [SPARK-10245] [SQL] Fix decimal literals with precision scale

2015-08-25 Thread yhuai 
Repository: spark
Updated Branches:
  refs/heads/branch-1.5 6f05b7aeb - 8925896b1


[SPARK-10245] [SQL] Fix decimal literals with precision  scale

In BigDecimal or java.math.BigDecimal, the precision could be smaller than 
scale, for example, BigDecimal(0.001) has precision = 1 and scale = 3. But 
DecimalType require that the precision should be larger than scale, so we 
should use the maximum of precision and scale when inferring the schema from 
decimal literal.

Author: Davies Liu dav...@databricks.com

Closes #8428 from davies/smaller_decimal.

(cherry picked from commit ec89bd840a6862751999d612f586a962cae63f6d)
Signed-off-by: Yin Huai yh...@databricks.com


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8925896b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8925896b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8925896b

Branch: refs/heads/branch-1.5
Commit: 8925896b1eb0a13d723d38fb263d3bec0a01ec10
Parents: 6f05b7a
Author: Davies Liu dav...@databricks.com
Authored: Tue Aug 25 14:55:34 2015 -0700
Committer: Yin Huai yh...@databricks.com
Committed: Tue Aug 25 14:55:45 2015 -0700

--
 .../apache/spark/sql/catalyst/expressions/literals.scala  |  7 ---
 .../sql/catalyst/expressions/LiteralExpressionSuite.scala |  8 +---
 .../test/scala/org/apache/spark/sql/SQLQuerySuite.scala   | 10 ++
 3 files changed, 19 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/8925896b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 34bad23..8c0c5d5 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -36,9 +36,10 @@ object Literal {
 case s: Short = Literal(s, ShortType)
 case s: String = Literal(UTF8String.fromString(s), StringType)
 case b: Boolean = Literal(b, BooleanType)
-case d: BigDecimal = Literal(Decimal(d), DecimalType(d.precision, 
d.scale))
-case d: java.math.BigDecimal = Literal(Decimal(d), 
DecimalType(d.precision(), d.scale()))
-case d: Decimal = Literal(d, DecimalType(d.precision, d.scale))
+case d: BigDecimal = Literal(Decimal(d), 
DecimalType(Math.max(d.precision, d.scale), d.scale))
+case d: java.math.BigDecimal =
+  Literal(Decimal(d), DecimalType(Math.max(d.precision, d.scale), 
d.scale()))
+case d: Decimal = Literal(d, DecimalType(Math.max(d.precision, d.scale), 
d.scale))
 case t: Timestamp = Literal(DateTimeUtils.fromJavaTimestamp(t), 
TimestampType)
 case d: Date = Literal(DateTimeUtils.fromJavaDate(d), DateType)
 case a: Array[Byte] = Literal(a, BinaryType)

http://git-wip-us.apache.org/repos/asf/spark/blob/8925896b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index f6404d2..015eb18 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -83,12 +83,14 @@ class LiteralExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
   }
 
   test(decimal) {
-List(0.0, 1.2, 1., 5).foreach { d =
+List(-0.0001, 0.0, 0.001, 1.2, 1., 5).foreach { d =
   checkEvaluation(Literal(Decimal(d)), Decimal(d))
   checkEvaluation(Literal(Decimal(d.toInt)), Decimal(d.toInt))
   checkEvaluation(Literal(Decimal(d.toLong)), Decimal(d.toLong))
-  checkEvaluation(Literal(Decimal((d * 1000L).toLong, 10, 1)),
-Decimal((d * 1000L).toLong, 10, 1))
+  checkEvaluation(Literal(Decimal((d * 1000L).toLong, 10, 3)),
+Decimal((d * 1000L).toLong, 10, 3))
+  checkEvaluation(Literal(BigDecimal(d.toString)), Decimal(d))
+  checkEvaluation(Literal(new java.math.BigDecimal(d.toString)), 
Decimal(d))
 }
   }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/8925896b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
--
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala