spark git commit: [SPARK-16602][SQL] `Nvl` function should support numeric-string cases
Repository: spark
Updated Branches:
refs/heads/branch-2.0 6ca1d941b -> f18f9ca5b
[SPARK-16602][SQL] `Nvl` function should support numeric-string cases
## What changes were proposed in this pull request?
`Nvl` function should support numeric-straing cases like Hive/Spark1.6.
Currently, `Nvl` finds the tightest common types among numeric types. This PR
extends that to consider `String` type, too.
```scala
- TypeCoercion.findTightestCommonTypeOfTwo(left.dataType, right.dataType).map {
dtype =>
+ TypeCoercion.findTightestCommonTypeToString(left.dataType,
right.dataType).map { dtype =>
```
**Before**
```scala
scala> sql("select nvl('0', 1)").collect()
org.apache.spark.sql.AnalysisException: cannot resolve `nvl("0", 1)` due to
data type mismatch:
input to function coalesce should all be the same type, but it's [string, int];
line 1 pos 7
```
**After**
```scala
scala> sql("select nvl('0', 1)").collect()
res0: Array[org.apache.spark.sql.Row] = Array([0])
```
## How was this patch tested?
Pass the Jenkins tests.
Author: Dongjoon Hyun
Closes #14251 from dongjoon-hyun/SPARK-16602.
(cherry picked from commit 162d04a30e38bb83d35865679145f8ea80b84c26)
Signed-off-by: Reynold Xin
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f18f9ca5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f18f9ca5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f18f9ca5
Branch: refs/heads/branch-2.0
Commit: f18f9ca5b22ca11712694b1106463ae6efc1d646
Parents: 6ca1d94
Author: Dongjoon Hyun
Authored: Tue Jul 19 10:28:17 2016 -0700
Committer: Reynold Xin
Committed: Tue Jul 19 10:28:24 2016 -0700
--
.../spark/sql/catalyst/analysis/TypeCoercion.scala | 2 +-
.../sql/catalyst/expressions/nullExpressions.scala | 2 +-
.../catalyst/expressions/NullFunctionsSuite.scala| 15 +++
3 files changed, 17 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f18f9ca5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index baec6d1..9a040f8 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -100,7 +100,7 @@ object TypeCoercion {
}
/** Similar to [[findTightestCommonType]], but can promote all the way to
StringType. */
- private def findTightestCommonTypeToString(left: DataType, right: DataType):
Option[DataType] = {
+ def findTightestCommonTypeToString(left: DataType, right: DataType):
Option[DataType] = {
findTightestCommonTypeOfTwo(left, right).orElse((left, right) match {
case (StringType, t2: AtomicType) if t2 != BinaryType && t2 !=
BooleanType => Some(StringType)
case (t1: AtomicType, StringType) if t1 != BinaryType && t1 !=
BooleanType => Some(StringType)
http://git-wip-us.apache.org/repos/asf/spark/blob/f18f9ca5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
index 523fb05..1c18265 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
@@ -134,7 +134,7 @@ case class Nvl(left: Expression, right: Expression) extends
RuntimeReplaceable {
override def replaceForTypeCoercion(): Expression = {
if (left.dataType != right.dataType) {
- TypeCoercion.findTightestCommonTypeOfTwo(left.dataType,
right.dataType).map { dtype =>
+ TypeCoercion.findTightestCommonTypeToString(left.dataType,
right.dataType).map { dtype =>
copy(left = Cast(left, dtype), right = Cast(right, dtype))
}.getOrElse(this)
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/f18f9ca5/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
spark git commit: [SPARK-16602][SQL] `Nvl` function should support numeric-string cases
Repository: spark
Updated Branches:
refs/heads/master 0bd76e872 -> 162d04a30
[SPARK-16602][SQL] `Nvl` function should support numeric-string cases
## What changes were proposed in this pull request?
`Nvl` function should support numeric-straing cases like Hive/Spark1.6.
Currently, `Nvl` finds the tightest common types among numeric types. This PR
extends that to consider `String` type, too.
```scala
- TypeCoercion.findTightestCommonTypeOfTwo(left.dataType, right.dataType).map {
dtype =>
+ TypeCoercion.findTightestCommonTypeToString(left.dataType,
right.dataType).map { dtype =>
```
**Before**
```scala
scala> sql("select nvl('0', 1)").collect()
org.apache.spark.sql.AnalysisException: cannot resolve `nvl("0", 1)` due to
data type mismatch:
input to function coalesce should all be the same type, but it's [string, int];
line 1 pos 7
```
**After**
```scala
scala> sql("select nvl('0', 1)").collect()
res0: Array[org.apache.spark.sql.Row] = Array([0])
```
## How was this patch tested?
Pass the Jenkins tests.
Author: Dongjoon Hyun
Closes #14251 from dongjoon-hyun/SPARK-16602.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/162d04a3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/162d04a3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/162d04a3
Branch: refs/heads/master
Commit: 162d04a30e38bb83d35865679145f8ea80b84c26
Parents: 0bd76e8
Author: Dongjoon Hyun
Authored: Tue Jul 19 10:28:17 2016 -0700
Committer: Reynold Xin
Committed: Tue Jul 19 10:28:17 2016 -0700
--
.../spark/sql/catalyst/analysis/TypeCoercion.scala | 2 +-
.../sql/catalyst/expressions/nullExpressions.scala | 2 +-
.../catalyst/expressions/NullFunctionsSuite.scala| 15 +++
3 files changed, 17 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/162d04a3/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index baec6d1..9a040f8 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -100,7 +100,7 @@ object TypeCoercion {
}
/** Similar to [[findTightestCommonType]], but can promote all the way to
StringType. */
- private def findTightestCommonTypeToString(left: DataType, right: DataType):
Option[DataType] = {
+ def findTightestCommonTypeToString(left: DataType, right: DataType):
Option[DataType] = {
findTightestCommonTypeOfTwo(left, right).orElse((left, right) match {
case (StringType, t2: AtomicType) if t2 != BinaryType && t2 !=
BooleanType => Some(StringType)
case (t1: AtomicType, StringType) if t1 != BinaryType && t1 !=
BooleanType => Some(StringType)
http://git-wip-us.apache.org/repos/asf/spark/blob/162d04a3/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
index 523fb05..1c18265 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
@@ -134,7 +134,7 @@ case class Nvl(left: Expression, right: Expression) extends
RuntimeReplaceable {
override def replaceForTypeCoercion(): Expression = {
if (left.dataType != right.dataType) {
- TypeCoercion.findTightestCommonTypeOfTwo(left.dataType,
right.dataType).map { dtype =>
+ TypeCoercion.findTightestCommonTypeToString(left.dataType,
right.dataType).map { dtype =>
copy(left = Cast(left, dtype), right = Cast(right, dtype))
}.getOrElse(this)
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/162d04a3/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
index ace6c15..712fe35 100644
---
