Repository: spark Updated Branches: refs/heads/branch-1.2 1ca39b723 -> 1a650e7d8
[SPARK-4425][SQL] Handle NaN or Infinity cast to Timestamp correctly. `Cast` from `NaN` or `Infinity` of `Double` or `Float` to `TimestampType` throws `NumberFormatException`. Author: Takuya UESHIN <[email protected]> Closes #3283 from ueshin/issues/SPARK-4425 and squashes the following commits: 14def0c [Takuya UESHIN] Fix Cast to be able to handle NaN or Infinity to TimestampType. (cherry picked from commit 566c791931645bfaaaf57ee5a15b9ffad534f81e) Signed-off-by: Michael Armbrust <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1a650e7d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1a650e7d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1a650e7d Branch: refs/heads/branch-1.2 Commit: 1a650e7d863b72025625c3140b038ab12ec86eca Parents: 1ca39b7 Author: Takuya UESHIN <[email protected]> Authored: Mon Nov 17 16:28:07 2014 -0800 Committer: Michael Armbrust <[email protected]> Committed: Mon Nov 17 16:28:39 2014 -0800 ---------------------------------------------------------------------- .../apache/spark/sql/catalyst/expressions/Cast.scala | 14 ++++++++++++-- .../expressions/ExpressionEvaluationSuite.scala | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/1a650e7d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index b401096..b47865f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -32,6 +32,8 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w override def nullable = (child.dataType, dataType) match { case (StringType, _: NumericType) => true case (StringType, TimestampType) => true + case (DoubleType, TimestampType) => true + case (FloatType, TimestampType) => true case (StringType, DateType) => true case (_: NumericType, DateType) => true case (BooleanType, DateType) => true @@ -117,10 +119,18 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w buildCast[Decimal](_, d => decimalToTimestamp(d)) // TimestampWritable.doubleToTimestamp case DoubleType => - buildCast[Double](_, d => decimalToTimestamp(Decimal(d))) + buildCast[Double](_, d => try { + decimalToTimestamp(Decimal(d)) + } catch { + case _: NumberFormatException => null + }) // TimestampWritable.floatToTimestamp case FloatType => - buildCast[Float](_, f => decimalToTimestamp(Decimal(f))) + buildCast[Float](_, f => try { + decimalToTimestamp(Decimal(f)) + } catch { + case _: NumberFormatException => null + }) } private[this] def decimalToTimestamp(d: Decimal) = { http://git-wip-us.apache.org/repos/asf/spark/blob/1a650e7d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala index 3a6a020..3f5b9f6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala @@ -450,6 +450,11 @@ class ExpressionEvaluationSuite extends FunSuite { // A test for higher precision than millis checkEvaluation(Cast(Cast(0.00000001, TimestampType), DoubleType), 0.00000001) + + checkEvaluation(Cast(Literal(Double.NaN), TimestampType), null) + checkEvaluation(Cast(Literal(1.0 / 0.0), TimestampType), null) + checkEvaluation(Cast(Literal(Float.NaN), TimestampType), null) + checkEvaluation(Cast(Literal(1.0f / 0.0f), TimestampType), null) } test("null checking") { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
