This is an automated email from the ASF dual-hosted git repository. yamamuro pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new cb6a0d0 [SPARK-32688][SQL][TEST] Add special values to LiteralGenerator for float and double cb6a0d0 is described below commit cb6a0d08cc020d9a2c19173c9023a9f5e565dd6c Author: Tanel Kiis <tanel.k...@gmail.com> AuthorDate: Wed Sep 16 12:13:15 2020 +0900 [SPARK-32688][SQL][TEST] Add special values to LiteralGenerator for float and double ### What changes were proposed in this pull request? The `LiteralGenerator` for float and double datatypes was supposed to yield special values (NaN, +-inf) among others, but the `Gen.chooseNum` method does not yield values that are outside the defined range. The `Gen.chooseNum` for a wide range of floats and doubles does not yield values in the "everyday" range as stated in https://github.com/typelevel/scalacheck/issues/113 . There is an similar class `RandomDataGenerator` that is used in some other tests. Added `-0.0` and `-0.0f` as special values to there too. These changes revealed an inconsistency with the equality check between `-0.0` and `0.0`. ### Why are the changes needed? The `LiteralGenerator` is mostly used in the `checkConsistencyBetweenInterpretedAndCodegen` method in `MathExpressionsSuite`. This change would have caught the bug fixed in #29495 . ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Locally reverted #29495 and verified that the existing test cases caught the bug. Closes #29515 from tanelk/SPARK-32688. Authored-by: Tanel Kiis <tanel.k...@gmail.com> Signed-off-by: Takeshi Yamamuro <yamam...@apache.org> (cherry picked from commit 6051755bfe23a0e4564bf19476ec34cd7fd6008d) Signed-off-by: Takeshi Yamamuro <yamam...@apache.org> --- .../org/apache/spark/sql/RandomDataGenerator.scala | 4 ++-- .../sql/catalyst/expressions/LiteralGenerator.scala | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala index 6a5bdc4..3e2dc3f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala @@ -260,10 +260,10 @@ object RandomDataGenerator { new MathContext(precision)).bigDecimal) case DoubleType => randomNumeric[Double]( rand, r => longBitsToDouble(r.nextLong()), Seq(Double.MinValue, Double.MinPositiveValue, - Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0)) + Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0, -0.0)) case FloatType => randomNumeric[Float]( rand, r => intBitsToFloat(r.nextInt()), Seq(Float.MinValue, Float.MinPositiveValue, - Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f)) + Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f, -0.0f)) case ByteType => randomNumeric[Byte]( rand, _.nextInt().toByte, Seq(Byte.MinValue, Byte.MaxValue, 0.toByte)) case IntegerType => randomNumeric[Int]( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala index d92eb01..c8e3b0e 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala @@ -68,16 +68,27 @@ object LiteralGenerator { lazy val longLiteralGen: Gen[Literal] = for { l <- Arbitrary.arbLong.arbitrary } yield Literal.create(l, LongType) + // The floatLiteralGen and doubleLiteralGen will 50% of the time yield arbitrary values + // and 50% of the time will yield some special values that are more likely to reveal + // corner cases. This behavior is similar to the integral value generators. lazy val floatLiteralGen: Gen[Literal] = for { - f <- Gen.chooseNum(Float.MinValue / 2, Float.MaxValue / 2, - Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity) + f <- Gen.oneOf( + Gen.oneOf( + Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity, Float.MinPositiveValue, + Float.MaxValue, -Float.MaxValue, 0.0f, -0.0f, 1.0f, -1.0f), + Arbitrary.arbFloat.arbitrary + ) } yield Literal.create(f, FloatType) lazy val doubleLiteralGen: Gen[Literal] = for { - f <- Gen.chooseNum(Double.MinValue / 2, Double.MaxValue / 2, - Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity) + f <- Gen.oneOf( + Gen.oneOf( + Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity, Double.MinPositiveValue, + Double.MaxValue, -Double.MaxValue, 0.0, -0.0, 1.0, -1.0), + Arbitrary.arbDouble.arbitrary + ) } yield Literal.create(f, DoubleType) // TODO cache the generated data --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org