This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new ed3c479 Revert "[SPARK-34233][SQL] FIX NPE for char padding in binary
comparison"
ed3c479 is described below
commit ed3c479a3664919515bc74a3212fc96f8d2feff7
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jan 27 20:07:08 2021 -0800
Revert "[SPARK-34233][SQL] FIX NPE for char padding in binary comparison"
This reverts commit cf21e8898ab484a833b6696d0cf4bb0c871e7ff6.
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 22 ++++-------
.../apache/spark/sql/CharVarcharTestSuite.scala | 43 +---------------------
2 files changed, 10 insertions(+), 55 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 6fd6901..fb95323a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -3888,15 +3888,13 @@ object ApplyCharTypePadding extends Rule[LogicalPlan] {
if attr.dataType == StringType && list.forall(_.foldable) =>
CharVarcharUtils.getRawType(attr.metadata).flatMap {
case CharType(length) =>
- val (nulls, literalChars) =
- list.map(_.eval().asInstanceOf[UTF8String]).partition(_ ==
null)
- val literalCharLengths = literalChars.map(_.numChars())
+ val literalCharLengths =
list.map(_.eval().asInstanceOf[UTF8String].numChars())
val targetLen = (length +: literalCharLengths).max
Some(i.copy(
value = addPadding(attr, length, targetLen),
list = list.zip(literalCharLengths).map {
case (lit, charLength) => addPadding(lit, charLength,
targetLen)
- } ++ nulls.map(Literal.create(_, StringType))))
+ }))
case _ => None
}.getOrElse(i)
@@ -3917,17 +3915,13 @@ object ApplyCharTypePadding extends Rule[LogicalPlan] {
CharVarcharUtils.getRawType(attr.metadata).flatMap {
case CharType(length) =>
val str = lit.eval().asInstanceOf[UTF8String]
- if (str == null) {
- None
+ val stringLitLen = str.numChars()
+ if (length < stringLitLen) {
+ Some(Seq(StringRPad(attr, Literal(stringLitLen)), lit))
+ } else if (length > stringLitLen) {
+ Some(Seq(attr, StringRPad(lit, Literal(length))))
} else {
- val stringLitLen = str.numChars()
- if (length < stringLitLen) {
- Some(Seq(StringRPad(attr, Literal(stringLitLen)), lit))
- } else if (length > stringLitLen) {
- Some(Seq(attr, StringRPad(lit, Literal(length))))
- } else {
- None
- }
+ None
}
case _ => None
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
index 744757b..ff8820a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
@@ -152,22 +152,6 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
}
}
- test("SPARK-34233: char/varchar with null value for partitioned columns") {
- Seq("CHAR(5)", "VARCHAR(5)").foreach { typ =>
- withTable("t") {
- sql(s"CREATE TABLE t(i STRING, c $typ) USING $format PARTITIONED BY
(c)")
- sql("INSERT INTO t VALUES ('1', null)")
- checkPlainResult(spark.table("t"), typ, null)
- sql("INSERT OVERWRITE t VALUES ('1', null)")
- checkPlainResult(spark.table("t"), typ, null)
- sql("INSERT OVERWRITE t PARTITION (c=null) VALUES ('1')")
- checkPlainResult(spark.table("t"), typ, null)
- sql("ALTER TABLE t DROP PARTITION(c=null)")
- checkAnswer(spark.table("t"), Nil)
- }
- }
- }
-
test("char/varchar type values length check: partitioned columns of other
types") {
// DSV2 doesn't support DROP PARTITION yet.
assume(format != "foo")
@@ -451,8 +435,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
("c1 IN ('a', 'b')", true),
("c1 = c2", true),
("c1 < c2", false),
- ("c1 IN (c2)", true),
- ("c1 <=> null", false)))
+ ("c1 IN (c2)", true)))
}
}
@@ -468,29 +451,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
("c1 IN ('a', 'b')", true),
("c1 = c2", true),
("c1 < c2", false),
- ("c1 IN (c2)", true),
- ("c1 <=> null", false)))
- }
- }
-
- private def testNullConditions(df: DataFrame, conditions: Seq[String]): Unit
= {
- conditions.foreach { cond =>
- checkAnswer(df.selectExpr(cond), Row(null))
- }
- }
-
- test("SPARK-34233: char type comparison with null values") {
- val conditions = Seq("c = null", "c IN ('e', null)", "c IN (null)")
- withTable("t") {
- sql(s"CREATE TABLE t(c CHAR(2)) USING $format")
- sql("INSERT INTO t VALUES ('a')")
- testNullConditions(spark.table("t"), conditions)
- }
-
- withTable("t") {
- sql(s"CREATE TABLE t(i INT, c CHAR(2)) USING $format PARTITIONED BY (c)")
- sql("INSERT INTO t VALUES (1, 'a')")
- testNullConditions(spark.table("t"), conditions)
+ ("c1 IN (c2)", true)))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
