This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b0a2ffc3e1c [SPARK-43357][SQL] Filter date type quote date
b0a2ffc3e1c is described below
commit b0a2ffc3e1c7697b289ffcd578e7d39925e6f3bb
Author: stijndehaes <[email protected]>
AuthorDate: Tue May 9 21:06:15 2023 +0800
[SPARK-43357][SQL] Filter date type quote date
### What changes were proposed in this pull request?
In this PR the date value is quoted before being sent to the hive metastore
### Why are the changes needed?
Glue for example expects the dates to be quoted, without quotes we get
errors.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Changed the result of unit tests, also tested by running the fix against
glue
Closes #41035 from stijndehaes/bugfix/hive-date-partition.
Authored-by: stijndehaes <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/hive/client/HiveShim.scala | 4 ++--
.../org/apache/spark/sql/hive/client/FiltersSuite.scala | 16 ++++++++--------
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 5e5d2757e9d..9defd87aa7d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -880,7 +880,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
case Literal(value, _: IntegralType) => Some(value.toString)
case Literal(value, _: StringType) =>
Some(quoteStringLiteral(value.toString))
case Literal(value, _: DateType) =>
- Some(dateFormatter.format(value.asInstanceOf[Int]))
+
Some(quoteStringLiteral(dateFormatter.format(value.asInstanceOf[Int])))
case _ => None
}
}
@@ -933,7 +933,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
object ExtractableDateValues {
private lazy val valueToLiteralString: PartialFunction[Any, String] = {
- case value: Int => dateFormatter.format(value)
+ case value: Int => quoteStringLiteral(dateFormatter.format(value))
}
def unapply(values: Set[Any]): Option[Seq[String]] = {
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
index e610218e1d7..627206c583e 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
@@ -66,17 +66,17 @@ class FiltersSuite extends SparkFunSuite with PlanTest {
filterTest("date filter",
(a("datecol", DateType) === Literal(Date.valueOf("2019-01-01"))) :: Nil,
- "datecol = 2019-01-01")
+ "datecol = \"2019-01-01\"")
filterTest("date filter with IN predicate",
(a("datecol", DateType) in
(Literal(Date.valueOf("2019-01-01")),
Literal(Date.valueOf("2019-01-07")))) :: Nil,
- "(datecol = 2019-01-01 or datecol = 2019-01-07)")
+ "(datecol = \"2019-01-01\" or datecol = \"2019-01-07\")")
filterTest("date and string filter",
(Literal(Date.valueOf("2019-01-01")) === a("datecol", DateType)) ::
(Literal("a") === a("strcol", IntegerType)) :: Nil,
- "2019-01-01 = datecol and \"a\" = strcol")
+ "\"2019-01-01\" = datecol and \"a\" = strcol")
filterTest("date filter with null",
(a("datecol", DateType) === Literal(null)) :: Nil,
@@ -105,7 +105,7 @@ class FiltersSuite extends SparkFunSuite with PlanTest {
filterTest("NOT: date filter",
(a("datecol", DateType) =!= Literal(Date.valueOf("2019-01-01"))) :: Nil,
- "datecol != 2019-01-01")
+ "datecol != \"2019-01-01\"")
filterTest("not-in, string filter",
(Not(In(a("strcol", StringType), Seq(Literal("a"), Literal("b"))))) :: Nil,
@@ -118,7 +118,7 @@ class FiltersSuite extends SparkFunSuite with PlanTest {
filterTest("not-in, date filter",
(Not(In(a("datecol", DateType),
Seq(Literal(Date.valueOf("2021-01-01")),
Literal(Date.valueOf("2021-01-02")))))) :: Nil,
- """(datecol != 2021-01-01 and datecol != 2021-01-02)""")
+ """(datecol != "2021-01-01" and datecol != "2021-01-02")""")
filterTest("not-in, date filter with null",
(Not(In(a("datecol", DateType),
@@ -139,7 +139,7 @@ class FiltersSuite extends SparkFunSuite with PlanTest {
(Not(InSet(a("datecol", DateType),
Set(Literal(Date.valueOf("2020-01-01")).eval(),
Literal(Date.valueOf("2020-01-02")).eval())))) :: Nil,
- """(datecol != 2020-01-01 and datecol != 2020-01-02)""")
+ """(datecol != "2020-01-01" and datecol != "2020-01-02")""")
filterTest("not-inset, date filter with null",
(Not(InSet(a("datecol", DateType),
@@ -215,7 +215,7 @@ class FiltersSuite extends SparkFunSuite with PlanTest {
checkConverted(
InSet(a("datecol", DateType),
Range(1, 20).map(d => Literal(d, DateType).eval(EmptyRow)).toSet),
- "(datecol >= 1970-01-02 and datecol <= 1970-01-20)")
+ "(datecol >= \"1970-01-02\" and datecol <= \"1970-01-20\")")
}
}
@@ -246,7 +246,7 @@ class FiltersSuite extends SparkFunSuite with PlanTest {
val dateFilter = InSet(a("p", DateType), Set(null,
Literal(Date.valueOf("2020-01-01")).eval(),
Literal(Date.valueOf("2021-01-01")).eval()))
val dateConverted = shim.convertFilters(testTable, Seq(dateFilter))
- assert(dateConverted == "(p = 2020-01-01 or p = 2021-01-01)")
+ assert(dateConverted == "(p = \"2020-01-01\" or p = \"2021-01-01\")")
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
