HyukjinKwon commented on a change in pull request #32583:
URL: https://github.com/apache/spark/pull/32583#discussion_r737028535
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
##########
@@ -577,6 +578,60 @@ class HivePartitionFilteringSuite(version: String)
dateStrValue)
}
+ test("getPartitionsByFilter: substr(chunk,0,1)=a") {
+ Seq("true" -> Seq("aa", "ab"), "false" -> chunkValue).foreach { t =>
+ withSQLConf(pruningFastFallback -> t._1) {
+ testMetastorePartitionFiltering(
+ Substring(attr("chunk"), Literal(0), Literal(1)) === "a",
+ dsValue,
+ hValue,
+ t._2,
+ dateValue,
+ dateStrValue)
+ }
+ }
+ }
+
+ test("getPartitionsByFilter: year(d)=2019") {
+ Seq("true" -> Seq("2019-01-01", "2019-01-02", "2019-01-03"),
+ "false" -> dateValue).foreach { t =>
+ withSQLConf(pruningFastFallback -> t._1) {
+ testMetastorePartitionFiltering(
+ Year(attr("d")) === 2019,
+ dsValue,
+ hValue,
+ chunkValue,
+ t._2,
+ dateStrValue)
+ }
+ }
+ }
+
+ test("getPartitionsByFilter: datestr=concat(2020-,01-,01)") {
+ Seq("true" -> Seq("2020-01-01"), "false" -> dateStrValue).foreach { t =>
+ withSQLConf(pruningFastFallback -> t._1) {
+ testMetastorePartitionFiltering(
+ attr("datestr") === Concat(Seq("2020-", "01-", "01")),
+ dsValue,
+ hValue,
+ chunkValue,
+ dateValue,
+ t._2)
+ }
+ }
+ }
+
+ test(s"getPartitionsByFilter: ds=20170101 when $fallbackKey=true") {
Review comment:
Hm, the tests added here are flaky:
```
sbt.ForkMain$ForkError: java.lang.reflect.InvocationTargetException: null
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:310)
at
org.apache.spark.sql.hive.client.HiveClientBuilder$.buildClient(HiveClientBuilder.scala:50)
at
org.apache.spark.sql.hive.client.HiveVersionSuite.buildClient(HiveVersionSuite.scala:47)
at
org.apache.spark.sql.hive.client.HivePartitionFilteringSuite.init(HivePartitionFilteringSuite.scala:70)
at
org.apache.spark.sql.hive.client.HivePartitionFilteringSuite.$anonfun$new$64(HivePartitionFilteringSuite.scala:626)
at
org.apache.spark.sql.catalyst.plans.SQLHelper.withSQLConf(SQLHelper.scala:54)
at
org.apache.spark.sql.catalyst.plans.SQLHelper.withSQLConf$(SQLHelper.scala:38)
at
org.apache.spark.sql.hive.client.HivePartitionFilteringSuite.withSQLConf(HivePartitionFilteringSuite.scala:39)
at
org.apache.spark.sql.hive.client.HivePartitionFilteringSuite.$anonfun$new$63(HivePartitionFilteringSuite.scala:625)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
```
https://github.com/apache/spark/runs/4001143506
https://github.com/apache/spark/runs/4001138656
https://github.com/apache/spark/runs/4002981360
https://github.com/apache/spark/runs/4006594260
Possibly related: https://github.com/apache/spark/runs/4008251656
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
