HyukjinKwon commented on a change in pull request #34451:
URL: https://github.com/apache/spark/pull/34451#discussion_r752830334
##########
File path:
external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
##########
@@ -284,4 +289,109 @@ private[v2] trait V2JDBCTest extends SharedSparkSession
with DockerIntegrationFu
testIndexUsingSQL(s"$catalogName.new_table")
}
}
+
+ def supportsTableSample: Boolean = false
+
+ private def samplePushed(df: DataFrame): Boolean = {
+ val sample = df.queryExecution.optimizedPlan.collect {
+ case s: Sample => s
+ }
+ sample.isEmpty
+ }
+
+ private def filterPushed(df: DataFrame): Boolean = {
+ val filter = df.queryExecution.optimizedPlan.collect {
+ case f: Filter => f
+ }
+ filter.isEmpty
+ }
+
+ private def limitPushed(df: DataFrame, limit: Int): Boolean = {
+ val filter = df.queryExecution.optimizedPlan.collect {
+ case relation: DataSourceV2ScanRelation => relation.scan match {
+ case v1: V1ScanWrapper =>
+ return v1.pushedDownOperators.limit == Some(limit)
+ }
+ }
+ false
+ }
+
+ private def columnPruned(df: DataFrame, col: String): Boolean = {
+ val scan = df.queryExecution.optimizedPlan.collectFirst {
+ case s: DataSourceV2ScanRelation => s
+ }.get
+ scan.schema.names.sameElements(Seq(col))
+ }
+
+ test("SPARK-37038: Test TABLESAMPLE") {
Review comment:
This test is flaky:
```
- SPARK-37038: Test TABLESAMPLE *** FAILED *** (972 milliseconds)
Array([18]) had length 1 instead of expected length 2
(V2JDBCTest.scala:330)
org.scalatest.exceptions.TestFailedException:
at
org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:472)
at
org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:471)
at
org.scalatest.Assertions$.newAssertionFailedException(Assertions.scala:1231)
at
org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:1295)
at
org.apache.spark.sql.jdbc.v2.V2JDBCTest.$anonfun$$init$$35(V2JDBCTest.scala:330)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1468)
at
org.apache.spark.sql.test.SQLTestUtilsBase.withTable(SQLTestUtils.scala:306)
at
org.apache.spark.sql.test.SQLTestUtilsBase.withTable$(SQLTestUtils.scala:304)
at
org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.withTable(DockerJDBCIntegrationSuite.scala:95)
at
org.apache.spark.sql.jdbc.v2.V2JDBCTest.$anonfun$$init$$34(V2JDBCTest.scala:280)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at
org.scalatest.funsuite.AnyFunSuiteLike$$anon$1.apply(AnyFunSuiteLike.scala:226)
at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:190)
at
org.scalatest.funsuite.AnyFunSuiteLike.invokeWithFixture$1(AnyFunSuiteLike.scala:224)
at
org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTest$1(AnyFunSuiteLike.scala:236)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
```
https://github.com/apache/spark/runs/4259414295?check_suite_focus=true
I saw this few times but retriggered all of them so I can't fine now 😢 . I
am going to monitor this a bit more but thought it's worth mentioning :-).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
