HyukjinKwon commented on code in PR #38422:
URL: https://github.com/apache/spark/pull/38422#discussion_r1016103772
##########
sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala:
##########
@@ -2322,15 +2322,21 @@ class DataFrameSuite extends QueryTest
test("SPARK-13774: Check error message for non existent path without globbed
paths") {
val uuid = UUID.randomUUID().toString
val baseDir = Utils.createTempDir()
+ val filePath1 = new File(baseDir, "file").getAbsolutePath
+ val filePath2 = new File(baseDir, "file2").getAbsolutePath
+ val filePath3 = new File(uuid, "file3").getAbsolutePath
try {
- val e = intercept[AnalysisException] {
- spark.read.format("csv").load(
- new File(baseDir, "file").getAbsolutePath,
- new File(baseDir, "file2").getAbsolutePath,
- new File(uuid, "file3").getAbsolutePath,
- uuid).rdd
- }
- assert(e.getMessage.startsWith("Path does not exist"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ spark.read.format("csv").load(
+ filePath1,
+ filePath2,
+ filePath3,
+ uuid).rdd
+ },
+ errorClass = "PATH_NOT_FOUND",
+ parameters = Map("path" -> s"file:$filePath1")
Review Comment:
This fails with Scala 2.13 because the order of files returned by glob can
be indeterministic:
https://github.com/apache/spark/actions/runs/3413245795/jobs/5679750240
```
- SPARK-13774: Check error message for non existent path without globbed
paths *** FAILED *** (25 milliseconds)
Map("path" ->
"file:/home/runner/work/spark/spark/sql/core/ca57e8c5-89dc-4b3e-b5ee-b5e3b2009168")
did not equal Map("path" ->
"file:/home/runner/work/spark/spark/target/tmp/spark-1d80d6a1-157c-4c91-84ca-c3016237aec6/file")
(SparkFunSuite.scala:317)
Analysis:
JavaCollectionWrappers$JMapWrapper(path:
file:/home/runner/work/spark/spark/sql/core/ca57e8c5-89dc-4b3e-b5ee-b5e3b2009168
->
file:/home/runner/work/spark/spark/target/tmp/spark-1d80d6a1-157c-4c91-84ca-c3016237aec6/file)
org.scalatest.exceptions.TestFailedException:
at
org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:472)
at
org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:471)
at
org.scalatest.Assertions$.newAssertionFailedException(Assertions.scala:1231)
at
org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:1295)
at org.apache.spark.SparkFunSuite.checkError(SparkFunSuite.scala:317)
at
org.apache.spark.sql.DataFrameSuite.$anonfun$new$465(DataFrameSuite.scala:2329)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at
org.scalatest.funsuite.AnyFunSuiteLike$$anon$1.apply(AnyFunSuiteLike.scala:226)
at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:207)
at
org.scalatest.funsuite.AnyFunSuiteLike.invokeWithFixture$1(AnyFunSuiteLike.scala:224)
at
org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTest$1(AnyFunSuiteLike.scala:236)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
at
org.scalatest.funsuite.AnyFunSuiteLike.runTest(AnyFunSuiteLike.scala:236)
at
org.scalatest.funsuite.AnyFunSuiteLike.runTest$(AnyFunSuiteLike.scala:218)
at
org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:66)
at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:234)
at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:227)
at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:66)
at
org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTests$1(AnyFunSuiteLike.scala:269)
at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:413)
at scala.collection.immutable.List.foreach(List.scala:333)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:396)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:475)
at
org.scalatest.funsuite.AnyFunSuiteLike.runTests(AnyFunSuiteLike.scala:269)
at
org.scalatest.funsuite.AnyFunSuiteLike.runTests$(AnyFunSuiteLike.scala:268)
at org.scalatest.funsuite.AnyFunSuite.runTests(AnyFunSuite.scala:1564)
at org.scalatest.Suite.run(Suite.scala:1114)
at org.scalatest.Suite.run$(Suite.scala:1096)
at
org.scalatest.funsuite.AnyFunSuite.org$scalatest$funsuite$AnyFunSuiteLike$$super$run(AnyFunSuite.scala:1564)
at
org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$run$1(AnyFunSuiteLike.scala:273)
at org.scalatest.SuperEngine.runImpl(Engine.scala:535)
at org.scalatest.funsuite.AnyFunSuiteLike.run(AnyFunSuiteLike.scala:273)
at org.scalatest.funsuite.AnyFunSuiteLike.run$(AnyFunSuiteLike.scala:272)
at
org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:66)
at
org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:66)
at
org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321)
at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517)
at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:413)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
