micheal-o commented on code in PR #47819:
URL: https://github.com/apache/spark/pull/47819#discussion_r1725936402
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala:
##########
@@ -142,8 +142,12 @@ class ForeachWriterSuite extends StreamTest with
SharedSparkSession with BeforeA
val e = intercept[StreamingQueryException] {
query.processAllAvailable()
}
- assert(e.getCause.isInstanceOf[SparkException])
- assert(e.getCause.getCause.getMessage === "ForeachSinkSuite error")
+
+ val errClass = "FOREACH_USER_FUNCTION_ERROR"
+
+ // verify that we classified the exception
+ assert(e.getMessage.contains(errClass))
Review Comment:
you can use `checkError()` func
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala:
##########
@@ -157,6 +161,21 @@ class ForeachWriterSuite extends StreamTest with
SharedSparkSession with BeforeA
assert(errorEvent.error.get.getMessage === "ForeachSinkSuite error")
// 'close' shouldn't be called with abort message if close with error
has been called
assert(allEvents(0).size == 3)
+
+ val e2 = intercept[StreamingQueryException] {
+ val query2 = input.toDS().repartition(1).writeStream
+ .foreach(new TestForeachWriter() {
+ override def process(value: Int): Unit = {
+ super.process(value)
+ throw ExecutorDeadException("network error")
+ }
+ }).start()
+ query2.processAllAvailable()
+ }
+
+ // we didn't wrap the spark exception
+ assert(!e2.getMessage.contains(errClass))
+ assert(e2.getCause.getMessage.contains("network error"))
Review Comment:
also check causing ex type
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala:
##########
@@ -142,8 +142,12 @@ class ForeachWriterSuite extends StreamTest with
SharedSparkSession with BeforeA
val e = intercept[StreamingQueryException] {
query.processAllAvailable()
}
- assert(e.getCause.isInstanceOf[SparkException])
- assert(e.getCause.getCause.getMessage === "ForeachSinkSuite error")
+
+ val errClass = "FOREACH_USER_FUNCTION_ERROR"
+
+ // verify that we classified the exception
+ assert(e.getMessage.contains(errClass))
+ assert(e.getCause.getMessage.contains("ForeachSinkSuite error"))
Review Comment:
lets also assert the causing error is the expected exception type
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
