kevin85421 commented on code in PR #37400:
URL: https://github.com/apache/spark/pull/37400#discussion_r948224828
##########
core/src/test/scala/org/apache/spark/SparkContextSuite.scala:
##########
@@ -1354,6 +1355,50 @@ class SparkContextSuite extends SparkFunSuite with
LocalSparkContext with Eventu
}.getMessage
assert(msg.contains("Cannot use the keyword 'proxy' or 'history' in
reverse proxy URL"))
}
+
+ test("ExitCode HEARTBEAT_FAILURE should be counted as network failure") {
+ // This test is used to prove that driver will receive executorExitCode
before onDisconnected
+ // removes the executor. If the executor is removed by onDisconnected, the
executor loss will be
+ // considered as a task failure. Spark will throw a SparkException because
TASK_MAX_FAILURES is
+ // 1. On the other hand, driver removes executor with exitCode
HEARTBEAT_FAILURE, the loss
+ // should be counted as network failure, and thus the job should not throw
SparkException.
+
+ val conf = new SparkConf().set(TASK_MAX_FAILURES, 1)
+ val sc = new SparkContext("local-cluster[1, 1, 1024]",
"test-exit-code-heartbeat", conf)
+ val result = sc.parallelize(1 to 10, 1).map { x =>
+ val context = org.apache.spark.TaskContext.get()
+ if (context.taskAttemptId() == 0) {
+ System.exit(ExecutorExitCode.HEARTBEAT_FAILURE)
+ } else {
+ x
+ }
+ }.count()
+ assert(result == 10L)
+ sc.stop()
+ }
+
+ test("ExitCode HEARTBEAT_FAILURE will be counted as task failure when
EXECUTOR_REMOVE_DELAY is" +
Review Comment:
Updated
https://github.com/apache/spark/pull/37400/commits/80b86910a59f5b409caa17d46305027ed89c3dd1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
