Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/18555#discussion_r126274206
--- Diff:
core/src/main/scala/org/apache/spark/internal/config/package.scala ---
@@ -88,50 +137,76 @@ package object config {
.createOptional
private[spark] val PY_FILES = ConfigBuilder("spark.submit.pyFiles")
+ .doc("Comma-separated list of .zip, .egg, or .py files to place on " +
+ "the PYTHONPATH for Python apps.")
.internal()
.stringConf
.toSequence
.createWithDefault(Nil)
private[spark] val MAX_TASK_FAILURES =
ConfigBuilder("spark.task.maxFailures")
+ .doc("Number of failures of any particular task before giving up on
the job. " +
+ "The total number of failures spread across different tasks " +
+ "will not cause the job to fail; " +
+ "a particular task has to fail this number of attempts. " +
+ "Should be greater than or equal to 1. Number of allowed retries =
this value - 1.")
.intConf
+ .checkValue(_ > 0, "The retry times of task should be greater than
or equal to 1")
.createWithDefault(4)
// Blacklist confs
private[spark] val BLACKLIST_ENABLED =
ConfigBuilder("spark.blacklist.enabled")
+ .doc("If set to 'true', prevent Spark from scheduling tasks on
executors " +
+ "that have been blacklisted due to too many task failures. " +
+ "The blacklisting algorithm can be further controlled by " +
+ "the other 'spark.blacklist' configuration options. ")
.booleanConf
.createOptional
private[spark] val MAX_TASK_ATTEMPTS_PER_EXECUTOR =
ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerExecutor")
+ .doc("For a given task, how many times it can be " +
+ "retried on one executor before the executor is blacklisted for
that task.")
.intConf
+ .checkValue(_ > 0, "The value should be greater than or equal to 1")
.createWithDefault(1)
private[spark] val MAX_TASK_ATTEMPTS_PER_NODE =
ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerNode")
+ .doc("For a given task, how many times it can be " +
+ "retried on one node, before the entire node is blacklisted for
that task.")
.intConf
+ .checkValue(_ > 0, "The value should be greater than or equal to 1")
.createWithDefault(2)
private[spark] val MAX_FAILURES_PER_EXEC =
ConfigBuilder("spark.blacklist.application.maxFailedTasksPerExecutor")
.intConf
+ .checkValue(_ > 0, "The value should be greater than or equal to 1")
.createWithDefault(2)
private[spark] val MAX_FAILURES_PER_EXEC_STAGE =
ConfigBuilder("spark.blacklist.stage.maxFailedTasksPerExecutor")
+ .doc("How many different tasks must fail on one executor, " +
+ "within one stage, before the executor is blacklisted for that
stage.")
.intConf
+ .checkValue(_ > 0, "The value should be greater than or equal to 1")
.createWithDefault(2)
private[spark] val MAX_FAILED_EXEC_PER_NODE =
ConfigBuilder("spark.blacklist.application.maxFailedExecutorsPerNode")
.intConf
+ .checkValue(_ > 0, "The value should be greater than or equal to 1")
.createWithDefault(2)
private[spark] val MAX_FAILED_EXEC_PER_NODE_STAGE =
ConfigBuilder("spark.blacklist.stage.maxFailedExecutorsPerNode")
+ .doc("How many different executors are marked as blacklisted for a
given stage, " +
+ "before the entire node is marked as failed for the stage.")
.intConf
+ .checkValue(_ > 0, "The value should be greater than or equal to 1")
--- End diff --
ditto
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
