Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/4505#discussion_r25010835
--- Diff:
core/src/main/scala/org/apache/spark/deploy/worker/WorkerArguments.scala ---
@@ -23,46 +23,56 @@ import org.apache.spark.util.{IntParam, MemoryParam,
Utils}
import org.apache.spark.SparkConf
/**
- * Command-line parser for the worker.
+ * Arguments parser for the worker.
*/
private[spark] class WorkerArguments(args: Array[String], conf: SparkConf)
{
- var host = Utils.localHostName()
- var port = 0
- var webUiPort = 8081
- var cores = inferDefaultCores()
- var memory = inferDefaultMemory()
+ var host: String = null
+ var port: Int = -1
+ var webUiPort: Int = -1
+ var cores: Int = -1
+ var memory: Int = -1
var masters: Array[String] = null
var workDir: String = null
var propertiesFile: String = null
- // Check for settings in environment variables
- if (System.getenv("SPARK_WORKER_PORT") != null) {
- port = System.getenv("SPARK_WORKER_PORT").toInt
- }
- if (System.getenv("SPARK_WORKER_CORES") != null) {
- cores = System.getenv("SPARK_WORKER_CORES").toInt
- }
- if (conf.getenv("SPARK_WORKER_MEMORY") != null) {
- memory = Utils.memoryStringToMb(conf.getenv("SPARK_WORKER_MEMORY"))
- }
- if (System.getenv("SPARK_WORKER_WEBUI_PORT") != null) {
- webUiPort = System.getenv("SPARK_WORKER_WEBUI_PORT").toInt
- }
- if (System.getenv("SPARK_WORKER_DIR") != null) {
- workDir = System.getenv("SPARK_WORKER_DIR")
- }
-
parse(args.toList)
// This mutates the SparkConf, so all accesses to it must be made after
this line
propertiesFile = Utils.loadDefaultSparkProperties(conf, propertiesFile)
- if (conf.contains("spark.worker.ui.port")) {
- webUiPort = conf.get("spark.worker.ui.port").toInt
- }
-
+ loadEnvironmentArguments()
checkWorkerMemory()
+ /**
+ * Load arguments from environment variables, Spark properties etc.
+ */
+ private def loadEnvironmentArguments(): Unit = {
+ host = Option(host)
+ .orElse(conf.getOption("spark.worker.host"))
+ .getOrElse(Utils.localHostName())
+ if (port < 0) {
+ port = conf.getOption("spark.worker.port").map(_.toInt)
+ .orElse(Option(conf.getenv("SPARK_WORKER_PORT")).map(_.toInt))
+ .getOrElse(0)
--- End diff --
why not just `map(_.toInt)` after the `orElse`? Less duplicate code there
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
