This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new bf893624161c [SPARK-50397][CORE] Remove deprecated `--ip` and `-i`
arguments from `Master/Worker`
bf893624161c is described below
commit bf893624161c017f80c0563b97a05fec455c9454
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Nov 24 18:26:04 2024 -0800
[SPARK-50397][CORE] Remove deprecated `--ip` and `-i` arguments from
`Master/Worker`
### What changes were proposed in this pull request?
This PR aims to remove `--ip` and `-i` arguments and `SPARK_MASTER_IP`
environment variable from `Master/Worker` from Apache Spark 4.0.0.
### Why are the changes needed?
These arguments were deprecated on Apr 15, 2013 before
[v0.8.0-incubating](https://github.com/apache/spark/releases/tag/v0.8.0-incubating)
and have never been used in Apache Spark code base.
-
https://github.com/apache/spark/commit/d90d2af1036e909f81cf77c85bfe589993c4f9f3#diff-6a66898887ff9c00a900384bdc2a0c098a4e97a302fecf484b46814a5371210a
```
Utils.checkHost(value, "ip no longer supported, please use hostname " +
value)
```
### Does this PR introduce _any_ user-facing change?
Yes, however, I believe these incubating-era arguments are not only
misleading (because IPs are not hostnames) but also not required in Apache
Spark 4.0.0 users.
### How was this patch tested?
Manual review.
**BEFORE**
```
$ sbin/start-master.sh --help
Usage: ./sbin/start-master.sh [options]
Options:
-i HOST, --ip HOST Hostname to listen on (deprecated, please use
--host or -h)
-h HOST, --host HOST Hostname to listen on
-p PORT, --port PORT Port to listen on (default: 7077)
--webui-port PORT Port for web UI (default: 8080)
--properties-file FILE Path to a custom Spark properties file.
Default is conf/spark-defaults.conf.
```
**AFTER**
```
$ sbin/start-master.sh --help
Usage: ./sbin/start-master.sh [options]
Options:
-h HOST, --host HOST Hostname to listen on
-p PORT, --port PORT Port to listen on (default: 7077)
--webui-port PORT Port for web UI (default: 8080)
--properties-file FILE Path to a custom Spark properties file.
Default is conf/spark-defaults.conf.
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48938 from dongjoon-hyun/SPARK-50397.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/deploy/master/MasterArguments.scala | 12 ------------
.../org/apache/spark/deploy/worker/WorkerArguments.scala | 6 ------
docs/spark-standalone.md | 4 ----
3 files changed, 22 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/MasterArguments.scala
b/core/src/main/scala/org/apache/spark/deploy/master/MasterArguments.scala
index 6647b11874d7..0904581d7236 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/MasterArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/MasterArguments.scala
@@ -33,12 +33,6 @@ private[master] class MasterArguments(args: Array[String],
conf: SparkConf) exte
var webUiPort = 8080
var propertiesFile: String = null
- // Check for settings in environment variables
- if (System.getenv("SPARK_MASTER_IP") != null) {
- logWarning("SPARK_MASTER_IP is deprecated, please use SPARK_MASTER_HOST")
- host = System.getenv("SPARK_MASTER_IP")
- }
-
if (System.getenv("SPARK_MASTER_HOST") != null) {
host = System.getenv("SPARK_MASTER_HOST")
}
@@ -63,11 +57,6 @@ private[master] class MasterArguments(args: Array[String],
conf: SparkConf) exte
@tailrec
private def parse(args: List[String]): Unit = args match {
- case ("--ip" | "-i") :: value :: tail =>
- Utils.checkHost(value)
- host = value
- parse(tail)
-
case ("--host" | "-h") :: value :: tail =>
Utils.checkHost(value)
host = value
@@ -103,7 +92,6 @@ private[master] class MasterArguments(args: Array[String],
conf: SparkConf) exte
"Usage: Master [options]\n" +
"\n" +
"Options:\n" +
- " -i HOST, --ip HOST Hostname to listen on (deprecated, please use
--host or -h) \n" +
" -h HOST, --host HOST Hostname to listen on\n" +
" -p PORT, --port PORT Port to listen on (default: 7077)\n" +
" --webui-port PORT Port for web UI (default: 8080)\n" +
diff --git
a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerArguments.scala
b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerArguments.scala
index f24cd5941830..87ca01fe82a9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerArguments.scala
@@ -70,11 +70,6 @@ private[worker] class WorkerArguments(args: Array[String],
conf: SparkConf) {
@tailrec
private def parse(args: List[String]): Unit = args match {
- case ("--ip" | "-i") :: value :: tail =>
- Utils.checkHost(value)
- host = value
- parse(tail)
-
case ("--host" | "-h") :: value :: tail =>
Utils.checkHost(value)
host = value
@@ -137,7 +132,6 @@ private[worker] class WorkerArguments(args: Array[String],
conf: SparkConf) {
" -c CORES, --cores CORES Number of cores to use\n" +
" -m MEM, --memory MEM Amount of memory to use (e.g. 1000M, 2G)\n" +
" -d DIR, --work-dir DIR Directory to run apps in (default:
SPARK_HOME/work)\n" +
- " -i HOST, --ip IP Hostname to listen on (deprecated, please
use --host or -h)\n" +
" -h HOST, --host HOST Hostname to listen on\n" +
" -p PORT, --port PORT Port to listen on (default: random)\n" +
" --webui-port PORT Port for web UI (default: 8081)\n" +
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 8bc7445d17c7..d828436e7734 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -59,10 +59,6 @@ Finally, the following configuration options can be passed
to the master and wor
<td><code>-h HOST</code>, <code>--host HOST</code></td>
<td>Hostname to listen on</td>
</tr>
- <tr>
- <td><code>-i HOST</code>, <code>--ip HOST</code></td>
- <td>Hostname to listen on (deprecated, use -h or --host)</td>
- </tr>
<tr>
<td><code>-p PORT</code>, <code>--port PORT</code></td>
<td>Port for service to listen on (default: 7077 for master, random for
worker)</td>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
