Repository: spark
Updated Branches:
refs/heads/master 2eaf05878 -> 26c1b959c
[SPARK-25711][CORE] Improve start-history-server.sh: show usage User-Friendly
and remove deprecated options
## What changes were proposed in this pull request?
Currently, if we try run
```
./start-history-server.sh -h
```
We will get such error
```
java.io.FileNotFoundException: File -h does not exist
```
1. This is not User-Friendly. For option `-h` or `--help`, it should be parsed
correctly and show the usage of the class/script.
2. We can remove deprecated options for setting event log directory through
command line options.
After fix, we can get following output:
```
Usage: ./sbin/start-history-server.sh [options]
Options:
--properties-file FILE Path to a custom Spark properties file.
Default is conf/spark-defaults.conf.
Configuration options can be set by setting the corresponding JVM system
property.
History Server options are always available; additional options depend on the
provider.
History Server options:
spark.history.ui.port Port where server will listen for
connections
(default 18080)
spark.history.acls.enable Whether to enable view acls for all
applications
(default false)
spark.history.provider Name of history provider class (defaults to
file system-based provider)
spark.history.retainedApplications Max number of application UIs to keep
loaded in memory
(default 50)
FsHistoryProvider options:
spark.history.fs.logDirectory Directory where app logs are stored
(default: file:/tmp/spark-events)
spark.history.fs.updateInterval How often to reload log data from storage
(in seconds, default: 10)
```
## How was this patch tested?
Manual test
Closes #22699 from gengliangwang/refactorSHSUsage.
Authored-by: Gengliang Wang <gengliang.w...@databricks.com>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/26c1b959
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/26c1b959
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/26c1b959
Branch: refs/heads/master
Commit: 26c1b959cf29b8552beb715cc5d39288d5298bdc
Parents: 2eaf058
Author: Gengliang Wang <gengliang.w...@databricks.com>
Authored: Sat Oct 13 13:34:31 2018 -0700
Committer: Dongjoon Hyun <dongj...@apache.org>
Committed: Sat Oct 13 13:34:31 2018 -0700
----------------------------------------------------------------------
.../deploy/history/HistoryServerArguments.scala | 34 ++++++--------------
.../history/HistoryServerArgumentsSuite.scala | 12 -------
sbin/start-history-server.sh | 17 +++++++++-
3 files changed, 25 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/26c1b959/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
----------------------------------------------------------------------
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
index 080ba12..49f00cb 100644
---
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
@@ -34,35 +34,21 @@ private[history] class HistoryServerArguments(conf:
SparkConf, args: Array[Strin
@tailrec
private def parse(args: List[String]): Unit = {
- if (args.length == 1) {
- setLogDirectory(args.head)
- } else {
- args match {
- case ("--dir" | "-d") :: value :: tail =>
- setLogDirectory(value)
- parse(tail)
+ args match {
+ case ("--help" | "-h") :: tail =>
+ printUsageAndExit(0)
- case ("--help" | "-h") :: tail =>
- printUsageAndExit(0)
+ case ("--properties-file") :: value :: tail =>
+ propertiesFile = value
+ parse(tail)
- case ("--properties-file") :: value :: tail =>
- propertiesFile = value
- parse(tail)
+ case Nil =>
- case Nil =>
-
- case _ =>
- printUsageAndExit(1)
- }
+ case _ =>
+ printUsageAndExit(1)
}
}
- private def setLogDirectory(value: String): Unit = {
- logWarning("Setting log directory through the command line is deprecated
as of " +
- "Spark 1.1.0. Please set this through spark.history.fs.logDirectory
instead.")
- conf.set("spark.history.fs.logDirectory", value)
- }
-
// This mutates the SparkConf, so all accesses to it must be made after
this line
Utils.loadDefaultSparkProperties(conf, propertiesFile)
@@ -73,8 +59,6 @@ private[history] class HistoryServerArguments(conf:
SparkConf, args: Array[Strin
|Usage: HistoryServer [options]
|
|Options:
- | DIR Deprecated; set
spark.history.fs.logDirectory directly
- | --dir DIR (-d DIR) Deprecated; set
spark.history.fs.logDirectory directly
| --properties-file FILE Path to a custom Spark properties file.
| Default is conf/spark-defaults.conf.
|
http://git-wip-us.apache.org/repos/asf/spark/blob/26c1b959/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerArgumentsSuite.scala
----------------------------------------------------------------------
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerArgumentsSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerArgumentsSuite.scala
index de321db..3795482 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerArgumentsSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerArgumentsSuite.scala
@@ -40,18 +40,6 @@ class HistoryServerArgumentsSuite extends SparkFunSuite {
assert(conf.get("spark.testing") === "true")
}
- test("Directory Arguments Parsing --dir or -d") {
- val argStrings = Array("--dir", "src/test/resources/spark-events1")
- val hsa = new HistoryServerArguments(conf, argStrings)
- assert(conf.get("spark.history.fs.logDirectory") ===
"src/test/resources/spark-events1")
- }
-
- test("Directory Param can also be set directly") {
- val argStrings = Array("src/test/resources/spark-events2")
- val hsa = new HistoryServerArguments(conf, argStrings)
- assert(conf.get("spark.history.fs.logDirectory") ===
"src/test/resources/spark-events2")
- }
-
test("Properties File Arguments Parsing --properties-file") {
val tmpDir = Utils.createTempDir()
val outFile = File.createTempFile("test-load-spark-properties", "test",
tmpDir)
http://git-wip-us.apache.org/repos/asf/spark/blob/26c1b959/sbin/start-history-server.sh
----------------------------------------------------------------------
diff --git a/sbin/start-history-server.sh b/sbin/start-history-server.sh
index 38a43b9..71dace4 100755
--- a/sbin/start-history-server.sh
+++ b/sbin/start-history-server.sh
@@ -28,7 +28,22 @@ if [ -z "${SPARK_HOME}" ]; then
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
fi
+# NOTE: This exact class name is matched downstream by SparkSubmit.
+# Any changes need to be reflected there.
+CLASS="org.apache.spark.deploy.history.HistoryServer"
+
+if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
+ echo "Usage: ./sbin/start-history-server.sh [options]"
+ pattern="Usage:"
+ pattern+="\|Using Spark's default log4j profile:"
+ pattern+="\|Started daemon with process name"
+ pattern+="\|Registered signal handler for"
+
+ "${SPARK_HOME}"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2
+ exit 1
+fi
+
. "${SPARK_HOME}/sbin/spark-config.sh"
. "${SPARK_HOME}/bin/load-spark-env.sh"
-exec "${SPARK_HOME}/sbin"/spark-daemon.sh start
org.apache.spark.deploy.history.HistoryServer 1 "$@"
+exec "${SPARK_HOME}/sbin"/spark-daemon.sh start $CLASS 1 "$@"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
