Repository: spark
Updated Branches:
refs/heads/master 70ffff21f -> 586d19822
[SPARK-15844][CORE] HistoryServer doesn't come up if spark.authenticate = true
## What changes were proposed in this pull request?
During history server startup, the spark configuration is examined. If
security.authentication is
set, log at debug and set the value to false, so that {{SecurityManager}} can
be created.
## How was this patch tested?
A new test in `HistoryServerSuite` sets the `spark.authenticate` property to
true, tries to create a security manager via a new package-private method
`HistoryServer.createSecurityManager(SparkConf)`. This is the method used in
`HistoryServer.main`. All other instantiations of a security manager in
`HistoryServerSuite` have been switched to the new method, for consistency with
the production code.
Author: Steve Loughran <[email protected]>
Closes #13579 from steveloughran/history/SPARK-15844-security.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/586d1982
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/586d1982
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/586d1982
Branch: refs/heads/master
Commit: 586d19822810129c6cb840c2bd4464ad34c88458
Parents: 70ffff2
Author: Steve Loughran <[email protected]>
Authored: Mon Dec 12 12:24:53 2016 -0800
Committer: Marcelo Vanzin <[email protected]>
Committed: Mon Dec 12 12:24:53 2016 -0800
----------------------------------------------------------------------
.../spark/deploy/history/HistoryServer.scala | 17 ++++++++++++++++-
.../spark/deploy/history/HistoryServerSuite.scala | 17 ++++++++++++++---
2 files changed, 30 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/586d1982/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
----------------------------------------------------------------------
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index 7e21fa6..2b00a4a 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -269,7 +269,7 @@ object HistoryServer extends Logging {
Utils.initDaemon(log)
new HistoryServerArguments(conf, argStrings)
initSecurity()
- val securityManager = new SecurityManager(conf)
+ val securityManager = createSecurityManager(conf)
val providerName = conf.getOption("spark.history.provider")
.getOrElse(classOf[FsHistoryProvider].getName())
@@ -289,6 +289,21 @@ object HistoryServer extends Logging {
while(true) { Thread.sleep(Int.MaxValue) }
}
+ /**
+ * Create a security manager.
+ * This turns off security in the SecurityManager, so that the the History
Server can start
+ * in a Spark cluster where security is enabled.
+ * @param config configuration for the SecurityManager constructor
+ * @return the security manager for use in constructing the History Server.
+ */
+ private[history] def createSecurityManager(config: SparkConf):
SecurityManager = {
+ if (config.getBoolean(SecurityManager.SPARK_AUTH_CONF, false)) {
+ logDebug(s"Clearing ${SecurityManager.SPARK_AUTH_CONF}")
+ config.set(SecurityManager.SPARK_AUTH_CONF, "false")
+ }
+ new SecurityManager(config)
+ }
+
def initSecurity() {
// If we are accessing HDFS and it has security enabled (Kerberos), we
have to login
// from a keytab file so that we can access HDFS beyond the kerberos
ticket expiration.
http://git-wip-us.apache.org/repos/asf/spark/blob/586d1982/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
----------------------------------------------------------------------
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 715811a..d3b79dd 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -75,7 +75,7 @@ class HistoryServerSuite extends SparkFunSuite with
BeforeAndAfter with Matchers
.set("spark.testing", "true")
provider = new FsHistoryProvider(conf)
provider.checkForLogs()
- val securityManager = new SecurityManager(conf)
+ val securityManager = HistoryServer.createSecurityManager(conf)
server = new HistoryServer(conf, provider, securityManager, 18080)
server.initialize()
@@ -288,7 +288,7 @@ class HistoryServerSuite extends SparkFunSuite with
BeforeAndAfter with Matchers
provider = new FsHistoryProvider(conf)
provider.checkForLogs()
- val securityManager = new SecurityManager(conf)
+ val securityManager = HistoryServer.createSecurityManager(conf)
server = new HistoryServer(conf, provider, securityManager, 18080)
server.initialize()
@@ -349,6 +349,17 @@ class HistoryServerSuite extends SparkFunSuite with
BeforeAndAfter with Matchers
}
+ /**
+ * Verify that the security manager needed for the history server can be
instantiated
+ * when `spark.authenticate` is `true`, rather than raise an
`IllegalArgumentException`.
+ */
+ test("security manager starts with spark.authenticate set") {
+ val conf = new SparkConf()
+ .set("spark.testing", "true")
+ .set(SecurityManager.SPARK_AUTH_CONF, "true")
+ HistoryServer.createSecurityManager(conf)
+ }
+
test("incomplete apps get refreshed") {
implicit val webDriver: WebDriver = new HtmlUnitDriver
@@ -368,7 +379,7 @@ class HistoryServerSuite extends SparkFunSuite with
BeforeAndAfter with Matchers
.set("spark.history.cache.window", "250ms")
.remove("spark.testing")
val provider = new FsHistoryProvider(myConf)
- val securityManager = new SecurityManager(myConf)
+ val securityManager = HistoryServer.createSecurityManager(myConf)
sc = new SparkContext("local", "test", myConf)
val logDirUri = logDir.toURI
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
