Repository: incubator-livy Updated Branches: refs/heads/master 8b3fcbc8d -> 02c5915f3
[LIVY-475] Support of Hadoop CredentialProvider API In this PR I've added following option to livy.conf: ``` ``` to allow to specify path to Hadoop Credential Provider, which than used in `WebServer.scala` to set Keystore password and Key password to JKS that used to enable SSL encryption. Also before trying to use Hadoop CredentialProvider API I'm checking if it available (as it was not available in Hadoop < 2.6) using the same [method that used in Oozie](https://github.com/apache/oozie/commit/6a731f9926158da38d1e3b518671ada95a544fe8#diff-800f95e605f21c5aaf5edef13039c9b9R124). To use this, you will need to generate Credential Provider containing "livy.keystore.password" and/or "livy.key-password" in the common way: ```bash hadoop credential create "livy.keystore.password" -value "keystore_secret" -provider jceks://hdfsnn1.example.com/my/path/livy_creds.jceks hadoop credential create "livy.key-password" -value "key_secret" -provider jceks://hdfsnn1.example.com/my/path/livy_creds.jceks ``` Author: Ivan Dzikovsky <idzikov...@protonmail.com> Closes #99 from idzikovsky/LIVY-475. Change-Id: Iae60550900067e61a37a6f74acb5e299005f3397 Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/02c5915f Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/02c5915f Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/02c5915f Branch: refs/heads/master Commit: 02c5915f38a0cd19acfc8222233cba37d73393f9 Parents: 8b3fcbc Author: Ivan Dzikovsky <idzikov...@protonmail.com> Authored: Thu May 31 19:53:25 2018 +0800 Committer: jerryshao <ss...@hortonworks.com> Committed: Thu May 31 19:53:25 2018 +0800 ---------------------------------------------------------------------- conf/livy.conf.template | 5 ++++ .../main/scala/org/apache/livy/LivyConf.scala | 2 ++ .../org/apache/livy/server/WebServer.scala | 24 ++++++++++++++++---- 3 files changed, 27 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/02c5915f/conf/livy.conf.template ---------------------------------------------------------------------- diff --git a/conf/livy.conf.template b/conf/livy.conf.template index 6f50e2f..fede70a 100644 --- a/conf/livy.conf.template +++ b/conf/livy.conf.template @@ -23,6 +23,11 @@ # Specify the key password. # livy.key-password = +# Hadoop Credential Provider Path to get "livy.keystore.password" and "livy.key-password". +# Credential Provider can be created using command as follow: +# hadoop credential create "livy.keystore.password" -value "secret" -provider jceks://hdfs/path/to/livy.jceks +# livy.hadoop.security.credential.provider.path = + # What host address to start the server on. By default, Livy will bind to all network interfaces. # livy.server.host = 0.0.0.0 http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/02c5915f/server/src/main/scala/org/apache/livy/LivyConf.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala index 3f6a86c..edee79a 100644 --- a/server/src/main/scala/org/apache/livy/LivyConf.scala +++ b/server/src/main/scala/org/apache/livy/LivyConf.scala @@ -85,6 +85,8 @@ object LivyConf { val SSL_KEYSTORE_PASSWORD = Entry("livy.keystore.password", null) val SSL_KEY_PASSWORD = Entry("livy.key-password", null) + val HADOOP_CREDENTIAL_PROVIDER_PATH = Entry("livy.hadoop.security.credential.provider.path", null) + val AUTH_TYPE = Entry("livy.server.auth.type", null) val AUTH_KERBEROS_PRINCIPAL = Entry("livy.server.auth.kerberos.principal", null) val AUTH_KERBEROS_KEYTAB = Entry("livy.server.auth.kerberos.keytab", null) http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/02c5915f/server/src/main/scala/org/apache/livy/server/WebServer.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/org/apache/livy/server/WebServer.scala b/server/src/main/scala/org/apache/livy/server/WebServer.scala index 549d6ab..1955974 100644 --- a/server/src/main/scala/org/apache/livy/server/WebServer.scala +++ b/server/src/main/scala/org/apache/livy/server/WebServer.scala @@ -20,6 +20,7 @@ package org.apache.livy.server import java.net.InetAddress import javax.servlet.ServletContextListener +import org.apache.hadoop.conf.Configuration import org.eclipse.jetty.server._ import org.eclipse.jetty.server.handler.{HandlerCollection, RequestLogHandler} import org.eclipse.jetty.servlet.{DefaultServlet, ServletContextHandler} @@ -48,10 +49,25 @@ class WebServer(livyConf: LivyConf, var host: String, var port: Int) extends Log val sslContextFactory = new SslContextFactory() sslContextFactory.setKeyStorePath(keystore) - Option(livyConf.get(LivyConf.SSL_KEYSTORE_PASSWORD)) - .foreach(sslContextFactory.setKeyStorePassword) - Option(livyConf.get(LivyConf.SSL_KEY_PASSWORD)) - .foreach(sslContextFactory.setKeyManagerPassword) + + val credentialProviderPath = livyConf.get(LivyConf.HADOOP_CREDENTIAL_PROVIDER_PATH) + val hadoopConf = new Configuration() + if (credentialProviderPath != null) { + hadoopConf.set("hadoop.security.credential.provider.path", credentialProviderPath) + } + + val keyStorePassword = Option(livyConf.get(LivyConf.SSL_KEYSTORE_PASSWORD)) + .orElse { + Option(hadoopConf.getPassword(LivyConf.SSL_KEYSTORE_PASSWORD.key)).map(_.mkString) + } + + val keyPassword = Option(livyConf.get(LivyConf.SSL_KEY_PASSWORD)) + .orElse { + Option(hadoopConf.getPassword(LivyConf.SSL_KEY_PASSWORD.key)).map(_.mkString) + } + + keyStorePassword.foreach(sslContextFactory.setKeyStorePassword) + keyPassword.foreach(sslContextFactory.setKeyManagerPassword) (new ServerConnector(server, new SslConnectionFactory(sslContextFactory, "http/1.1"),