This is an automated email from the ASF dual-hosted git repository.
gyogal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/livy.git
The following commit(s) were added to refs/heads/master by this push:
new a676d999 [LIVY-1060] Livy Spark sessions fail to connect to Hive
Metastore over SSL with keystore password error
a676d999 is described below
commit a676d999c7101ef4bb12c863536ff244e10cf420
Author: nileshrathi345 <[email protected]>
AuthorDate: Tue Aug 4 19:37:31 2026 +0530
[LIVY-1060] Livy Spark sessions fail to connect to Hive Metastore over SSL
with keystore password error
## What changes were proposed in this pull request?
Resolves Hive Metastore SSL credential handling for Spark sessions/jobs
submitted by Livy.
- Added `enrichHiveMetastoreSslConf` in `SparkApp.prepareSparkConf` to read
`hive.metastore.keystore.password` and `hive.metastore.truststore.password`
from Livy’s local credential provider
(`livy.hadoop.security.credential.provider.path`) and inject them as
`spark.hadoop.*` properties for the Spark app.
- Ensures YARN/K8s driver and executors get explicit HMS SSL passwords
without needing access to the Livy host’s local JCEKS file.
- Preserves the existing credential provider path in Spark conf (fix for
interactive session creation regression from blanking the path).
JIRA: https://issues.apache.org/jira/browse/LIVY-1060
## How was this patch tested?
- Added `SparkAppSpec` unit tests for no-op, successful HMS SSL resolution,
unrelated credentials, and unreadable provider path cases.
- Verified credential resolution from a local JCEKS test store and no-op
paths when provider is missing, unreadable, or has no HMS aliases.
## Was this patch authored or co-authored using generative AI tooling?
No
---
.../scala/org/apache/livy/utils/SparkApp.scala | 56 +++++++++-
.../scala/org/apache/livy/utils/SparkAppSpec.scala | 114 +++++++++++++++++++++
2 files changed, 167 insertions(+), 3 deletions(-)
diff --git a/server/src/main/scala/org/apache/livy/utils/SparkApp.scala
b/server/src/main/scala/org/apache/livy/utils/SparkApp.scala
index e424f80f..acb2b0e1 100644
--- a/server/src/main/scala/org/apache/livy/utils/SparkApp.scala
+++ b/server/src/main/scala/org/apache/livy/utils/SparkApp.scala
@@ -17,9 +17,13 @@
package org.apache.livy.utils
+import java.io.IOException
+
import scala.collection.JavaConverters._
-import org.apache.livy.LivyConf
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.livy.{LivyConf, Logging}
object AppInfo {
val DRIVER_LOG_URL_NAME = "driverLogUrl"
@@ -54,7 +58,7 @@ trait SparkAppListener {
/**
* Provide factory methods for SparkApp.
*/
-object SparkApp {
+object SparkApp extends Logging {
private val SPARK_YARN_TAG_KEY = "spark.yarn.tags"
object State extends Enumeration {
@@ -73,7 +77,7 @@ object SparkApp {
uniqueAppTag: String,
livyConf: LivyConf,
sparkConf: Map[String, String]): Map[String, String] = {
- if (livyConf.isRunningOnYarn()) {
+ val baseConf = if (livyConf.isRunningOnYarn()) {
val userYarnTags = sparkConf.get(SPARK_YARN_TAG_KEY).map("," +
_).getOrElse("")
val mergedYarnTags = uniqueAppTag + userYarnTags
sparkConf ++ Map(
@@ -89,6 +93,52 @@ object SparkApp {
} else {
sparkConf
}
+ enrichHiveMetastoreSslConf(baseConf, livyConf)
+ }
+
+ /**
+ * Reads HMS SSL passwords from LivyConf's local JCEKS provider and injects
them as
+ * spark.hadoop.* properties for YARN/K8s drivers and executors. The
incoming Spark
+ * conf (including any user-supplied credential provider path) is preserved;
only
+ * HMS SSL properties are merged via `conf ++ sslProps`.
+ *
+ * No-op if no credential provider path is configured, or if it resolves to
+ * unrelated (non-HMS) secrets; in either case the original conf is returned
+ * untouched so any user-supplied provider path (e.g. an HDFS-backed one used
+ * for other credentials) is preserved.
+ */
+ private def enrichHiveMetastoreSslConf(
+ conf: Map[String, String],
+ livyConf: LivyConf): Map[String, String] = {
+ val credentialProviderPath =
livyConf.get(LivyConf.HADOOP_CREDENTIAL_PROVIDER_PATH)
+ if (credentialProviderPath == null || credentialProviderPath.isEmpty) {
+ return conf
+ }
+ val hadoopConf = new Configuration()
+ hadoopConf.set("hadoop.security.credential.provider.path",
credentialProviderPath)
+ val hmsSslKeys = Seq("hive.metastore.keystore.password",
"hive.metastore.truststore.password")
+ val sslProps =
+ try {
+ hmsSslKeys.flatMap { key =>
+ Option(hadoopConf.getPassword(key)).map(_.mkString) match {
+ case Some(password) if password.nonEmpty =>
Some(s"spark.hadoop.$key" -> password)
+ case _ => None
+ }
+ }.toMap
+ } catch {
+ case e: IOException =>
+ warn(s"Could not resolve Hive Metastore SSL credentials from
provider path " +
+ s"$credentialProviderPath: ${e.getMessage}")
+ Map.empty[String, String]
+ }
+
+ if (sslProps.isEmpty) {
+ // Nothing HMS-related found (or resolution failed); leave the user's
+ // provider path config exactly as it was.
+ conf
+ } else {
+ conf ++ sslProps
+ }
}
/**
diff --git a/server/src/test/scala/org/apache/livy/utils/SparkAppSpec.scala
b/server/src/test/scala/org/apache/livy/utils/SparkAppSpec.scala
new file mode 100644
index 00000000..94026029
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/utils/SparkAppSpec.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.utils
+
+import java.nio.file.{Files, Path}
+
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.security.alias.CredentialProviderFactory
+import org.scalatest.FunSpec
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+
+class SparkAppSpec extends FunSpec with LivyBaseUnitTestSuite {
+
+ private val providerPathKey =
"spark.hadoop.hadoop.security.credential.provider.path"
+ private val truststorePasswordKey =
"spark.hadoop.hive.metastore.truststore.password"
+
+ private def deleteRecursively(path: Path): Unit = {
+ if (Files.exists(path)) {
+
Files.walk(path).iterator().asScala.toSeq.reverse.foreach(Files.deleteIfExists)
+ }
+ }
+
+ private def withCredentialStore(secrets: Map[String, String])
+ (f: LivyConf => Unit): Unit = {
+ val jceksDir = Files.createTempDirectory("livy-cred-test")
+ val jceksPath = jceksDir.resolve("credentials.jceks")
+ val providerPath = s"jceks://file${jceksPath.toAbsolutePath}"
+
+ val hadoopConf = new Configuration()
+ hadoopConf.set("hadoop.security.credential.provider.path", providerPath)
+ val provider = CredentialProviderFactory.getProviders(hadoopConf).get(0)
+ secrets.foreach { case (alias, value) =>
+ provider.createCredentialEntry(alias, value.toCharArray)
+ }
+ provider.flush()
+
+ val livyConf = new LivyConf(false)
+ livyConf.set(LivyConf.HADOOP_CREDENTIAL_PROVIDER_PATH, providerPath)
+ try {
+ f(livyConf)
+ } finally {
+ deleteRecursively(jceksDir)
+ }
+ }
+
+ describe("SparkApp.prepareSparkConf") {
+ it("should leave conf unchanged when no credential provider path is
configured") {
+ val livyConf = new LivyConf(false)
+ livyConf.set(LivyConf.LIVY_SPARK_MASTER, "local")
+ val inputConf = Map("spark.app.name" -> "test-app")
+
+ val result = SparkApp.prepareSparkConf("livy-test-tag", livyConf,
inputConf)
+
+ assert(result === inputConf)
+ }
+
+ it("should resolve HMS SSL credentials locally and preserve the provider
path") {
+ withCredentialStore(Map("hive.metastore.truststore.password" ->
"trust-secret")) { livyConf =>
+ livyConf.set(LivyConf.LIVY_SPARK_MASTER, "local")
+ val inputConf = Map(
+ "spark.app.name" -> "test-app",
+ providerPathKey -> "jceks://file/local-only-path")
+
+ val result = SparkApp.prepareSparkConf("livy-test-tag", livyConf,
inputConf)
+
+ assert(result(truststorePasswordKey) === "trust-secret")
+ assert(result(providerPathKey) === "jceks://file/local-only-path") //
preserved
+ assert(result("spark.app.name") === "test-app")
+ }
+ }
+
+ it("should leave conf unchanged when JCEKS has no HMS SSL aliases") {
+ withCredentialStore(Map("livy.keystore.password" -> "livy-secret")) {
livyConf =>
+ livyConf.set(LivyConf.LIVY_SPARK_MASTER, "local")
+ val inputConf = Map(
+ "spark.app.name" -> "test-app",
+ providerPathKey -> "jceks://file/user-supplied")
+
+ val result = SparkApp.prepareSparkConf("livy-test-tag", livyConf,
inputConf)
+
+ assert(result === inputConf)
+ }
+ }
+
+ it("should leave conf unchanged when credential provider path cannot be
read") {
+ val livyConf = new LivyConf(false)
+ livyConf.set(LivyConf.LIVY_SPARK_MASTER, "local")
+ livyConf.set(LivyConf.HADOOP_CREDENTIAL_PROVIDER_PATH,
"jceks://file/does-not-exist.jceks")
+ val inputConf = Map("spark.app.name" -> "test-app")
+
+ val result = SparkApp.prepareSparkConf("livy-test-tag", livyConf,
inputConf)
+
+ assert(result === inputConf)
+ }
+ }
+}
