Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/20679#discussion_r170666428
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala ---
@@ -115,7 +115,9 @@ private[sql] class SessionState(
private[sql] object SessionState {
def newHadoopConf(hadoopConf: Configuration, sqlConf: SQLConf):
Configuration = {
val newHadoopConf = new Configuration(hadoopConf)
- sqlConf.getAllConfs.foreach { case (k, v) => if (v ne null)
newHadoopConf.set(k, v) }
+ sqlConf.getAllConfs.foreach { case (k, v) =>
+ if (v ne null) newHadoopConf.set(k, v.stripPrefix("spark.hadoop"))
--- End diff --
Could we add a test case for this feature?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
