Github user yhuai commented on a diff in the pull request:
https://github.com/apache/spark/pull/13679#discussion_r67107457
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala ---
@@ -66,6 +67,30 @@ private[sql] class SharedState(val sparkContext:
SparkContext) {
val jarClassLoader = new NonClosableMutableURLClassLoader(
org.apache.spark.util.Utils.getContextOrSparkClassLoader)
+ {
+ // Set the Hive metastore warehouse path to the one we use
+ val tempConf = new SQLConf
+ sparkContext.conf.getAll.foreach { case (k, v) =>
tempConf.setConfString(k, v) }
+ val hiveWarehouseDir = hadoopConf.get("hive.metastore.warehouse.dir")
+ if (hiveWarehouseDir != null &&
!tempConf.contains(SQLConf.WAREHOUSE_PATH.key)) {
+ // If hive.metastore.warehouse.dir is set and
spark.sql.warehouse.dir is not set,
+ // we will respect the value of hive.metastore.warehouse.dir.
+ tempConf.setConfString(SQLConf.WAREHOUSE_PATH.key, hiveWarehouseDir)
+ sparkContext.conf.set(SQLConf.WAREHOUSE_PATH.key, hiveWarehouseDir)
+ logInfo(s"${SQLConf.WAREHOUSE_PATH.key} is not set, but
hive.metastore.warehouse.dir " +
+ s"is set. Setting ${SQLConf.WAREHOUSE_PATH.key} to the value of " +
+ s"hive.metastore.warehouse.dir ('$hiveWarehouseDir').")
+ } else {
+ // If spark.sql.warehouse.dir is set, we will override
hive.metastore.warehouse.dir using
+ // the value of spark.sql.warehouse.dir.
+ // When neither spark.sql.warehouse.dir nor
hive.metastore.warehouse.dir is set,
+ // we will set hive.metastore.warehouse.dir to the default value of
spark.sql.warehouse.dir.
+ sparkContext.conf.set("hive.metastore.warehouse.dir",
tempConf.warehousePath)
+ }
+
+ logInfo(s"Warehouse path is '${tempConf.warehousePath}'.")
+ }
--- End diff --
@rxin How about this version? This block is moved to SharedState from
HiveSharedState.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
