attilapiros commented on code in PR #38084:
URL: https://github.com/apache/spark/pull/38084#discussion_r989341069
##########
core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala:
##########
@@ -500,14 +523,35 @@ private[spark] object SparkHadoopUtil extends Logging {
hadoopConf.get("fs.s3a.endpoint.region") == null) {
// set to US central endpoint which can also connect to buckets
// in other regions at the expense of a HEAD request during fs creation
- hadoopConf.set("fs.s3a.endpoint", "s3.amazonaws.com")
+ hadoopConf.set("fs.s3a.endpoint", "s3.amazonaws.com", setBySpark)
}
}
private def appendSparkHiveConfigs(conf: SparkConf, hadoopConf:
Configuration): Unit = {
// Copy any "spark.hive.foo=bar" spark properties into conf as
"hive.foo=bar"
for ((key, value) <- conf.getAll if key.startsWith("spark.hive.")) {
- hadoopConf.set(key.substring("spark.".length), value)
+ hadoopConf.set(key.substring("spark.".length), value,
+ "Set by Spark from keys starting with 'spark.hive'")
+ }
+ }
+
+ /**
+ * Extract the origin of a configuration key, or a default value if
+ * the key is not found or its origin is not known.
+ * Note that options provided by credential providers (JCEKS stores etc)
+ * are not resolved, so values retrieved by Configuration.getPassword()
+ * may not be recorded as having an origin.
+ * @param hadoopConf hadoop configuration to examine.
+ * @param key key to look up
+ * @param defVal default value
+ * @return the origin of the current entry in the configuration, or the
default.
+ */
+ def extractOrigin(hadoopConf: Configuration, key: String, defVal: String):
String = {
Review Comment:
If we have the answer we should fix the documentation (in the Hadoop code).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
