Github user jerryshao commented on a diff in the pull request:
https://github.com/apache/spark/pull/21216#discussion_r189240503
--- Diff:
resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala
---
@@ -200,7 +200,31 @@ object YarnSparkHadoopUtil {
.map(new Path(_).getFileSystem(hadoopConf))
.getOrElse(FileSystem.get(hadoopConf))
- filesystemsToAccess + stagingFS
+ // Add the list of available namenodes for all namespaces in HDFS
federation.
+ // If ViewFS is enabled, this is skipped as ViewFS already handles
delegation tokens for its
+ // namespaces.
+ val hadoopFilesystems = if (stagingFS.getScheme == "viewfs") {
+ Set.empty
+ } else {
+ val nameservices = hadoopConf.getTrimmedStrings("dfs.nameservices")
+ // Retrieving the filesystem for the nameservices where HA is not
enabled
+ val filesystemsWithoutHA = nameservices.flatMap { ns =>
+ hadoopConf.get(s"dfs.namenode.rpc-address.$ns") match {
+ case null => None
+ case nameNode => Some(new
Path(s"hdfs://$nameNode").getFileSystem(hadoopConf))
+ }
+ }
+ // Retrieving the filesystem for the nameservices where HA is enabled
+ val filesystemsWithHA = nameservices.flatMap { ns =>
+ hadoopConf.get(s"dfs.ha.namenodes.$ns") match {
+ case null => None
+ case _ => Some(new Path(s"hdfs://$ns").getFileSystem(hadoopConf))
--- End diff --
Also here.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
