xkrogen commented on a change in pull request #31591:
URL: https://github.com/apache/spark/pull/31591#discussion_r613539257
##########
File path:
resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
##########
@@ -518,6 +518,32 @@ private[spark] class Client(
require(localizedPath != null, "Keytab file already distributed.")
}
+ // If we passed in a ivySettings file, make sure we copy the file to the
distributed cache
+ // in cluster mode so that the driver can access it
+ val ivySettings = sparkConf.getOption("spark.jars.ivySettings")
+ val ivySettingsLocalizedPath: Option[String] = ivySettings match {
+ case Some(ivySettingsPath) if isClusterMode =>
+ val uri = new URI(ivySettingsPath)
+ Option(uri.getScheme).getOrElse("file") match {
+ case "file" =>
+ val ivySettingsFile = new File(ivySettingsPath)
+ require(ivySettingsFile.exists(), s"Ivy settings file
$ivySettingsFile not found")
+ require(ivySettingsFile.isFile(), s"Ivy settings file
$ivySettingsFile is not a" +
+ "normal file")
+ // Generate a file name that can be used for the ivySettings file,
that does not
Review comment:
SGTM thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
