AngersZhuuuu commented on a change in pull request #24909: [SPARK-28106][SQL]
When Spark SQL use "add jar" , before add to SparkContext, check jar path
exist first.
URL: https://github.com/apache/spark/pull/24909#discussion_r298027099
##########
File path: core/src/main/scala/org/apache/spark/SparkContext.scala
##########
@@ -1800,21 +1804,26 @@ class SparkContext(config: SparkConf) extends Logging {
addJarFile(new File(path))
} else {
val uri = new URI(path)
- // SPARK-17650: Make sure this is a valid URL before adding it to the
list of dependencies
Utils.validateURL(uri)
uri.getScheme match {
- // A JAR file which exists only on the driver node
+ // A JAR file which exists only on the driver node
case null =>
- // SPARK-22585 path without schema is not url encoded
+ // SPARK-22585 path without schema is not url encoded
addJarFile(new File(uri.getRawPath))
- // A JAR file which exists only on the driver node
- case "file" => addJarFile(new File(uri.getPath))
- // A JAR file which exists locally on every worker node
+ case "file" => addJarFile(new File(uri.getRawPath))
case "local" => "file:" + uri.getPath
case _ => path
}
}
if (key != null) {
+ val hadoopPath = new Path(key)
+ val scheme = new URI(key).getScheme
+ if (check && !Array("http", "https", "ftp", "spark").contains(scheme))
{
+ val fs = hadoopPath.getFileSystem(hadoopConfiguration)
+ if (!fs.exists(hadoopPath)) {
+ throw new FileNotFoundException(s"Jar ${key} not found")
Review comment:
@jerryshao Done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
