AngersZhuuuu commented on a change in pull request #29881:
URL: https://github.com/apache/spark/pull/29881#discussion_r506787887
##########
File path: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
##########
@@ -396,23 +440,43 @@ private[spark] object HiveUtils extends Logging {
config = configurations,
barrierPrefixes = hiveMetastoreBarrierPrefixes,
sharedPrefixes = hiveMetastoreSharedPrefixes)
+ } else if (hiveMetastoreJars == "path") {
+ // Convert to files and expand any directories.
+ val jars =
+ HiveUtils.hiveMetastoreJarsPath(sqlConf)
+ .flatMap {
+ case path if path.contains("\\") && Utils.isWindows =>
+ addLocalHiveJars(new File(path))
+ case path =>
+ DataSource.checkAndGlobPathIfNecessary(
+ pathStrings = Seq(path),
+ hadoopConf = hadoopConf,
+ checkEmptyGlobPath = true,
+ checkFilesExist = false,
+ enableGlobbing = true
+ ).map(_.toUri.toURL)
+ }
+
+ logInfo(
+ s"Initializing HiveMetastoreConnection version $hiveMetastoreVersion "
+
+ s"using path: ${jars.mkString(";")}")
+ new IsolatedClientLoader(
+ version = metaVersion,
+ sparkConf = conf,
+ hadoopConf = hadoopConf,
+ execJars = jars.toSeq,
+ config = configurations,
+ isolationOn = true,
+ barrierPrefixes = hiveMetastoreBarrierPrefixes,
+ sharedPrefixes = hiveMetastoreSharedPrefixes)
} else {
// Convert to files and expand any directories.
val jars =
hiveMetastoreJars
.split(File.pathSeparator)
- .flatMap {
- case path if new File(path).getName == "*" =>
- val files = new File(path).getParentFile.listFiles()
- if (files == null) {
- logWarning(s"Hive jar path '$path' does not exist.")
- Nil
- } else {
-
files.filter(_.getName.toLowerCase(Locale.ROOT).endsWith(".jar")).toSeq
- }
- case path =>
- new File(path) :: Nil
- }
+ .flatMap { path =>
+ addLocalHiveJars(new File(path))
+ }
.map(_.toURI.toURL)
Review comment:
> This line can be removed, as `addLocalHiveJars` already returns URL
Done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
