Github user squito commented on a diff in the pull request:
https://github.com/apache/spark/pull/20399#discussion_r163989326
--- Diff:
core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
---
@@ -110,7 +111,64 @@ class HadoopDelegationTokenManagerSuite extends
SparkFunSuite with Matchers {
creds.getAllTokens.size should be (0)
}
+ test("SPARK-23209: obtain tokens when Hive classes are not available") {
+ // This test needs a custom class loader to hide Hive classes which
are in the classpath.
+ // Because the manager code loads the Hive provider directly instead
of using reflection, we
+ // need to drive the test through the custom class loader so a new
copy that cannot find
+ // Hive classes is loaded.
+ val currentLoader = Thread.currentThread().getContextClassLoader()
+ val noHive = new ClassLoader() {
+ override def loadClass(name: String, resolve: Boolean): Class[_] = {
+ if (name.startsWith("org.apache.hive") ||
name.startsWith("org.apache.hadoop.hive")) {
+ throw new ClassNotFoundException(name)
+ }
+
+ if (name.startsWith("java") || name.startsWith("scala")) {
+ currentLoader.loadClass(name)
+ } else {
+ val classFileName = name.replaceAll("\\.", "/") + ".class"
+ val in = currentLoader.getResourceAsStream(classFileName)
+ if (in != null) {
+ val bytes = IOUtils.toByteArray(in)
+ defineClass(name, bytes, 0, bytes.length)
+ } else {
+ throw new ClassNotFoundException(name)
+ }
+ }
+ }
+ }
+
+ try {
+ Thread.currentThread().setContextClassLoader(noHive)
+ val test =
noHive.loadClass(NoHiveTest.getClass.getName().stripSuffix("$"))
+ test.getMethod("main", classOf[Array[String]]).invoke(null,
Array[String]())
+ } finally {
+ Thread.currentThread().setContextClassLoader(currentLoader)
+ }
+ }
+
private[spark] def hadoopFSsToAccess(hadoopConf: Configuration):
Set[FileSystem] = {
Set(FileSystem.get(hadoopConf))
}
}
+
+/** Test code for SPARK-23209 to avoid using too much reflection above. */
+private object NoHiveTest extends Matchers {
+
+ def main(args: Array[String]): Unit = {
--- End diff --
super minor: can you name this something other than "main"? it makes it
seem like you're launching it as seperate process (maybe leftover from earlier
attempt?)
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
