zsxwing commented on a change in pull request #24683: [SPARK-20547][REPL]Throw
RemoteClassLoadedError for transient errors in ExecutorClassLoader
URL: https://github.com/apache/spark/pull/24683#discussion_r310374648
##########
File path:
repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
##########
@@ -218,4 +228,131 @@ class ExecutorClassLoaderSuite
}
}
+ test("nonexistent class and transient errors should cause different errors")
{
+ val conf = new SparkConf()
+ .setMaster("local")
+ .setAppName("executor-class-loader-test")
+ .set("spark.network.timeout", "11s")
+ .set("spark.repl.class.outputDir", tempDir1.getAbsolutePath)
+ val sc = new SparkContext(conf)
+ try {
+ val replClassUri = sc.conf.get("spark.repl.class.uri")
+
+ // Create an RpcEnv for executor
+ val rpcEnv = RpcEnv.create(
+ SparkEnv.executorSystemName,
+ "localhost",
+ "localhost",
+ 0,
+ sc.conf,
+ new SecurityManager(conf), 0, clientMode = true)
+
+ try {
+ val env = mock[SparkEnv]
+ when(env.rpcEnv).thenReturn(rpcEnv)
+
+ val classLoader = new ExecutorClassLoader(
+ conf,
+ env,
+ replClassUri,
+ getClass().getClassLoader(),
+ false)
+
+ // Test loading a nonexistent class
+ intercept[java.lang.ClassNotFoundException] {
+ classLoader.loadClass("NonexistentClass")
+ }
+
+ // Stop SparkContext to simulate transient errors in executors
+ sc.stop()
+
+ val e = intercept[RemoteClassLoaderError] {
+ classLoader.loadClass("ThisIsAClassName")
+ }
+ assert(e.getMessage.contains("ThisIsAClassName"))
+ // RemoteClassLoaderError must not be LinkageError nor
ClassNotFoundException. Otherwise,
+ // JVM will cache it and doesn't retry to load a class.
+ assert(!e.isInstanceOf[LinkageError] &&
!e.isInstanceOf[ClassNotFoundException])
Review comment:
@srowen this line is to ensure we will never make `RemoteClassLoaderError`
extend `LinkageError` or `ClassNotFoundException` in future. Can we turn the
warning off for this check, or just rewrite it to something like
`assert(e.getClass.getSimpleName != "LinkageError" && e.getClass.getSimpleName
!= "ClassNotFoundException")`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
