zhengruifeng commented on code in PR #43739:
URL: https://github.com/apache/spark/pull/43739#discussion_r1389712978
##########
mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala:
##########
@@ -101,4 +102,46 @@ class FunctionsSuite extends MLTest {
val resultVec3 =
df3.select(array_to_vector(col("c1"))).collect()(0)(0).asInstanceOf[Vector]
assert(resultVec3 === Vectors.dense(Array(1.0, 2.0)))
}
+
+ test("SPARK-45859: 'functions$' should not be affected by a broken class
loader") {
+ quietly {
+ // Only one SparkContext should be running in this JVM (see SPARK-2243)
+ sc.stop()
+
+ val conf = new SparkConf()
+ .setAppName("FunctionsSuite")
+ .setMaster("local-cluster[1,1,1024]")
+ sc = new SparkContext(conf)
+ // Make `functions$` be loaded by a broken class loader
+ intercept[SparkException] {
+ sc.parallelize(1 to 1).foreach { _ =>
+ val originalClassLoader = Thread.currentThread.getContextClassLoader
+ try {
+ Thread.currentThread.setContextClassLoader(new BrokenClassLoader)
+ vector_to_array(col("vector"))
+ array_to_vector(col("array"))
+ } finally {
+ Thread.currentThread.setContextClassLoader(originalClassLoader)
+ }
+ }
+ }
+
+ // We should be able to use `functions$` even it was loaded by a broken
class loader
+ sc.parallelize(1 to 1).foreach { _ =>
+ vector_to_array(col("vector"))
+ array_to_vector(col("array"))
+ }
+
+ // this UT should be the last one in this test suite, since it uses
+ // a different `sc` from the standard one.
+ // stop it here in case new UTs are added after this one.
Review Comment:
sounds good, let me move it to a separate file
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
