HyukjinKwon closed pull request #15591: [SPARK-17922] [SQL] ClassCastException
..GeneratedClass$GeneratedIterator cannot be cast to
...expressions.UnsafeProjection
URL: https://github.com/apache/spark/pull/15591
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 6cab50ae1bf8d..62b9751bfc5d4 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -40,6 +40,7 @@ import org.apache.spark.sql.types._
import org.apache.spark.unsafe.Platform
import org.apache.spark.unsafe.types._
import org.apache.spark.util.{ParentClassLoader, Utils}
+import org.apache.spark.sql.catalyst.util.DelegateClassLoader
/**
* Java source for evaluating an [[Expression]] given a [[InternalRow]] of
input.
@@ -874,7 +875,7 @@ object CodeGenerator extends Logging {
// find other possible classes (see
org.codehaus.janinoClassLoaderIClassLoader's
// findIClass method). Please also see
https://issues.apache.org/jira/browse/SPARK-15622 and
// https://issues.apache.org/jira/browse/SPARK-11636.
- val parentClassLoader = new
ParentClassLoader(Utils.getContextOrSparkClassLoader)
+ val parentClassLoader = new
DelegateClassLoader(Utils.getContextOrSparkClassLoader,
"org.apache.spark.sql.catalyst.expressions.GeneratedClass")
evaluator.setParentClassLoader(parentClassLoader)
// Cannot be under package codegen, or fail with
java.lang.InstantiationException
evaluator.setClassName("org.apache.spark.sql.catalyst.expressions.GeneratedClass")
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DelegateClassLoader.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DelegateClassLoader.scala
new file mode 100644
index 0000000000000..dc4d31d024388
--- /dev/null
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DelegateClassLoader.scala
@@ -0,0 +1,37 @@
+package org.apache.spark.sql.catalyst.util
+/*
+ * See : https://issues.apache.org/jira/browse/SPARK-17922.
+ * Janino compiler internally creates a byteclassloader
(http://grepcode.com/file/repo1.maven.org/maven2/org.codehaus.janino/janino/2.5.15/org/codehaus/janino/ByteArrayClassLoader.java#ByteArrayClassLoader)
+ * to load the compiled generated class. But this class loader doesnot
override load class to load the class from byte array for the generated class.
+ * Instead the call first goes to parent class loader and if somehow the
classloader finds the old generatedClass( all the generated class names are
same)
+ * it will incorrectly load the old generated class. This class loader will be
used to intercept delegation to parent if the class has to be loaded by the
current byte class loader.
+ * This will be set as the parent class loader for janino compiler in
CodeGenerator.doCompile
+ * Special classloader to skip delegating to parent class loader when the
class name is same as the generated class name.
+ * Because that class should be loaded by the current class loader
+ */
+class DelegateClassLoader(parent:ClassLoader, skipClass: String) extends
ClassLoader(parent) {
+ override def findClass(name: String): Class[_] = {
+ if(checkClassName(name)) {
+ return null
+ }
+ super.findClass(name)
+ }
+
+ override def loadClass(name: String): Class[_] = {
+ if(checkClassName(name)) {
+ return null
+ }
+ super.loadClass(name)
+ }
+
+ override def loadClass(name: String, resolve: Boolean): Class[_] = {
+ if(checkClassName(name)) {
+ return null
+ }
+ super.loadClass(name, resolve)
+ }
+
+ def checkClassName(name: String): Boolean = {
+ skipClass.equals(name)
+ }
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
