Repository: spark
Updated Branches:
refs/heads/master d38887b8a -> 2245c87af
Use the Executor's ClassLoader in sc.objectFile().
This makes it possible to read classes from the object file which were
specified in the user-provided jars. (By default ObjectInputStream uses
latestUserDefinedLoader, which may or may not be the right one.)
I created this because I ran into the following problem. I have x:RDD[X] with X
being defined in the jar that I provide to SparkContext. I save it with
x.saveAsObjectFile("x"). I try to load it with sc.objectFile\[X\]("x"). It
fails with ClassNotFoundException.
After a good while of debugging I figured out that Utils.deserialize() most
likely uses the ClassLoader of Utils. This is the bootstrap ClassLoader, so it
is not aware of the dynamically added jars. This patch fixes the issue.
A more robust fix would be to always default to
Thread.currentThread.getContextClassLoader. This would prevent this problem
from biting anyone in the future. It would be a bit harder to test though. On
the topic of testing, if you'd like to see tests for this, I will need some
hand-holding. Thanks!
Author: Daniel Darabos <[email protected]>
Closes #181 from darabos/master and squashes the following commits:
45a011a [Daniel Darabos] Add test for SPARK-1877. (Fixed in 52eb54d.)
e13e090 [Daniel Darabos] Merge branch 'master' of
https://github.com/apache/spark
61fe0d0 [Daniel Darabos] Fix style (line too long).
1b5df2c [Daniel Darabos] Use the Executor's ClassLoader in sc.objectFile().
This makes it possible to read classes from the object file which were
specified in the user-provided jars. (By default ObjectInputStream uses
latestUserDefinedLoader, which may or may not be the right one.)
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2245c87a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2245c87a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2245c87a
Branch: refs/heads/master
Commit: 2245c87af4f507cda361e16f322a14eac25b38fd
Parents: d38887b
Author: Daniel Darabos <[email protected]>
Authored: Sat Jul 12 00:07:42 2014 -0700
Committer: Patrick Wendell <[email protected]>
Committed: Sat Jul 12 00:07:42 2014 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/TestUtils.scala | 4 ++--
.../test/scala/org/apache/spark/FileSuite.scala | 25 ++++++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/2245c87a/core/src/main/scala/org/apache/spark/TestUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala
b/core/src/main/scala/org/apache/spark/TestUtils.scala
index 885c682..8ca7310 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -92,8 +92,8 @@ private[spark] object TestUtils {
def createCompiledClass(className: String, destDir: File, value: String =
""): File = {
val compiler = ToolProvider.getSystemJavaCompiler
val sourceFile = new JavaSourceFromString(className,
- "public class " + className + " { @Override public String toString() { "
+
- "return \"" + value + "\";}}")
+ "public class " + className + " implements java.io.Serializable {" +
+ " @Override public String toString() { return \"" + value + "\"; }}")
// Calling this outputs a class file in pwd. It's easier to just rename
the file than
// build a custom FileManager that controls the output location.
http://git-wip-us.apache.org/repos/asf/spark/blob/2245c87a/core/src/test/scala/org/apache/spark/FileSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/FileSuite.scala
b/core/src/test/scala/org/apache/spark/FileSuite.scala
index 070e974..c70e22c 100644
--- a/core/src/test/scala/org/apache/spark/FileSuite.scala
+++ b/core/src/test/scala/org/apache/spark/FileSuite.scala
@@ -177,6 +177,31 @@ class FileSuite extends FunSuite with LocalSparkContext {
assert(output.collect().toList === List((1, "a"), (2, "aa"), (3, "aaa")))
}
+ test("object files of classes from a JAR") {
+ val original = Thread.currentThread().getContextClassLoader
+ val className = "FileSuiteObjectFileTest"
+ val jar = TestUtils.createJarWithClasses(Seq(className))
+ val loader = new java.net.URLClassLoader(Array(jar),
Utils.getContextOrSparkClassLoader)
+ Thread.currentThread().setContextClassLoader(loader)
+ try {
+ sc = new SparkContext("local", "test")
+ val objs = sc.makeRDD(1 to 3).map { x =>
+ val loader = Thread.currentThread().getContextClassLoader
+ Class.forName(className, true, loader).newInstance()
+ }
+ val outputDir = new File(tempDir, "output").getAbsolutePath
+ objs.saveAsObjectFile(outputDir)
+ // Try reading the output back as an object file
+ val ct = reflect.ClassTag[Any](Class.forName(className, true, loader))
+ val output = sc.objectFile[Any](outputDir)
+ assert(output.collect().size === 3)
+ assert(output.collect().head.getClass.getName === className)
+ }
+ finally {
+ Thread.currentThread().setContextClassLoader(original)
+ }
+ }
+
test("write SequenceFile using new Hadoop API") {
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat
sc = new SparkContext("local", "test")
