Marius, Could you please help merging this to master, I believe this is the last thing we need to get the 0.2.0 release out.
Thanks On Mon, Jan 23, 2017 at 8:37 AM, <chipsenkb...@apache.org> wrote: > Repository: incubator-toree > Updated Branches: > refs/heads/0.1.x 2eb26cd3c -> f0bb5d238 > > > Removed compiled assets from codebase. > > Generate them during testing instead. This is adapted from code that is > part of Apache Spark. > > > Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo > Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/ > commit/f0bb5d23 > Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/f0bb5d23 > Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/f0bb5d23 > > Branch: refs/heads/0.1.x > Commit: f0bb5d238dc10376db3935c15e007ea46076e548 > Parents: 2eb26cd > Author: mariusvniekerk <marius.v.niek...@gmail.com> > Authored: Mon Jan 23 10:47:42 2017 -0500 > Committer: mariusvniekerk <marius.v.niek...@gmail.com> > Committed: Mon Jan 23 10:53:30 2017 -0500 > > ---------------------------------------------------------------------- > .../src/test/resources/ScalaTestJar.jar | Bin 1313 -> 0 bytes > .../src/test/resources/TestJar.jar | Bin 849 -> 0 bytes > .../src/test/resources/TestJar2.jar | Bin 736 -> 0 bytes > .../AddExternalJarMagicSpecForIntegration.scala | 60 +++++++++++-------- > .../apache/spark/toree/testutils/JarUtils.scala | 41 +++++++++++++ > 5 files changed, 75 insertions(+), 26 deletions(-) > ---------------------------------------------------------------------- > > > http://git-wip-us.apache.org/repos/asf/incubator-toree/ > blob/f0bb5d23/scala-interpreter/src/test/resources/ScalaTestJar.jar > ---------------------------------------------------------------------- > diff --git a/scala-interpreter/src/test/resources/ScalaTestJar.jar > b/scala-interpreter/src/test/resources/ScalaTestJar.jar > deleted file mode 100644 > index de9bed6..0000000 > Binary files a/scala-interpreter/src/test/resources/ScalaTestJar.jar and > /dev/null differ > > http://git-wip-us.apache.org/repos/asf/incubator-toree/ > blob/f0bb5d23/scala-interpreter/src/test/resources/TestJar.jar > ---------------------------------------------------------------------- > diff --git a/scala-interpreter/src/test/resources/TestJar.jar > b/scala-interpreter/src/test/resources/TestJar.jar > deleted file mode 100644 > index 153ea3a..0000000 > Binary files a/scala-interpreter/src/test/resources/TestJar.jar and > /dev/null differ > > http://git-wip-us.apache.org/repos/asf/incubator-toree/ > blob/f0bb5d23/scala-interpreter/src/test/resources/TestJar2.jar > ---------------------------------------------------------------------- > diff --git a/scala-interpreter/src/test/resources/TestJar2.jar > b/scala-interpreter/src/test/resources/TestJar2.jar > deleted file mode 100644 > index 1e40e7e..0000000 > Binary files a/scala-interpreter/src/test/resources/TestJar2.jar and > /dev/null differ > > http://git-wip-us.apache.org/repos/asf/incubator-toree/ > blob/f0bb5d23/scala-interpreter/src/test/scala/ > integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala > ---------------------------------------------------------------------- > diff --git a/scala-interpreter/src/test/scala/integration/interpreter/ > scala/AddExternalJarMagicSpecForIntegration.scala > b/scala-interpreter/src/test/scala/integration/interpreter/scala/ > AddExternalJarMagicSpecForIntegration.scala > index 2863bec..8fdafe7 100644 > --- a/scala-interpreter/src/test/scala/integration/interpreter/scala/ > AddExternalJarMagicSpecForIntegration.scala > +++ b/scala-interpreter/src/test/scala/integration/interpreter/scala/ > AddExternalJarMagicSpecForIntegration.scala > @@ -17,13 +17,14 @@ > > package integration.interpreter.scala > > -import java.io.{ByteArrayOutputStream, OutputStream} > +import java.io.{ByteArrayOutputStream, File, OutputStream} > > +import org.apache.spark.toree.testutils.JarUtils > import org.apache.toree.global.StreamState > import org.apache.toree.interpreter._ > import org.apache.toree.kernel.api.KernelLike > import org.apache.toree.kernel.interpreter.scala.{ScalaInterpreter, > StandardSettingsProducer, StandardSparkIMainProducer, > StandardTaskManagerProducer} > -import org.apache.toree.utils.{TaskManager, MultiOutputStream} > +import org.apache.toree.utils.{MultiOutputStream, TaskManager} > import org.scalatest.mock.MockitoSugar > import org.scalatest.{BeforeAndAfter, FunSpec, Matchers} > > @@ -33,6 +34,7 @@ class AddExternalJarMagicSpecForIntegration > > private val outputResult = new ByteArrayOutputStream() > private var interpreter: Interpreter = _ > + private var tempdir: File = _ > > before { > interpreter = new ScalaInterpreter { > @@ -44,6 +46,8 @@ class AddExternalJarMagicSpecForIntegration > interpreter.init(mock[KernelLike]) > > StreamState.setStreams(outputStream = outputResult) > + > + tempdir = JarUtils.createTemporaryDir() > } > > after { > @@ -53,7 +57,8 @@ class AddExternalJarMagicSpecForIntegration > describe("ScalaInterpreter") { > describe("#addJars") { > it("should be able to load an external jar") { > - val testJarUrl = this.getClass.getClassLoader. > getResource("TestJar.jar") > + > + val testJarUrl = JarUtils.createDummyJar(tempdir.toString, > "testing.external", "TestClass") > > // > // NOTE: This can be done with any jar. I have tested it > previously by > @@ -62,14 +67,14 @@ class AddExternalJarMagicSpecForIntegration > > // Should fail since jar was not added to paths > interpreter.interpret( > - "import com.ibm.testjar.TestClass")._1 should be (Results.Error) > + "import testing.external.TestClass")._1 should be > (Results.Error) > > // Add jar to paths > interpreter.addJars(testJarUrl) > > // Should now succeed > interpreter.interpret( > - "import com.ibm.testjar.TestClass")._1 should be > (Results.Success) > + "import testing.external.TestClass")._1 should be > (Results.Success) > > // Should now run > interpreter.interpret( > @@ -78,7 +83,9 @@ class AddExternalJarMagicSpecForIntegration > outputResult.toString should be ("Hello, Chip\n") > } > > - it("should support Scala jars") { > + // Test disabled due to removal of statically compiled Scala Jar. > + // Should replace with something that fetches a jar using coursier > instead. > + ignore("should support Scala jars") { > val testJarUrl = this.getClass.getClassLoader. > getResource("ScalaTestJar.jar") > > // Should fail since jar was not added to paths > @@ -101,9 +108,9 @@ class AddExternalJarMagicSpecForIntegration > > it("should be able to add multiple jars at once") { > val testJar1Url = > - this.getClass.getClassLoader.getResource("TestJar.jar") > + JarUtils.createDummyJar(tempdir.toString, "testing.testjar1", > "TestClass") > val testJar2Url = > - this.getClass.getClassLoader.getResource("TestJar2.jar") > + JarUtils.createDummyJar(tempdir.toString, "testing.testjar2", > "TestClass") > // val interpreter = new ScalaInterpreter(List(), > mock[OutputStream]) > // with StandardSparkIMainProducer > // with StandardTaskManagerProducer > @@ -112,37 +119,37 @@ class AddExternalJarMagicSpecForIntegration > > // Should fail since jars were not added to paths > interpreter.interpret( > - "import com.ibm.testjar.TestClass")._1 should be (Results.Error) > + "import testing.testjar.TestClass")._1 should be (Results.Error) > interpreter.interpret( > - "import com.ibm.testjar2.TestClass")._1 should be > (Results.Error) > + "import testing.testjar2.TestClass")._1 should be > (Results.Error) > > // Add jars to paths > interpreter.addJars(testJar1Url, testJar2Url) > > // Should now succeed > interpreter.interpret( > - "import com.ibm.testjar.TestClass")._1 should be > (Results.Success) > + "import testing.testjar.TestClass")._1 should be > (Results.Success) > interpreter.interpret( > - "import com.ibm.testjar2.TestClass")._1 should be > (Results.Success) > + "import testing.testjar2.TestClass")._1 should be > (Results.Success) > > // Should now run > interpreter.interpret( > - """println(new com.ibm.testjar.TestClass().sayHello("Chip"))""" > + """println(new testing.testjar.TestClass().sayHello("Chip"))""" > ) should be ((Results.Success, Left(""))) > outputResult.toString should be ("Hello, Chip\n") > outputResult.reset() > > interpreter.interpret( > - """println(new com.ibm.testjar2.TestClass().CallMe())""" > + """println(new testing.testjar.TestClass().addStuff(1, 2))""" > ) should be ((Results.Success, Left(""))) > outputResult.toString should be ("3\n") > } > > it("should be able to add multiple jars in consecutive calls to > addjar") { > val testJar1Url = > - this.getClass.getClassLoader.getResource("TestJar.jar") > + JarUtils.createDummyJar(tempdir.toString, "testing.testjar1", > "TestClass") > val testJar2Url = > - this.getClass.getClassLoader.getResource("TestJar2.jar") > + JarUtils.createDummyJar(tempdir.toString, "testing.testjar2", > "TestClass") > // val interpreter = new ScalaInterpreter(List(), > mock[OutputStream]) > // with StandardSparkIMainProducer > // with StandardTaskManagerProducer > @@ -151,45 +158,46 @@ class AddExternalJarMagicSpecForIntegration > > // Should fail since jars were not added to paths > interpreter.interpret( > - "import com.ibm.testjar.TestClass")._1 should be (Results.Error) > + "import testing.testjar1.TestClass")._1 should be > (Results.Error) > interpreter.interpret( > - "import com.ibm.testjar2.TestClass")._1 should be > (Results.Error) > + "import testing.testjar2.TestClass")._1 should be > (Results.Error) > > // Add jars to paths > interpreter.addJars(testJar1Url) > interpreter.addJars(testJar2Url) > > // Should now succeed > + // Should now succeed > interpreter.interpret( > - "import com.ibm.testjar.TestClass")._1 should be > (Results.Success) > + "import testing.testjar1.TestClass")._1 should be > (Results.Success) > interpreter.interpret( > - "import com.ibm.testjar2.TestClass")._1 should be > (Results.Success) > + "import testing.testjar2.TestClass")._1 should be > (Results.Success) > > // Should now run > interpreter.interpret( > - """println(new com.ibm.testjar.TestClass().sayHello("Chip"))""" > + """println(new testing.testjar1.TestClass(). > sayHello("Chip"))""" > ) should be ((Results.Success, Left(""))) > outputResult.toString should be ("Hello, Chip\n") > outputResult.reset() > > interpreter.interpret( > - """println(new com.ibm.testjar2.TestClass().CallMe())""" > + """println(new testing.testjar2.TestClass().addStuff(1, 2))""" > ) should be ((Results.Success, Left(""))) > outputResult.toString should be ("3\n") > } > > it("should not have issues with previous variables") { > val testJar1Url = > - this.getClass.getClassLoader.getResource("TestJar.jar") > + JarUtils.createDummyJar(tempdir.toString, "testing.testjar1", > "TestClass") > val testJar2Url = > - this.getClass.getClassLoader.getResource("TestJar2.jar") > + JarUtils.createDummyJar(tempdir.toString, "testing.testjar2", > "TestClass") > > // Add a jar, which reinitializes the symbols > interpreter.addJars(testJar1Url) > > interpreter.interpret( > """ > - |val t = new com.ibm.testjar.TestClass() > + |val t = new testing.testjar1.TestClass() > """.stripMargin)._1 should be (Results.Success) > > // Add a second jar, which reinitializes the symbols and breaks > the > @@ -198,7 +206,7 @@ class AddExternalJarMagicSpecForIntegration > > interpreter.interpret( > """ > - |def runMe(testClass: com.ibm.testjar.TestClass) = > + |def runMe(testClass: testing.testjar1.TestClass) = > |testClass.sayHello("Hello") > """.stripMargin)._1 should be (Results.Success) > > > http://git-wip-us.apache.org/repos/asf/incubator-toree/ > blob/f0bb5d23/scala-interpreter/src/test/scala/org/apache/spark/toree/ > testutils/JarUtils.scala > ---------------------------------------------------------------------- > diff --git > a/scala-interpreter/src/test/scala/org/apache/spark/toree/testutils/JarUtils.scala > b/scala-interpreter/src/test/scala/org/apache/spark/toree/ > testutils/JarUtils.scala > new file mode 100644 > index 0000000..572aa4a > --- /dev/null > +++ b/scala-interpreter/src/test/scala/org/apache/spark/toree/ > testutils/JarUtils.scala > @@ -0,0 +1,41 @@ > +package org.apache.spark.toree.testutils > + > +import java.io.File > +import java.util.UUID > + > +import org.apache.spark.TestUtils.{JavaSourceFromString, _} > + > +object JarUtils { > + > + def createTemporaryDir() = { > + val tempDir = System.getProperty("java.io.tmpdir") > + val dir = new File(tempDir, UUID.randomUUID.toString) > + dir.mkdirs() > + dir.deleteOnExit() > + dir.getCanonicalFile > + } > + > + def createDummyJar(destDir: String, packageName: String, className: > String) = { > + val srcDir = new File(destDir, packageName) > + srcDir.mkdirs() > + val excSource = new JavaSourceFromString( > + new File(srcDir, className).getAbsolutePath, > + s"""package $packageName; > + | > + |public class $className implements java.io.Serializable { > + | public static String sayHello(String arg) { return "Hello, " + > arg; } > + | public static int addStuff(int arg1, int arg2) { return arg1 + > arg2; } > + |} > + """. > + stripMargin) > + val excFile = createCompiledClass(className, srcDir, excSource, > Seq.empty) > + val jarFile = new File(destDir, > + s"$packageName-$className-%s.jar".format(System. > currentTimeMillis())) > + val jarURL = createJar(Seq(excFile), jarFile, directoryPrefix = > Some(packageName)) > + jarFile.deleteOnExit() > + jarURL > + } > + > + > + > +} > > -- Luciano Resende http://twitter.com/lresende1975 http://lresende.blogspot.com/
