Repository: incubator-toree Updated Branches: refs/heads/master c064a0d97 -> 5d26bdabe
[TOREE-469] Remove binary jars used to test addJar functionality Apache source releases does not allow binaries on releases and these test artifacts were stopping releases to be approved. We now create a test jar as part of the test execution and use those to test the addJar functionality. Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/5d26bdab Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/5d26bdab Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/5d26bdab Branch: refs/heads/master Commit: 5d26bdabe466f3aa855d8ec74bf268baafd0db31 Parents: c064a0d Author: Luciano Resende <lrese...@apache.org> Authored: Wed Mar 28 08:41:32 2018 -0700 Committer: Luciano Resende <lrese...@apache.org> Committed: Thu Mar 29 23:08:42 2018 -0700 ---------------------------------------------------------------------- .travis.yml | 6 +- .../src/test/resources/ScalaTestJar.jar | Bin 1313 -> 0 bytes .../src/test/resources/TestJar.jar | Bin 849 -> 0 bytes .../src/test/resources/TestJar2.jar | Bin 736 -> 0 bytes .../AddExternalJarMagicSpecForIntegration.scala | 92 ++++++++++--------- .../spark/toree/test/utils/JarUtils.scala | 73 +++++++++++++++ test_toree.py | 2 +- 7 files changed, 126 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 98039a7..c5b8180 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,7 @@ cache: - $HOME/.sbt/boot/ - $HOME/.coursier/cache/v1 -branches: - only: - - master +#branches: +# only: +# - master http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/resources/ScalaTestJar.jar ---------------------------------------------------------------------- diff --git a/scala-interpreter/src/test/resources/ScalaTestJar.jar b/scala-interpreter/src/test/resources/ScalaTestJar.jar deleted file mode 100644 index de9bed6..0000000 Binary files a/scala-interpreter/src/test/resources/ScalaTestJar.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/resources/TestJar.jar ---------------------------------------------------------------------- diff --git a/scala-interpreter/src/test/resources/TestJar.jar b/scala-interpreter/src/test/resources/TestJar.jar deleted file mode 100644 index 153ea3a..0000000 Binary files a/scala-interpreter/src/test/resources/TestJar.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/resources/TestJar2.jar ---------------------------------------------------------------------- diff --git a/scala-interpreter/src/test/resources/TestJar2.jar b/scala-interpreter/src/test/resources/TestJar2.jar deleted file mode 100644 index 1e40e7e..0000000 Binary files a/scala-interpreter/src/test/resources/TestJar2.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala ---------------------------------------------------------------------- diff --git a/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala b/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala index 871afde..b6834e1 100644 --- a/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala +++ b/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala @@ -17,14 +17,15 @@ package integration.interpreter.scala -import java.io.{ByteArrayOutputStream, OutputStream} +import java.io.{ByteArrayOutputStream, File} +import java.net.URL +import org.apache.spark.toree.test.utils.JarUtils import org.apache.toree.annotations.SbtForked import org.apache.toree.global.StreamState import org.apache.toree.interpreter._ import org.apache.toree.kernel.api.KernelLike import org.apache.toree.kernel.interpreter.scala.ScalaInterpreter -import org.apache.toree.utils.{MultiOutputStream, TaskManager} import org.scalatest.mock.MockitoSugar import org.scalatest.{BeforeAndAfter, FunSpec, Matchers} @@ -35,6 +36,8 @@ class AddExternalJarMagicSpecForIntegration private val outputResult = new ByteArrayOutputStream() private var interpreter: Interpreter = _ + private var tempdir: File = _ + before { interpreter = new ScalaInterpreter { @@ -44,6 +47,8 @@ class AddExternalJarMagicSpecForIntegration interpreter.init(mock[KernelLike]) StreamState.setStreams(outputStream = outputResult) + + tempdir = JarUtils.createTemporaryDir() } after { @@ -53,108 +58,109 @@ class AddExternalJarMagicSpecForIntegration describe("ScalaInterpreter") { describe("#addJars") { - it("should be able to load an external jar") { - val testJarUrl = this.getClass.getClassLoader.getResource("TestJar.jar") + it("should do something") { + interpreter.interpret("1+1") + } - // - // NOTE: This can be done with any jar. I have tested it previously by - // downloading jgoodies, placing it in /tmp/... and loading it. - // + it("should be able to load Java jars") { + val testJar1Url = + JarUtils.createDummyJar(tempdir.toString, "test1", "TestClass") - // Should fail since jar was not added to paths + // Should fail since jars were not added to paths interpreter.interpret( - "import com.ibm.testjar.TestClass")._1 should be (Results.Error) + "import test1.TestClass")._1 should be (Results.Error) - // Add jar to paths - interpreter.addJars(testJarUrl) + // Add jars to paths + interpreter.addJars(testJar1Url) // Should now succeed interpreter.interpret( - "import com.ibm.testjar.TestClass")._1 should be (Results.Success) + "import test1.TestClass")._1 should be (Results.Success) // Should now run interpreter.interpret( - """println(new TestClass().sayHello("Chip"))""" + """println(test1.TestClass.sayHello("Chip"))""" ) should be ((Results.Success, Left(Map()))) outputResult.toString should be ("Hello, Chip\n") + outputResult.reset() + + interpreter.interpret( + """println(test1.TestClass.addStuff(2,1))""" + ) should be ((Results.Success, Left(Map()))) + outputResult.toString should be ("3\n") + outputResult.reset() } it("should support Scala jars") { - val testJarUrl = this.getClass.getClassLoader.getResource("ScalaTestJar.jar") + val locationURL = "http://repo1.maven.org/maven2/org/scala-rules/rule-engine-core_2.11/0.5.1/rule-engine-core_2.11-0.5.1.jar" + val testJarUrl = JarUtils.downloadJar(tempdir.toString, locationURL) // Should fail since jar was not added to paths interpreter.interpret( - "import com.ibm.scalatestjar.TestClass")._1 should be (Results.Error) + "import org.scalarules.utils._")._1 should be (Results.Error) // Add jar to paths interpreter.addJars(testJarUrl) // Should now succeed interpreter.interpret( - "import com.ibm.scalatestjar.TestClass")._1 should be (Results.Success) + "import org.scalarules.utils._")._1 should be (Results.Success) // Should now run + /* interpreter.interpret( """println(new TestClass().runMe())""" ) should be ((Results.Success, Left(Map()))) outputResult.toString should be ("You ran me!\n") + */ } it("should be able to add multiple jars at once") { val testJar1Url = - this.getClass.getClassLoader.getResource("TestJar.jar") + JarUtils.createDummyJar(tempdir.toString, "test1", "TestClass") val testJar2Url = - this.getClass.getClassLoader.getResource("TestJar2.jar") -// val interpreter = new ScalaInterpreter(List(), mock[OutputStream]) -// with StandardSparkIMainProducer -// with StandardTaskManagerProducer -// with StandardSettingsProducer -// interpreter.start() + JarUtils.createDummyJar(tempdir.toString, "test2", "TestClass") // Should fail since jars were not added to paths interpreter.interpret( - "import com.ibm.testjar.TestClass")._1 should be (Results.Error) + "import test1.TestClass")._1 should be (Results.Error) interpreter.interpret( - "import com.ibm.testjar2.TestClass")._1 should be (Results.Error) + "import test2.TestClass")._1 should be (Results.Error) // Add jars to paths interpreter.addJars(testJar1Url, testJar2Url) // Should now succeed interpreter.interpret( - "import com.ibm.testjar.TestClass")._1 should be (Results.Success) + "import test1.TestClass")._1 should be (Results.Success) interpreter.interpret( - "import com.ibm.testjar2.TestClass")._1 should be (Results.Success) + "import test2.TestClass")._1 should be (Results.Success) // Should now run interpreter.interpret( - """println(new com.ibm.testjar.TestClass().sayHello("Chip"))""" + """println(test1.TestClass.sayHello("Chip"))""" ) should be ((Results.Success, Left(Map()))) outputResult.toString should be ("Hello, Chip\n") outputResult.reset() interpreter.interpret( - """println(new com.ibm.testjar2.TestClass().CallMe())""" + """println(test2.TestClass.addStuff(2,1))""" ) should be ((Results.Success, Left(Map()))) outputResult.toString should be ("3\n") + outputResult.reset() } it("should be able to add multiple jars in consecutive calls to addjar") { val testJar1Url = - this.getClass.getClassLoader.getResource("TestJar.jar") + JarUtils.createDummyJar(tempdir.toString, "test1", "TestClass") val testJar2Url = - this.getClass.getClassLoader.getResource("TestJar2.jar") -// val interpreter = new ScalaInterpreter(List(), mock[OutputStream]) -// with StandardSparkIMainProducer -// with StandardTaskManagerProducer -// with StandardSettingsProducer -// interpreter.start() + JarUtils.createDummyJar(tempdir.toString, "test2", "TestClass") // Should fail since jars were not added to paths interpreter.interpret( - "import com.ibm.testjar.TestClass")._1 should be (Results.Error) + "import test1.TestClass")._1 should be (Results.Error) interpreter.interpret( - "import com.ibm.testjar2.TestClass")._1 should be (Results.Error) + "import test2.TestClass")._1 should be (Results.Error) // Add jars to paths interpreter.addJars(testJar1Url) @@ -162,19 +168,19 @@ class AddExternalJarMagicSpecForIntegration // Should now succeed interpreter.interpret( - "import com.ibm.testjar.TestClass")._1 should be (Results.Success) + "import test1.TestClass")._1 should be (Results.Success) interpreter.interpret( - "import com.ibm.testjar2.TestClass")._1 should be (Results.Success) + "import test2.TestClass")._1 should be (Results.Success) // Should now run interpreter.interpret( - """println(new com.ibm.testjar.TestClass().sayHello("Chip"))""" + """println(test1.TestClass.sayHello("Chip"))""" ) should be ((Results.Success, Left(Map()))) outputResult.toString should be ("Hello, Chip\n") outputResult.reset() interpreter.interpret( - """println(new com.ibm.testjar2.TestClass().CallMe())""" + """println(test2.TestClass.addStuff(2,1))""" ) should be ((Results.Success, Left(Map()))) outputResult.toString should be ("3\n") } http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/scala/org/apache/spark/toree/test/utils/JarUtils.scala ---------------------------------------------------------------------- diff --git a/scala-interpreter/src/test/scala/org/apache/spark/toree/test/utils/JarUtils.scala b/scala-interpreter/src/test/scala/org/apache/spark/toree/test/utils/JarUtils.scala new file mode 100644 index 0000000..0e1c82c --- /dev/null +++ b/scala-interpreter/src/test/scala/org/apache/spark/toree/test/utils/JarUtils.scala @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +package org.apache.spark.toree.test.utils + +import sys.process._ +import java.net.URL +import java.io.File +import java.util.UUID + +import scala.language.postfixOps + +import org.apache.spark.TestUtils.{JavaSourceFromString, _} + +object JarUtils { + + def createTemporaryDir() = { + val tempDir = System.getProperty("java.io.tmpdir") + val dir = new File(tempDir, UUID.randomUUID.toString) + dir.mkdirs() + dir.deleteOnExit() + dir.getCanonicalFile + } + + def createDummyJar(destDir: String, packageName: String, className: String) : URL = { + val srcDir = new File(destDir, packageName) + srcDir.mkdirs() + val source = + s"""package $packageName; + | + |public class $className implements java.io.Serializable { + | public static String sayHello(String arg) { return "Hello, " + arg; } + | public static int addStuff(int arg1, int arg2) { return arg1 + arg2; } + |} + """.stripMargin + + val sourceFile = + new JavaSourceFromString(new File(srcDir, className).toURI.getPath, source) + val compiledFile = createCompiledClass(className, srcDir, sourceFile, Seq.empty) + val jarFile = new File(destDir, + s"$packageName-$className-%s.jar".format(System.currentTimeMillis())) + val jarURL = createJar(Seq(compiledFile), jarFile, directoryPrefix = Some(packageName)) + jarFile.deleteOnExit() + jarURL + } + + def downloadJar(destDir: String, artifactURL: String) : URL = { + val fileName = getFileName(artifactURL). + replace(".jar", s"%s.jar".format(System.currentTimeMillis())) + val jarFile = new File(destDir, fileName) + jarFile.deleteOnExit() + (new URL(artifactURL) #> jarFile !!) + jarFile.toURI.toURL + } + + private def getFileName(artifactURL: String) = { + artifactURL.split("/").last + } +} http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/test_toree.py ---------------------------------------------------------------------- diff --git a/test_toree.py b/test_toree.py index 0f90db4..4644126 100644 --- a/test_toree.py +++ b/test_toree.py @@ -72,7 +72,7 @@ class ToreeScalaKernelTests(jupyter_kernel_test.KernelTests): ] test_statements_stdout = [ - {'code': '%AddJar http://0.0.0.0:8000/TestJar.jar'}, + {'code': '%AddJar http://home.apache.org/~lresende/toree/TestJar.jar'}, {'code': 'import com.ibm.testjar.TestClass\nprintln(new TestClass().sayHello("Person"))', 'result': 'Hello, Person\n'} ]