Repository: incubator-livy Updated Branches: refs/heads/master f99cc32df -> d39ab356b
[LIVY-510] Remove support for JDK7. With this change Livy will only build and run with JDK8. While this doesn't necessarily preclude using Java 7 for running Spark itself, that is not encouraged due to possible issues with serialization. There are a few small changes elsewhere that I made so tests would pass, but aren't related to the java 8 change: - a small tweak to the fallback code that initializes the python interpreter, which was failing with a weird reflection error. - expose the user's home directory in the MiniYarnCluster, to allow the tests to see pip packages installed in the user's home. - tweak the python API tests so that the pytest cache does not pollute the source directory. - changed Travis to use Oracle's JDK, since the Open JDK path was mixing Java 8 and 9 and causing errors. - updated jacoco to latest stable. Author: Marcelo Vanzin <van...@cloudera.com> Closes #111 from vanzin/LIVY-510. Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/d39ab356 Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/d39ab356 Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/d39ab356 Branch: refs/heads/master Commit: d39ab356b4e36d8780bd44b9d5020bfeec90cb2f Parents: f99cc32 Author: Marcelo Vanzin <van...@cloudera.com> Authored: Thu Sep 20 11:49:22 2018 +0800 Committer: jerryshao <sai.sai.s...@gmail.com> Committed: Thu Sep 20 11:49:22 2018 +0800 ---------------------------------------------------------------------- .travis.yml | 6 +- README.md | 6 +- .../livy/test/framework/MiniCluster.scala | 7 +- .../src/test/resources/test_python_api.py | 3 +- .../scala/org/apache/livy/test/JobApiIT.scala | 16 ++-- pom.xml | 82 ++------------------ python-api/setup.cfg | 1 + .../apache/livy/repl/PythonInterpreter.scala | 8 +- 8 files changed, 34 insertions(+), 95 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 6c5eee3..d38cf4f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,17 +33,13 @@ matrix: # Spark 2.2+ will only be verified using JDK8 # Thriftserver requires JDK8 - env: MVN_FLAG='-Pthriftserver -Pspark-2.2-it -DskipTests' - jdk: oraclejdk8 - env: MVN_FLAG='-Pthriftserver -Pspark-2.2 -DskipITs' - jdk: oraclejdk8 - env: MVN_FLAG='-Pthriftserver -Pspark-2.3-it -DskipTests' - jdk: oraclejdk8 - env: MVN_FLAG='-Pthriftserver -Pspark-2.3 -DskipITs' - jdk: oraclejdk8 jdk: - - openjdk7 + - oraclejdk8 addons: apt: http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 2dc33c2..ff6ae16 100644 --- a/README.md +++ b/README.md @@ -28,19 +28,19 @@ To build Livy, you will need: Debian/Ubuntu: * mvn (from ``maven`` package or maven3 tarball) - * openjdk-7-jdk (or Oracle Java7 jdk) + * openjdk-8-jdk (or Oracle JDK 8) * Python 2.6+ * R 3.x Redhat/CentOS: * mvn (from ``maven`` package or maven3 tarball) - * java-1.7.0-openjdk (or Oracle Java7 jdk) + * java-1.8.0-openjdk (or Oracle JDK 8) * Python 2.6+ * R 3.x MacOS: * Xcode command line tools - * Oracle's JDK 1.7+ + * Oracle's JDK 1.8 * Maven (Homebrew) * Python 2.6+ * R 3.x http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala ---------------------------------------------------------------------- diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala index c4d4eb5..b58abe2 100644 --- a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala +++ b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala @@ -101,6 +101,10 @@ object MiniYarnMain extends MiniClusterBase { config.localDirCount, config.logDirCount) yarnCluster.init(baseConfig) + // This allows applications run by YARN during the integration tests to find PIP modules + // installed in the user's home directory (instead of just the global ones). + baseConfig.set(YarnConfiguration.NM_USER_HOME_DIR, sys.env("HOME")) + // Install a shutdown hook for stop the service and kill all running applications. Runtime.getRuntime().addShutdownHook(new Thread() { override def run(): Unit = yarnCluster.stop() @@ -283,8 +287,7 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU sys.props("java.home") + "/bin/java", "-Dtest.appender=console", "-Djava.io.tmpdir=" + procTmp.getAbsolutePath(), - "-cp", childClasspath + File.pathSeparator + configDir.getAbsolutePath(), - "-XX:MaxPermSize=256m") ++ + "-cp", childClasspath + File.pathSeparator + configDir.getAbsolutePath()) ++ extraJavaArgs ++ Seq( klass.getName().stripSuffix("$"), http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/integration-test/src/test/resources/test_python_api.py ---------------------------------------------------------------------- diff --git a/integration-test/src/test/resources/test_python_api.py b/integration-test/src/test/resources/test_python_api.py index 3749e69..d3023c0 100644 --- a/integration-test/src/test/resources/test_python_api.py +++ b/integration-test/src/test/resources/test_python_api.py @@ -217,7 +217,6 @@ def test_upload_pyfile(): if __name__ == '__main__': - test_dir_path = os.getcwd() + "/src" - value = pytest.main(test_dir_path) + value = pytest.main([os.path.dirname(__file__)]) if value != 0: raise Exception("One or more test cases have failed.") http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala ---------------------------------------------------------------------- diff --git a/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala b/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala index 7c0e560..93b2524 100644 --- a/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala +++ b/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala @@ -249,7 +249,12 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg val uploadPyFilePath = createTempFilesForTest("upload_pyfile", ".py", uploadPyFileContent, false) - val builder = new ProcessBuilder(Seq("python", createPyTestsForPythonAPI().toString).asJava) + val tmpDir = new File(sys.props("java.io.tmpdir")).getAbsoluteFile() + val testDir = Files.createTempDirectory(tmpDir.toPath(), "python-tests-").toFile() + val testFile = createPyTestsForPythonAPI(testDir) + + val builder = new ProcessBuilder(Seq("python", testFile.getAbsolutePath()).asJava) + builder.directory(testDir) val env = builder.environment() env.put("LIVY_END_POINT", livyEndpoint) @@ -258,8 +263,8 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg env.put("UPLOAD_FILE_URL", uploadFilePath) env.put("UPLOAD_PYFILE_URL", uploadPyFilePath) - builder.redirectOutput(new File(sys.props("java.io.tmpdir") + "/pytest_results.log")) builder.redirectErrorStream(true) + builder.redirectOutput(new File(tmpDir, "pytest_results.log")) val process = builder.start() @@ -268,11 +273,10 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg assert(process.exitValue() === 0) } - private def createPyTestsForPythonAPI(): File = { - var source: InputStream = null + private def createPyTestsForPythonAPI(testDir: File): File = { + val file = Files.createTempFile(testDir.toPath(), "test_python_api-", ".py").toFile() + val source = getClass().getClassLoader().getResourceAsStream("test_python_api.py") try { - source = getClass.getClassLoader.getResourceAsStream("test_python_api.py") - val file = Files.createTempFile("", "").toFile Files.copy(source, file.toPath, StandardCopyOption.REPLACE_EXISTING) file } finally { http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e07f39d..b49c524 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,7 @@ <httpcore.version>4.4.4</httpcore.version> <jackson.version>2.9.5</jackson.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> - <jetty.version>9.2.16.v20160414</jetty.version> + <jetty.version>9.3.8.v20160314</jetty.version> <json4s.version>3.2.10</json4s.version> <junit.version>4.11</junit.version> <libthrift.version>0.9.3</libthrift.version> @@ -108,9 +108,7 @@ <scala.version>${scala-2.11.version}</scala.version> <scalatest.version>2.2.4</scalatest.version> <scalatra.version>2.3.0</scalatra.version> - <java.version>1.7</java.version> - <minJavaVersion>1.7</minJavaVersion> - <maxJavaVersion>1.8</maxJavaVersion> + <java.version>1.8</java.version> <test.redirectToFile>true</test.redirectToFile> <execution.root>${user.dir}</execution.root> <spark.home>${execution.root}/dev/spark</spark.home> @@ -580,26 +578,6 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <version>1.4.1</version> - <executions> - <execution> - <id>enforce-versions</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <requireJavaVersion> - <version>1.7</version> - </requireJavaVersion> - </rules> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <version>3.3</version> </plugin> @@ -675,7 +653,6 @@ <jvmArgs> <jvmArg>-Xms1024m</jvmArg> <jvmArg>-Xmx1024m</jvmArg> - <jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg> <jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg> </jvmArgs> </configuration> @@ -714,7 +691,7 @@ </systemProperties> <redirectTestOutputToFile>${test.redirectToFile}</redirectTestOutputToFile> <useFile>${test.redirectToFile}</useFile> - <argLine>${argLine} -Xmx2g -XX:MaxPermSize=512m</argLine> + <argLine>${argLine} -Xmx2g</argLine> <failIfNoTests>false</failIfNoTests> </configuration> </plugin> @@ -743,7 +720,7 @@ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory> <junitxml>.</junitxml> <filereports>WDF TestSuite.txt</filereports> - <argLine>${argLine} -Xmx2g -XX:MaxPermSize=512m</argLine> + <argLine>${argLine} -Xmx2g</argLine> </configuration> <executions> <execution> @@ -818,12 +795,12 @@ <inherited>false</inherited> <configuration> <rules> + <requireJavaVersion> + <version>${java.version}</version> + </requireJavaVersion> <requireMavenVersion> <version>[3.0.0,)</version> </requireMavenVersion> - <requireJavaVersion> - <version>[${minJavaVersion}.0,${maxJavaVersion}.1000}]</version> - </requireJavaVersion> <requireOS> <family>unix</family> </requireOS> @@ -831,26 +808,12 @@ </configuration> <executions> <execution> - <id>clean</id> - <goals> - <goal>enforce</goal> - </goals> - <phase>pre-clean</phase> - </execution> - <execution> <id>default</id> <goals> <goal>enforce</goal> </goals> <phase>validate</phase> </execution> - <execution> - <id>site</id> - <goals> - <goal>enforce</goal> - </goals> - <phase>pre-site</phase> - </execution> </executions> </plugin> @@ -996,7 +959,7 @@ <plugin> <groupId>org.jacoco</groupId> <artifactId>jacoco-maven-plugin</artifactId> - <version>0.7.7.201606060606</version> + <version>0.8.2</version> <executions> <execution> <goals> @@ -1065,33 +1028,6 @@ <modules> <module>thriftserver/server</module> </modules> - <properties> - <jetty.version>9.3.8.v20160314</jetty.version> - </properties> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <executions> - <execution> - <id>enforce-java</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <requireJavaVersion> - <version>1.8</version> - </requireJavaVersion> - </rules> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> </profile> <!-- Spark version profiles --> @@ -1184,7 +1120,6 @@ <spark.scala-2.11.version>2.2.0</spark.scala-2.11.version> <spark.scala-2.10.version>2.2.0</spark.scala-2.10.version> <spark.version>${spark.scala-2.11.version}</spark.version> - <java.version>1.8</java.version> <py4j.version>0.10.4</py4j.version> <json4s.version>3.2.11</json4s.version> </properties> @@ -1218,7 +1153,6 @@ <spark.version>${spark.scala-2.11.version}</spark.version> <netty.spark-2.11.version>4.1.17.Final</netty.spark-2.11.version> <netty.spark-2.10.version>4.0.37.Final</netty.spark-2.10.version> - <java.version>1.8</java.version> <py4j.version>0.10.7</py4j.version> <json4s.version>3.2.11</json4s.version> </properties> http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/python-api/setup.cfg ---------------------------------------------------------------------- diff --git a/python-api/setup.cfg b/python-api/setup.cfg index e2121f9..8a8799b 100644 --- a/python-api/setup.cfg +++ b/python-api/setup.cfg @@ -4,3 +4,4 @@ test=pytest [tool:pytest] addopts = --verbose python_files = src/test/python/*/*.py +cache_dir = target/.pytest_cache http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala ---------------------------------------------------------------------- diff --git a/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala b/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala index 97e746f..bab2bfa 100644 --- a/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala +++ b/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala @@ -201,9 +201,11 @@ object PythonInterpreter extends Logging { .newInstance(parts(0), gateway) } catch { case NonFatal(e) => - classOf[PythonProxyHandler].getConstructor(classOf[String], - Class.forName("py4j.CallbackClient"), classOf[Gateway]) - .newInstance(parts(0), gateway.getCallbackClient, gateway) + val cbClient = gateway.getClass().getMethod("getCallbackClient").invoke(gateway) + val cbClass = Class.forName("py4j.CallbackClient") + classOf[PythonProxyHandler] + .getConstructor(classOf[String], cbClass, classOf[Gateway]) + .newInstance(parts(0), cbClient, gateway) } Proxy.newProxyInstance(Thread.currentThread.getContextClassLoader,