This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 295f13f9a BIGTOP-4017. Upgrade Livy to 0.8.0. (#1192)
295f13f9a is described below
commit 295f13f9a75f2f47c5a0bb23ced9a1862b8af75c
Author: Kengo Seki <[email protected]>
AuthorDate: Wed Oct 18 11:49:59 2023 +0900
BIGTOP-4017. Upgrade Livy to 0.8.0. (#1192)
---
bigtop-packages/src/common/livy/do-component-build | 11 +-
.../src/common/livy/patch1-LIVY-756-0.7.1.diff | 1603 --------------------
bigtop.bom | 2 +-
3 files changed, 9 insertions(+), 1607 deletions(-)
diff --git a/bigtop-packages/src/common/livy/do-component-build
b/bigtop-packages/src/common/livy/do-component-build
index 5ee67b721..aca61c005 100644
--- a/bigtop-packages/src/common/livy/do-component-build
+++ b/bigtop-packages/src/common/livy/do-component-build
@@ -23,12 +23,17 @@ LIVY_BUILD_OPTS="-Drat.skip=true \
-Dskip=true \
-DskipITs=true \
-Dscala-${SCALA_VERSION%.*}.version=${SCALA_VERSION} \
- -Dscala-2.12.version=2.12.15 \
+ -Dscala.binary.version=${SCALA_VERSION%.*} \
+ -Dscala.version=${SCALA_VERSION} \
-Dzookeeper.version=${ZOOKEEPER_VERSION} \
-Dhadoop.version=${HADOOP_VERSION} \
+ -Dhadoop.major-minor.version=${HADOOP_VERSION%%.*} \
+ -Dhive.version=${HIVE_VERSION} \
-Dspark.scala-${SCALA_VERSION%.*}.version=${SPARK_VERSION} \
- -Pspark-3.0"
+ -Dspark.version=${SPARK_VERSION} \
+ -Pspark3"
mvn install $LIVY_BUILD_OPTS "$@"
-unzip assembly/target/apache-livy-$LIVY_VERSION-incubating-bin.zip && mv
apache-livy-$LIVY_VERSION-incubating-bin build
+unzip
assembly/target/apache-livy-$LIVY_VERSION-incubating_${SCALA_VERSION%.*}-bin.zip
+mv apache-livy-$LIVY_VERSION-incubating_${SCALA_VERSION%.*}-bin build
diff --git a/bigtop-packages/src/common/livy/patch1-LIVY-756-0.7.1.diff
b/bigtop-packages/src/common/livy/patch1-LIVY-756-0.7.1.diff
deleted file mode 100644
index c214b4aab..000000000
--- a/bigtop-packages/src/common/livy/patch1-LIVY-756-0.7.1.diff
+++ /dev/null
@@ -1,1603 +0,0 @@
-commit 840078c09b77f0d1d676d3f7c042ce154b0575a4
-Author: jerryshao <[email protected]>
-Date: Thu Jul 2 15:44:12 2020 +0800
-
- [LIVY-756] Add Spark 3.0 and Scala 2.12 support
-
- This PR is based tprelle 's PR #289 , and address all the left issues in
that PR:
-
- 1. multi-scala version support in one build (Scala 2.11 and 2.12 support).
- 2. make SparkR work.
-
- Also reverts most of the unnecessary changes. Besides this PR remove the
build below 2.4 (2.2, 2.3), since Spark 2.2 and 2.3 only ships with Scala 2.11,
hard to maintain multiple version. But user could still use 2.2 and 2.3 without
changes.
-
- All credits to tprelle.
-
- Run UT and IT with Spark 2.4.5 and 3.0.0 locally.
-
- Author: jerryshao <[email protected]>
-
- Closes #300 from jerryshao/LIVY-756.
-
- (cherry picked from commit 97cf2f75929ef6c152afc468adbead269bd0758f)
-
- Conflicts:
- integration-test/pom.xml
-
integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
- integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
- pom.xml
-
-diff --git a/README.md b/README.md
-index d5219e5..d454cbc 100644
---- a/README.md
-+++ b/README.md
-@@ -57,7 +57,7 @@ Required python packages for building Livy:
- To run Livy, you will also need a Spark installation. You can get Spark
releases at
- https://spark.apache.org/downloads.html.
-
--Livy requires Spark 2.2+. You can switch to a different version of Spark by
setting the
-+Livy requires Spark 2.4+. You can switch to a different version of Spark by
setting the
- ``SPARK_HOME`` environment variable in the Livy server process, without
needing to rebuild Livy.
-
-
-@@ -71,7 +71,7 @@ cd incubator-livy
- mvn package
- ```
-
--By default Livy is built against Apache Spark 2.2.0, but the version of Spark
used when running
-+By default Livy is built against Apache Spark 2.4.5, but the version of Spark
used when running
- Livy does not need to match the version used to build Livy. Livy internally
handles the differences
- between different Spark versions.
-
-diff --git a/assembly/assembly.xml b/assembly/assembly.xml
-index de61aee..eaefbb5 100644
---- a/assembly/assembly.xml
-+++ b/assembly/assembly.xml
-@@ -62,6 +62,13 @@
- <include>*</include>
- </includes>
- </fileSet>
-+ <fileSet>
-+
<directory>${project.parent.basedir}/repl/scala-2.12/target/jars</directory>
-+ <outputDirectory>${assembly.name}/repl_2.12-jars</outputDirectory>
-+ <includes>
-+ <include>*</include>
-+ </includes>
-+ </fileSet>
- <fileSet>
- <directory>${project.parent.basedir}/server/target/jars</directory>
- <outputDirectory>${assembly.name}/jars</outputDirectory>
-diff --git a/assembly/pom.xml b/assembly/pom.xml
-index 36bb48c..6928354 100644
---- a/assembly/pom.xml
-+++ b/assembly/pom.xml
-@@ -47,6 +47,12 @@
- <version>${project.version}</version>
- </dependency>
-
-+ <dependency>
-+ <groupId>${project.groupId}</groupId>
-+ <artifactId>livy-repl_2.12</artifactId>
-+ <version>${project.version}</version>
-+ </dependency>
-+
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>livy-server</artifactId>
-@@ -75,6 +81,23 @@
- </execution>
- </executions>
- </plugin>
-+
-+ <plugin>
-+ <groupId>org.apache.maven.plugins</groupId>
-+ <artifactId>maven-surefire-plugin</artifactId>
-+ <configuration>
-+ <skipTests>true</skipTests>
-+ </configuration>
-+ </plugin>
-+
-+ <plugin>
-+ <groupId>org.scalatest</groupId>
-+ <artifactId>scalatest-maven-plugin</artifactId>
-+ <configuration>
-+ <skipTests>true</skipTests>
-+ </configuration>
-+ </plugin>
-+
- </plugins>
- </build>
-
-diff --git a/client-common/pom.xml b/client-common/pom.xml
-index 3897c2b..3864527 100644
---- a/client-common/pom.xml
-+++ b/client-common/pom.xml
-@@ -36,7 +36,7 @@
- </dependency>
-
- <dependency>
-- <groupId>com.esotericsoftware.kryo</groupId>
-+ <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo</artifactId>
- </dependency>
- <dependency>
-diff --git
a/client-common/src/main/java/org/apache/livy/client/common/Serializer.java
b/client-common/src/main/java/org/apache/livy/client/common/Serializer.java
-index 3ea3f56..2f879ac 100644
---- a/client-common/src/main/java/org/apache/livy/client/common/Serializer.java
-+++ b/client-common/src/main/java/org/apache/livy/client/common/Serializer.java
-@@ -23,7 +23,8 @@ import java.nio.ByteBuffer;
- import com.esotericsoftware.kryo.Kryo;
- import com.esotericsoftware.kryo.io.Input;
- import com.esotericsoftware.kryo.io.Output;
--import
com.esotericsoftware.shaded.org.objenesis.strategy.StdInstantiatorStrategy;
-+import com.esotericsoftware.kryo.serializers.ClosureSerializer;
-+import org.objenesis.strategy.StdInstantiatorStrategy;
-
- import org.apache.livy.annotations.Private;
-
-@@ -49,7 +50,10 @@ public class Serializer {
- kryo.register(klass, REG_ID_BASE + count);
- count++;
- }
-- kryo.setInstantiatorStrategy(new StdInstantiatorStrategy());
-+ kryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(
-+ new StdInstantiatorStrategy()));
-+ kryo.register(java.lang.invoke.SerializedLambda.class);
-+ kryo.register(ClosureSerializer.Closure.class, new
ClosureSerializer());
- kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
- return kryo;
- }
-diff --git a/core/scala-2.12/pom.xml b/core/scala-2.12/pom.xml
-new file mode 100644
-index 0000000..5034774
---- /dev/null
-+++ b/core/scala-2.12/pom.xml
-@@ -0,0 +1,53 @@
-+<?xml version="1.0" encoding="UTF-8"?>
-+<!--
-+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
-+ ~ contributor license agreements. See the NOTICE file distributed with
-+ ~ this work for additional information regarding copyright ownership.
-+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
-+ ~ (the "License"); you may not use this file except in compliance with
-+ ~ the License. You may obtain a copy of the License at
-+ ~
-+ ~ http://www.apache.org/licenses/LICENSE-2.0
-+ ~
-+ ~ Unless required by applicable law or agreed to in writing, software
-+ ~ distributed under the License is distributed on an "AS IS" BASIS,
-+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ ~ See the License for the specific language governing permissions and
-+ ~ limitations under the License.
-+-->
-+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
-+ <modelVersion>4.0.0</modelVersion>
-+ <groupId>org.apache.livy</groupId>
-+ <artifactId>livy-core_2.12</artifactId>
-+ <version>0.7.1-incubating</version>
-+ <packaging>jar</packaging>
-+
-+ <parent>
-+ <groupId>org.apache.livy</groupId>
-+ <artifactId>livy-core-parent</artifactId>
-+ <version>0.7.1-incubating</version>
-+ <relativePath>../pom.xml</relativePath>
-+ </parent>
-+
-+ <properties>
-+ <scala.version>${scala-2.12.version}</scala.version>
-+ <scala.binary.version>2.12</scala.binary.version>
-+ </properties>
-+
-+ <build>
-+ <plugins>
-+ <plugin>
-+ <groupId>org.apache.maven.plugins</groupId>
-+ <artifactId>maven-jar-plugin</artifactId>
-+ <executions>
-+ <execution>
-+ <goals>
-+ <goal>test-jar</goal>
-+ </goals>
-+ </execution>
-+ </executions>
-+ </plugin>
-+ </plugins>
-+ </build>
-+
-+</project>
-diff --git a/core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala
b/core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala
-index 908172b..65c5cd8 100644
---- a/core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala
-+++ b/core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala
-@@ -17,9 +17,9 @@
-
- package org.apache.livy
-
--import org.scalatest.{Outcome, Suite}
-+import org.scalatest.{Outcome, TestSuite}
-
--trait LivyBaseUnitTestSuite extends Suite with Logging {
-+trait LivyBaseUnitTestSuite extends TestSuite with Logging {
-
- protected override def withFixture(test: NoArgTest): Outcome = {
- val testName = test.name
-diff --git a/coverage/pom.xml b/coverage/pom.xml
-index 9c23dca..495b86c 100644
---- a/coverage/pom.xml
-+++ b/coverage/pom.xml
-@@ -56,12 +56,24 @@
- <version>${project.version}</version>
- </dependency>
-
-+ <dependency>
-+ <groupId>${project.groupId}</groupId>
-+ <artifactId>livy-core_2.12</artifactId>
-+ <version>${project.version}</version>
-+ </dependency>
-+
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>livy-repl_2.11</artifactId>
- <version>${project.version}</version>
- </dependency>
-
-+ <dependency>
-+ <groupId>${project.groupId}</groupId>
-+ <artifactId>livy-repl_2.12</artifactId>
-+ <version>${project.version}</version>
-+ </dependency>
-+
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>livy-rsc</artifactId>
-@@ -80,6 +92,12 @@
- <version>${project.version}</version>
- </dependency>
-
-+ <dependency>
-+ <groupId>${project.groupId}</groupId>
-+ <artifactId>livy-scala-api_2.12</artifactId>
-+ <version>${project.version}</version>
-+ </dependency>
-+
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>livy-integration-test</artifactId>
-@@ -116,6 +134,23 @@
- </execution>
- </executions>
- </plugin>
-+
-+ <plugin>
-+ <groupId>org.apache.maven.plugins</groupId>
-+ <artifactId>maven-surefire-plugin</artifactId>
-+ <configuration>
-+ <skipTests>true</skipTests>
-+ </configuration>
-+ </plugin>
-+
-+ <plugin>
-+ <groupId>org.scalatest</groupId>
-+ <artifactId>scalatest-maven-plugin</artifactId>
-+ <configuration>
-+ <skipTests>true</skipTests>
-+ </configuration>
-+ </plugin>
-+
- </plugins>
- </build>
-
-diff --git
a/examples/src/main/scala/org/apache/livy/examples/WordCountApp.scala
b/examples/src/main/scala/org/apache/livy/examples/WordCountApp.scala
-index da30a76..2285f52 100644
---- a/examples/src/main/scala/org/apache/livy/examples/WordCountApp.scala
-+++ b/examples/src/main/scala/org/apache/livy/examples/WordCountApp.scala
-@@ -119,7 +119,7 @@ object WordCountApp {
- scalaClient.submit { context =>
- val sqlctx = context.sqlctx
- val rdd = sqlctx.read.json(inputPath)
-- rdd.registerTempTable("words")
-+ rdd.createOrReplaceTempView("words")
- val result = sqlctx.sql("select word, count(word) as word_count from
words " +
- "group by word order by word_count desc limit 1")
- result.first().toString()
-diff --git a/integration-test/pom.xml b/integration-test/pom.xml
-index 80a9c29..e36531d 100644
---- a/integration-test/pom.xml
-+++ b/integration-test/pom.xml
-@@ -297,6 +297,7 @@
- <LIVY_INTEGRATION_TEST>true</LIVY_INTEGRATION_TEST>
-
<SPARK_HOME>${project.build.directory}/${spark.bin.name}</SPARK_HOME>
-
<LIVY_TEST_THRIFT_ENABLED>${livy.test.thrift.enabled}</LIVY_TEST_THRIFT_ENABLED>
-+
<PYSPARK_ROW_FIELD_SORTING_ENABLED>true</PYSPARK_ROW_FIELD_SORTING_ENABLED>
- </environmentVariables>
- <systemProperties>
- <cluster.spec>${cluster.spec}</cluster.spec>
-diff --git a/integration-test/src/test/resources/rtest.R
b/integration-test/src/test/resources/rtest.R
-index a026a10..d955b7f 100644
---- a/integration-test/src/test/resources/rtest.R
-+++ b/integration-test/src/test/resources/rtest.R
-@@ -17,18 +17,17 @@
-
- library(SparkR)
-
--# Initialize SparkContext and SQLContext
--sc <- sparkR.init(appName="SparkR-DataFrame-example")
--sqlContext <- sparkRSQL.init(sc)
-+# Initialize SparkSession
-+sparkR.session(appName = "SparkR-DataFrame-example")
-
- # Create a simple local data.frame
- localDF <- data.frame(name=c("John", "Smith", "Sarah"), age=c(19, 23, 18))
-
- # Convert local data frame to a SparkDataFrame
--df <- createDataFrame(sqlContext, localDF)
-+df <- createDataFrame(localDF)
-
- # Print its schema
- printSchema(df)
-
- # Stop the SparkContext now
--sparkR.stop()
-+sparkR.session.stop()
-diff --git
a/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
b/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
-index 0c3d632..d1fa5e8 100644
---- a/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
-+++ b/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
-@@ -37,7 +37,8 @@ class InteractiveIT extends BaseIntegrationTestSuite {
- s.run("val scalaVersion =
util.Properties.versionString").result().left.foreach(info(_))
- s.run("1+1").verifyResult("res0: Int = 2\n")
-
s.run("""sc.getConf.get("spark.executor.instances")""").verifyResult("res1:
String = 1\n")
-- s.run("val sql = new org.apache.spark.sql.SQLContext(sc)").verifyResult(
-+
-+ s.run("val sql = spark.sqlContext").verifyResult(
- ".*" + Pattern.quote(
- "sql: org.apache.spark.sql.SQLContext =
org.apache.spark.sql.SQLContext") + ".*")
- s.run("abcde").verifyError(evalue = ".*?:[0-9]+: error: not found:
value abcde.*")
-@@ -47,7 +48,7 @@ class InteractiveIT extends BaseIntegrationTestSuite {
- // Verify query submission
- s.run(s"""val df = spark.createDataFrame(Seq(("jerry", 20), ("michael",
21)))""")
- .verifyResult(".*" + Pattern.quote("df:
org.apache.spark.sql.DataFrame") + ".*")
-- s.run("df.registerTempTable(\"people\")").result()
-+ s.run("df.createOrReplaceTempView(\"people\")").result()
- s.run("SELECT * FROM people",
Some(SQL)).verifyResult(".*\"jerry\",20.*\"michael\",21.*")
-
- // Verify Livy internal configurations are not exposed.
-@@ -104,7 +105,7 @@ class InteractiveIT extends BaseIntegrationTestSuite {
- s.run("1+1").verifyResult(startsWith(s"[$count] 2"))
- s.run("""localDF <- data.frame(name=c("John", "Smith", "Sarah"),
age=c(19, 23, 18))""")
- .verifyResult(null)
-- s.run("df <- createDataFrame(sqlContext, localDF)").verifyResult(null)
-+ s.run("df <- createDataFrame(localDF)").verifyResult(null)
- s.run("printSchema(df)").verifyResult(literal(
- """|root
- | |-- name: string (nullable = true)
-diff --git a/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
b/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
-index 20ebf3d..441b103 100644
---- a/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
-+++ b/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
-@@ -22,6 +22,8 @@ import java.net.URI
- import java.util.concurrent.{TimeUnit, Future => JFuture}
- import javax.servlet.http.HttpServletResponse
-
-+import scala.util.Properties
-+
- import com.fasterxml.jackson.databind.ObjectMapper
- import com.fasterxml.jackson.module.scala.DefaultScalaModule
- import org.scalatest.BeforeAndAfterAll
-@@ -31,6 +33,7 @@ import org.apache.livy.client.common.HttpMessages._
- import org.apache.livy.sessions.SessionKindModule
- import org.apache.livy.test.framework.BaseIntegrationTestSuite
- import org.apache.livy.test.jobs.spark2._
-+import org.apache.livy.utils.LivySparkUtils
-
- class Spark2JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll
with Logging {
-
-@@ -50,7 +53,8 @@ class Spark2JobApiIT extends BaseIntegrationTestSuite with
BeforeAndAfterAll wit
- livyClient.connectSession(sessionId).stop()
- }
-
-- test("create a new session and upload test jar") {
-+ scalaTest("create a new session and upload test jar") {
-+ val prevSessionCount = sessionList().total
- val tempClient = createClient(livyEndpoint)
-
- try {
-@@ -78,13 +82,13 @@ class Spark2JobApiIT extends BaseIntegrationTestSuite with
BeforeAndAfterAll wit
- }
- }
-
-- test("run spark2 job") {
-+ scalaTest("run spark2 job") {
- assume(client != null, "Client not active.")
- val result = waitFor(client.submit(new SparkSessionTest()))
- assert(result === 3)
- }
-
-- test("run spark2 dataset job") {
-+ scalaTest("run spark2 dataset job") {
- assume(client != null, "Client not active.")
- val result = waitFor(client.submit(new DatasetTest()))
- assert(result === 2)
-@@ -103,4 +107,18 @@ class Spark2JobApiIT extends BaseIntegrationTestSuite
with BeforeAndAfterAll wit
- private def createClient(uri: String): LivyClient = {
- new LivyClientBuilder().setURI(new URI(uri)).build()
- }
-+
-+ protected def scalaTest(desc: String)(testFn: => Unit): Unit = {
-+ test(desc) {
-+ val livyConf = new LivyConf()
-+ val (sparkVersion, scalaVersion) =
LivySparkUtils.sparkSubmitVersion(livyConf)
-+ val formattedSparkVersion =
LivySparkUtils.formatSparkVersion(sparkVersion)
-+ val versionString =
-+ LivySparkUtils.sparkScalaVersion(formattedSparkVersion, scalaVersion,
livyConf)
-+
-+ assume(versionString ==
LivySparkUtils.formatScalaVersion(Properties.versionNumberString),
-+ s"Scala test can only be run with ${Properties.versionString}")
-+ testFn
-+ }
-+ }
- }
-diff --git a/pom.xml b/pom.xml
-index 20b1a55..9b76e78 100644
---- a/pom.xml
-+++ b/pom.xml
-@@ -78,40 +78,46 @@
- </mailingLists>
-
- <properties>
-+ <asynchttpclient.version>2.10.1</asynchttpclient.version>
- <hadoop.version>2.7.3</hadoop.version>
- <hadoop.scope>compile</hadoop.scope>
-- <spark.scala-2.11.version>2.2.3</spark.scala-2.11.version>
-+ <spark.scala-2.11.version>2.4.5</spark.scala-2.11.version>
-+ <spark.scala-2.12.version>2.4.5</spark.scala-2.12.version>
- <spark.version>${spark.scala-2.11.version}</spark.version>
- <hive.version>3.0.0</hive.version>
- <commons-codec.version>1.9</commons-codec.version>
- <httpclient.version>4.5.3</httpclient.version>
- <httpcore.version>4.4.4</httpcore.version>
-- <jackson.version>2.9.9</jackson.version>
-+ <jackson.version>2.10.1</jackson.version>
- <javax.servlet-api.version>3.1.0</javax.servlet-api.version>
- <jetty.version>9.3.24.v20180605</jetty.version>
-- <json4s.version>3.2.11</json4s.version>
-+ <json4s.spark-2.11.version>3.5.3</json4s.spark-2.11.version>
-+ <json4s.spark-2.12.version>3.5.3</json4s.spark-2.12.version>
-+ <json4s.version>${json4s.spark-2.11.version}</json4s.version>
- <junit.version>4.11</junit.version>
- <libthrift.version>0.9.3</libthrift.version>
-- <kryo.version>2.22</kryo.version>
-+ <kryo.version>4.0.2</kryo.version>
- <metrics.version>3.1.0</metrics.version>
-- <mockito.version>1.9.5</mockito.version>
-- <netty.spark-2.11.version>4.0.37.Final</netty.spark-2.11.version>
-+ <mockito.version>1.10.19</mockito.version>
-+ <netty.spark-2.11.version>4.1.17.Final</netty.spark-2.11.version>
-+ <netty.spark-2.12.version>4.1.17.Final</netty.spark-2.12.version>
- <netty.version>${netty.spark-2.11.version}</netty.version>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <py4j.version>0.10.7</py4j.version>
- <scala-2.11.version>2.11.12</scala-2.11.version>
-+ <scala-2.12.version>2.12.10</scala-2.12.version>
- <scala.binary.version>2.11</scala.binary.version>
- <scala.version>${scala-2.11.version}</scala.version>
-- <scalatest.version>2.2.4</scalatest.version>
-- <scalatra.version>2.3.0</scalatra.version>
-+ <scalatest.version>3.0.8</scalatest.version>
-+ <scalatra.version>2.6.5</scalatra.version>
- <java.version>1.8</java.version>
- <test.redirectToFile>true</test.redirectToFile>
- <execution.root>${user.dir}</execution.root>
- <spark.home>${execution.root}/dev/spark</spark.home>
- <spark.bin.download.url>
--
https://archive.apache.org/dist/spark/spark-2.2.3/spark-2.2.3-bin-hadoop2.7.tgz
-+
https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz
- </spark.bin.download.url>
-- <spark.bin.name>spark-2.2.3-bin-hadoop2.7</spark.bin.name>
-+ <spark.bin.name>spark-2.4.5-bin-hadoop2.7</spark.bin.name>
- <!-- used for testing, NCSARequestLog use it for access log -->
- <livy.log.dir>${basedir}/target</livy.log.dir>
-
-@@ -199,15 +205,18 @@
- <module>client-http</module>
- <module>core</module>
- <module>core/scala-2.11</module>
-+ <module>core/scala-2.12</module>
- <module>coverage</module>
- <module>examples</module>
- <module>python-api</module>
- <module>repl</module>
- <module>repl/scala-2.11</module>
-+ <module>repl/scala-2.12</module>
- <module>rsc</module>
- <module>scala</module>
- <module>scala-api</module>
- <module>scala-api/scala-2.11</module>
-+ <module>scala-api/scala-2.12</module>
- <module>server</module>
- <module>test-lib</module>
- <module>integration-test</module>
-@@ -240,6 +249,12 @@
- <artifactId>scalatra-scalatest_${scala.binary.version}</artifactId>
- <version>${scalatra.version}</version>
- <scope>test</scope>
-+ <exclusions>
-+ <exclusion>
-+ <groupId>org.mockito</groupId>
-+ <artifactId>*</artifactId>
-+ </exclusion>
-+ </exclusions>
- </dependency>
-
- </dependencies>
-@@ -254,7 +269,7 @@
- </dependency>
-
- <dependency>
-- <groupId>com.esotericsoftware.kryo</groupId>
-+ <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo</artifactId>
- <version>${kryo.version}</version>
- </dependency>
-@@ -545,6 +560,12 @@
- <version>${scalatra.version}</version>
- </dependency>
-
-+ <dependency>
-+ <groupId>org.scalatra</groupId>
-+ <artifactId>scalatra-metrics_${scala.binary.version}</artifactId>
-+ <version>${scalatra.version}</version>
-+ </dependency>
-+
- <dependency>
- <groupId>org.scalatra</groupId>
- <artifactId>scalatra-test_${scala.binary.version}</artifactId>
-@@ -557,6 +578,12 @@
- <version>${py4j.version}</version>
- </dependency>
-
-+ <dependency>
-+ <groupId>org.asynchttpclient</groupId>
-+ <artifactId>async-http-client</artifactId>
-+ <version>${asynchttpclient.version}</version>
-+ </dependency>
-+
- <!-- we need a version > 1.7.13 because of SLF4J-324 -->
- <dependency>
- <groupId>org.slf4j</groupId>
-@@ -611,7 +638,7 @@
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
-- <version>3.2.2</version>
-+ <version>4.3.0</version>
- <executions>
- <execution>
- <goals>
-@@ -630,7 +657,6 @@
- <configuration>
- <scalaVersion>${scala.version}</scalaVersion>
- <recompileMode>incremental</recompileMode>
-- <useZincServer>true</useZincServer>
- <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
- <args>
- <arg>-unchecked</arg>
-@@ -661,7 +687,7 @@
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
-- <version>2.4.2</version>
-+ <version>3.2.1</version>
- </plugin>
-
- <plugin>
-@@ -693,7 +719,7 @@
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
-- <version>1.0</version>
-+ <version>2.0.0</version>
- <configuration>
- <environmentVariables>
- <LIVY_TEST>true</LIVY_TEST>
-@@ -1025,37 +1051,29 @@
- </modules>
- </profile>
-
-- <!-- Spark version profiles -->
-- <profile>
-- <id>spark-2.3</id>
-- <properties>
-- <spark.scala-2.11.version>2.3.3</spark.scala-2.11.version>
-- <spark.version>${spark.scala-2.11.version}</spark.version>
-- <netty.spark-2.11.version>4.1.17.Final</netty.spark-2.11.version>
-- <spark.bin.download.url>
--
https://archive.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz
-- </spark.bin.download.url>
-- <spark.bin.name>spark-2.3.3-bin-hadoop2.7</spark.bin.name>
-- </properties>
-- </profile>
--
- <profile>
-- <id>spark-2.4</id>
-+ <id>spark-3.0</id>
- <activation>
- <property>
-- <name>spark-2.4</name>
-+ <name>spark-3.0</name>
- </property>
- </activation>
- <properties>
-- <spark.scala-2.11.version>2.4.3</spark.scala-2.11.version>
-+ <spark.scala-2.12.version>3.0.0</spark.scala-2.12.version>
-+ <spark.scala-2.11.version>2.4.5</spark.scala-2.11.version>
- <spark.version>${spark.scala-2.11.version}</spark.version>
-- <netty.spark-2.11.version>4.1.17.Final</netty.spark-2.11.version>
-+ <netty.spark-2.12.version>4.1.47.Final</netty.spark-2.12.version>
-+ <netty.spark-2.11.version>4.1.47.Final</netty.spark-2.11.version>
-+ <netty.version>${netty.spark-2.11.version}</netty.version>
- <java.version>1.8</java.version>
-- <py4j.version>0.10.7</py4j.version>
-+ <py4j.version>0.10.9</py4j.version>
-+ <json4s.spark-2.11.version>3.5.3</json4s.spark-2.11.version>
-+ <json4s.spark-2.12.version>3.6.6</json4s.spark-2.12.version>
-+ <json4s.version>${json4s.spark-2.11.version}</json4s.version>
- <spark.bin.download.url>
--
https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz
-+
https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz
- </spark.bin.download.url>
-- <spark.bin.name>spark-2.4.3-bin-hadoop2.7</spark.bin.name>
-+ <spark.bin.name>spark-3.1.2-bin-hadoop3.2</spark.bin.name>
- </properties>
- </profile>
-
-diff --git a/repl/pom.xml b/repl/pom.xml
-index e01f7ab..b7e7fc5 100644
---- a/repl/pom.xml
-+++ b/repl/pom.xml
-@@ -175,6 +175,7 @@
-
<include>org.json4s:json4s-ast_${scala.binary.version}</include>
-
<include>org.json4s:json4s-core_${scala.binary.version}</include>
-
<include>org.json4s:json4s-jackson_${scala.binary.version}</include>
-+
<include>org.json4s:json4s-scalap_${scala.binary.version}</include>
- </includes>
- </artifactSet>
- <filters>
-@@ -211,6 +212,8 @@
- json4s-ast_${scala.binary.version},
- json4s-core_${scala.binary.version},
- json4s-jackson_${scala.binary.version},
-+ json4s-scalap_${scala.binary.version},
-+ scala-xml_${scala.binary.version},
- paranamer,
- scalap
- </excludeArtifactIds>
-diff --git a/repl/scala-2.11/pom.xml b/repl/scala-2.11/pom.xml
-index 4b85232..9bacd33 100644
---- a/repl/scala-2.11/pom.xml
-+++ b/repl/scala-2.11/pom.xml
-@@ -36,6 +36,7 @@
- <scala.binary.version>2.11</scala.binary.version>
- <spark.version>${spark.scala-2.11.version}</spark.version>
- <netty.version>${netty.spark-2.11.version}</netty.version>
-+ <json4s.version>${json4s.spark-2.11.version}</json4s.version>
- </properties>
-
- </project>
-diff --git
a/repl/scala-2.11/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
b/repl/scala-2.11/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
-index 7de2859..98c478f 100644
----
a/repl/scala-2.11/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
-+++
b/repl/scala-2.11/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
-@@ -27,15 +27,12 @@ import scala.tools.nsc.interpreter.IMain
- import scala.tools.nsc.interpreter.JLineCompletion
- import scala.tools.nsc.interpreter.JPrintWriter
- import scala.tools.nsc.interpreter.Results.Result
--import scala.util.control.NonFatal
-
- import org.apache.spark.SparkConf
- import org.apache.spark.repl.SparkILoop
-
--import org.apache.livy.rsc.driver.SparkEntries
--
- /**
-- * This represents a Spark interpreter. It is not thread safe.
-+ * This represents a Scala 2.11 Spark interpreter. It is not thread safe.
- */
- class SparkInterpreter(protected override val conf: SparkConf) extends
AbstractSparkInterpreter {
-
-diff --git a/repl/scala-2.12/pom.xml b/repl/scala-2.12/pom.xml
-new file mode 100644
-index 0000000..d17a521
---- /dev/null
-+++ b/repl/scala-2.12/pom.xml
-@@ -0,0 +1,42 @@
-+<?xml version="1.0" encoding="UTF-8"?>
-+<!--
-+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
-+ ~ contributor license agreements. See the NOTICE file distributed with
-+ ~ this work for additional information regarding copyright ownership.
-+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
-+ ~ (the "License"); you may not use this file except in compliance with
-+ ~ the License. You may obtain a copy of the License at
-+ ~
-+ ~ http://www.apache.org/licenses/LICENSE-2.0
-+ ~
-+ ~ Unless required by applicable law or agreed to in writing, software
-+ ~ distributed under the License is distributed on an "AS IS" BASIS,
-+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ ~ See the License for the specific language governing permissions and
-+ ~ limitations under the License.
-+-->
-+<project xmlns="http://maven.apache.org/POM/4.0.0";
-+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
-+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
-+ <modelVersion>4.0.0</modelVersion>
-+ <groupId>org.apache.livy</groupId>
-+ <artifactId>livy-repl_2.12</artifactId>
-+ <version>0.7.1-incubating</version>
-+ <packaging>jar</packaging>
-+
-+ <parent>
-+ <groupId>org.apache.livy</groupId>
-+ <artifactId>livy-repl-parent</artifactId>
-+ <version>0.7.1-incubating</version>
-+ <relativePath>../pom.xml</relativePath>
-+ </parent>
-+
-+ <properties>
-+ <scala.version>${scala-2.12.version}</scala.version>
-+ <scala.binary.version>2.12</scala.binary.version>
-+ <spark.version>${spark.scala-2.12.version}</spark.version>
-+ <netty.version>${netty.spark-2.12.version}</netty.version>
-+ <json4s.version>${json4s.spark-2.12.version}</json4s.version>
-+ </properties>
-+
-+</project>
-diff --git
a/repl/scala-2.12/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
b/repl/scala-2.12/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
-new file mode 100644
-index 0000000..bb8f7e5
---- /dev/null
-+++
b/repl/scala-2.12/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
-@@ -0,0 +1,132 @@
-+/*
-+ * Licensed to the Apache Software Foundation (ASF) under one or more
-+ * contributor license agreements. See the NOTICE file distributed with
-+ * this work for additional information regarding copyright ownership.
-+ * The ASF licenses this file to You under the Apache License, Version 2.0
-+ * (the "License"); you may not use this file except in compliance with
-+ * the License. You may obtain a copy of the License at
-+ *
-+ * http://www.apache.org/licenses/LICENSE-2.0
-+ *
-+ * Unless required by applicable law or agreed to in writing, software
-+ * distributed under the License is distributed on an "AS IS" BASIS,
-+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ * See the License for the specific language governing permissions and
-+ * limitations under the License.
-+ */
-+
-+package org.apache.livy.repl
-+
-+import java.io.File
-+import java.net.URLClassLoader
-+import java.nio.file.{Files, Paths}
-+
-+import scala.tools.nsc.Settings
-+import scala.tools.nsc.interpreter.Completion
-+import scala.tools.nsc.interpreter.IMain
-+import scala.tools.nsc.interpreter.JPrintWriter
-+import scala.tools.nsc.interpreter.NoCompletion
-+import scala.tools.nsc.interpreter.Results.Result
-+
-+import org.apache.spark.SparkConf
-+import org.apache.spark.repl.SparkILoop
-+
-+/**
-+ * This represents a Spark interpreter. It is not thread safe.
-+ */
-+class SparkInterpreter(protected override val conf: SparkConf) extends
AbstractSparkInterpreter {
-+
-+ private var sparkILoop: SparkILoop = _
-+
-+ override def start(): Unit = {
-+ require(sparkILoop == null)
-+
-+ val rootDir = conf.get("spark.repl.classdir",
System.getProperty("java.io.tmpdir"))
-+ val outputDir = Files.createTempDirectory(Paths.get(rootDir),
"spark").toFile
-+ outputDir.deleteOnExit()
-+ conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath)
-+
-+ val settings = new Settings()
-+ settings.processArguments(List("-Yrepl-class-based",
-+ "-Yrepl-outdir", s"${outputDir.getAbsolutePath}"), true)
-+ settings.usejavacp.value = true
-+ settings.embeddedDefaults(Thread.currentThread().getContextClassLoader())
-+
-+ sparkILoop = new SparkILoop(None, new JPrintWriter(outputStream, true))
-+ sparkILoop.settings = settings
-+ sparkILoop.createInterpreter()
-+ sparkILoop.initializeSynchronous()
-+
-+ restoreContextClassLoader {
-+ sparkILoop.compilerClasspath
-+ sparkILoop.ensureClassLoader
-+ var classLoader = Thread.currentThread().getContextClassLoader
-+ while (classLoader != null) {
-+ if (classLoader.getClass.getCanonicalName ==
-+ "org.apache.spark.util.MutableURLClassLoader") {
-+ val extraJarPath =
classLoader.asInstanceOf[URLClassLoader].getURLs()
-+ // Check if the file exists. Otherwise an exception will be
thrown.
-+ .filter { u => u.getProtocol == "file" && new
File(u.getPath).isFile }
-+ // Livy rsc and repl are also in the extra jars list. Filter them
out.
-+ .filterNot { u =>
Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
-+ // Some bad spark packages depend on the wrong version of
scala-reflect. Blacklist it.
-+ .filterNot { u =>
-+
Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
-+ }
-+
-+ extraJarPath.foreach { p => debug(s"Adding $p to Scala
interpreter's class path...") }
-+ sparkILoop.addUrlsToClassPath(extraJarPath: _*)
-+ classLoader = null
-+ } else {
-+ classLoader = classLoader.getParent
-+ }
-+ }
-+
-+ postStart()
-+ }
-+ }
-+
-+ override def close(): Unit = synchronized {
-+ super.close()
-+
-+ if (sparkILoop != null) {
-+ sparkILoop.closeInterpreter()
-+ sparkILoop = null
-+ }
-+ }
-+
-+ override protected def isStarted(): Boolean = {
-+ sparkILoop != null
-+ }
-+
-+ override protected def interpret(code: String): Result = {
-+ sparkILoop.interpret(code)
-+ }
-+
-+ override protected def completeCandidates(code: String, cursor: Int) :
Array[String] = {
-+ val completer : Completion = {
-+ try {
-+ val cls =
Class.forName("scala.tools.nsc.interpreter.PresentationCompilerCompleter")
-+
cls.getDeclaredConstructor(classOf[IMain]).newInstance(sparkILoop.intp)
-+ .asInstanceOf[Completion]
-+ } catch {
-+ case e : ClassNotFoundException => NoCompletion
-+ }
-+ }
-+ completer.complete(code, cursor).candidates.toArray
-+ }
-+
-+ override protected def valueOfTerm(name: String): Option[Any] = {
-+ // IMain#valueOfTerm will always return None, so use other way instead.
-+ Option(sparkILoop.lastRequest.lineRep.call("$result"))
-+ }
-+
-+ override protected def bind(name: String,
-+ tpe: String,
-+ value: Object,
-+ modifier: List[String]): Unit = {
-+ sparkILoop.beQuietDuring {
-+ sparkILoop.bind(name, tpe, value, modifier)
-+ }
-+ }
-+}
-diff --git
a/repl/scala-2.12/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
b/repl/scala-2.12/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
-new file mode 100644
-index 0000000..d922034
---- /dev/null
-+++
b/repl/scala-2.12/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
-@@ -0,0 +1,68 @@
-+/*
-+ * Licensed to the Apache Software Foundation (ASF) under one or more
-+ * contributor license agreements. See the NOTICE file distributed with
-+ * this work for additional information regarding copyright ownership.
-+ * The ASF licenses this file to You under the Apache License, Version 2.0
-+ * (the "License"); you may not use this file except in compliance with
-+ * the License. You may obtain a copy of the License at
-+ *
-+ * http://www.apache.org/licenses/LICENSE-2.0
-+ *
-+ * Unless required by applicable law or agreed to in writing, software
-+ * distributed under the License is distributed on an "AS IS" BASIS,
-+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ * See the License for the specific language governing permissions and
-+ * limitations under the License.
-+ */
-+
-+package org.apache.livy.repl
-+
-+import org.scalatest._
-+
-+import org.apache.livy.LivyBaseUnitTestSuite
-+
-+class SparkInterpreterSpec extends FunSpec with Matchers with
LivyBaseUnitTestSuite {
-+ describe("SparkInterpreter") {
-+ val interpreter = new SparkInterpreter(null)
-+
-+ it("should parse Scala compile error.") {
-+ // Regression test for LIVY-.
-+ val error =
-+ """<console>:27: error: type mismatch;
-+ | found : Int
-+ | required: String
-+ | sc.setJobGroup(groupName, groupName, true)
-+ | ^
-+ |<console>:27: error: type mismatch;
-+ | found : Int
-+ | required: String
-+ | sc.setJobGroup(groupName, groupName, true)
-+ | ^
-+ |""".stripMargin
-+
-+ val parsedError =
AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(error)
-+
-+ val expectedTraceback = parsedError.tail
-+
-+ val (ename, traceback) = interpreter.parseError(error)
-+ ename shouldBe "<console>:27: error: type mismatch;"
-+ traceback shouldBe expectedTraceback
-+ }
-+
-+ it("should parse Scala runtime error.") {
-+ val error =
-+ """java.lang.RuntimeException: message
-+ | ... 48 elided
-+ |
-+ |Tailing message""".stripMargin
-+
-+ val parsedError =
AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(error)
-+
-+ val expectedTraceback = parsedError.tail
-+
-+ val (ename, traceback) = interpreter.parseError(error)
-+ ename shouldBe "java.lang.RuntimeException: message"
-+ traceback shouldBe expectedTraceback
-+ }
-+ }
-+}
-diff --git a/repl/src/main/scala/org/apache/livy/repl/Session.scala
b/repl/src/main/scala/org/apache/livy/repl/Session.scala
-index ea8a761..262c811 100644
---- a/repl/src/main/scala/org/apache/livy/repl/Session.scala
-+++ b/repl/src/main/scala/org/apache/livy/repl/Session.scala
-@@ -348,8 +348,10 @@ class Session(
- case "1" =>
- (s"""setJobGroup(sc, "$jobGroup", "Job group for statement
$jobGroup", FALSE)""",
- SparkR)
-- case "2" =>
-+ case "2" | "3" =>
- (s"""setJobGroup("$jobGroup", "Job group for statement
$jobGroup", FALSE)""", SparkR)
-+ case v =>
-+ throw new IllegalArgumentException(s"Unknown Spark major version
[$v]")
- }
- }
- // Set the job group
-diff --git a/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
b/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
-index 781ed72..3d9d4ac 100644
---- a/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
-+++ b/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
-@@ -113,7 +113,7 @@ class SQLInterpreterSpec extends BaseInterpreterSpec {
- it should "execute sql queries" in withInterpreter { interpreter =>
- val rdd = sparkEntries.sc().parallelize(Seq(People("Jerry", 20),
People("Michael", 21)))
- val df = sparkEntries.sqlctx().createDataFrame(rdd)
-- df.registerTempTable("people")
-+ df.createOrReplaceTempView("people")
-
- // Test normal behavior
- val resp1 = interpreter.execute(
-@@ -159,7 +159,7 @@ class SQLInterpreterSpec extends BaseInterpreterSpec {
- ("1", new java.math.BigDecimal(1.0)),
- ("2", new java.math.BigDecimal(2.0))))
- val df = sparkEntries.sqlctx().createDataFrame(rdd).selectExpr("_1 as
col1", "_2 as col2")
-- df.registerTempTable("test")
-+ df.createOrReplaceTempView("test")
-
- val resp1 = interpreter.execute(
- """
-diff --git a/rsc/pom.xml b/rsc/pom.xml
-index 057a592..97462b2 100644
---- a/rsc/pom.xml
-+++ b/rsc/pom.xml
-@@ -58,7 +58,7 @@
- </dependency>
-
- <dependency>
-- <groupId>com.esotericsoftware.kryo</groupId>
-+ <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo</artifactId>
- </dependency>
- <dependency>
-@@ -140,7 +140,9 @@
- <artifactSet>
- <includes>
- <include>org.apache.livy:livy-client-common</include>
-- <include>com.esotericsoftware.kryo:kryo</include>
-+ <include>com.esotericsoftware:kryo</include>
-+ <include>com.esotericsoftware:minlog</include>
-+ <include>com.esotericsoftware:reflectasm</include>
- </includes>
- </artifactSet>
- <filters>
-diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/SparkEntries.java
b/rsc/src/main/java/org/apache/livy/rsc/driver/SparkEntries.java
-index c64fc72..6726bb1 100644
---- a/rsc/src/main/java/org/apache/livy/rsc/driver/SparkEntries.java
-+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/SparkEntries.java
-@@ -17,7 +17,6 @@
-
- package org.apache.livy.rsc.driver;
-
--import java.lang.reflect.Method;
- import java.util.concurrent.TimeUnit;
-
- import org.apache.spark.SparkConf;
-@@ -60,7 +59,7 @@ public class SparkEntries {
- return sc;
- }
-
-- public SparkSession sparkSession() throws Exception {
-+ public SparkSession sparkSession() {
- if (sparksession == null) {
- synchronized (this) {
- if (sparksession == null) {
-@@ -100,7 +99,7 @@ public class SparkEntries {
- if (sqlctx == null) {
- synchronized (this) {
- if (sqlctx == null) {
-- sqlctx = new SQLContext(sc());
-+ sqlctx = sparkSession().sqlContext();
- LOG.info("Created SQLContext.");
- }
- }
-@@ -112,7 +111,7 @@ public class SparkEntries {
- if (hivectx == null) {
- synchronized (this) {
- if (hivectx == null) {
-- SparkConf conf = sc.getConf();
-+ SparkConf conf = sc().getConf();
- if (conf.getBoolean("spark.repl.enableHiveContext", false) ||
- conf.get("spark.sql.catalogImplementation",
"in-memory").toLowerCase()
- .equals("hive")) {
-diff --git a/rsc/src/main/java/org/apache/livy/rsc/rpc/KryoMessageCodec.java
b/rsc/src/main/java/org/apache/livy/rsc/rpc/KryoMessageCodec.java
-index b860e65..191ea50 100644
---- a/rsc/src/main/java/org/apache/livy/rsc/rpc/KryoMessageCodec.java
-+++ b/rsc/src/main/java/org/apache/livy/rsc/rpc/KryoMessageCodec.java
-@@ -17,17 +17,10 @@
-
- package org.apache.livy.rsc.rpc;
-
--import java.io.ByteArrayOutputStream;
- import java.io.IOException;
- import java.nio.ByteBuffer;
--import java.util.Arrays;
- import java.util.List;
-
--import com.esotericsoftware.kryo.Kryo;
--import com.esotericsoftware.kryo.io.ByteBufferInputStream;
--import com.esotericsoftware.kryo.io.Input;
--import com.esotericsoftware.kryo.io.Output;
--import
com.esotericsoftware.shaded.org.objenesis.strategy.StdInstantiatorStrategy;
- import io.netty.buffer.ByteBuf;
- import io.netty.channel.ChannelHandlerContext;
- import io.netty.handler.codec.ByteToMessageCodec;
-diff --git a/scala-api/scala-2.12/pom.xml b/scala-api/scala-2.12/pom.xml
-new file mode 100644
-index 0000000..c5d8d7f
---- /dev/null
-+++ b/scala-api/scala-2.12/pom.xml
-@@ -0,0 +1,38 @@
-+<?xml version="1.0" encoding="UTF-8"?>
-+<!--
-+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
-+ ~ contributor license agreements. See the NOTICE file distributed with
-+ ~ this work for additional information regarding copyright ownership.
-+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
-+ ~ (the "License"); you may not use this file except in compliance with
-+ ~ the License. You may obtain a copy of the License at
-+ ~
-+ ~ http://www.apache.org/licenses/LICENSE-2.0
-+ ~
-+ ~ Unless required by applicable law or agreed to in writing, software
-+ ~ distributed under the License is distributed on an "AS IS" BASIS,
-+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ ~ See the License for the specific language governing permissions and
-+ ~ limitations under the License.
-+-->
-+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
-+ <modelVersion>4.0.0</modelVersion>
-+ <groupId>org.apache.livy</groupId>
-+ <artifactId>livy-scala-api_2.12</artifactId>
-+ <version>0.7.1-incubating</version>
-+ <packaging>jar</packaging>
-+
-+ <parent>
-+ <groupId>org.apache.livy</groupId>
-+ <artifactId>livy-scala-api-parent</artifactId>
-+ <version>0.7.1-incubating</version>
-+ <relativePath>../pom.xml</relativePath>
-+ </parent>
-+
-+ <properties>
-+ <scala.version>${scala-2.12.version}</scala.version>
-+ <scala.binary.version>2.12</scala.binary.version>
-+ <spark.version>${spark.scala-2.12.version}</spark.version>
-+ <netty.version>${netty.spark-2.12.version}</netty.version>
-+ </properties>
-+</project>
-diff --git a/scala-api/src/main/resources/build.marker
b/scala-api/src/main/resources/build.marker
-new file mode 100644
-index 0000000..e69de29
-diff --git
a/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobHandle.scala
b/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobHandle.scala
-index d1cf29d..a04cbfa 100644
---- a/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobHandle.scala
-+++ b/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobHandle.scala
-@@ -190,6 +190,14 @@ class ScalaJobHandle[T] private[livy] (jobHandle:
JobHandle[T]) extends Future[T
- getJavaFutureResult(jobHandle, atMost)
- this
- }
-+
-+ // These two methods must be implemented in Scala 2.12. They are
implemented as unsupported
-+ // operations here.
-+ def transform[S](f: (Try[T]) => Try[S])(implicit executor:
ExecutionContext): Future[S] =
-+ throw new UnsupportedOperationException()
-+
-+ def transformWith[S](f: (Try[T]) => Future[S])(implicit executor:
ExecutionContext): Future[S] =
-+ throw new UnsupportedOperationException()
- }
-
- private abstract class AbstractScalaJobHandleListener[T] extends Listener[T] {
-diff --git a/server/pom.xml b/server/pom.xml
-index 773d51f..4fe452f 100644
---- a/server/pom.xml
-+++ b/server/pom.xml
-@@ -200,7 +200,7 @@
- <dependency>
- <groupId>org.scalatra</groupId>
- <artifactId>scalatra-metrics_${scala.binary.version}</artifactId>
-- <version>2.4.0.M3</version>
-+ <version>${scalatra.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.typesafe.akka</groupId>
-@@ -255,6 +255,12 @@
- <groupId>org.scalatra</groupId>
- <artifactId>scalatra-test_${scala.binary.version}</artifactId>
- <scope>test</scope>
-+ <exclusions>
-+ <exclusion>
-+ <groupId>org.mockito</groupId>
-+ <artifactId>*</artifactId>
-+ </exclusion>
-+ </exclusions>
- </dependency>
-
- <dependency>
-@@ -368,4 +374,3 @@
- </build>
-
- </project>
--
-diff --git a/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
b/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
-index 1fc27a5..a726e7d 100644
---- a/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
-+++ b/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
-@@ -47,7 +47,7 @@ abstract class SessionServlet[S <: Session, R <:
RecoveryMetadata](
- with ApiVersioningSupport
- with MethodOverride
- with UrlGeneratorSupport
-- with GZipSupport
-+ with ContentEncodingSupport
- {
- /**
- * Creates a new session based on the current request. The implementation
is responsible for
-diff --git
a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
-index cdeddda..9f39372 100644
----
a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
-+++
b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
-@@ -204,11 +204,13 @@ object InteractiveSession extends Logging {
- } else {
- val sparkHome = livyConf.sparkHome().get
- val libdir = sparkMajorVersion match {
-- case 2 =>
-+ case 2 | 3 =>
- if (new File(sparkHome, "RELEASE").isFile) {
- new File(sparkHome, "jars")
-- } else {
-+ } else if (new File(sparkHome,
"assembly/target/scala-2.11/jars").isDirectory) {
- new File(sparkHome, "assembly/target/scala-2.11/jars")
-+ } else {
-+ new File(sparkHome, "assembly/target/scala-2.12/jars")
- }
- case v =>
- throw new RuntimeException(s"Unsupported Spark major version:
$sparkMajorVersion")
-diff --git a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
-index dc98b0d..c94b199 100644
---- a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
-+++ b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
-@@ -30,6 +30,8 @@ object LivySparkUtils extends Logging {
- // For each Spark version we supported, we need to add this mapping
relation in case Scala
- // version cannot be detected from "spark-submit --version".
- private val _defaultSparkScalaVersion = SortedMap(
-+ // Spark 3.0 + Scala 2.12
-+ (3, 0) -> "2.12",
- // Spark 2.4 + Scala 2.11
- (2, 4) -> "2.11",
- // Spark 2.3 + Scala 2.11
-@@ -40,7 +42,7 @@ object LivySparkUtils extends Logging {
-
- // Supported Spark version
- private val MIN_VERSION = (2, 2)
-- private val MAX_VERSION = (3, 0)
-+ private val MAX_VERSION = (3, 1)
-
- private val sparkVersionRegex = """version (.*)""".r.unanchored
- private val scalaVersionRegex = """Scala version (.*), Java""".r.unanchored
-diff --git
a/server/src/test/scala/org/apache/livy/server/BaseJsonServletSpec.scala
b/server/src/test/scala/org/apache/livy/server/BaseJsonServletSpec.scala
-index 959707a..a96ae2b 100644
---- a/server/src/test/scala/org/apache/livy/server/BaseJsonServletSpec.scala
-+++ b/server/src/test/scala/org/apache/livy/server/BaseJsonServletSpec.scala
-@@ -108,7 +108,8 @@ abstract class BaseJsonServletSpec extends ScalatraSuite
- // Only try to parse the body if response is in the "OK" range (20x).
- if ((status / 100) * 100 == SC_OK) {
- val result =
-- if (header("Content-Type").startsWith("application/json")) {
-+ if (klass.runtimeClass != classOf[Unit] &&
-+ header("Content-Type").startsWith("application/json")) {
- // Sometimes there's an empty body with no "Content-Length" header.
So read the whole
- // body first, and only send it to Jackson if there's content.
- val in = response.inputStream
-diff --git
a/server/src/test/scala/org/apache/livy/server/SessionServletSpec.scala
b/server/src/test/scala/org/apache/livy/server/SessionServletSpec.scala
-index cdd1783..466c4a2 100644
---- a/server/src/test/scala/org/apache/livy/server/SessionServletSpec.scala
-+++ b/server/src/test/scala/org/apache/livy/server/SessionServletSpec.scala
-@@ -20,7 +20,7 @@ package org.apache.livy.server
- import javax.servlet.http.HttpServletRequest
- import javax.servlet.http.HttpServletResponse._
-
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.LivyConf
- import org.apache.livy.server.recovery.SessionStore
-diff --git
a/server/src/test/scala/org/apache/livy/server/batch/BatchServletSpec.scala
b/server/src/test/scala/org/apache/livy/server/batch/BatchServletSpec.scala
-index 1999caa..5a84035 100644
---- a/server/src/test/scala/org/apache/livy/server/batch/BatchServletSpec.scala
-+++ b/server/src/test/scala/org/apache/livy/server/batch/BatchServletSpec.scala
-@@ -26,7 +26,7 @@ import javax.servlet.http.HttpServletResponse._
- import scala.concurrent.duration.Duration
-
- import org.mockito.Mockito._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{LivyConf, Utils}
- import org.apache.livy.server.{AccessManager, BaseSessionServletSpec}
-diff --git
a/server/src/test/scala/org/apache/livy/server/batch/BatchSessionSpec.scala
b/server/src/test/scala/org/apache/livy/server/batch/BatchSessionSpec.scala
-index bc9ddc4..20e6136 100644
---- a/server/src/test/scala/org/apache/livy/server/batch/BatchSessionSpec.scala
-+++ b/server/src/test/scala/org/apache/livy/server/batch/BatchSessionSpec.scala
-@@ -26,8 +26,8 @@ import scala.concurrent.duration.Duration
- import org.mockito.Matchers
- import org.mockito.Matchers.anyObject
- import org.mockito.Mockito._
--import org.scalatest.{BeforeAndAfter, FunSpec, ShouldMatchers}
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatest.{BeforeAndAfter, FunSpec}
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf, Utils}
- import org.apache.livy.server.AccessManager
-@@ -38,7 +38,7 @@ import org.apache.livy.utils.{AppInfo, Clock, SparkApp}
- class BatchSessionSpec
- extends FunSpec
- with BeforeAndAfter
-- with ShouldMatchers
-+ with org.scalatest.Matchers
- with LivyBaseUnitTestSuite {
-
- val script: Path = {
-diff --git
a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionServletSpec.scala
b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionServletSpec.scala
-index c97aa19..78407d5 100644
----
a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionServletSpec.scala
-+++
b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionServletSpec.scala
-@@ -23,6 +23,7 @@ import javax.servlet.http.{HttpServletRequest,
HttpServletResponse}
- import scala.collection.JavaConverters._
- import scala.concurrent.duration._
- import scala.concurrent.Future
-+import scala.language.postfixOps
-
- import org.json4s.jackson.Json4sScalaModule
- import org.mockito.Matchers._
-@@ -31,7 +32,7 @@ import org.mockito.invocation.InvocationOnMock
- import org.mockito.stubbing.Answer
- import org.scalatest.Entry
- import org.scalatest.concurrent.Eventually._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{ExecuteRequest, LivyConf}
- import org.apache.livy.client.common.HttpMessages.SessionInfo
-diff --git
a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
-index 55f0e21..d13e682 100644
----
a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
-+++
b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
-@@ -31,7 +31,7 @@ import org.mockito.Matchers._
- import org.mockito.Mockito.{atLeastOnce, verify, when}
- import org.scalatest.{BeforeAndAfterAll, FunSpec, Matchers}
- import org.scalatest.concurrent.Eventually._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{ExecuteRequest, JobHandle, LivyBaseUnitTestSuite,
LivyConf}
- import org.apache.livy.rsc.{PingJob, RSCClient, RSCConf}
-diff --git
a/server/src/test/scala/org/apache/livy/server/interactive/JobApiSpec.scala
b/server/src/test/scala/org/apache/livy/server/interactive/JobApiSpec.scala
-index 1646492..8ec0b19 100644
---- a/server/src/test/scala/org/apache/livy/server/interactive/JobApiSpec.scala
-+++ b/server/src/test/scala/org/apache/livy/server/interactive/JobApiSpec.scala
-@@ -29,7 +29,7 @@ import scala.language.postfixOps
-
- import org.apache.hadoop.security.UserGroupInformation
- import org.scalatest.concurrent.Eventually._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{Job, JobHandle, LivyConf}
- import org.apache.livy.client.common.{BufferUtils, Serializer}
-diff --git
a/server/src/test/scala/org/apache/livy/server/interactive/SessionHeartbeatSpec.scala
b/server/src/test/scala/org/apache/livy/server/interactive/SessionHeartbeatSpec.scala
-index 074b9c2..c9ca9d5 100644
----
a/server/src/test/scala/org/apache/livy/server/interactive/SessionHeartbeatSpec.scala
-+++
b/server/src/test/scala/org/apache/livy/server/interactive/SessionHeartbeatSpec.scala
-@@ -24,7 +24,7 @@ import scala.language.postfixOps
- import org.mockito.Mockito.{never, verify, when}
- import org.scalatest.{FunSpec, Matchers}
- import org.scalatest.concurrent.Eventually._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.LivyConf
- import org.apache.livy.server.recovery.SessionStore
-diff --git
a/server/src/test/scala/org/apache/livy/server/recovery/FileSystemStateStoreSpec.scala
b/server/src/test/scala/org/apache/livy/server/recovery/FileSystemStateStoreSpec.scala
-index 4758c85..082a80a 100644
----
a/server/src/test/scala/org/apache/livy/server/recovery/FileSystemStateStoreSpec.scala
-+++
b/server/src/test/scala/org/apache/livy/server/recovery/FileSystemStateStoreSpec.scala
-@@ -32,7 +32,7 @@ import org.mockito.invocation.InvocationOnMock
- import org.mockito.stubbing.Answer
- import org.scalatest.FunSpec
- import org.scalatest.Matchers._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
-
-diff --git
a/server/src/test/scala/org/apache/livy/server/recovery/SessionStoreSpec.scala
b/server/src/test/scala/org/apache/livy/server/recovery/SessionStoreSpec.scala
-index 5eeb2cf..8861051 100644
----
a/server/src/test/scala/org/apache/livy/server/recovery/SessionStoreSpec.scala
-+++
b/server/src/test/scala/org/apache/livy/server/recovery/SessionStoreSpec.scala
-@@ -22,7 +22,7 @@ import scala.util.Success
- import org.mockito.Mockito._
- import org.scalatest.FunSpec
- import org.scalatest.Matchers._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
- import org.apache.livy.sessions.Session.RecoveryMetadata
-diff --git
a/server/src/test/scala/org/apache/livy/server/recovery/StateStoreSpec.scala
b/server/src/test/scala/org/apache/livy/server/recovery/StateStoreSpec.scala
-index c7040a5..8c2d4b3 100644
----
a/server/src/test/scala/org/apache/livy/server/recovery/StateStoreSpec.scala
-+++
b/server/src/test/scala/org/apache/livy/server/recovery/StateStoreSpec.scala
-@@ -17,8 +17,6 @@
-
- package org.apache.livy.server.recovery
-
--import scala.reflect.classTag
--
- import org.scalatest.{BeforeAndAfter, FunSpec}
- import org.scalatest.Matchers._
-
-diff --git
a/server/src/test/scala/org/apache/livy/server/recovery/ZooKeeperStateStoreSpec.scala
b/server/src/test/scala/org/apache/livy/server/recovery/ZooKeeperStateStoreSpec.scala
-index 88e530f..6937e05 100644
----
a/server/src/test/scala/org/apache/livy/server/recovery/ZooKeeperStateStoreSpec.scala
-+++
b/server/src/test/scala/org/apache/livy/server/recovery/ZooKeeperStateStoreSpec.scala
-@@ -26,7 +26,7 @@ import org.apache.zookeeper.data.Stat
- import org.mockito.Mockito._
- import org.scalatest.FunSpec
- import org.scalatest.Matchers._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
-
-diff --git
a/server/src/test/scala/org/apache/livy/sessions/SessionManagerSpec.scala
b/server/src/test/scala/org/apache/livy/sessions/SessionManagerSpec.scala
-index 7f5e31e..8014d4a 100644
---- a/server/src/test/scala/org/apache/livy/sessions/SessionManagerSpec.scala
-+++ b/server/src/test/scala/org/apache/livy/sessions/SessionManagerSpec.scala
-@@ -25,7 +25,7 @@ import scala.util.{Failure, Try}
- import org.mockito.Mockito.{doReturn, never, verify, when}
- import org.scalatest.{FunSpec, Matchers}
- import org.scalatest.concurrent.Eventually._
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
- import org.apache.livy.server.batch.{BatchRecoveryMetadata, BatchSession}
-diff --git
a/server/src/test/scala/org/apache/livy/utils/LivySparkUtilsSuite.scala
b/server/src/test/scala/org/apache/livy/utils/LivySparkUtilsSuite.scala
-index ab3e715..ea62d0c 100644
---- a/server/src/test/scala/org/apache/livy/utils/LivySparkUtilsSuite.scala
-+++ b/server/src/test/scala/org/apache/livy/utils/LivySparkUtilsSuite.scala
-@@ -46,6 +46,8 @@ class LivySparkUtilsSuite extends FunSuite with Matchers
with LivyBaseUnitTestSu
- test("should recognize supported Spark versions") {
- testSparkVersion("2.2.0")
- testSparkVersion("2.3.0")
-+ testSparkVersion("2.4.0")
-+ testSparkVersion("3.0.0")
- }
-
- test("should complain about unsupported Spark versions") {
-@@ -85,6 +87,8 @@ class LivySparkUtilsSuite extends FunSuite with Matchers
with LivyBaseUnitTestSu
- test("defaultSparkScalaVersion() should return default Scala version") {
- defaultSparkScalaVersion(formatSparkVersion("2.2.1")) shouldBe "2.11"
- defaultSparkScalaVersion(formatSparkVersion("2.3.0")) shouldBe "2.11"
-+ defaultSparkScalaVersion(formatSparkVersion("2.4.0")) shouldBe "2.11"
-+ defaultSparkScalaVersion(formatSparkVersion("3.0.0")) shouldBe "2.12"
- }
-
- test("sparkScalaVersion() should use spark-submit detected Scala version.")
{
-@@ -104,5 +108,6 @@ class LivySparkUtilsSuite extends FunSuite with Matchers
with LivyBaseUnitTestSu
- test("sparkScalaVersion() should use default Spark Scala version.") {
- sparkScalaVersion(formatSparkVersion("2.2.0"), None, livyConf) shouldBe
"2.11"
- sparkScalaVersion(formatSparkVersion("2.3.1"), None, livyConf) shouldBe
"2.11"
-+ sparkScalaVersion(formatSparkVersion("3.0.0"), None, livyConf) shouldBe
"2.12"
- }
- }
-diff --git
a/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
b/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
-index d43125d..ddd9767 100644
---- a/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
-+++ b/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
-@@ -35,7 +35,7 @@ import org.mockito.invocation.InvocationOnMock
- import org.mockito.stubbing.Answer
- import org.scalatest.concurrent.Eventually
- import org.scalatest.FunSpec
--import org.scalatest.mock.MockitoSugar.mock
-+import org.scalatestplus.mockito.MockitoSugar.mock
-
- import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf, Utils}
- import org.apache.livy.utils.SparkApp._
-diff --git
a/test-lib/src/main/java/org/apache/livy/test/jobs/SQLGetTweets.java
b/test-lib/src/main/java/org/apache/livy/test/jobs/SQLGetTweets.java
-index a9660c4..30c87f0 100644
---- a/test-lib/src/main/java/org/apache/livy/test/jobs/SQLGetTweets.java
-+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/SQLGetTweets.java
-@@ -61,7 +61,7 @@ public class SQLGetTweets implements Job<List<String>> {
- }
-
- SQLContext sqlctx = useHiveContext ? jc.hivectx() : jc.sqlctx();
-- sqlctx.read().json(input.toString()).registerTempTable("tweets");
-+ sqlctx.read().json(input.toString()).createOrReplaceTempView("tweets");
-
- List<String> tweetList = new ArrayList<>();
- Row[] result =
-diff --git
a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala
b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala
-index 8220c27..90799f7 100644
----
a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala
-+++
b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala
-@@ -17,7 +17,7 @@
-
- package org.apache.livy.thriftserver.types
-
--import org.json4s.{DefaultFormats, JValue}
-+import org.json4s.{DefaultFormats, JValue, StringInput}
- import org.json4s.JsonAST.{JObject, JString}
- import org.json4s.jackson.JsonMethods.parse
-
-@@ -52,6 +52,7 @@ object DataTypeUtils {
- }
- }
-
-+
- /**
- * Converts a JSON representing the Spark schema (the one returned by
`df.schema.json`) into
- * a [[Schema]] instance.
-@@ -60,7 +61,7 @@ object DataTypeUtils {
- * @return a [[Schema]] representing the schema provided as input
- */
- def schemaFromSparkJson(sparkJson: String): Schema = {
-- val schema = parse(sparkJson) \ "fields"
-+ val schema = parse(StringInput(sparkJson), false) \ "fields"
- val fields = schema.children.map { field =>
- val name = (field \ "name").extract[String]
- val hiveType = toFieldType(field \ "type")
-diff --git
a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
-index 19abb0d..780a318 100644
----
a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
-+++
b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
-@@ -516,7 +516,8 @@ class BinaryThriftServerSuite extends ThriftServerBaseTest
with CommonThriftTest
- statement.close()
- }
- }
-- assert(caught.getMessage.contains("Table or view not found:
`global_temp`.`invalid_table`"))
-+ assert(caught.getMessage.replaceAll("`", "")
-+ .contains("Table or view not found: global_temp.invalid_table"))
- }
- }
-
-diff --git a/thriftserver/session/pom.xml b/thriftserver/session/pom.xml
-index d56f7c2..d0018ae 100644
---- a/thriftserver/session/pom.xml
-+++ b/thriftserver/session/pom.xml
-@@ -94,17 +94,4 @@
- </plugins>
- </build>
-
-- <profiles>
-- <!--
-- Override the json4s version to match Spark 2.4's. This module doesn't
use json4s, but the
-- Spark APIs called in the tests require a different version of json4s
than Livy's (and Livy
-- doesn't really work with Spark's version yet).
-- -->
-- <profile>
-- <id>spark-2.4</id>
-- <properties>
-- <json4s.version>3.5.3</json4s.version>
-- </properties>
-- </profile>
-- </profiles>
- </project>
-diff --git
a/thriftserver/session/src/test/java/org/apache/livy/thriftserver/session/ColumnBufferTest.java
b/thriftserver/session/src/test/java/org/apache/livy/thriftserver/session/ColumnBufferTest.java
-index 71ed020..b8832fb 100644
----
a/thriftserver/session/src/test/java/org/apache/livy/thriftserver/session/ColumnBufferTest.java
-+++
b/thriftserver/session/src/test/java/org/apache/livy/thriftserver/session/ColumnBufferTest.java
-@@ -27,7 +27,6 @@ import java.util.List;
- import java.util.Map;
- import java.util.Properties;
-
--import org.apache.spark.SparkConf;
- import org.apache.spark.SparkContext;
- import org.apache.spark.launcher.SparkLauncher;
- import org.apache.spark.sql.Dataset;
-@@ -36,6 +35,7 @@ import org.apache.spark.sql.Encoders;
- import org.apache.spark.sql.Row;
- import org.apache.spark.sql.RowFactory;
- import org.apache.spark.sql.SQLContext;
-+import org.apache.spark.sql.SparkSession;
- import org.apache.spark.sql.types.StructField;
- import org.junit.Test;
- import static org.junit.Assert.*;
-@@ -46,14 +46,14 @@ public class ColumnBufferTest {
- public void testColumnBuffer() throws Exception {
- String warehouse =
Files.createTempDirectory("spark-warehouse-").toFile().getAbsolutePath();
-
-- SparkConf conf = new SparkConf()
-- .set(SparkLauncher.SPARK_MASTER, "local")
-- .set("spark.app.name", getClass().getName())
-- .set("spark.sql.warehouse.dir", warehouse);
-- SparkContext sc = new SparkContext(conf);
-+ SparkSession session = SparkSession.builder()
-+ .master("local")
-+ .appName(getClass().getName())
-+ .config("spark.sql.warehouse.dir", warehouse)
-+ .getOrCreate();
-
- try {
-- SQLContext spark = SQLContext.getOrCreate(sc);
-+ SQLContext spark = session.sqlContext();
-
- TestBean tb = new TestBean();
- tb.setId(1);
-@@ -144,7 +144,7 @@ public class ColumnBufferTest {
- }
- }
- } finally {
-- sc.stop();
-+ session.stop();
- }
- }
-
diff --git a/bigtop.bom b/bigtop.bom
index c6d91914c..70ff2c1da 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -360,7 +360,7 @@ bigtop {
'livy' {
name = 'livy'
relNotes = 'Apache Livy'
- version { base = '0.7.1'; pkg = base; release = 1 }
+ version { base = '0.8.0'; pkg = base; release = 1 }
tarball { destination = "${name}-${version.base}.zip"
source = "apache-livy-${version.base}-incubating-src.zip"
}
url { download_path = "incubator/livy/${version.base}-incubating/"
