Repository: spark Updated Branches: refs/heads/master 12f0d2422 -> 12ab7f7e8
[SPARK-14280][BUILD][WIP] Update change-version.sh and pom.xml to add Scala 2.12 profiles and enable 2.12 compilation â¦build; fix some things that will be warnings or errors in 2.12; restore Scala 2.12 profile infrastructure ## What changes were proposed in this pull request? This change adds back the infrastructure for a Scala 2.12 build, but does not enable it in the release or Python test scripts. In order to make that meaningful, it also resolves compile errors that the code hits in 2.12 only, in a way that still works with 2.11. It also updates dependencies to the earliest minor release of dependencies whose current version does not yet support Scala 2.12. This is in a sense covered by other JIRAs under the main umbrella, but implemented here. The versions below still work with 2.11, and are the _latest_ maintenance release in the _earliest_ viable minor release. - Scalatest 2.x -> 3.0.3 - Chill 0.8.0 -> 0.8.4 - Clapper 1.0.x -> 1.1.2 - json4s 3.2.x -> 3.4.2 - Jackson 2.6.x -> 2.7.9 (required by json4s) This change does _not_ fully enable a Scala 2.12 build: - It will also require dropping support for Kafka before 0.10. Easy enough, just didn't do it yet here - It will require recreating `SparkILoop` and `Main` for REPL 2.12, which is SPARK-14650. Possible to do here too. What it does do is make changes that resolve much of the remaining gap without affecting the current 2.11 build. ## How was this patch tested? Existing tests and build. Manually tested with `./dev/change-scala-version.sh 2.12` to verify it compiles, modulo the exceptions above. Author: Sean Owen <[email protected]> Closes #18645 from srowen/SPARK-14280. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/12ab7f7e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/12ab7f7e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/12ab7f7e Branch: refs/heads/master Commit: 12ab7f7e89ec9e102859ab3b710815d3058a2e8d Parents: 12f0d24 Author: Sean Owen <[email protected]> Authored: Fri Sep 1 19:21:21 2017 +0100 Committer: Sean Owen <[email protected]> Committed: Fri Sep 1 19:21:21 2017 +0100 ---------------------------------------------------------------------- bin/load-spark-env.cmd | 22 ++++----- bin/load-spark-env.sh | 22 ++++----- .../scala/org/apache/spark/FutureAction.scala | 8 ++++ .../org/apache/spark/api/java/JavaUtils.scala | 6 +-- .../deploy/history/ApplicationCacheSuite.scala | 2 +- .../deploy/history/HistoryServerSuite.scala | 2 +- .../apache/spark/executor/ExecutorSuite.scala | 2 +- .../netty/NettyBlockTransferSecuritySuite.scala | 6 +-- .../netty/NettyBlockTransferServiceSuite.scala | 2 +- .../spark/rpc/netty/NettyRpcEnvSuite.scala | 2 +- .../spark/scheduler/BlacklistTrackerSuite.scala | 2 +- .../spark/scheduler/TaskContextSuite.scala | 48 +++++++++++++++----- .../scheduler/TaskSchedulerImplSuite.scala | 2 +- .../PartiallyUnrolledIteratorSuite.scala | 2 +- dev/create-release/release-build.sh | 2 + dev/deps/spark-deps-hadoop-2.6 | 6 +-- dev/deps/spark-deps-hadoop-2.7 | 6 +-- .../kinesis/KinesisCheckpointerSuite.scala | 2 +- .../KinesisInputDStreamBuilderSuite.scala | 5 +- .../kinesis/KinesisReceiverSuite.scala | 2 +- .../spark/launcher/AbstractCommandBuilder.java | 20 ++++---- .../apache/spark/ml/linalg/MatricesSuite.scala | 2 +- mllib/pom.xml | 4 ++ .../org/apache/spark/ml/PipelineSuite.scala | 2 +- .../spark/mllib/linalg/MatricesSuite.scala | 2 +- pom.xml | 24 ++++------ project/MimaExcludes.scala | 6 ++- project/SparkBuild.scala | 16 ++++++- python/run-tests.py | 1 + .../spark/repl/ExecutorClassLoaderSuite.scala | 2 +- .../mesos/MesosClusterSchedulerSuite.scala | 2 +- ...esosCoarseGrainedSchedulerBackendSuite.scala | 2 +- .../MesosFineGrainedSchedulerBackendSuite.scala | 2 +- .../mesos/MesosSchedulerBackendUtilSuite.scala | 3 -- .../mesos/MesosSchedulerUtilsSuite.scala | 2 +- .../ClientDistributedCacheManagerSuite.scala | 2 +- .../cluster/YarnSchedulerBackendSuite.scala | 2 +- sql/catalyst/pom.xml | 4 ++ .../sql/catalyst/analysis/AnalysisSuite.scala | 4 +- .../scala/org/apache/spark/sql/Dataset.scala | 5 +- .../execution/aggregate/typedaggregators.scala | 8 ++-- .../spark/sql/DatasetPrimitiveSuite.scala | 2 +- .../org/apache/spark/sql/DatasetSuite.scala | 2 +- .../execution/ExchangeCoordinatorSuite.scala | 4 +- .../parquet/ParquetFilterSuite.scala | 2 +- .../sql/streaming/StreamingQuerySuite.scala | 2 +- sql/hive/pom.xml | 5 ++ .../sql/hive/client/HiveVersionSuite.scala | 4 +- .../apache/spark/streaming/ReceiverSuite.scala | 6 ++- .../spark/streaming/StreamingContextSuite.scala | 6 ++- .../ExecutorAllocationManagerSuite.scala | 2 +- .../streaming/util/WriteAheadLogSuite.scala | 4 +- tools/pom.xml | 2 +- 53 files changed, 184 insertions(+), 123 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/bin/load-spark-env.cmd ---------------------------------------------------------------------- diff --git a/bin/load-spark-env.cmd b/bin/load-spark-env.cmd index 993aa31..f946197 100644 --- a/bin/load-spark-env.cmd +++ b/bin/load-spark-env.cmd @@ -35,21 +35,21 @@ if [%SPARK_ENV_LOADED%] == [] ( rem Setting SPARK_SCALA_VERSION if not already set. -rem set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-2.11" -rem set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-2.12" +set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-2.11" +set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-2.12" if [%SPARK_SCALA_VERSION%] == [] ( - rem if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% ( - rem echo "Presence of build for multiple Scala versions detected." - rem echo "Either clean one of them or, set SPARK_SCALA_VERSION=2.11 in spark-env.cmd." - rem exit 1 - rem ) - rem if exist %ASSEMBLY_DIR2% ( + if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% ( + echo "Presence of build for multiple Scala versions detected." + echo "Either clean one of them or, set SPARK_SCALA_VERSION in spark-env.cmd." + exit 1 + ) + if exist %ASSEMBLY_DIR2% ( set SPARK_SCALA_VERSION=2.11 - rem ) else ( - rem set SPARK_SCALA_VERSION=2.12 - rem ) + ) else ( + set SPARK_SCALA_VERSION=2.12 + ) ) exit /b 0 http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/bin/load-spark-env.sh ---------------------------------------------------------------------- diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh index 9de6203..d05d94e 100644 --- a/bin/load-spark-env.sh +++ b/bin/load-spark-env.sh @@ -46,18 +46,18 @@ fi if [ -z "$SPARK_SCALA_VERSION" ]; then - #ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11" - #ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.12" + ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11" + ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.12" - #if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then - # echo -e "Presence of build for multiple Scala versions detected." 1>&2 - # echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2 - # exit 1 - #fi + if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then + echo -e "Presence of build for multiple Scala versions detected." 1>&2 + echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION in spark-env.sh.' 1>&2 + exit 1 + fi - #if [ -d "$ASSEMBLY_DIR2" ]; then + if [ -d "$ASSEMBLY_DIR2" ]; then export SPARK_SCALA_VERSION="2.11" - #else - # export SPARK_SCALA_VERSION="2.12" - #fi + else + export SPARK_SCALA_VERSION="2.12" + fi fi http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/main/scala/org/apache/spark/FutureAction.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/FutureAction.scala b/core/src/main/scala/org/apache/spark/FutureAction.scala index 0899693..1034fdc 100644 --- a/core/src/main/scala/org/apache/spark/FutureAction.scala +++ b/core/src/main/scala/org/apache/spark/FutureAction.scala @@ -89,6 +89,14 @@ trait FutureAction[T] extends Future[T] { */ override def value: Option[Try[T]] + // These two methods must be implemented in Scala 2.12, but won't be used by Spark + + def transform[S](f: (Try[T]) => Try[S])(implicit executor: ExecutionContext): Future[S] = + throw new UnsupportedOperationException() + + def transformWith[S](f: (Try[T]) => Future[S])(implicit executor: ExecutionContext): Future[S] = + throw new UnsupportedOperationException() + /** * Blocks and returns the result of this job. */ http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala index f820401..d650623 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala @@ -56,9 +56,9 @@ private[spark] object JavaUtils { val ui = underlying.iterator var prev : Option[A] = None - def hasNext: Boolean = ui.hasNext + override def hasNext: Boolean = ui.hasNext - def next(): Entry[A, B] = { + override def next(): Entry[A, B] = { val (k, v) = ui.next() prev = Some(k) new ju.Map.Entry[A, B] { @@ -74,7 +74,7 @@ private[spark] object JavaUtils { } } - def remove() { + override def remove() { prev match { case Some(k) => underlying match { http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala index 871c874..c175ed3 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala @@ -33,7 +33,7 @@ import org.mockito.Mockito._ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.Matchers -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.SparkFunSuite import org.apache.spark.internal.Logging http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 95acb9a..18da8c1 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -39,7 +39,7 @@ import org.openqa.selenium.WebDriver import org.openqa.selenium.htmlunit.HtmlUnitDriver import org.scalatest.{BeforeAndAfter, Matchers} import org.scalatest.concurrent.Eventually -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.scalatest.selenium.WebBrowser import org.apache.spark._ http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala index 601dde6..884a275 100644 --- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala +++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala @@ -33,7 +33,7 @@ import org.mockito.Mockito.{inOrder, verify, when} import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.concurrent.Eventually -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark._ import org.apache.spark.TaskState.TaskState http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala index e4c133c..21138bd 100644 --- a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala +++ b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala @@ -28,8 +28,8 @@ import scala.util.{Failure, Success, Try} import com.google.common.io.CharStreams import org.mockito.Mockito._ -import org.scalatest.ShouldMatchers -import org.scalatest.mock.MockitoSugar +import org.scalatest.Matchers +import org.scalatest.mockito.MockitoSugar import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite} import org.apache.spark.network.{BlockDataManager, BlockTransferService} @@ -38,7 +38,7 @@ import org.apache.spark.network.shuffle.BlockFetchingListener import org.apache.spark.storage.{BlockId, ShuffleBlockId} import org.apache.spark.util.ThreadUtils -class NettyBlockTransferSecuritySuite extends SparkFunSuite with MockitoSugar with ShouldMatchers { +class NettyBlockTransferSecuritySuite extends SparkFunSuite with MockitoSugar with Matchers { test("security default off") { val conf = new SparkConf() .set("spark.app.id", "app-id") http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala index 9825930..f7bc372 100644 --- a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala +++ b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.network.BlockDataManager class NettyBlockTransferServiceSuite extends SparkFunSuite with BeforeAndAfterEach - with ShouldMatchers { + with Matchers { private var service0: NettyBlockTransferService = _ private var service1: NettyBlockTransferService = _ http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala b/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala index 7771637..f9481f8 100644 --- a/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala +++ b/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.rpc.netty -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark._ import org.apache.spark.network.client.TransportClient http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala index 520d85a..a136d69 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala @@ -22,7 +22,7 @@ import org.mockito.Mockito.{never, verify, when} import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.BeforeAndAfterEach -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark._ import org.apache.spark.internal.config http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala index 992d339..a1d9085 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala @@ -54,7 +54,10 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark val rdd = new RDD[String](sc, List()) { override def getPartitions = Array[Partition](StubPartition(0)) override def compute(split: Partition, context: TaskContext) = { - context.addTaskCompletionListener(context => TaskContextSuite.completed = true) + context.addTaskCompletionListener(new TaskCompletionListener { + override def onTaskCompletion(context: TaskContext): Unit = + TaskContextSuite.completed = true + }) sys.error("failed") } } @@ -95,9 +98,13 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark test("all TaskCompletionListeners should be called even if some fail") { val context = TaskContext.empty() val listener = mock(classOf[TaskCompletionListener]) - context.addTaskCompletionListener(_ => throw new Exception("blah")) + context.addTaskCompletionListener(new TaskCompletionListener { + override def onTaskCompletion(context: TaskContext): Unit = throw new Exception("blah") + }) context.addTaskCompletionListener(listener) - context.addTaskCompletionListener(_ => throw new Exception("blah")) + context.addTaskCompletionListener(new TaskCompletionListener { + override def onTaskCompletion(context: TaskContext): Unit = throw new Exception("blah") + }) intercept[TaskCompletionListenerException] { context.markTaskCompleted(None) @@ -109,9 +116,15 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark test("all TaskFailureListeners should be called even if some fail") { val context = TaskContext.empty() val listener = mock(classOf[TaskFailureListener]) - context.addTaskFailureListener((_, _) => throw new Exception("exception in listener1")) + context.addTaskFailureListener(new TaskFailureListener { + override def onTaskFailure(context: TaskContext, error: Throwable): Unit = + throw new Exception("exception in listener1") + }) context.addTaskFailureListener(listener) - context.addTaskFailureListener((_, _) => throw new Exception("exception in listener3")) + context.addTaskFailureListener(new TaskFailureListener { + override def onTaskFailure(context: TaskContext, error: Throwable): Unit = + throw new Exception("exception in listener3") + }) val e = intercept[TaskCompletionListenerException] { context.markTaskFailed(new Exception("exception in task")) @@ -232,7 +245,10 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark var invocations = 0 val context = TaskContext.empty() context.markTaskCompleted(None) - context.addTaskCompletionListener(_ => invocations += 1) + context.addTaskCompletionListener(new TaskCompletionListener { + override def onTaskCompletion(context: TaskContext): Unit = + invocations += 1 + }) assert(invocations == 1) context.markTaskCompleted(None) assert(invocations == 1) @@ -244,10 +260,12 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark val error = new RuntimeException val context = TaskContext.empty() context.markTaskFailed(error) - context.addTaskFailureListener { (_, e) => - lastError = e - invocations += 1 - } + context.addTaskFailureListener(new TaskFailureListener { + override def onTaskFailure(context: TaskContext, e: Throwable): Unit = { + lastError = e + invocations += 1 + } + }) assert(lastError == error) assert(invocations == 1) context.markTaskFailed(error) @@ -267,9 +285,15 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark test("all TaskCompletionListeners should be called even if some fail or a task") { val context = TaskContext.empty() val listener = mock(classOf[TaskCompletionListener]) - context.addTaskCompletionListener(_ => throw new Exception("exception in listener1")) + context.addTaskCompletionListener(new TaskCompletionListener { + override def onTaskCompletion(context: TaskContext): Unit = + throw new Exception("exception in listener1") + }) context.addTaskCompletionListener(listener) - context.addTaskCompletionListener(_ => throw new Exception("exception in listener3")) + context.addTaskCompletionListener(new TaskCompletionListener { + override def onTaskCompletion(context: TaskContext): Unit = + throw new Exception("exception in listener3") + }) val e = intercept[TaskCompletionListenerException] { context.markTaskCompleted(Some(new Exception("exception in task"))) http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala index ab67a39..b8626bf 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala @@ -24,7 +24,7 @@ import scala.collection.mutable.HashMap import org.mockito.Matchers.{anyInt, anyObject, anyString, eq => meq} import org.mockito.Mockito.{atLeast, atMost, never, spy, times, verify, when} import org.scalatest.BeforeAndAfterEach -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark._ import org.apache.spark.internal.Logging http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/core/src/test/scala/org/apache/spark/storage/PartiallyUnrolledIteratorSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/storage/PartiallyUnrolledIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/PartiallyUnrolledIteratorSuite.scala index 4253cc8..cbc903f 100644 --- a/core/src/test/scala/org/apache/spark/storage/PartiallyUnrolledIteratorSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/PartiallyUnrolledIteratorSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.storage import org.mockito.Matchers import org.mockito.Mockito._ -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.SparkFunSuite import org.apache.spark.memory.MemoryMode.ON_HEAP http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/dev/create-release/release-build.sh ---------------------------------------------------------------------- diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 9bf2899..ee2407a 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -349,6 +349,8 @@ if [[ "$1" == "publish-release" ]]; then # Clean-up Zinc nailgun process /usr/sbin/lsof -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill + #./dev/change-scala-version.sh 2.11 + pushd $tmp_repo/org/apache/spark # Remove any extra files generated during install http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/dev/deps/spark-deps-hadoop-2.6 ---------------------------------------------------------------------- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index e481b4d..1535103 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -28,8 +28,8 @@ breeze_2.11-0.13.2.jar calcite-avatica-1.2.0-incubating.jar calcite-core-1.2.0-incubating.jar calcite-linq4j-1.2.0-incubating.jar -chill-java-0.8.0.jar -chill_2.11-0.8.0.jar +chill-java-0.8.4.jar +chill_2.11-0.8.4.jar commons-beanutils-1.7.0.jar commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar @@ -168,7 +168,7 @@ scala-compiler-2.11.8.jar scala-library-2.11.8.jar scala-parser-combinators_2.11-1.0.4.jar scala-reflect-2.11.8.jar -scala-xml_2.11-1.0.2.jar +scala-xml_2.11-1.0.5.jar scalap-2.11.8.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/dev/deps/spark-deps-hadoop-2.7 ---------------------------------------------------------------------- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index b8046b1..deaa288 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -28,8 +28,8 @@ breeze_2.11-0.13.2.jar calcite-avatica-1.2.0-incubating.jar calcite-core-1.2.0-incubating.jar calcite-linq4j-1.2.0-incubating.jar -chill-java-0.8.0.jar -chill_2.11-0.8.0.jar +chill-java-0.8.4.jar +chill_2.11-0.8.4.jar commons-beanutils-1.7.0.jar commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar @@ -169,7 +169,7 @@ scala-compiler-2.11.8.jar scala-library-2.11.8.jar scala-parser-combinators_2.11-1.0.4.jar scala-reflect-2.11.8.jar -scala-xml_2.11-1.0.2.jar +scala-xml_2.11-1.0.5.jar scalap-2.11.8.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala ---------------------------------------------------------------------- diff --git a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala index 8d56d4b..e26f447 100644 --- a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala @@ -30,7 +30,7 @@ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.{BeforeAndAfterEach, PrivateMethodTester} import org.scalatest.concurrent.Eventually -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.streaming.{Duration, TestSuiteBase} import org.apache.spark.util.ManualClock http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisInputDStreamBuilderSuite.scala ---------------------------------------------------------------------- diff --git a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisInputDStreamBuilderSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisInputDStreamBuilderSuite.scala index 1c13065..afa1a7f 100644 --- a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisInputDStreamBuilderSuite.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisInputDStreamBuilderSuite.scala @@ -17,13 +17,10 @@ package org.apache.spark.streaming.kinesis -import java.lang.IllegalArgumentException - import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream import org.scalatest.BeforeAndAfterEach -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar -import org.apache.spark.SparkFunSuite import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.{Seconds, StreamingContext, TestSuiteBase} http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala ---------------------------------------------------------------------- diff --git a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala index 3b14c84..2fadda2 100644 --- a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala @@ -28,7 +28,7 @@ import org.mockito.Matchers._ import org.mockito.Matchers.{eq => meq} import org.mockito.Mockito._ import org.scalatest.{BeforeAndAfter, Matchers} -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.streaming.{Duration, TestSuiteBase} http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java ---------------------------------------------------------------------- diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index c32974a..ce24400 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -231,17 +231,17 @@ abstract class AbstractCommandBuilder { return scala; } String sparkHome = getSparkHome(); - //File scala212 = new File(sparkHome, "launcher/target/scala-2.12"); + File scala212 = new File(sparkHome, "launcher/target/scala-2.12"); File scala211 = new File(sparkHome, "launcher/target/scala-2.11"); - //checkState(!scala210.isDirectory() || !scala211.isDirectory(), - // "Presence of build for multiple Scala versions detected.\n" + - // "Either clean one of them or set SPARK_SCALA_VERSION in your environment."); - //if (scala212.isDirectory()) { - // return "2.12"; - //} else { - checkState(scala211.isDirectory(), "Cannot find any build directories."); - return "2.11"; - //} + checkState(!scala212.isDirectory() || !scala211.isDirectory(), + "Presence of build for multiple Scala versions detected.\n" + + "Either clean one of them or set SPARK_SCALA_VERSION in your environment."); + if (scala212.isDirectory()) { + return "2.12"; + } else { + checkState(scala211.isDirectory(), "Cannot find any build directories."); + return "2.11"; + } } String getSparkHome() { http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala ---------------------------------------------------------------------- diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala index 7fb9034..ace4416 100644 --- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala +++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala @@ -21,7 +21,7 @@ import java.util.Random import breeze.linalg.{CSCMatrix, Matrix => BM} import org.mockito.Mockito.when -import org.scalatest.mock.MockitoSugar._ +import org.scalatest.mockito.MockitoSugar._ import scala.collection.mutable.{Map => MutableMap} import org.apache.spark.ml.SparkMLFunSuite http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/mllib/pom.xml ---------------------------------------------------------------------- diff --git a/mllib/pom.xml b/mllib/pom.xml index c72a16a..925b542 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -35,6 +35,10 @@ <dependencies> <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parser-combinators_${scala.binary.version}</artifactId> + </dependency> + <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> <version>${project.version}</version> http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/mllib/src/test/scala/org/apache/spark/ml/PipelineSuite.scala ---------------------------------------------------------------------- diff --git a/mllib/src/test/scala/org/apache/spark/ml/PipelineSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/PipelineSuite.scala index 4a7e4dd..7848eae 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/PipelineSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/PipelineSuite.scala @@ -22,7 +22,7 @@ import scala.collection.JavaConverters._ import org.apache.hadoop.fs.Path import org.mockito.Matchers.{any, eq => meq} import org.mockito.Mockito.when -import org.scalatest.mock.MockitoSugar.mock +import org.scalatest.mockito.MockitoSugar.mock import org.apache.spark.SparkFunSuite import org.apache.spark.ml.Pipeline.SharedReadWrite http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala ---------------------------------------------------------------------- diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala index 6736e7d..c8ac92e 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala @@ -23,7 +23,7 @@ import scala.collection.mutable.{Map => MutableMap} import breeze.linalg.{CSCMatrix, Matrix => BM} import org.mockito.Mockito.when -import org.scalatest.mock.MockitoSugar._ +import org.scalatest.mockito.MockitoSugar._ import org.apache.spark.SparkFunSuite import org.apache.spark.ml.{linalg => newlinalg} http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index fffd70e..eab8056 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ <hive.parquet.version>1.6.0</hive.parquet.version> <jetty.version>9.3.20.v20170531</jetty.version> <javaxservlet.version>3.1.0</javaxservlet.version> - <chill.version>0.8.0</chill.version> + <chill.version>0.8.4</chill.version> <ivy.version>2.4.0</ivy.version> <oro.version>2.0.8</oro.version> <codahale.metrics.version>3.1.2</codahale.metrics.version> @@ -727,6 +727,11 @@ <version>${scala.version}</version> </dependency> <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parser-combinators_${scala.binary.version}</artifactId> + <version>1.0.4</version> + </dependency> + <dependency> <groupId>org.scala-lang</groupId> <artifactId>scalap</artifactId> <version>${scala.version}</version> @@ -734,7 +739,7 @@ <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_${scala.binary.version}</artifactId> - <version>2.2.6</version> + <version>3.0.3</version> <scope>test</scope> </dependency> <dependency> @@ -746,7 +751,7 @@ <dependency> <groupId>org.scalacheck</groupId> <artifactId>scalacheck_${scala.binary.version}</artifactId> - <version>1.12.5</version> <!-- 1.13.0 appears incompatible with scalatest 2.2.6 --> + <version>1.13.5</version> <scope>test</scope> </dependency> <dependency> @@ -2029,13 +2034,6 @@ <goal>testCompile</goal> </goals> </execution> - <execution> - <id>attach-scaladocs</id> - <phase>verify</phase> - <goals> - <goal>doc-jar</goal> - </goals> - </execution> </executions> <configuration> <scalaVersion>${scala.version}</scalaVersion> @@ -2670,12 +2668,10 @@ <id>scala-2.11</id> </profile> - <!-- Draft of Scala 2.12 profile for later --> - <!-- <profile> <id>scala-2.12</id> <properties> - <scala.version>2.12.1</scala.version> + <scala.version>2.12.3</scala.version> <scala.binary.version>2.12</scala.binary.version> </properties> <build> @@ -2694,6 +2690,7 @@ <bannedDependencies> <excludes combine.children="append"> <exclude>*:*_2.11</exclude> + <exclude>*:*_2.10</exclude> </excludes> </bannedDependencies> </rules> @@ -2704,7 +2701,6 @@ </plugins> </build> </profile> - --> <!-- This is a profile to enable the use of the ASF snapshot and staging repositories http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/project/MimaExcludes.scala ---------------------------------------------------------------------- diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 27e4183..dd299e0 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -64,7 +64,11 @@ object MimaExcludes { ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedFMeasure"), ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.asBinary"), ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.org$apache$spark$ml$classification$LogisticRegressionSummary$$multiclassMetrics"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.org$apache$spark$ml$classification$LogisticRegressionSummary$_setter_$org$apache$spark$ml$classification$LogisticRegressionSummary$$multiclassMetrics_=") + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.org$apache$spark$ml$classification$LogisticRegressionSummary$_setter_$org$apache$spark$ml$classification$LogisticRegressionSummary$$multiclassMetrics_="), + + // [SPARK-14280] Support Scala 2.12 + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.FutureAction.transformWith"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.FutureAction.transform") ) // Exclude rules for 2.2.x http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/project/SparkBuild.scala ---------------------------------------------------------------------- diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 18059ad..9d903ed 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -87,11 +87,18 @@ object SparkBuild extends PomBuild { val projectsMap: Map[String, Seq[Setting[_]]] = Map.empty override val profiles = { - Properties.envOrNone("SBT_MAVEN_PROFILES") match { + val profiles = Properties.envOrNone("SBT_MAVEN_PROFILES") match { case None => Seq("sbt") case Some(v) => v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq } + if (System.getProperty("scala-2.12") == "") { + // To activate scala-2.10 profile, replace empty property value to non-empty value + // in the same way as Maven which handles -Dname as -Dname=true before executes build process. + // see: https://github.com/apache/maven/blob/maven-3.0.4/maven-embedder/src/main/java/org/apache/maven/cli/MavenCli.java#L1082 + System.setProperty("scala-2.12", "true") + } + profiles } Properties.envOrNone("SBT_MAVEN_PROPERTIES") match { @@ -747,7 +754,12 @@ object CopyDependencies { object TestSettings { import BuildCommons._ - private val scalaBinaryVersion = "2.11" + private val scalaBinaryVersion = + if (System.getProperty("scala-2.12") == "true") { + "2.12" + } else { + "2.11" + } lazy val settings = Seq ( // Fork new JVMs for tests and set Java options for those fork := true, http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/python/run-tests.py ---------------------------------------------------------------------- diff --git a/python/run-tests.py b/python/run-tests.py index afd3d29..1341086 100755 --- a/python/run-tests.py +++ b/python/run-tests.py @@ -55,6 +55,7 @@ LOGGER = logging.getLogger() # Find out where the assembly jars are located. # Later, add back 2.12 to this list: +# for scala in ["2.11", "2.12"]: for scala in ["2.11"]: build_dir = os.path.join(SPARK_HOME, "assembly", "target", "scala-" + scala) if os.path.isdir(build_dir): http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala ---------------------------------------------------------------------- diff --git a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala index 092d3c2..e5e2094 100644 --- a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala +++ b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala @@ -35,7 +35,7 @@ import org.mockito.Mockito._ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.BeforeAndAfterAll -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark._ import org.apache.spark.internal.Logging http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala ---------------------------------------------------------------------- diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala index f0f99e9..ff63e3f 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala @@ -27,7 +27,7 @@ import org.apache.mesos.SchedulerDriver import org.apache.mesos.protobuf.ByteString import org.mockito.{ArgumentCaptor, Matchers} import org.mockito.Mockito._ -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.{LocalSparkContext, SparkConf, SparkFunSuite} import org.apache.spark.deploy.Command http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala ---------------------------------------------------------------------- diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala index ab29c29..f6bae01 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala @@ -30,7 +30,7 @@ import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.BeforeAndAfter import org.scalatest.concurrent.ScalaFutures -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.{LocalSparkContext, SecurityManager, SparkConf, SparkContext, SparkFunSuite} import org.apache.spark.deploy.mesos.config._ http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala ---------------------------------------------------------------------- diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala index 4ee85b9..2d2f90c 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala @@ -33,7 +33,7 @@ import org.apache.mesos.Protos.Value.Scalar import org.mockito.{ArgumentCaptor, Matchers} import org.mockito.Matchers._ import org.mockito.Mockito._ -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite} import org.apache.spark.executor.MesosExecutorBackend http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala ---------------------------------------------------------------------- diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala index caf9d89..f49d7c2 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala @@ -17,9 +17,6 @@ package org.apache.spark.scheduler.cluster.mesos -import org.scalatest._ -import org.scalatest.mock.MockitoSugar - import org.apache.spark.{SparkConf, SparkFunSuite} class MesosSchedulerBackendUtilSuite extends SparkFunSuite { http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala ---------------------------------------------------------------------- diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala index 5d4bf6d..7df7389 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala @@ -23,7 +23,7 @@ import scala.language.reflectiveCalls import org.apache.mesos.Protos.{Resource, Value} import org.mockito.Mockito._ import org.scalatest._ -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite} import org.apache.spark.internal.config._ http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManagerSuite.scala ---------------------------------------------------------------------- diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManagerSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManagerSuite.scala index b696e08..b091fec 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManagerSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManagerSuite.scala @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType import org.apache.hadoop.yarn.api.records.LocalResourceVisibility import org.apache.hadoop.yarn.util.ConverterUtils import org.mockito.Mockito.when -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.yarn.config._ http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala ---------------------------------------------------------------------- diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala index 0a413b2..7fac57f 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.scheduler.cluster import scala.language.reflectiveCalls import org.mockito.Mockito.when -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.{LocalSparkContext, SparkContext, SparkFunSuite} import org.apache.spark.scheduler.TaskSchedulerImpl http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/catalyst/pom.xml ---------------------------------------------------------------------- diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index fce8149..1a75c7e 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -39,6 +39,10 @@ <groupId>org.scala-lang</groupId> <artifactId>scala-reflect</artifactId> </dependency> + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parser-combinators_${scala.binary.version}</artifactId> + </dependency> <dependency> <groupId>org.apache.spark</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index e5fcd60..e56a5d6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.analysis import java.util.TimeZone -import org.scalatest.ShouldMatchers +import org.scalatest.Matchers import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.dsl.expressions._ @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.types._ -class AnalysisSuite extends AnalysisTest with ShouldMatchers { +class AnalysisSuite extends AnalysisTest with Matchers { import org.apache.spark.sql.catalyst.analysis.TestRelations._ test("union project *") { http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index 5d8a183..6db6aa3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -2576,8 +2576,9 @@ class Dataset[T] private[sql]( * @group action * @since 1.6.0 */ - def foreachPartition(func: ForeachPartitionFunction[T]): Unit = - foreachPartition(it => func.call(it.asJava)) + def foreachPartition(func: ForeachPartitionFunction[T]): Unit = { + foreachPartition((it: Iterator[T]) => func.call(it.asJava)) + } /** * Returns the first `n` rows in the Dataset. http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/typedaggregators.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/typedaggregators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/typedaggregators.scala index 1dae5f6..b6550bf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/typedaggregators.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/typedaggregators.scala @@ -37,7 +37,7 @@ class TypedSumDouble[IN](val f: IN => Double) extends Aggregator[IN, Double, Dou override def outputEncoder: Encoder[Double] = ExpressionEncoder[Double]() // Java api support - def this(f: MapFunction[IN, java.lang.Double]) = this(x => f.call(x).asInstanceOf[Double]) + def this(f: MapFunction[IN, java.lang.Double]) = this((x: IN) => f.call(x).asInstanceOf[Double]) def toColumnJava: TypedColumn[IN, java.lang.Double] = { toColumn.asInstanceOf[TypedColumn[IN, java.lang.Double]] @@ -55,7 +55,7 @@ class TypedSumLong[IN](val f: IN => Long) extends Aggregator[IN, Long, Long] { override def outputEncoder: Encoder[Long] = ExpressionEncoder[Long]() // Java api support - def this(f: MapFunction[IN, java.lang.Long]) = this(x => f.call(x).asInstanceOf[Long]) + def this(f: MapFunction[IN, java.lang.Long]) = this((x: IN) => f.call(x).asInstanceOf[Long]) def toColumnJava: TypedColumn[IN, java.lang.Long] = { toColumn.asInstanceOf[TypedColumn[IN, java.lang.Long]] @@ -75,7 +75,7 @@ class TypedCount[IN](val f: IN => Any) extends Aggregator[IN, Long, Long] { override def outputEncoder: Encoder[Long] = ExpressionEncoder[Long]() // Java api support - def this(f: MapFunction[IN, Object]) = this(x => f.call(x)) + def this(f: MapFunction[IN, Object]) = this((x: IN) => f.call(x).asInstanceOf[Any]) def toColumnJava: TypedColumn[IN, java.lang.Long] = { toColumn.asInstanceOf[TypedColumn[IN, java.lang.Long]] } @@ -94,7 +94,7 @@ class TypedAverage[IN](val f: IN => Double) extends Aggregator[IN, (Double, Long override def outputEncoder: Encoder[Double] = ExpressionEncoder[Double]() // Java api support - def this(f: MapFunction[IN, java.lang.Double]) = this(x => f.call(x).asInstanceOf[Double]) + def this(f: MapFunction[IN, java.lang.Double]) = this((x: IN) => f.call(x).asInstanceOf[Double]) def toColumnJava: TypedColumn[IN, java.lang.Double] = { toColumn.asInstanceOf[TypedColumn[IN, java.lang.Double]] } http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala index f62f9e2..edcdd77 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala @@ -151,7 +151,7 @@ class DatasetPrimitiveSuite extends QueryTest with SharedSQLContext { test("foreachPartition") { val ds = Seq(1, 2, 3).toDS() val acc = sparkContext.longAccumulator - ds.foreachPartition(_.foreach(acc.add(_))) + ds.foreachPartition((it: Iterator[Int]) => it.foreach(acc.add(_))) assert(acc.value == 6) } http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index 6245b2e..5015f37 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -364,7 +364,7 @@ class DatasetSuite extends QueryTest with SharedSQLContext { test("foreachPartition") { val ds = Seq(("a", 1), ("b", 2), ("c", 3)).toDS() val acc = sparkContext.longAccumulator - ds.foreachPartition(_.foreach(v => acc.add(v._2))) + ds.foreachPartition((it: Iterator[(String, Int)]) => it.foreach(v => acc.add(v._2))) assert(acc.value == 6) } http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeCoordinatorSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeCoordinatorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeCoordinatorSuite.scala index ab4042a..f1b5e3b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeCoordinatorSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeCoordinatorSuite.scala @@ -377,7 +377,7 @@ class ExchangeCoordinatorSuite extends SparkFunSuite with BeforeAndAfterAll { } test(s"determining the number of reducers: complex query 1$testNameNote") { - val test = { spark: SparkSession => + val test: (SparkSession) => Unit = { spark: SparkSession => val df1 = spark .range(0, 1000, 1, numInputPartitions) @@ -429,7 +429,7 @@ class ExchangeCoordinatorSuite extends SparkFunSuite with BeforeAndAfterAll { } test(s"determining the number of reducers: complex query 2$testNameNote") { - val test = { spark: SparkSession => + val test: (SparkSession) => Unit = { spark: SparkSession => val df1 = spark .range(0, 1000, 1, numInputPartitions) http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala index 28e8521..90f6620 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala @@ -506,7 +506,7 @@ class ParquetFilterSuite extends QueryTest with ParquetTest with SharedSQLContex sparkContext.register(accu) val df = spark.read.parquet(path).filter("a < 100") - df.foreachPartition(_.foreach(v => accu.add(0))) + df.foreachPartition((it: Iterator[Row]) => it.foreach(v => accu.add(0))) df.collect if (enablePushDown) { http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala index 969f594..bf7c448 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala @@ -25,7 +25,7 @@ import org.scalactic.TolerantNumerics import org.scalatest.BeforeAndAfter import org.scalatest.concurrent.Eventually._ import org.scalatest.concurrent.PatienceConfiguration.Timeout -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.SparkException import org.apache.spark.internal.Logging http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/hive/pom.xml ---------------------------------------------------------------------- diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 5508a00..a649daf 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -177,6 +177,11 @@ <artifactId>libfb303</artifactId> </dependency> <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-compiler</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.scalacheck</groupId> <artifactId>scalacheck_${scala.binary.version}</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala index 986c667..ed475a0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hive.client import org.apache.hadoop.conf.Configuration +import org.scalactic.source.Position import org.scalatest.Tag import org.apache.spark.SparkFunSuite @@ -40,7 +41,8 @@ private[client] abstract class HiveVersionSuite(version: String) extends SparkFu override def suiteName: String = s"${super.suiteName}($version)" - override protected def test(testName: String, testTags: Tag*)(testFun: => Unit): Unit = { + override protected def test(testName: String, testTags: Tag*)(testFun: => Any) + (implicit pos: Position): Unit = { super.test(s"$version: $testName", testTags: _*)(testFun) } } http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/streaming/src/test/scala/org/apache/spark/streaming/ReceiverSuite.scala ---------------------------------------------------------------------- diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ReceiverSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ReceiverSuite.scala index 1b1e21f..5fc626c 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/ReceiverSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/ReceiverSuite.scala @@ -24,8 +24,8 @@ import java.util.concurrent.Semaphore import scala.collection.mutable import scala.collection.mutable.ArrayBuffer +import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits} import org.scalatest.concurrent.Eventually._ -import org.scalatest.concurrent.Timeouts import org.scalatest.time.SpanSugar._ import org.apache.spark.SparkConf @@ -36,7 +36,7 @@ import org.apache.spark.streaming.receiver.WriteAheadLogBasedBlockHandler._ import org.apache.spark.util.Utils /** Testsuite for testing the network receiver behavior */ -class ReceiverSuite extends TestSuiteBase with Timeouts with Serializable { +class ReceiverSuite extends TestSuiteBase with TimeLimits with Serializable { test("receiver life cycle") { @@ -60,6 +60,8 @@ class ReceiverSuite extends TestSuiteBase with Timeouts with Serializable { // Verify that the receiver intercept[Exception] { + // Necessary to make failAfter interrupt awaitTermination() in ScalaTest 3.x + implicit val signaler: Signaler = ThreadSignaler failAfter(200 millis) { executingThread.join() } http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala ---------------------------------------------------------------------- diff --git a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala index eb996c9..96ab5a2 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala @@ -27,8 +27,8 @@ import scala.collection.mutable.Queue import org.apache.commons.io.FileUtils import org.scalatest.{Assertions, BeforeAndAfter, PrivateMethodTester} +import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits} import org.scalatest.concurrent.Eventually._ -import org.scalatest.concurrent.Timeouts import org.scalatest.exceptions.TestFailedDueToTimeoutException import org.scalatest.time.SpanSugar._ @@ -42,7 +42,7 @@ import org.apache.spark.streaming.receiver.Receiver import org.apache.spark.util.Utils -class StreamingContextSuite extends SparkFunSuite with BeforeAndAfter with Timeouts with Logging { +class StreamingContextSuite extends SparkFunSuite with BeforeAndAfter with TimeLimits with Logging { val master = "local[2]" val appName = this.getClass.getSimpleName @@ -406,6 +406,8 @@ class StreamingContextSuite extends SparkFunSuite with BeforeAndAfter with Timeo // test whether awaitTermination() does not exit if not time is given val exception = intercept[Exception] { + // Necessary to make failAfter interrupt awaitTermination() in ScalaTest 3.x + implicit val signaler: Signaler = ThreadSignaler failAfter(1000 millis) { ssc.awaitTermination() throw new Exception("Did not wait for stop") http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala ---------------------------------------------------------------------- diff --git a/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala index 1d2bf35..8d81b58 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala @@ -21,7 +21,7 @@ import org.mockito.Matchers.{eq => meq} import org.mockito.Mockito._ import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, PrivateMethodTester} import org.scalatest.concurrent.Eventually.{eventually, timeout} -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.scalatest.time.SpanSugar._ import org.apache.spark.{ExecutorAllocationClient, SparkConf, SparkFunSuite} http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala ---------------------------------------------------------------------- diff --git a/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala index ede1539..4a2549f 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala @@ -36,7 +36,7 @@ import org.mockito.Mockito._ import org.scalatest.{BeforeAndAfter, BeforeAndAfterEach, PrivateMethodTester} import org.scalatest.concurrent.Eventually import org.scalatest.concurrent.Eventually._ -import org.scalatest.mock.MockitoSugar +import org.scalatest.mockito.MockitoSugar import org.apache.spark.{SparkConf, SparkException, SparkFunSuite} import org.apache.spark.streaming.scheduler._ @@ -484,7 +484,7 @@ class BatchedWriteAheadLogSuite extends CommonWriteAheadLogTests( // we make the write requests in separate threads so that we don't block the test thread private def writeAsync(wal: WriteAheadLog, event: String, time: Long): Promise[Unit] = { val p = Promise[Unit]() - p.completeWith(Future { + p.completeWith(Future[Unit] { val v = wal.write(event, time) assert(v === walHandle) }(walBatchingExecutionContext)) http://git-wip-us.apache.org/repos/asf/spark/blob/12ab7f7e/tools/pom.xml ---------------------------------------------------------------------- diff --git a/tools/pom.xml b/tools/pom.xml index 7ba4dc9..37427e8 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -44,7 +44,7 @@ <dependency> <groupId>org.clapper</groupId> <artifactId>classutil_${scala.binary.version}</artifactId> - <version>1.0.6</version> + <version>1.1.2</version> </dependency> </dependencies> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
