Repository: predictionio Updated Branches: refs/heads/develop e48e93bed -> 718ebe2e8
[PIO-171] Drop Scala 2.10 and Spark 1.6 support Closes #471 Project: http://git-wip-us.apache.org/repos/asf/predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/predictionio/commit/718ebe2e Tree: http://git-wip-us.apache.org/repos/asf/predictionio/tree/718ebe2e Diff: http://git-wip-us.apache.org/repos/asf/predictionio/diff/718ebe2e Branch: refs/heads/develop Commit: 718ebe2e8e98c5462d14ad2663c65288f0409972 Parents: e48e93b Author: Naoki Takezoe <[email protected]> Authored: Wed Sep 26 11:25:01 2018 -0700 Committer: Donald Szeto <[email protected]> Committed: Wed Sep 26 11:25:30 2018 -0700 ---------------------------------------------------------------------- .travis.yml | 36 +++++++++++++------- bin/pio-class | 2 +- build.sbt | 21 ++---------- .../data/SparkVersionDependent.scala | 30 ---------------- docs/manual/source/install/index.html.md.erb | 6 ++-- .../tools/commands/Management.scala | 2 +- 6 files changed, 31 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 49cbb65..bdceb7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,32 +47,44 @@ env: matrix: - BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL - PIO_SCALA_VERSION=2.10.6 - PIO_SPARK_VERSION=1.6.3 + PIO_SCALA_VERSION=2.11.8 + PIO_SPARK_VERSION=2.0.2 + PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL - PIO_SCALA_VERSION=2.10.6 - PIO_SPARK_VERSION=1.6.3 + PIO_SCALA_VERSION=2.11.8 + PIO_SPARK_VERSION=2.0.2 + PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS - PIO_SCALA_VERSION=2.10.6 - PIO_SPARK_VERSION=1.6.3 + PIO_SCALA_VERSION=2.11.8 + PIO_SPARK_VERSION=2.0.2 PIO_ELASTICSEARCH_VERSION=5.5.2 + PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS - PIO_SCALA_VERSION=2.10.6 - PIO_SPARK_VERSION=1.6.3 + PIO_SCALA_VERSION=2.11.8 + PIO_SPARK_VERSION=2.0.2 PIO_ELASTICSEARCH_VERSION=5.5.2 + PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS - PIO_SCALA_VERSION=2.10.6 - PIO_SPARK_VERSION=1.6.3 + PIO_SCALA_VERSION=2.11.8 + PIO_SPARK_VERSION=2.0.2 PIO_ELASTICSEARCH_VERSION=1.7.3 + PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS - PIO_SCALA_VERSION=2.10.6 - PIO_SPARK_VERSION=1.6.3 + PIO_SCALA_VERSION=2.11.8 + PIO_SPARK_VERSION=2.0.2 PIO_ELASTICSEARCH_VERSION=1.7.3 + PIO_HADOOP_VERSION=2.6.5 + - BUILD_TYPE=Integration + METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3 + PIO_SCALA_VERSION=2.11.8 + PIO_SPARK_VERSION=2.0.2 + PIO_ELASTICSEARCH_VERSION=5.5.2 + PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/bin/pio-class ---------------------------------------------------------------------- diff --git a/bin/pio-class b/bin/pio-class index 9e01a40..67ae562 100755 --- a/bin/pio-class +++ b/bin/pio-class @@ -44,7 +44,7 @@ fi # Make sure the Apache Spark version meets the prerequisite if it is a binary # distribution -MIN_SPARK_VERSION="1.6.3" +MIN_SPARK_VERSION="2.0.2" if [ -z "$SPARK_HOME" ]; then echo -e "\033[0;31mSPARK_HOME must be set in conf/pio-env.sh, or in the environment!\033[0m" exit 1 http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index a067a3b..6a61521 100644 --- a/build.sbt +++ b/build.sbt @@ -17,24 +17,7 @@ import PIOBuild._ lazy val scalaSparkDepsVersion = Map( - "2.10" -> Map( - "1.6" -> Map( - "akka" -> "2.3.15", - "hadoop" -> "2.6.5", - "json4s" -> "3.2.10"), - "2.0" -> Map( - "akka" -> "2.3.16", - "hadoop" -> "2.7.3", - "json4s" -> "3.2.11"), - "2.1" -> Map( - "akka" -> "2.3.16", - "hadoop" -> "2.7.3", - "json4s" -> "3.2.11")), "2.11" -> Map( - "1.6" -> Map( - "akka" -> "2.3.15", - "hadoop" -> "2.6.5", - "json4s" -> "3.2.10"), "2.0" -> Map( "akka" -> "2.4.17", "hadoop" -> "2.7.3", @@ -62,7 +45,7 @@ scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.11.8") scalaBinaryVersion in ThisBuild := binaryVersion(scalaVersion.value) -crossScalaVersions in ThisBuild := Seq("2.10.6", "2.11.8") +crossScalaVersions in ThisBuild := Seq("2.11.8") scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature") @@ -73,7 +56,7 @@ javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint:deprecation", "-Xlint:unchecked") // Ignore differentiation of Spark patch levels -sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", (if (scalaBinaryVersion.value == "2.10") "1.6.3" else "2.1.1")) +sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "2.1.1") sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value) http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala ---------------------------------------------------------------------- diff --git a/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala b/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala deleted file mode 100644 index 0652e0b..0000000 --- a/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.predictionio.data - -import org.apache.spark.SparkContext -import org.apache.spark.sql.SQLContext - -object SparkVersionDependent { - - def sqlSession(sc: SparkContext): SQLContext = { - return new SQLContext(sc) - } - -} http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/docs/manual/source/install/index.html.md.erb ---------------------------------------------------------------------- diff --git a/docs/manual/source/install/index.html.md.erb b/docs/manual/source/install/index.html.md.erb index e666b69..c8bd44a 100644 --- a/docs/manual/source/install/index.html.md.erb +++ b/docs/manual/source/install/index.html.md.erb @@ -25,7 +25,7 @@ It is **very important** to meet the minimum version of the following technologies that power Apache PredictionIO®. * Apache Hadoop 2.6.5 (optional, required only if YARN and HDFS are needed) -* Apache Spark 1.6.3 for Hadoop 2.6 +* Apache Spark 2.0.2 for Hadoop 2.6 * Java SE Development Kit 8 and one of the following sets: @@ -41,8 +41,8 @@ or * Apache HBase 0.98.5 * Elasticsearch 1.7.6 -WARNING: **Note that support for Scala 2.10 and Spark 1.6.x are deprecated as of PredictionIO 0.13.0, -and may be removed in a future release.** +WARNING: **Note that support for Scala 2.10 and Spark 1.6.x has been dropped in PredictionIO 0.14.0. +They are no longer supported.** If you are running on a single machine, we recommend a minimum of 2GB memory. http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala index cd71fdd..8c4c6ae 100644 --- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala +++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala @@ -109,7 +109,7 @@ object Management extends EitherLogging { val sparkHomePath = Common.getSparkHome(sparkHome) if (new File(s"$sparkHomePath/bin/spark-submit").exists) { info(s"Apache Spark is installed at $sparkHomePath") - val sparkMinVersion = "1.6.3" + val sparkMinVersion = "2.0.2" pioStatus = pioStatus.copy( sparkHome = sparkHomePath, sparkMinVersion = sparkMinVersion)
