Grand unification
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/be8397fc Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/be8397fc Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/be8397fc Branch: refs/heads/feature/xbuild Commit: be8397fc63a967a5cabf838c52096bd806991db7 Parents: b6c90a0 Author: Donald Szeto <[email protected]> Authored: Thu Mar 16 00:08:15 2017 -0700 Committer: Donald Szeto <[email protected]> Committed: Thu Mar 16 00:08:15 2017 -0700 ---------------------------------------------------------------------- build.sbt | 177 +++++++++---------- common/build.sbt | 2 + conf/pio-vendors.sh | 26 +++ conf/set_build_profile.sh | 31 ---- core/build.sbt | 2 + data/build.sbt | 2 + e2/build.sbt | 2 + make-distribution.sh | 35 +--- project/Build.scala | 57 ------ project/PIOBuild.scala | 33 ++++ storage/elasticsearch/build.sbt | 2 + storage/elasticsearch1/build.sbt | 2 + storage/hbase/build.sbt | 2 + storage/hdfs/build.sbt | 2 + storage/jdbc/build.sbt | 2 + storage/localfs/build.sbt | 2 + tests/Dockerfile | 7 +- tests/Dockerfile-es1 | 57 ------ tests/after_script.travis.sh | 8 +- tests/build_docker.sh | 38 ++-- tests/docker-compose-es1.yml | 38 ---- tests/docker-compose.yml | 2 +- tests/docker-files/env-conf/pio-env-es1.sh | 103 ----------- tests/docker-files/env-conf/pio-env.sh | 10 +- .../engines/recommendation-engine/build.sbt | 4 +- .../engines/recommendation-engine/manifest.json | 1 - .../engines/recommendation-engine/pio.sbt | 4 - .../recommendation-engine/project/pio-build.sbt | 1 - tests/run_docker.sh | 19 +- tests/script.travis.sh | 2 +- tests/unit.sh | 13 +- tools/build.sbt | 1 + 32 files changed, 222 insertions(+), 465 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index 330dd06..16ce10e 100644 --- a/build.sbt +++ b/build.sbt @@ -15,55 +15,36 @@ * limitations under the License. */ +import PIOBuild._ import UnidocKeys._ -import scala.math.Ordering.Implicits._ - -lazy val profiles: Map[String, Profile] = - Map( - "scala-2.10" -> Profile( - name="scala-2.10", - scalaVersion="2.10.5", - sparkVersion="1.6.3", - hadoopVersion="2.6.4", - akkaVersion="2.3.15"), - - "scala-2.11" -> Profile( - name="scala-2.11", - scalaVersion="2.11.8", - sparkVersion="2.0.2", - hadoopVersion="2.7.3", - akkaVersion="2.4.10")) - -lazy val defaultProfile = "scala-2.10" - -buildProfile := { - val profileName = sys.props.get("build.profile").getOrElse(defaultProfile) - val profile = profiles(profileName) - val scalaVersion = profile.scalaVersion - - val sparkVersion = sys.props.get("spark.version") map { sv => - if ((versionMajor(sv), versionMinor(sv)) < (1, 6)) { - throw new IllegalArgumentException("Spark versions below 1.6 are no longer supported") - } else if (versionMajor(sv) >= 2 && - (versionMajor(scalaVersion), versionMinor(scalaVersion)) < (2, 11)) { - throw new IllegalArgumentException("Spark 2.x requires Scala 2.11 and above") - } else { - sv - } - } getOrElse(profile.sparkVersion) - - val hadoopVersion = sys.props.get("hadoop.version").getOrElse(profile.hadoopVersion) - - if (hadoopVersion != profile.hadoopVersion || sparkVersion != profile.sparkVersion) { - profile.copy( - name = profile.name + "-custom", - sparkVersion = sparkVersion, - hadoopVersion = hadoopVersion) - } else { - profile - } -} +lazy val scalaSparkDependenciesVersion = Map( + "2.10" -> Map( + "1.6" -> Map( + "akka" -> "2.3.15", + "hadoop" -> "2.6.5", + "json4s" -> "3.2.10"), + "2.0" -> Map( + "akka" -> "2.3.16", + "hadoop" -> "2.7.3", + "json4s" -> "3.2.11"), + "2.1" -> Map( + "akka" -> "2.3.16", + "hadoop" -> "2.7.3", + "json4s" -> "3.2.11")), + "2.11" -> Map( + "1.6" -> Map( + "akka" -> "2.3.15", + "hadoop" -> "2.6.5", + "json4s" -> "3.2.10"), + "2.0" -> Map( + "akka" -> "2.4.17", + "hadoop" -> "2.7.3", + "json4s" -> "3.2.11"), + "2.1" -> Map( + "akka" -> "2.4.17", + "hadoop" -> "2.7.3", + "json4s" -> "3.2.11"))) name := "apache-predictionio-parent" @@ -71,13 +52,9 @@ version in ThisBuild := "0.11.0-SNAPSHOT" organization in ThisBuild := "org.apache.predictionio" -scalaVersion in ThisBuild := { - val version = buildProfile.value.scalaVersion - if (versionMinor(version) < 11) { - sLog.value.warn(s"Scala version ${version} is deprecated!") - } - version -} +scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.10.6") + +crossScalaVersions in ThisBuild := Seq("2.10.6", "2.11.8") scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature") @@ -87,13 +64,22 @@ fork in (ThisBuild, run) := true javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.7", "-target", "1.7", "-Xlint:deprecation", "-Xlint:unchecked") -akkaVersion in ThisBuild := buildProfile.value.akkaVersion +// Ignore differentiation of Spark patch levels +sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "1.6.3") + +akkaVersion in ThisBuild := sys.props.getOrElse( + "akka.version", + scalaSparkDependenciesVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("akka")) + +lazy val es = sys.props.getOrElse("elasticsearch.version", "1.7.6") -json4sVersion in ThisBuild := "3.2.10" +elasticsearchVersion in ThisBuild := es -sparkVersion in ThisBuild := buildProfile.value.sparkVersion +json4sVersion in ThisBuild := scalaSparkDependenciesVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("json4s") -hadoopVersion in ThisBuild := buildProfile.value.hadoopVersion +hadoopVersion in ThisBuild := sys.props.getOrElse( + "hadoop.version", + scalaSparkDependenciesVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("hadoop")) val pioBuildInfoSettings = buildInfoSettings ++ Seq( sourceGenerators in Compile <+= buildInfo, @@ -115,7 +101,7 @@ val genjavadocSettings: Seq[sbt.Def.Setting[_]] = Seq( // in unit tests of data subproject are used later in spark jobs executed in core. // They need to have properly configured classloaders to load core classes for spark // in subsequent tests. -def coreClasses(baseDirectory: java.io.File, scalaVersion: String) = Seq( +def coreClasses(baseDirectory: java.io.File, scalaVersion: String): Seq[File] = Seq( baseDirectory / s"../core/target/scala-${versionPrefix(scalaVersion)}/classes", baseDirectory / s"../core/target/scala-${versionPrefix(scalaVersion)}/test-classes") @@ -127,45 +113,54 @@ val commonSettings = Seq( val common = (project in file("common")). settings(commonSettings: _*). - settings(genjavadocSettings: _*) + settings(genjavadocSettings: _*). + disablePlugins(sbtassembly.AssemblyPlugin) val data = (project in file("data")). dependsOn(common). settings(commonSettings: _*). settings(genjavadocSettings: _*). settings(unmanagedSourceDirectories in Compile += - sourceDirectory.value / s"main/spark-${versionMajor(sparkVersion.value)}") + sourceDirectory.value / s"main/spark-${versionMajor(sparkVersion.value)}"). + disablePlugins(sbtassembly.AssemblyPlugin) val dataElasticsearch1 = (project in file("storage/elasticsearch1")). settings(commonSettings: _*). - settings(genjavadocSettings: _*) + settings(genjavadocSettings: _*). + settings(publishArtifact := false) val dataElasticsearch = (project in file("storage/elasticsearch")). settings(commonSettings: _*). - settings(genjavadocSettings: _*) + settings(genjavadocSettings: _*). + settings(publishArtifact := false) val dataHbase = (project in file("storage/hbase")). settings(commonSettings: _*). - settings(genjavadocSettings: _*) + settings(genjavadocSettings: _*). + settings(publishArtifact := false) val dataHdfs = (project in file("storage/hdfs")). settings(commonSettings: _*). - settings(genjavadocSettings: _*) + settings(genjavadocSettings: _*). + settings(publishArtifact := false) val dataJdbc = (project in file("storage/jdbc")). settings(commonSettings: _*). - settings(genjavadocSettings: _*) + settings(genjavadocSettings: _*). + settings(publishArtifact := false) val dataLocalfs = (project in file("storage/localfs")). settings(commonSettings: _*). - settings(genjavadocSettings: _*) + settings(genjavadocSettings: _*). + settings(publishArtifact := false) val core = (project in file("core")). dependsOn(data). settings(commonSettings: _*). settings(genjavadocSettings: _*). settings(pioBuildInfoSettings: _*). - enablePlugins(SbtTwirl) + enablePlugins(SbtTwirl). + disablePlugins(sbtassembly.AssemblyPlugin) val tools = (project in file("tools")). dependsOn(core). @@ -173,12 +168,28 @@ val tools = (project in file("tools")). settings(commonSettings: _*). settings(genjavadocSettings: _*). enablePlugins(SbtTwirl). - settings(fullClasspath in Test ++= coreClasses(baseDirectory.value, scalaVersion.value)) + settings(fullClasspath in Test ++= coreClasses(baseDirectory.value, scalaVersion.value)). + settings(publishArtifact := false) val e2 = (project in file("e2")). settings(commonSettings: _*). settings(genjavadocSettings: _*). - settings(fullClasspath in Test ++= coreClasses(baseDirectory.value, scalaVersion.value)) + settings(fullClasspath in Test ++= coreClasses(baseDirectory.value, scalaVersion.value)). + disablePlugins(sbtassembly.AssemblyPlugin) + +val dataEs = if (versionMajor(es) == 1) dataElasticsearch1 else dataElasticsearch + +val subprojects = Seq( + common, + core, + data, + dataEs, + dataHbase, + dataHdfs, + dataJdbc, + dataLocalfs, + tools, + e2) val root = (project in file(".")). settings(commonSettings: _*). @@ -237,18 +248,8 @@ val root = (project in file(".")). "docs/javadoc/javadoc-overview.html", "-noqualifier", "java.lang")). - aggregate( - common, - core, - data, - dataElasticsearch1, - dataElasticsearch, - dataHbase, - dataHdfs, - dataJdbc, - dataLocalfs, - tools, - e2) + aggregate(subprojects map Project.projectToRef: _*). + disablePlugins(sbtassembly.AssemblyPlugin) val pioUnidoc = taskKey[Unit]("Builds PredictionIO ScalaDoc") @@ -308,11 +309,9 @@ parallelExecution in Global := false testOptions in Test += Tests.Argument("-oDF") -printProfile := { - val profile = buildProfile.value - println(s"PIO_PROFILE_VERSION=${profile.name}") - println(s"PIO_SCALA_VERSION=${profile.scalaVersion}") - println(s"PIO_SPARK_VERSION=${profile.sparkVersion}") - println(s"PIO_HADOOP_VERSION=${profile.hadoopVersion}") - println(s"PIO_AKKA_VERSION=${profile.akkaVersion}") +printBuildInfo := { + println(s"PIO_SCALA_VERSION=${scalaVersion.value}") + println(s"PIO_SPARK_VERSION=${sparkVersion.value}") + println(s"PIO_ELASTICSEARCH_VERSION=${elasticsearchVersion.value}") + println(s"PIO_HADOOP_VERSION=${hadoopVersion.value}") } http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/common/build.sbt ---------------------------------------------------------------------- diff --git a/common/build.sbt b/common/build.sbt index 47fba3b..5a0fba1 100644 --- a/common/build.sbt +++ b/common/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-common" libraryDependencies ++= Seq( http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/conf/pio-vendors.sh ---------------------------------------------------------------------- diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh index 00e7783..830b576 100644 --- a/conf/pio-vendors.sh +++ b/conf/pio-vendors.sh @@ -19,6 +19,32 @@ # IMPORTANT: PIO_*_VERSION for dependencies must be set before envoking this script. # `source conf/set_build_profile.sh $BUILD_PROFILE` to get the proper versions +if [ -z "$PIO_SCALA_VERSION" ]; then + PIO_SCALA_VERSION="2.10.6" +fi + +if [ -z "$PIO_SPARK_VERSION" ]; then + PIO_SPARK_VERSION="1.6.3" +fi + +if [ -z "$PIO_HADOOP_VERSION" ]; then + PIO_HADOOP_VERSION="2.6.5" +fi + +if [ -z "$PIO_ELASTICSEARCH_VERSION" ]; then + PIO_ELASTICSEARCH_VERSION="1.7.3" +fi + +ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'` + +if [ "$ES_MAJOR" = "1" ]; then + export ES_IMAGE="elasticsearch" + export ES_TAG="1" +else + export ES_IMAGE="docker.elastic.co/elasticsearch/elasticsearch" + export ES_TAG="5.2.2" +fi + PGSQL_JAR=postgresql-9.4-1204.jdbc41.jar PGSQL_DOWNLOAD=https://jdbc.postgresql.org/download/${PGSQL_JAR} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/conf/set_build_profile.sh ---------------------------------------------------------------------- diff --git a/conf/set_build_profile.sh b/conf/set_build_profile.sh deleted file mode 100755 index 397e824..0000000 --- a/conf/set_build_profile.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Sets version of profile dependencies from sbt configuration. -# eg. Run `source ./set_build_profile.sh scala-2.11` - -set -e - -if [[ "$#" -ne 1 ]]; then - echo "Usage: set-build-profile.sh <build-profile>" - exit 1 -fi - -set -a -eval `sbt/sbt --error 'set showSuccess := false' -Dbuild.profile=$1 printProfile | grep '.*_VERSION=.*'` -set +a http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/core/build.sbt ---------------------------------------------------------------------- diff --git a/core/build.sbt b/core/build.sbt index 584133c..2d2a381 100644 --- a/core/build.sbt +++ b/core/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-core" libraryDependencies ++= Seq( http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/data/build.sbt ---------------------------------------------------------------------- diff --git a/data/build.sbt b/data/build.sbt index 27eeb78..778a8f1 100644 --- a/data/build.sbt +++ b/data/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-data" libraryDependencies ++= Seq( http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/e2/build.sbt ---------------------------------------------------------------------- diff --git a/e2/build.sbt b/e2/build.sbt index 50de5e5..f30536a 100644 --- a/e2/build.sbt +++ b/e2/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-e2" parallelExecution in Test := false http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/make-distribution.sh ---------------------------------------------------------------------- diff --git a/make-distribution.sh b/make-distribution.sh index 2000340..70ba4a4 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -21,15 +21,12 @@ set -e usage () { - echo "Usage: $0 [-h|--help] [--with-es=x]" + echo "Usage: $0 [-h|--help]" echo "" echo " -h|--help Show usage" - echo "" - echo " --with-es=1 Build distribution with Elasticsearch 1 support as default" - echo " --with-es=5 Build distribution with Elasticsearch 5 support as default" } -ES_VERSION=1 +JAVA_PROPS=() for i in "$@" do @@ -39,11 +36,8 @@ case $i in shift exit ;; - --with-es=*) - ES_VERSION="${i#*=}" - shift - ;; -D*) + JAVA_PROPS+=("$i") shift ;; *) @@ -53,14 +47,6 @@ case $i in esac done -if [ "$ES_VERSION" = "1" ] || [ "$ES_VERSION" = "5" ] -then - echo -e "\033[0;32mBuilding with Elasticsearch $ES_VERSION support as the default choice\033[0m" -else - usage - exit 1 -fi - FWDIR="$(cd `dirname $0`; pwd)" DISTDIR="${FWDIR}/dist" @@ -68,10 +54,10 @@ VERSION=$(grep ^version ${FWDIR}/build.sbt | grep ThisBuild | grep -o '".*"' | s echo "Building binary distribution for PredictionIO $VERSION..." +sbt/sbt printBuildInfo + cd ${FWDIR} -sbt/sbt "$@" common/publishLocal data/publishLocal core/publishLocal e2/publishLocal \ -dataElasticsearch1/assembly dataElasticsearch/assembly dataHbase/assembly dataHdfs/assembly \ -dataJdbc/assembly dataLocalfs/assembly tools/assembly +sbt/sbt "$JAVA_PROPS" publishLocal assembly cd ${FWDIR} rm -rf ${DISTDIR} @@ -79,8 +65,8 @@ mkdir -p ${DISTDIR}/bin mkdir -p ${DISTDIR}/conf mkdir -p ${DISTDIR}/lib mkdir -p ${DISTDIR}/lib/spark -mkdir -p ${DISTDIR}/lib/extra mkdir -p ${DISTDIR}/project + mkdir -p ${DISTDIR}/sbt cp ${FWDIR}/bin/* ${DISTDIR}/bin || : @@ -90,13 +76,6 @@ cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark -if [ "$ES_VERSION" = "5" ] -then - mv ${DISTDIR}/lib/spark/pio-data-elasticsearch1-assembly-*.jar ${DISTDIR}/lib/extra -else - mv ${DISTDIR}/lib/spark/pio-data-elasticsearch-assembly-*.jar ${DISTDIR}/lib/extra -fi - rm -f ${DISTDIR}/lib/*javadoc.jar rm -f ${DISTDIR}/lib/*sources.jar rm -f ${DISTDIR}/conf/pio-env.sh http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/project/Build.scala ---------------------------------------------------------------------- diff --git a/project/Build.scala b/project/Build.scala deleted file mode 100644 index cc37912..0000000 --- a/project/Build.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import sbt._ -import Keys._ - -case class Profile( - name: String, - scalaVersion: String, - sparkVersion: String, - hadoopVersion: String, - akkaVersion: String) - -object PIOBuild extends Build { - val elasticsearchVersion = SettingKey[String]( - "elasticsearch-version", - "The version of Elasticsearch used for building.") - val json4sVersion = SettingKey[String]( - "json4s-version", - "The version of JSON4S used for building.") - val sparkVersion = SettingKey[String]( - "spark-version", - "The version of Apache Spark used for building.") - val hadoopVersion = SettingKey[String]( - "hadoop-version", - "The version of Apache Hadoop used for building") - val akkaVersion = SettingKey[String]( - "akka-version", - "The version of Akka used for building") - val buildProfile = SettingKey[Profile]( - "build-profile", - "The dependency profile used for the build") - val childrenPomExtra = SettingKey[scala.xml.NodeSeq]( - "children-pom-extra", - "Extra POM data for children projects.") - - def versionPrefix(versionString: String) = - versionString.split('.').take(2).mkString(".") - def versionMajor(versionString: String) = versionString.split('.')(0).toInt - def versionMinor(versionString: String) = versionString.split('.')(1).toInt - - lazy val printProfile = taskKey[Unit]("Print settings for the chosen profile") -} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/project/PIOBuild.scala ---------------------------------------------------------------------- diff --git a/project/PIOBuild.scala b/project/PIOBuild.scala new file mode 100644 index 0000000..69ac189 --- /dev/null +++ b/project/PIOBuild.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sbt._ + +object PIOBuild { + val elasticsearchVersion = settingKey[String]("The version of Elasticsearch used for building") + val json4sVersion = settingKey[String]("The version of JSON4S used for building") + val sparkVersion = settingKey[String]("The version of Apache Spark used for building") + val hadoopVersion = settingKey[String]("The version of Apache Hadoop used for building") + val akkaVersion = settingKey[String]("The version of Akka used for building") + val childrenPomExtra = settingKey[scala.xml.NodeSeq]("Extra POM data for children projects") + + def versionPrefix(versionString: String): String = versionString.split('.').take(2).mkString(".") + def versionMajor(versionString: String): Int = versionString.split('.')(0).toInt + def versionMinor(versionString: String): Int = versionString.split('.')(1).toInt + + lazy val printBuildInfo = taskKey[Unit]("Print build information") +} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/elasticsearch/build.sbt ---------------------------------------------------------------------- diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt index 7d9b2ad..a7d3ab5 100644 --- a/storage/elasticsearch/build.sbt +++ b/storage/elasticsearch/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-data-elasticsearch" elasticsearchVersion := "5.2.1" http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/elasticsearch1/build.sbt ---------------------------------------------------------------------- diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt index dde7285..c78c75d 100644 --- a/storage/elasticsearch1/build.sbt +++ b/storage/elasticsearch1/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-data-elasticsearch1" elasticsearchVersion := "1.7.3" http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/hbase/build.sbt ---------------------------------------------------------------------- diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt index 513e294..22b53d8 100644 --- a/storage/hbase/build.sbt +++ b/storage/hbase/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-data-hbase" libraryDependencies ++= Seq( http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/hdfs/build.sbt ---------------------------------------------------------------------- diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt index 7ddc86c..2942e56 100644 --- a/storage/hdfs/build.sbt +++ b/storage/hdfs/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-data-hdfs" libraryDependencies ++= Seq( http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/jdbc/build.sbt ---------------------------------------------------------------------- diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt index 2011722..905ed53 100644 --- a/storage/jdbc/build.sbt +++ b/storage/jdbc/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-data-jdbc" libraryDependencies ++= Seq( http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/localfs/build.sbt ---------------------------------------------------------------------- diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt index f9e101d..90a61db 100644 --- a/storage/localfs/build.sbt +++ b/storage/localfs/build.sbt @@ -15,6 +15,8 @@ * limitations under the License. */ +import PIOBuild._ + name := "apache-predictionio-data-localfs" libraryDependencies ++= Seq( http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/Dockerfile ---------------------------------------------------------------------- diff --git a/tests/Dockerfile b/tests/Dockerfile index a04b79c..619bdf8 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -20,13 +20,15 @@ FROM predictionio/pio ARG SPARK_ARCHIVE ARG SPARK_DIR ARG PGSQL_JAR -ARG BUILD_PROFILE ARG PIO_SCALA_VERSION ARG PIO_SPARK_VERSION +ARG PIO_ELASTICSEARCH_VERSION -ENV BUILD_PROFILE=$BUILD_PROFILE ENV PIO_SCALA_VERSION=$PIO_SCALA_VERSION ENV PIO_SPARK_VERSION=$PIO_SPARK_VERSION +ENV PIO_ELASTICSEARCH_VERSION=$PIO_ELASTICSEARCH_VERSION + +ENV PGSQL_JAR=$PGSQL_JAR # WORKAROUND: es-hadoop stops on RDD#take(1) ADD docker-files/${SPARK_ARCHIVE} /vendors @@ -35,7 +37,6 @@ ENV SPARK_HOME /vendors/${SPARK_DIR} COPY docker-files/${PGSQL_JAR} /drivers/${PGSQL_JAR} COPY docker-files/init.sh init.sh -COPY docker-files/set_build_profile.sh set_build_profile.sh COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml COPY docker-files/env-conf/pio-env.sh ${PIO_HOME}/conf/pio-env.sh COPY docker-files/pgpass /root/.pgpass http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/Dockerfile-es1 ---------------------------------------------------------------------- diff --git a/tests/Dockerfile-es1 b/tests/Dockerfile-es1 deleted file mode 100644 index 73d4c36..0000000 --- a/tests/Dockerfile-es1 +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM predictionio/pio - -ENV SPARK_VERSION 1.6.3 -ENV ELASTICSEARCH_VERSION 1.7 -ENV HBASE_VERSION 1.0.0 - -ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors -# WORKAROUND: es-hadoop stops on RDD#take(1) -RUN echo "spark.locality.wait.node 0s" > /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6/conf/spark-defaults.conf -ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6 - -COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar -COPY docker-files/init.sh init.sh -COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml -COPY docker-files/env-conf/pio-env-es1.sh ${PIO_HOME}/conf/pio-env.sh -COPY docker-files/pgpass /root/.pgpass -RUN chmod 600 /root/.pgpass - -# Python -RUN pip install python-dateutil -RUN pip install pytz - -# Default repositories setup -ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL -ENV PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE PGSQL -ENV PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE PGSQL - -# JVM settings -ENV JVM_OPTS '-Dfile.encoding=UTF8 -Xms2048M -Xmx2048M -Xss8M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=256M' - -# Expose relevant ports -# pio engine -EXPOSE 8000 -# eventserver -EXPOSE 7070 - -ENV SLEEP_TIME 30 - -ENTRYPOINT ["/init.sh"] -CMD 'bash' http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/after_script.travis.sh ---------------------------------------------------------------------- diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh index 8cbe2c0..e9d5792 100755 --- a/tests/after_script.travis.sh +++ b/tests/after_script.travis.sh @@ -18,12 +18,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $DIR/../conf/pio-vendors.sh + # Print a summary of containers used docker ps -a # Clean up used containers -if [ "$ES_VERSION" = "1" ]; then - docker-compose -f $DIR/docker-compose-es1.yml down -else - docker-compose -f $DIR/docker-compose.yml down -fi +docker-compose -f $DIR/docker-compose.yml down http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/build_docker.sh ---------------------------------------------------------------------- diff --git a/tests/build_docker.sh b/tests/build_docker.sh index 13a6f6a..9b6c2c6 100755 --- a/tests/build_docker.sh +++ b/tests/build_docker.sh @@ -22,7 +22,6 @@ docker pull predictionio/pio-testing-base pushd $DIR/.. -source conf/set_build_profile.sh ${BUILD_PROFILE} source conf/pio-vendors.sh if [ ! -f $DIR/docker-files/${PGSQL_JAR} ]; then wget $PGSQL_DOWNLOAD @@ -33,36 +32,23 @@ if [ ! -f $DIR/docker-files/${SPARK_ARCHIVE} ]; then mv $SPARK_ARCHIVE $DIR/docker-files/ fi -if [ -z "$ES_VERSION" ]; then - ./make-distribution.sh -Dbuild.profile=${BUILD_PROFILE} -else - ./make-distribution.sh --with-es=$ES_VERSION -Dbuild.profile=${BUILD_PROFILE} -fi +./make-distribution.sh \ + -Dscala.version=$PIO_SCALA_VERSION \ + -Dspark.version=$PIO_SPARK_VERSION \ + -Dhadoop.version=$PIO_HADOOP_VERSION \ + -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION sbt/sbt clean mkdir assembly cp dist/lib/*.jar assembly/ mkdir -p lib/spark -if [ -e dist/lib/spark/*.jar ]; then - cp dist/lib/spark/*.jar lib/spark -fi +cp dist/lib/spark/*.jar lib/spark rm *.tar.gz docker build -t predictionio/pio . popd -if [ "$ES_VERSION" = "1" ]; then - docker build -t predictionio/pio-testing-es1 -f $DIR/Dockerfile-es1 $DIR \ - --build-arg SPARK_ARCHIVE=$SPARK_ARCHIVE \ - --build-arg SPARK_DIR=$SPARK_DIR \ - --build-arg PGSQL_JAR=$PGSQL_JAR \ - --build-arg BUILD_PROFILE=$BUILD_PROFILE \ - --build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \ - --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION -else - docker build -t predictionio/pio-testing $DIR \ - --build-arg SPARK_ARCHIVE=$SPARK_ARCHIVE \ - --build-arg SPARK_DIR=$SPARK_DIR \ - --build-arg PGSQL_JAR=$PGSQL_JAR \ - --build-arg BUILD_PROFILE=$BUILD_PROFILE \ - --build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \ - --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION -fi +docker build -t predictionio/pio-testing $DIR \ + --build-arg SPARK_ARCHIVE=$SPARK_ARCHIVE \ + --build-arg SPARK_DIR=$SPARK_DIR \ + --build-arg PGSQL_JAR=$PGSQL_JAR \ + --build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \ + --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/docker-compose-es1.yml ---------------------------------------------------------------------- diff --git a/tests/docker-compose-es1.yml b/tests/docker-compose-es1.yml deleted file mode 100644 index 381f41c..0000000 --- a/tests/docker-compose-es1.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "2" -services: - elasticsearch: - image: elasticsearch:1 - environment: - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - hbase: - image: harisekhon/hbase:1.0 - postgres: - image: postgres:9 - environment: - POSTGRES_USER: pio - POSTGRES_PASSWORD: pio - POSTGRES_INITDB_ARGS: --encoding=UTF8 - pio-testing: - image: predictionio/pio-testing-es1:latest - depends_on: - - elasticsearch - - hbase - - postgres - volumes: - - ~/.ivy2:/root/.ivy2 - - ~/.sbt:/root/.sbt http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/docker-compose.yml ---------------------------------------------------------------------- diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 06126a5..ac1d91d 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -16,7 +16,7 @@ version: "2" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2 + image: ${ES_IMAGE}:${ES_TAG} environment: - xpack.security.enabled=false - "ES_JAVA_OPTS=-Xms512m -Xmx512m" http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/docker-files/env-conf/pio-env-es1.sh ---------------------------------------------------------------------- diff --git a/tests/docker-files/env-conf/pio-env-es1.sh b/tests/docker-files/env-conf/pio-env-es1.sh deleted file mode 100644 index e1076ba..0000000 --- a/tests/docker-files/env-conf/pio-env-es1.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# -# Copy this file as pio-env.sh and edit it for your site's configuration. -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# PredictionIO Main Configuration -# -# This section controls core behavior of PredictionIO. It is very likely that -# you need to change these to fit your site. - -# SPARK_HOME: Apache Spark is a hard dependency and must be configured. -SPARK_HOME=$SPARK_HOME - -POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar -MYSQL_JDBC_DRIVER= - -# ES_CONF_DIR: You must configure this if you have advanced configuration for -# your Elasticsearch setup. -# ES_CONF_DIR=/opt/elasticsearch - -# HADOOP_CONF_DIR: You must configure this if you intend to run PredictionIO -# with Hadoop 2. -# HADOOP_CONF_DIR=/opt/hadoop - -# HBASE_CONF_DIR: You must configure this if you intend to run PredictionIO -# with HBase on a remote cluster. -HBASE_CONF_DIR=$PIO_HOME/conf - -# Filesystem paths where PredictionIO uses as block storage. -PIO_FS_BASEDIR=$HOME/.pio_store -PIO_FS_ENGINESDIR=$PIO_FS_BASEDIR/engines -PIO_FS_TMPDIR=$PIO_FS_BASEDIR/tmp - -# PredictionIO Storage Configuration -# -# This section controls programs that make use of PredictionIO's built-in -# storage facilities. Default values are shown below. -# -# For more information on storage configuration please refer to -# https://docs.prediction.io/system/anotherdatastore/ - -# Storage Repositories - -# Default is to use PostgreSQL -PIO_STORAGE_REPOSITORIES_METADATA_NAME=pio_meta -PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$PIO_STORAGE_REPOSITORIES_METADATA_SOURCE - -PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME=pio_event -PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE - -PIO_STORAGE_REPOSITORIES_MODELDATA_NAME=pio_model -PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE - -# Storage Data Sources - -# PostgreSQL Default Settings -# Please change "pio" to your database name in PIO_STORAGE_SOURCES_PGSQL_URL -# Please change PIO_STORAGE_SOURCES_PGSQL_USERNAME and -# PIO_STORAGE_SOURCES_PGSQL_PASSWORD accordingly -PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc -PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://postgres/pio -PIO_STORAGE_SOURCES_PGSQL_USERNAME=pio -PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio - -# MySQL Example -# PIO_STORAGE_SOURCES_MYSQL_TYPE=jdbc -# PIO_STORAGE_SOURCES_MYSQL_URL=jdbc:mysql://localhost/pio -# PIO_STORAGE_SOURCES_MYSQL_USERNAME=pio -# PIO_STORAGE_SOURCES_MYSQL_PASSWORD=pio - -# Elasticsearch Example -PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch -#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio -PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch -PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300 -#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME - -# Local File System Example -PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs -PIO_STORAGE_SOURCES_LOCALFS_PATH=$PIO_FS_BASEDIR/local_models - -# HBase Example -PIO_STORAGE_SOURCES_HBASE_TYPE=hbase -#PIO_STORAGE_SOURCES_HBASE_HOME=$HBASE_HOME - -# HDFS config -PIO_STORAGE_SOURCES_HDFS_TYPE=hdfs -PIO_STORAGE_SOURCES_HDFS_PATH=/hdfs_models http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/docker-files/env-conf/pio-env.sh ---------------------------------------------------------------------- diff --git a/tests/docker-files/env-conf/pio-env.sh b/tests/docker-files/env-conf/pio-env.sh index 0acf3a7..51d2e05 100644 --- a/tests/docker-files/env-conf/pio-env.sh +++ b/tests/docker-files/env-conf/pio-env.sh @@ -24,9 +24,9 @@ # you need to change these to fit your site. # SPARK_HOME: Apache Spark is a hard dependency and must be configured. -SPARK_HOME=$SPARK_HOME +# SPARK_HOME=$SPARK_HOME -POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar +POSTGRES_JDBC_DRIVER=/drivers/$PGSQL_JAR MYSQL_JDBC_DRIVER= # ES_CONF_DIR: You must configure this if you have advanced configuration for @@ -88,7 +88,11 @@ PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch #PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http -PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200 +if [ "$PIO_ELASTICSEARCH_VERSION" == "1" ]; then + PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300 +else + PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200 +fi #PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME # Local File System Example http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/pio_tests/engines/recommendation-engine/build.sbt ---------------------------------------------------------------------- diff --git a/tests/pio_tests/engines/recommendation-engine/build.sbt b/tests/pio_tests/engines/recommendation-engine/build.sbt index dcb9fb6..13fe354 100644 --- a/tests/pio_tests/engines/recommendation-engine/build.sbt +++ b/tests/pio_tests/engines/recommendation-engine/build.sbt @@ -27,5 +27,5 @@ organization := "org.apache.predictionio" libraryDependencies ++= Seq( "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-SNAPSHOT" % "provided", - "org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.2") % "provided", - "org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.2") % "provided") + "org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided", + "org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/pio_tests/engines/recommendation-engine/manifest.json ---------------------------------------------------------------------- diff --git a/tests/pio_tests/engines/recommendation-engine/manifest.json b/tests/pio_tests/engines/recommendation-engine/manifest.json deleted file mode 100644 index 9c545ce..0000000 --- a/tests/pio_tests/engines/recommendation-engine/manifest.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"yDBpzjz39AjIxlOAh8W4t3QSc75uPCuJ","version":"ee98ff9c009ef0d9fb1bc6b78750b83a0ceb37b2","name":"recommendation-engine","description":"pio-autogen-manifest","files":[],"engineFactory":""} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/pio_tests/engines/recommendation-engine/pio.sbt ---------------------------------------------------------------------- diff --git a/tests/pio_tests/engines/recommendation-engine/pio.sbt b/tests/pio_tests/engines/recommendation-engine/pio.sbt deleted file mode 100644 index 5d8c366..0000000 --- a/tests/pio_tests/engines/recommendation-engine/pio.sbt +++ /dev/null @@ -1,4 +0,0 @@ -// Generated automatically by pio build. -// Changes in this file will be overridden. - -pioVersion := "0.11.0-SNAPSHOT" http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt ---------------------------------------------------------------------- diff --git a/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt b/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt deleted file mode 100644 index 8346a96..0000000 --- a/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("io.prediction" % "pio-build" % "0.9.0") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/run_docker.sh ---------------------------------------------------------------------- diff --git a/tests/run_docker.sh b/tests/run_docker.sh index 6257fa2..ad7e189 100755 --- a/tests/run_docker.sh +++ b/tests/run_docker.sh @@ -46,17 +46,10 @@ shift DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -if [ "$ES_VERSION" = "1" ]; then - docker-compose -f $DIR/docker-compose-es1.yml run \ - -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \ - -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \ - -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \ - pio-testing $@ -else - docker-compose -f $DIR/docker-compose.yml run \ - -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \ - -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \ - -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \ - pio-testing $@ -fi +source $DIR/../conf/pio-vendors.sh +docker-compose -f $DIR/docker-compose.yml run \ + -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \ + -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \ + -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \ + pio-testing $@ http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/script.travis.sh ---------------------------------------------------------------------- diff --git a/tests/script.travis.sh b/tests/script.travis.sh index efdbb9e..1d4985e 100755 --- a/tests/script.travis.sh +++ b/tests/script.travis.sh @@ -18,7 +18,7 @@ if [[ $BUILD_TYPE == Unit ]]; then tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \ - "/PredictionIO/tests/unit.sh $BUILD_PROFILE" + "/PredictionIO/tests/unit.sh" else tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \ "python3 /PredictionIO/tests/pio_tests/tests.py" http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/unit.sh ---------------------------------------------------------------------- diff --git a/tests/unit.sh b/tests/unit.sh index 724b878..d6572c2 100755 --- a/tests/unit.sh +++ b/tests/unit.sh @@ -25,11 +25,20 @@ pushd /PredictionIO set -a source ./conf/pio-env.sh set +a +source ./conf/pio-vendors.sh # Run stylecheck -sbt/sbt -Dbuild.profile=$1 scalastyle +sbt/sbt scalastyle \ + -Dscala.version=$PIO_SCALA_VERSION \ + -Dspark.version=$PIO_SPARK_VERSION \ + -Dhadoop.version=$PIO_HADOOP_VERSION \ + -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION # Run all unit tests -sbt/sbt -Dbuild.profile=$1 test +sbt/sbt test \ + -Dscala.version=$PIO_SCALA_VERSION \ + -Dspark.version=$PIO_SPARK_VERSION \ + -Dhadoop.version=$PIO_HADOOP_VERSION \ + -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION popd http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tools/build.sbt ---------------------------------------------------------------------- diff --git a/tools/build.sbt b/tools/build.sbt index d1ce2b9..0971b62 100644 --- a/tools/build.sbt +++ b/tools/build.sbt @@ -15,6 +15,7 @@ * limitations under the License. */ +import PIOBuild._ import sbtassembly.AssemblyPlugin.autoImport._ name := "apache-predictionio-tools"
