This is an automated email from the ASF dual-hosted git repository. donald pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/predictionio.git
The following commit(s) were added to refs/heads/develop by this push: new c5703d7 [PIO-188] Update the build matrix on travis (#489) c5703d7 is described below commit c5703d7067dac7e4a38bb4b85a065d562ff73bb9 Author: takako shimamoto <chiboch...@gmail.com> AuthorDate: Sat Oct 27 03:33:43 2018 +0900 [PIO-188] Update the build matrix on travis (#489) --- .travis.yml | 121 ++++++++++------------------------------------- build.sbt | 5 +- conf/pio-vendors.sh | 15 ++++-- tests/build_docker.sh | 3 +- tests/docker-compose.yml | 2 +- tests/unit.sh | 6 ++- 6 files changed, 46 insertions(+), 106 deletions(-) diff --git a/.travis.yml b/.travis.yml index abd4ab0..2a756e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,131 +47,60 @@ env: matrix: - BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL - PIO_SCALA_VERSION=2.11.8 - PIO_SPARK_VERSION=2.0.2 - PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration - METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL - PIO_SCALA_VERSION=2.11.8 - PIO_SPARK_VERSION=2.0.2 - PIO_HADOOP_VERSION=2.6.5 + METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=S3 + PIO_ELASTICSEARCH_VERSION=1.7.3 - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS - PIO_SCALA_VERSION=2.11.8 - PIO_SPARK_VERSION=2.0.2 + METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3 PIO_ELASTICSEARCH_VERSION=5.6.9 - PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS - PIO_SCALA_VERSION=2.11.8 - PIO_SPARK_VERSION=2.0.2 - PIO_ELASTICSEARCH_VERSION=5.6.9 - PIO_HADOOP_VERSION=2.6.5 + METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3 + PIO_ELASTICSEARCH_VERSION=6.4.2 - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS - PIO_SCALA_VERSION=2.11.8 - PIO_SPARK_VERSION=2.0.2 - PIO_ELASTICSEARCH_VERSION=1.7.3 - PIO_HADOOP_VERSION=2.6.5 + PIO_HBASE_VERSION=1.2.6 + - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS - PIO_SCALA_VERSION=2.11.8 + METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL + PIO_SCALA_VERSION=2.11.12 PIO_SPARK_VERSION=2.0.2 - PIO_ELASTICSEARCH_VERSION=1.7.3 PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3 - PIO_SCALA_VERSION=2.11.8 - PIO_SPARK_VERSION=2.0.2 - PIO_ELASTICSEARCH_VERSION=5.6.9 - PIO_HADOOP_VERSION=2.6.5 - - - BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.1.1 + PIO_SPARK_VERSION=2.1.3 + PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.1.1 - - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.1.1 - PIO_ELASTICSEARCH_VERSION=5.6.9 - - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.1.1 - PIO_ELASTICSEARCH_VERSION=5.6.9 - - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.1.1 - PIO_ELASTICSEARCH_VERSION=1.7.3 - - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.1.1 - PIO_ELASTICSEARCH_VERSION=1.7.3 + PIO_SPARK_VERSION=2.2.2 + PIO_HADOOP_VERSION=2.6.5 - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3 + METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.1.1 - PIO_ELASTICSEARCH_VERSION=5.6.9 + PIO_SPARK_VERSION=2.3.2 + PIO_HADOOP_VERSION=2.6.5 - - BUILD_TYPE=Unit - METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.2.0 - BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.2.0 - - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.2.0 - PIO_ELASTICSEARCH_VERSION=1.7.3 - - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.2.0 - PIO_ELASTICSEARCH_VERSION=5.6.9 + PIO_SPARK_VERSION=2.0.2 + PIO_HADOOP_VERSION=2.7.3 - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3 - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.2.0 - PIO_ELASTICSEARCH_VERSION=5.6.9 - - - BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.3.1 - PIO_HADOOP_VERSION=2.7.7 + PIO_SPARK_VERSION=2.1.3 + PIO_HADOOP_VERSION=2.7.3 - BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.3.1 - PIO_HADOOP_VERSION=2.7.7 + PIO_SPARK_VERSION=2.2.2 + PIO_HADOOP_VERSION=2.7.3 - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS + METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.3.1 - PIO_ELASTICSEARCH_VERSION=1.7.3 - PIO_HADOOP_VERSION=2.7.7 - - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.3.1 - PIO_ELASTICSEARCH_VERSION=5.6.9 - PIO_HADOOP_VERSION=2.7.7 - - BUILD_TYPE=Integration - METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3 - PIO_SCALA_VERSION=2.11.12 - PIO_SPARK_VERSION=2.3.1 - PIO_ELASTICSEARCH_VERSION=5.6.9 - PIO_HADOOP_VERSION=2.7.7 + PIO_SPARK_VERSION=2.3.2 + PIO_HADOOP_VERSION=2.7.3 - BUILD_TYPE=LicenseCheck diff --git a/build.sbt b/build.sbt index 94739ff..5027fd9 100644 --- a/build.sbt +++ b/build.sbt @@ -56,7 +56,7 @@ javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint:deprecation", "-Xlint:unchecked") // Ignore differentiation of Spark patch levels -sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "2.1.2") +sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "2.1.3") sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value) @@ -314,6 +314,7 @@ testOptions in Test += Tests.Argument("-oDF") printBuildInfo := { println(s"PIO_SCALA_VERSION=${scalaVersion.value}") println(s"PIO_SPARK_VERSION=${sparkVersion.value}") - println(s"PIO_ELASTICSEARCH_VERSION=${elasticsearchVersion.value}") println(s"PIO_HADOOP_VERSION=${hadoopVersion.value}") + println(s"PIO_ELASTICSEARCH_VERSION=${elasticsearchVersion.value}") + println(s"PIO_HBASE_VERSION=${hbaseVersion.value}") } diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh index 0489e85..f657a7c 100644 --- a/conf/pio-vendors.sh +++ b/conf/pio-vendors.sh @@ -20,21 +20,25 @@ # `source conf/set_build_profile.sh $BUILD_PROFILE` to get the proper versions if [ -z "$PIO_SCALA_VERSION" ]; then - PIO_SCALA_VERSION="2.11.8" + PIO_SCALA_VERSION="2.11.12" fi if [ -z "$PIO_SPARK_VERSION" ]; then - PIO_SPARK_VERSION="2.1.1" + PIO_SPARK_VERSION="2.1.3" fi if [ -z "$PIO_HADOOP_VERSION" ]; then - PIO_HADOOP_VERSION="2.6.5" + PIO_HADOOP_VERSION="2.7.3" fi if [ -z "$PIO_ELASTICSEARCH_VERSION" ]; then PIO_ELASTICSEARCH_VERSION="5.6.9" fi +if [ -z "$PIO_HBASE_VERSION" ]; then + PIO_HBASE_VERSION="1.2.6" +fi + ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'` if [ "$ES_MAJOR" = "1" ]; then @@ -42,9 +46,12 @@ if [ "$ES_MAJOR" = "1" ]; then export ES_TAG="1" else export ES_IMAGE="docker.elastic.co/elasticsearch/elasticsearch" - export ES_TAG="5.6.9" + export ES_TAG="$PIO_ELASTICSEARCH_VERSION" fi +HBASE_MAJOR=`echo $PIO_HBASE_VERSION | awk -F. '{print $1 "." $2}'` +export HBASE_TAG="$HBASE_MAJOR" + PGSQL_JAR=postgresql-9.4-1204.jdbc41.jar PGSQL_DOWNLOAD=https://jdbc.postgresql.org/download/${PGSQL_JAR} diff --git a/tests/build_docker.sh b/tests/build_docker.sh index 5879a58..0adf3b0 100755 --- a/tests/build_docker.sh +++ b/tests/build_docker.sh @@ -41,7 +41,8 @@ set -e -Dscala.version=$PIO_SCALA_VERSION \ -Dspark.version=$PIO_SPARK_VERSION \ -Dhadoop.version=$PIO_HADOOP_VERSION \ - -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION + -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION \ + -Dhbase.version=$PIO_HBASE_VERSION sbt/sbt clean storage/clean assembly_folder=assembly/src/universal/lib diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index e0eda34..0c196f1 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -21,7 +21,7 @@ services: - xpack.security.enabled=false - "ES_JAVA_OPTS=-Xms512m -Xmx512m" hbase: - image: harisekhon/hbase:1.0 + image: harisekhon/hbase:${HBASE_TAG} postgres: image: postgres:9 environment: diff --git a/tests/unit.sh b/tests/unit.sh index 1421dce..a9c84fc 100755 --- a/tests/unit.sh +++ b/tests/unit.sh @@ -32,13 +32,15 @@ sbt/sbt scalastyle \ -Dscala.version=$PIO_SCALA_VERSION \ -Dspark.version=$PIO_SPARK_VERSION \ -Dhadoop.version=$PIO_HADOOP_VERSION \ - -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION + -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION \ + -Dhbase.version=$PIO_HBASE_VERSION # Run all unit tests sbt/sbt dataJdbc/compile test storage/test \ -Dscala.version=$PIO_SCALA_VERSION \ -Dspark.version=$PIO_SPARK_VERSION \ -Dhadoop.version=$PIO_HADOOP_VERSION \ - -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION + -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION \ + -Dhbase.version=$PIO_HBASE_VERSION popd