[PIO-161] Spark 2.3 support Closes #461
Project: http://git-wip-us.apache.org/repos/asf/predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/predictionio/commit/eac83c5b Tree: http://git-wip-us.apache.org/repos/asf/predictionio/tree/eac83c5b Diff: http://git-wip-us.apache.org/repos/asf/predictionio/diff/eac83c5b Branch: refs/heads/livedoc Commit: eac83c5b3c52317ac5124021c8cd95671badb7ff Parents: a18475a Author: Donald Szeto <don...@apache.org> Authored: Sun Sep 2 20:13:20 2018 -0700 Committer: Donald Szeto <don...@apache.org> Committed: Sun Sep 2 20:13:20 2018 -0700 ---------------------------------------------------------------------- .travis.yml | 29 +++++++++++++++++++++++++++++ build.sbt | 4 ++++ conf/pio-vendors.sh | 3 ++- tests/build_docker.sh | 7 +++++-- 4 files changed, 40 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/predictionio/blob/eac83c5b/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index d2a8d2f..49cbb65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -132,6 +132,35 @@ env: PIO_SPARK_VERSION=2.2.0 PIO_ELASTICSEARCH_VERSION=5.5.2 + - BUILD_TYPE=Unit + METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL + PIO_SCALA_VERSION=2.11.12 + PIO_SPARK_VERSION=2.3.1 + PIO_HADOOP_VERSION=2.7.7 + - BUILD_TYPE=Integration + METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL + PIO_SCALA_VERSION=2.11.12 + PIO_SPARK_VERSION=2.3.1 + PIO_HADOOP_VERSION=2.7.7 + - BUILD_TYPE=Integration + METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS + PIO_SCALA_VERSION=2.11.12 + PIO_SPARK_VERSION=2.3.1 + PIO_ELASTICSEARCH_VERSION=1.7.3 + PIO_HADOOP_VERSION=2.7.7 + - BUILD_TYPE=Integration + METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS + PIO_SCALA_VERSION=2.11.12 + PIO_SPARK_VERSION=2.3.1 + PIO_ELASTICSEARCH_VERSION=5.5.2 + PIO_HADOOP_VERSION=2.7.7 + - BUILD_TYPE=Integration + METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3 + PIO_SCALA_VERSION=2.11.12 + PIO_SPARK_VERSION=2.3.1 + PIO_ELASTICSEARCH_VERSION=5.5.2 + PIO_HADOOP_VERSION=2.7.7 + - BUILD_TYPE=LicenseCheck before_install: http://git-wip-us.apache.org/repos/asf/predictionio/blob/eac83c5b/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index 0243610..c753f76 100644 --- a/build.sbt +++ b/build.sbt @@ -46,6 +46,10 @@ lazy val scalaSparkDepsVersion = Map( "2.2" -> Map( "akka" -> "2.4.17", "hadoop" -> "2.7.3", + "json4s" -> "3.2.11"), + "2.3" -> Map( + "akka" -> "2.4.17", + "hadoop" -> "2.7.3", "json4s" -> "3.2.11"))) name := "apache-predictionio-parent" http://git-wip-us.apache.org/repos/asf/predictionio/blob/eac83c5b/conf/pio-vendors.sh ---------------------------------------------------------------------- diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh index 6420207..162372f 100644 --- a/conf/pio-vendors.sh +++ b/conf/pio-vendors.sh @@ -51,7 +51,8 @@ PGSQL_DOWNLOAD=https://jdbc.postgresql.org/download/${PGSQL_JAR} HADOOP_MAJOR=`echo $PIO_HADOOP_VERSION | awk -F. '{print $1 "." $2}'` SPARK_DIR=spark-${PIO_SPARK_VERSION}-bin-hadoop${HADOOP_MAJOR} SPARK_ARCHIVE=${SPARK_DIR}.tgz -SPARK_DOWNLOAD=http://d3kbcqa49mib13.cloudfront.net/${SPARK_ARCHIVE} +SPARK_DOWNLOAD_MIRROR=https://www.apache.org/dyn/closer.lua\?action=download\&filename=spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE} +SPARK_DOWNLOAD_ARCHIVE=https://archive.apache.org/dist/spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE} # ELASTICSEARCH_DOWNLOAD # 5.x https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz # 1.x https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz http://git-wip-us.apache.org/repos/asf/predictionio/blob/eac83c5b/tests/build_docker.sh ---------------------------------------------------------------------- diff --git a/tests/build_docker.sh b/tests/build_docker.sh index b17a304..2c6f8f8 100755 --- a/tests/build_docker.sh +++ b/tests/build_docker.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -28,7 +28,10 @@ if [ ! -f $DIR/docker-files/${PGSQL_JAR} ]; then mv ${PGSQL_JAR} $DIR/docker-files/ fi if [ ! -f $DIR/docker-files/${SPARK_ARCHIVE} ]; then - wget $SPARK_DOWNLOAD + curl -fLo $SPARK_ARCHIVE $SPARK_DOWNLOAD_MIRROR + if [[ $? -ne 0 ]]; then + curl -fLo $SPARK_ARCHIVE $SPARK_DOWNLOAD_ARCHIVE + fi mv $SPARK_ARCHIVE $DIR/docker-files/ fi
