[PIO-49] Additional improvement to ES5 support * Add option to build ES1 or ES5-default distribution * Aggregate all storage subprojects * Update deprecated SBT code * Update Docker Compose to use official Elastic images * Update Travis to use new build option
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/31c4bd19 Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/31c4bd19 Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/31c4bd19 Branch: refs/heads/develop Commit: 31c4bd1927eb9ae753f82b39b71ff3df7ca779de Parents: d78b3cb Author: Donald Szeto <[email protected]> Authored: Tue Mar 7 23:16:33 2017 -0800 Committer: Donald Szeto <[email protected]> Committed: Tue Mar 7 23:16:33 2017 -0800 ---------------------------------------------------------------------- .travis.yml | 4 +-- build.sbt | 6 +++++ make-distribution.sh | 47 ++++++++++++++++++++++++++++++++++- storage/elasticsearch/build.sbt | 3 +-- storage/elasticsearch1/build.sbt | 2 +- storage/hbase/build.sbt | 2 +- storage/hdfs/build.sbt | 2 +- storage/jdbc/build.sbt | 2 +- storage/localfs/build.sbt | 2 +- tests/build-docker.sh | 8 ++++-- tests/docker-compose.yml | 4 ++- 11 files changed, 69 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index b763e2d..799ec3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,8 +46,8 @@ env: matrix: - BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL - BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL - - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS - - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS + - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS ES_VERSION=5 + - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS ES_VERSION=5 before_install: - unset SBT_OPTS JVM_OPTS http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index 98444b9..fa5ba29 100644 --- a/build.sbt +++ b/build.sbt @@ -166,6 +166,12 @@ val root = (project in file(".")). common, core, data, + dataElasticsearch1, + dataElasticsearch, + dataHbase, + dataHdfs, + dataJdbc, + dataLocalfs, tools, e2) http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/make-distribution.sh ---------------------------------------------------------------------- diff --git a/make-distribution.sh b/make-distribution.sh index 31954c0..c360c0e 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -19,6 +19,45 @@ set -e +usage () +{ + echo "Usage: $0 [-h|--help] [--with-es=x]" + echo "" + echo " -h|--help Show usage" + echo "" + echo " --with-es=1 Build distribution with Elasticsearch 1 support as default" + echo " --with-es=5 Build distribution with Elasticsearch 5 support as default" +} + +ES_VERSION=1 + +for i in "$@" +do +case $i in + -h|--help) + usage + shift + exit + ;; + --with-es=*) + ES_VERSION="${i#*=}" + shift + ;; + *) + usage + exit 1 + ;; +esac +done + +if [ "$ES_VERSION" = "1" ] || [ "$ES_VERSION" = "5" ] +then + echo -e "\033[0;32mBuilding with Elasticsearch $ES_VERSION support as the default choice\033[0m" +else + usage + exit 1 +fi + FWDIR="$(cd `dirname $0`; pwd)" DISTDIR="${FWDIR}/dist" @@ -45,7 +84,13 @@ cp ${FWDIR}/project/build.properties ${DISTDIR}/project cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark -cp ${FWDIR}/assembly/extra/*jar ${DISTDIR}/lib/extra + +if [ "$ES_VERSION" = "5" ] +then + mv ${DISTDIR}/lib/spark/pio-data-elasticsearch1-assembly-*.jar ${DISTDIR}/lib/extra +else + mv ${DISTDIR}/lib/spark/pio-data-elasticsearch-assembly-*.jar ${DISTDIR}/lib/extra +fi rm -f ${DISTDIR}/lib/*javadoc.jar rm -f ${DISTDIR}/lib/*sources.jar http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/elasticsearch/build.sbt ---------------------------------------------------------------------- diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt index b22cbd8..925b66f 100644 --- a/storage/elasticsearch/build.sbt +++ b/storage/elasticsearch/build.sbt @@ -53,5 +53,4 @@ assemblyShadeRules in assembly := Seq( // skip test in assembly test in assembly := {} -outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar") - +assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/elasticsearch1/build.sbt ---------------------------------------------------------------------- diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt index 8c29b84..457aba2 100644 --- a/storage/elasticsearch1/build.sbt +++ b/storage/elasticsearch1/build.sbt @@ -43,5 +43,5 @@ assemblyMergeStrategy in assembly := { // skip test in assembly test in assembly := {} -outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "extra" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar") +assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/hbase/build.sbt ---------------------------------------------------------------------- diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt index 5856a5e..673697b 100644 --- a/storage/hbase/build.sbt +++ b/storage/hbase/build.sbt @@ -52,5 +52,5 @@ assemblyMergeStrategy in assembly := { // skip test in assembly test in assembly := {} -outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar") +assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/hdfs/build.sbt ---------------------------------------------------------------------- diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt index 9f064c6..35c9153 100644 --- a/storage/hdfs/build.sbt +++ b/storage/hdfs/build.sbt @@ -40,5 +40,5 @@ assemblyMergeStrategy in assembly := { // skip test in assembly test in assembly := {} -outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar") +assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/jdbc/build.sbt ---------------------------------------------------------------------- diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt index 63d420b..0a07ee0 100644 --- a/storage/jdbc/build.sbt +++ b/storage/jdbc/build.sbt @@ -43,5 +43,5 @@ assemblyMergeStrategy in assembly := { // skip test in assembly test in assembly := {} -outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar") +assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/localfs/build.sbt ---------------------------------------------------------------------- diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt index 2cf9977..f94b355 100644 --- a/storage/localfs/build.sbt +++ b/storage/localfs/build.sbt @@ -40,5 +40,5 @@ assemblyMergeStrategy in assembly := { // skip test in assembly test in assembly := {} -outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar") +assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/tests/build-docker.sh ---------------------------------------------------------------------- diff --git a/tests/build-docker.sh b/tests/build-docker.sh index 459b929..a899a18 100755 --- a/tests/build-docker.sh +++ b/tests/build-docker.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/usr/bin/env bash -ex # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -29,7 +29,11 @@ fi docker pull predictionio/pio-testing-base pushd $DIR/.. -./make-distribution.sh +if [ -z "$ES_VERSION" ]; then + ./make-distribution.sh +else + ./make-distribution.sh --with-es=$ES_VERSION +fi sbt/sbt clean mkdir assembly cp dist/lib/*.jar assembly/ http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/tests/docker-compose.yml ---------------------------------------------------------------------- diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index b556f7b..b6774d3 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -16,7 +16,9 @@ version: "2" services: elasticsearch: - image: elasticsearch:5-alpine + image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2 + environment: + - xpack.security.enabled=false hbase: image: harisekhon/hbase:1.0 postgres:
