Integration tests for both ES1 and ES5
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/3da809c7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/3da809c7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/3da809c7 Branch: refs/heads/master Commit: 3da809c7f6b82add7007d19707bfd312ec68bb1a Parents: 6d92b45 Author: Donald Szeto <[email protected]> Authored: Mon Mar 13 14:52:24 2017 -0700 Committer: Donald Szeto <[email protected]> Committed: Mon Mar 13 14:52:24 2017 -0700 ---------------------------------------------------------------------- tests/Dockerfile-es1 | 57 +++++++++++++ tests/after_script.travis.sh | 8 +- tests/before_script.travis.sh | 4 +- tests/build-docker.sh | 8 +- tests/docker-compose-es1.yml | 38 +++++++++ tests/docker-files/env-conf/pio-env-es1.sh | 103 ++++++++++++++++++++++++ tests/run_docker.sh | 19 +++-- 7 files changed, 229 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/Dockerfile-es1 ---------------------------------------------------------------------- diff --git a/tests/Dockerfile-es1 b/tests/Dockerfile-es1 new file mode 100644 index 0000000..73d4c36 --- /dev/null +++ b/tests/Dockerfile-es1 @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM predictionio/pio + +ENV SPARK_VERSION 1.6.3 +ENV ELASTICSEARCH_VERSION 1.7 +ENV HBASE_VERSION 1.0.0 + +ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors +# WORKAROUND: es-hadoop stops on RDD#take(1) +RUN echo "spark.locality.wait.node 0s" > /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6/conf/spark-defaults.conf +ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6 + +COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar +COPY docker-files/init.sh init.sh +COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml +COPY docker-files/env-conf/pio-env-es1.sh ${PIO_HOME}/conf/pio-env.sh +COPY docker-files/pgpass /root/.pgpass +RUN chmod 600 /root/.pgpass + +# Python +RUN pip install python-dateutil +RUN pip install pytz + +# Default repositories setup +ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL +ENV PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE PGSQL +ENV PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE PGSQL + +# JVM settings +ENV JVM_OPTS '-Dfile.encoding=UTF8 -Xms2048M -Xmx2048M -Xss8M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=256M' + +# Expose relevant ports +# pio engine +EXPOSE 8000 +# eventserver +EXPOSE 7070 + +ENV SLEEP_TIME 30 + +ENTRYPOINT ["/init.sh"] +CMD 'bash' http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/after_script.travis.sh ---------------------------------------------------------------------- diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh index 005d410..8cbe2c0 100755 --- a/tests/after_script.travis.sh +++ b/tests/after_script.travis.sh @@ -16,8 +16,14 @@ # limitations under the License. # +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + # Print a summary of containers used docker ps -a # Clean up used containers -docker-compose -f tests/docker-compose.yml down +if [ "$ES_VERSION" = "1" ]; then + docker-compose -f $DIR/docker-compose-es1.yml down +else + docker-compose -f $DIR/docker-compose.yml down +fi http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/before_script.travis.sh ---------------------------------------------------------------------- diff --git a/tests/before_script.travis.sh b/tests/before_script.travis.sh index b6d3966..9ec7a4a 100755 --- a/tests/before_script.travis.sh +++ b/tests/before_script.travis.sh @@ -16,4 +16,6 @@ # limitations under the License. # -tests/build-docker.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +$DIR/build-docker.sh http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/build-docker.sh ---------------------------------------------------------------------- diff --git a/tests/build-docker.sh b/tests/build-docker.sh index 4438396..ed43715 100755 --- a/tests/build-docker.sh +++ b/tests/build-docker.sh @@ -39,6 +39,12 @@ mkdir assembly cp dist/lib/*.jar assembly/ mkdir -p lib/spark cp dist/lib/spark/*.jar lib/spark +rm *.tar.gz docker build -t predictionio/pio . popd -docker build -t predictionio/pio-testing $DIR + +if [ "$ES_VERSION" = "1" ]; then + docker build -t predictionio/pio-testing-es1 -f $DIR/Dockerfile-es1 $DIR +else + docker build -t predictionio/pio-testing $DIR +fi http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/docker-compose-es1.yml ---------------------------------------------------------------------- diff --git a/tests/docker-compose-es1.yml b/tests/docker-compose-es1.yml new file mode 100644 index 0000000..381f41c --- /dev/null +++ b/tests/docker-compose-es1.yml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2" +services: + elasticsearch: + image: elasticsearch:1 + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + hbase: + image: harisekhon/hbase:1.0 + postgres: + image: postgres:9 + environment: + POSTGRES_USER: pio + POSTGRES_PASSWORD: pio + POSTGRES_INITDB_ARGS: --encoding=UTF8 + pio-testing: + image: predictionio/pio-testing-es1:latest + depends_on: + - elasticsearch + - hbase + - postgres + volumes: + - ~/.ivy2:/root/.ivy2 + - ~/.sbt:/root/.sbt http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/docker-files/env-conf/pio-env-es1.sh ---------------------------------------------------------------------- diff --git a/tests/docker-files/env-conf/pio-env-es1.sh b/tests/docker-files/env-conf/pio-env-es1.sh new file mode 100644 index 0000000..e1076ba --- /dev/null +++ b/tests/docker-files/env-conf/pio-env-es1.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# +# Copy this file as pio-env.sh and edit it for your site's configuration. +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# PredictionIO Main Configuration +# +# This section controls core behavior of PredictionIO. It is very likely that +# you need to change these to fit your site. + +# SPARK_HOME: Apache Spark is a hard dependency and must be configured. +SPARK_HOME=$SPARK_HOME + +POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar +MYSQL_JDBC_DRIVER= + +# ES_CONF_DIR: You must configure this if you have advanced configuration for +# your Elasticsearch setup. +# ES_CONF_DIR=/opt/elasticsearch + +# HADOOP_CONF_DIR: You must configure this if you intend to run PredictionIO +# with Hadoop 2. +# HADOOP_CONF_DIR=/opt/hadoop + +# HBASE_CONF_DIR: You must configure this if you intend to run PredictionIO +# with HBase on a remote cluster. +HBASE_CONF_DIR=$PIO_HOME/conf + +# Filesystem paths where PredictionIO uses as block storage. +PIO_FS_BASEDIR=$HOME/.pio_store +PIO_FS_ENGINESDIR=$PIO_FS_BASEDIR/engines +PIO_FS_TMPDIR=$PIO_FS_BASEDIR/tmp + +# PredictionIO Storage Configuration +# +# This section controls programs that make use of PredictionIO's built-in +# storage facilities. Default values are shown below. +# +# For more information on storage configuration please refer to +# https://docs.prediction.io/system/anotherdatastore/ + +# Storage Repositories + +# Default is to use PostgreSQL +PIO_STORAGE_REPOSITORIES_METADATA_NAME=pio_meta +PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$PIO_STORAGE_REPOSITORIES_METADATA_SOURCE + +PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME=pio_event +PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE + +PIO_STORAGE_REPOSITORIES_MODELDATA_NAME=pio_model +PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE + +# Storage Data Sources + +# PostgreSQL Default Settings +# Please change "pio" to your database name in PIO_STORAGE_SOURCES_PGSQL_URL +# Please change PIO_STORAGE_SOURCES_PGSQL_USERNAME and +# PIO_STORAGE_SOURCES_PGSQL_PASSWORD accordingly +PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc +PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://postgres/pio +PIO_STORAGE_SOURCES_PGSQL_USERNAME=pio +PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio + +# MySQL Example +# PIO_STORAGE_SOURCES_MYSQL_TYPE=jdbc +# PIO_STORAGE_SOURCES_MYSQL_URL=jdbc:mysql://localhost/pio +# PIO_STORAGE_SOURCES_MYSQL_USERNAME=pio +# PIO_STORAGE_SOURCES_MYSQL_PASSWORD=pio + +# Elasticsearch Example +PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch +#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio +PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch +PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300 +#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME + +# Local File System Example +PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs +PIO_STORAGE_SOURCES_LOCALFS_PATH=$PIO_FS_BASEDIR/local_models + +# HBase Example +PIO_STORAGE_SOURCES_HBASE_TYPE=hbase +#PIO_STORAGE_SOURCES_HBASE_HOME=$HBASE_HOME + +# HDFS config +PIO_STORAGE_SOURCES_HDFS_TYPE=hdfs +PIO_STORAGE_SOURCES_HDFS_PATH=/hdfs_models http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/run_docker.sh ---------------------------------------------------------------------- diff --git a/tests/run_docker.sh b/tests/run_docker.sh index fe07957..6257fa2 100755 --- a/tests/run_docker.sh +++ b/tests/run_docker.sh @@ -46,8 +46,17 @@ shift DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -docker-compose -f $DIR/docker-compose.yml run \ - -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \ - -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \ - -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \ - pio-testing $@ +if [ "$ES_VERSION" = "1" ]; then + docker-compose -f $DIR/docker-compose-es1.yml run \ + -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \ + -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \ + -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \ + pio-testing $@ +else + docker-compose -f $DIR/docker-compose.yml run \ + -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \ + -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \ + -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \ + pio-testing $@ +fi +
