Repository: incubator-predictionio Updated Branches: refs/heads/develop 9b629800b -> e72d66e20
[PIO-53] Docker compose based integration tests Remove dependency on Docker hub to enable per commit integration tests Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/e72d66e2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/e72d66e2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/e72d66e2 Branch: refs/heads/develop Commit: e72d66e201ddd70cfb679b82e332b851eb7297e1 Parents: 9b62980 Author: Donald Szeto <[email protected]> Authored: Tue Feb 14 23:09:25 2017 -0800 Committer: Donald Szeto <[email protected]> Committed: Tue Feb 14 23:09:25 2017 -0800 ---------------------------------------------------------------------- .gitignore | 3 ++ .travis.yml | 1 - make-distribution.sh | 1 + tests/Dockerfile | 56 ++++++------------------- tests/after_script.travis.sh | 2 + tests/before_script.travis.sh | 1 + tests/build-docker.sh | 15 +++++++ tests/docker-compose.yml | 31 ++++++++++++++ tests/docker-files/env-conf/hbase-site.xml | 6 +++ tests/docker-files/env-conf/pio-env.sh | 4 +- tests/docker-files/init.sh | 51 +++++++++++----------- tests/docker-files/pgpass | 1 + tests/docker-files/wait-for-postgres.sh | 16 +++++++ tests/run_docker.sh | 16 ++----- tests/script.travis.sh | 2 +- 15 files changed, 122 insertions(+), 84 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 4e3460f..68ad5c4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ quickstartapp/ /docs/manual/source/gallery/template-gallery.html.md test-reports/ apache-rat-0.11.jar +tests/dist +tests/docker-files/*.jar +tests/docker-files/*.tgz http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 68dee42..5e11d1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,7 +68,6 @@ env: before_install: - unset SBT_OPTS JVM_OPTS - - docker pull predictionio/pio-testing - service haveged start before_script: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/make-distribution.sh ---------------------------------------------------------------------- diff --git a/make-distribution.sh b/make-distribution.sh index 23769fe..a6accdb 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -54,6 +54,7 @@ touch ${DISTDIR}/RELEASE TARNAME="PredictionIO-$VERSION.tar.gz" TARDIR="PredictionIO-$VERSION" cp -r ${DISTDIR} ${TARDIR} +cp -r ${DISTDIR} tests/ tar zcvf ${TARNAME} ${TARDIR} rm -rf ${TARDIR} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/Dockerfile ---------------------------------------------------------------------- diff --git a/tests/Dockerfile b/tests/Dockerfile index 41ef344..fda6682 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -15,65 +15,49 @@ # limitations under the License. # -from ubuntu +FROM ubuntu:xenial ENV SPARK_VERSION 1.4.0 ENV ELASTICSEARCH_VERSION 1.4.4 ENV HBASE_VERSION 1.0.0 -RUN echo "== Updating system ==" -RUN apt-get update -y -RUN echo "== Downloading packages ==" -RUN apt-get install -y \ +RUN apt-get update && apt-get install -y \ wget curl \ python-pip \ python3-pip \ - postgresql postgresql-contrib \ + postgresql-client \ openjdk-8-jdk \ openssh-client openssh-server \ git RUN pip install predictionio -RUN pip3 install --upgrade pip -RUN pip3 install xmlrunner -RUN pip3 install --upgrade requests -RUN pip3 install --upgrade urllib3 +RUN pip3 install --upgrade \ + pip \ + xmlrunner \ + requests \ + urllib3 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/jre -RUN echo "== Installing Spark ==" -RUN mkdir vendors -RUN wget http://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz -RUN tar zxvfC spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors -RUN rm spark-${SPARK_VERSION}-bin-hadoop2.6.tgz +ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6 -RUN echo "== Installing Elasticsearch ==" -RUN wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz -RUN tar zxvfC elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz /vendors -RUN rm elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz ENV ELASTICSEARCH_HOME /vendors/elasticsearch-${ELASTICSEARCH_VERSION} -RUN echo "== Installing HBase ==" -RUN wget http://archive.apache.org/dist/hbase/hbase-${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz -RUN tar zxvfC hbase-${HBASE_VERSION}-bin.tar.gz /vendors -RUN rm hbase-${HBASE_VERSION}-bin.tar.gz ENV HBASE_HOME /vendors/hbase-${HBASE_VERSION} -RUN echo "== Downloading database drivers ==" -RUN mkdir drivers -RUN wget https://jdbc.postgresql.org/download/postgresql-9.4-1204.jdbc41.jar -P /drivers +COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers -RUN mkdir PredictionIO ENV PIO_HOME /PredictionIO ENV PATH ${PIO_HOME}/bin/:${PATH} -ENV HOST_PIO_HOME /pio_host +COPY dist ${PIO_HOME} -RUN echo "== Setting configs ==" COPY docker-files/init.sh init.sh COPY docker-files/env-conf/spark-env.sh ${SPARK_HOME}/conf/spark-env.sh COPY docker-files/env-conf/hbase-site.xml ${HBASE_HOME}/conf/hbase-site.xml COPY docker-files/env-conf/pio-env.sh /pio-env.sh +COPY docker-files/wait-for-postgres.sh /wait-for-postgres.sh +COPY docker-files/pgpass /root/.pgpass # Default repositories setup ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL @@ -88,20 +72,6 @@ ENV JVM_OPTS '-Dfile.encoding=UTF8 -Xms2048M -Xmx2048M -Xss8M -XX:MaxPermSize=51 EXPOSE 8000 # eventserver EXPOSE 7070 -# spark master UI -EXPOSE 8080 -# spark worker UI -EXPOSE 8081 -# spark context UI -EXPOSE 4040 -# HMaster -EXPOSE 60000 -# HMaster Info Web UI -EXPOSE 60010 -# Region Server -Expose 60020 -# Region Server Http -EXPOSE 60030 ENTRYPOINT ["/init.sh"] CMD 'bash' http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/after_script.travis.sh ---------------------------------------------------------------------- diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh index 69e28e0..f41e929 100755 --- a/tests/after_script.travis.sh +++ b/tests/after_script.travis.sh @@ -20,4 +20,6 @@ set -e if [[ $BUILD_TYPE == Unit ]]; then ./bin/travis/pio-stop-travis +else + docker-compose -f tests/docker-compose.yml down fi http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/before_script.travis.sh ---------------------------------------------------------------------- diff --git a/tests/before_script.travis.sh b/tests/before_script.travis.sh index 9bfecdc..fbc6b0a 100755 --- a/tests/before_script.travis.sh +++ b/tests/before_script.travis.sh @@ -37,4 +37,5 @@ if [[ $BUILD_TYPE == Unit ]]; then else # Integration Tests ./make-distribution.sh + tests/build-docker.sh fi http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/build-docker.sh ---------------------------------------------------------------------- diff --git a/tests/build-docker.sh b/tests/build-docker.sh new file mode 100755 index 0000000..fe783cb --- /dev/null +++ b/tests/build-docker.sh @@ -0,0 +1,15 @@ +#!/bin/bash -x + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +if [ ! -f $DIR/docker-files/spark-1.4.0-bin-hadoop2.6.tgz ]; then + wget http://d3kbcqa49mib13.cloudfront.net/spark-1.4.0-bin-hadoop2.6.tgz + mv spark-1.4.0-bin-hadoop2.6.tgz $DIR/docker-files/ +fi + +if [ ! -f $DIR/docker-files/postgresql-9.4-1204.jdbc41.jar ]; then + wget https://jdbc.postgresql.org/download/postgresql-9.4-1204.jdbc41.jar + mv postgresql-9.4-1204.jdbc41.jar $DIR/docker-files/ +fi + +docker build -t predictionio/pio-testing $DIR http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-compose.yml ---------------------------------------------------------------------- diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..d55834d --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,31 @@ +version: "3" +services: + elasticsearch: + image: elasticsearch:1-alpine + ports: + - 9200:9200 + - 9300:9300 + hbase: + image: harisekhon/hbase:1.0 + ports: + - 16000:16000 + - 16010:16010 + - 16020:16020 + - 16030:16030 + postgres: + image: postgres:9 + ports: + - 5432:5432 + environment: + POSTGRES_USER: pio + POSTGRES_PASSWORD: pio + POSTGRES_INITDB_ARGS: --encoding=UTF8 + pio-testing: + image: predictionio/pio-testing:latest + depends_on: + - elasticsearch + - hbase + - postgres + volumes: + - ./pio_tests:/tests/pio_tests + - ~/.ivy2:/root/.ivy2 http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/env-conf/hbase-site.xml ---------------------------------------------------------------------- diff --git a/tests/docker-files/env-conf/hbase-site.xml b/tests/docker-files/env-conf/hbase-site.xml index 12185f1..3c4e24c 100644 --- a/tests/docker-files/env-conf/hbase-site.xml +++ b/tests/docker-files/env-conf/hbase-site.xml @@ -17,6 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. --> <configuration> + <!-- <property> <name>hbase.rootdir</name> <value>file:///hbase-files/data</value> @@ -25,4 +26,9 @@ limitations under the License. <name>hbase.zookeeper.property.dataDir</name> <value>/hbase-files/zookeeper</value> </property> + --> + <property> + <name>hbase.zookeeper.quorum</name> + <value>hbase</value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/env-conf/pio-env.sh ---------------------------------------------------------------------- diff --git a/tests/docker-files/env-conf/pio-env.sh b/tests/docker-files/env-conf/pio-env.sh index 4ffec46..5417d24 100644 --- a/tests/docker-files/env-conf/pio-env.sh +++ b/tests/docker-files/env-conf/pio-env.sh @@ -73,7 +73,7 @@ PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_MODELDATA_SO # Please change PIO_STORAGE_SOURCES_PGSQL_USERNAME and # PIO_STORAGE_SOURCES_PGSQL_PASSWORD accordingly PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc -PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://localhost/pio +PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://postgres/pio PIO_STORAGE_SOURCES_PGSQL_USERNAME=pio PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio @@ -86,7 +86,7 @@ PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio # Elasticsearch Example PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch #PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio -PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=localhost +PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300 PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/init.sh ---------------------------------------------------------------------- diff --git a/tests/docker-files/init.sh b/tests/docker-files/init.sh index 69fd86c..dc74ada 100755 --- a/tests/docker-files/init.sh +++ b/tests/docker-files/init.sh @@ -16,40 +16,41 @@ # limitations under the License. # -set -e +#set -e -echo '== Setting up Postgres... ==' -service postgresql start -runuser postgres -c 'createuser -s root' -createdb root +#echo '== Setting up Postgres... ==' +#service postgresql start +#runuser postgres -c 'createuser -s root' +#createdb root +/wait-for-postgres.sh postgres -psql -c "create user pio with password 'pio'" && createdb pio +#psql -c "create user pio with password 'pio'" && createdb pio -echo '== Starting SSH... ==' -service ssh start -ssh-keygen -b 2048 -t rsa -q -f /root/.ssh/id_rsa -N "" -cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys +#echo '== Starting SSH... ==' +#/etc/init.d/sshd start +#ssh-keygen -b 2048 -t rsa -q -f /root/.ssh/id_rsa -N "" +#cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys -echo '== Starting HBase... ==' -$HBASE_HOME/bin/start-hbase.sh +#echo '== Starting HBase... ==' +#$HBASE_HOME/bin/start-hbase.sh -echo '== Starting standalone Spark cluster... ==' -$SPARK_HOME/sbin/start-all.sh +#echo '== Starting standalone Spark cluster... ==' +#$SPARK_HOME/sbin/start-all.sh -echo '== Starting Elasticsearch... ==' -$ELASTICSEARCH_HOME/bin/elasticsearch -d -p $PIO_HOME/es.pid +#echo '== Starting Elasticsearch... ==' +#$ELASTICSEARCH_HOME/bin/elasticsearch -d -p $PIO_HOME/es.pid -echo '== Copying distribution to PIO_HOME... ==' -DISTRIBUTION_TAR=`find /pio_host -maxdepth 1 -name PredictionIO-*.tar.gz | head -1` -tar zxvfC $DISTRIBUTION_TAR / -DIR_NAME=/`basename $DISTRIBUTION_TAR` -DIR_NAME=${DIR_NAME%.tar.gz} -mv $DIR_NAME/* $PIO_HOME/ +#echo '== Copying distribution to PIO_HOME... ==' +#DISTRIBUTION_TAR=`find /pio_host -maxdepth 1 -name PredictionIO-*.tar.gz | head -1` +#tar zxvfC $DISTRIBUTION_TAR / +#DIR_NAME=/`basename $DISTRIBUTION_TAR` +#DIR_NAME=${DIR_NAME%.tar.gz} +#mv $DIR_NAME/* $PIO_HOME/ mv /pio-env.sh $PIO_HOME/conf/pio-env.sh -echo '== Copying tests to a separate directory ==' -mkdir /tests -cp -r /pio_host/tests/pio_tests /tests/pio_tests +#echo '== Copying tests to a separate directory ==' +#mkdir /tests +#cp -r /pio_host/tests/pio_tests /tests/pio_tests export PYTHONPATH=/tests:$PYTHONPATH # after initialization run given command http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/pgpass ---------------------------------------------------------------------- diff --git a/tests/docker-files/pgpass b/tests/docker-files/pgpass new file mode 100644 index 0000000..dcbcb23 --- /dev/null +++ b/tests/docker-files/pgpass @@ -0,0 +1 @@ +postgres:5432:pio:pio:pio http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/wait-for-postgres.sh ---------------------------------------------------------------------- diff --git a/tests/docker-files/wait-for-postgres.sh b/tests/docker-files/wait-for-postgres.sh new file mode 100755 index 0000000..3fcc27c --- /dev/null +++ b/tests/docker-files/wait-for-postgres.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# wait-for-postgres.sh + +set -e + +host="$1" +shift +cmd="$@" + +until psql -h "$host" -U "pio" -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - executing command" +exec $cmd http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/run_docker.sh ---------------------------------------------------------------------- diff --git a/tests/run_docker.sh b/tests/run_docker.sh index a8a79f3..c60fb7e 100755 --- a/tests/run_docker.sh +++ b/tests/run_docker.sh @@ -16,12 +16,11 @@ # limitations under the License. # -USAGE=$"Usage: run_docer <meta> <event> <model> <pio> <command> +USAGE=$"Usage: run_docker <meta> <event> <model> <command> Where: meta = [PGSQL,ELASTICSEARCH] event = [PGSQL,HBASE] model = [PGSQL,LOCALFS,HDFS] - pio = path to PredictionIO directory command = command to run in the container" if ! [[ "$1" =~ ^(PGSQL|ELASTICSEARCH)$ ]]; then @@ -39,17 +38,10 @@ if ! [[ "$3" =~ ^(PGSQL|LOCALFS|HDFS)$ ]]; then exit 1 fi -if [ ! -d "$4" ]; then - echo "Directory $4 does not exist" - echo "$USAGE" - exit 1 -fi +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -docker run -it -h localhost \ - -v $4:/pio_host \ - -v ~/.ivy2:/root/.ivy2 \ +docker-compose -f $DIR/docker-compose.yml run \ -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$1 \ -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$2 \ -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$3 \ - -p 8000:8000 -p 7070:7070 -p 8080:8080 -p 8081:8081 -p 4040:4040 \ - -p 60000:60000 -p 60010:60010 -p 60020:60020 -p 60030:60030 predictionio/pio-testing $5 + pio-testing $4 http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/script.travis.sh ---------------------------------------------------------------------- diff --git a/tests/script.travis.sh b/tests/script.travis.sh index 4adb135..f7f81ac 100755 --- a/tests/script.travis.sh +++ b/tests/script.travis.sh @@ -36,5 +36,5 @@ else REPO=`pwd` ./tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \ - $REPO 'python3 /tests/pio_tests/tests.py' + 'python3 /tests/pio_tests/tests.py' fi
