Repository: incubator-gearpump Updated Branches: refs/heads/master 2d2f27953 -> 835d9814b
[GEARPUMP-361] Build gearpump docker image locally Author: manuzhang <[email protected]> Closes #233 from manuzhang/fix-it. Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/835d9814 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/835d9814 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/835d9814 Branch: refs/heads/master Commit: 835d9814b1112daf9e623a52e78672dfd2146eac Parents: 2d2f279 Author: manuzhang <[email protected]> Authored: Sat Dec 9 15:38:28 2017 +0800 Committer: manuzhang <[email protected]> Committed: Sat Dec 9 15:38:38 2017 +0800 ---------------------------------------------------------------------- integrationtest/README.md | 12 ++++++------ .../integrationtest/minicluster/BaseContainer.scala | 11 ++--------- integrationtest/docker/gearpump/Dockerfile | 2 +- integrationtest/docker/gearpump/README.md | 4 ++-- integrationtest/docker/gearpump/start.sh | 11 ----------- 5 files changed, 11 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/835d9814/integrationtest/README.md ---------------------------------------------------------------------- diff --git a/integrationtest/README.md b/integrationtest/README.md index b626c6a..28ca6b9 100644 --- a/integrationtest/README.md +++ b/integrationtest/README.md @@ -3,8 +3,6 @@ To run the integration test, you need a Linux with Kernel version >= 3.10 and Docker 1.7 (or higher). The test framework will use several Docker images. These docker image **NEED** to be prepared **BEFOREHAND** to avoid timeout during testing: - * [The Gearpump Cluster Launcher and Storm Client](https://hub.docker.com/r/grubykarol/gearpump-launcher/) - `docker pull grubykarol/gearpump-launcher` * [The standalone single node Kafka cluster with Zookeeper](https://hub.docker.com/r/grubykarol/kafka/) `docker pull grubykarol/kafka:0.8.2.1` * [The Hadoop image](https://hub.docker.com/r/sequenceiq/hadoop-docker/) @@ -29,7 +27,9 @@ The integration test framework use docker to simulate a real cluster. The test s `git clone https://github.com/apache/incubator-gearpump.git` 2. Build Gearpump project `sbt assembly pack` -3. Run Integration test +3. Build Gearpump docker image + `docker build -t gearpump/gearpump-launcher integrationtest/docker/gearpump` +4. Run Integration test `sbt it:test` The test will launch a Gearpump cluster with 1 master and 2 worker nodes as 3 Docker containers. It might take 10-20 minutes to go through all the test cases. It depends on, how powerful your machine is. The Docker container itself does not have a Gearpump distribution. It will link your local build to the Docker container. When tests are finished, you can see the test result on the screen, or you can save them to a file with this command `sbt it:test > test_report.out`. To investigate Gearpump log, please check the directory `output/target/pack/logs`. @@ -69,15 +69,15 @@ You can launch as many worker containers as you wish, but only one master for th export GEARPUMP_HOME=/path/to/gearpump/dist ## Start Master node -docker run -d -h master0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --name master0 grubykarol/gearpump-launcher master -ip master0 -port 3000 +docker run -d -h master0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --name master0 gearpump/gearpump-launcher master -ip master0 -port 3000 ## Start Worker0 node -docker run -d -h worker0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 grubykarol/gearpump-launcher worker +docker run -d -h worker0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 gearpump/gearpump-launcher worker ## ... ## Start Worker1 node -docker run -d -h worker1 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 grubykarol/gearpump-launcher worker +docker run -d -h worker1 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 gearpump/gearpump-launcher worker ``` http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/835d9814/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala ---------------------------------------------------------------------- diff --git a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala index 6e32f63..f1819df 100644 --- a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala +++ b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala @@ -28,15 +28,9 @@ class BaseContainer(val host: String, command: String, masterAddrs: List[(String, Int)], tunnelPorts: Set[Int] = Set.empty) { - private val IMAGE_NAME = "grubykarol/gearpump-launcher" + private val IMAGE_NAME = "gearpump/gearpump-launcher" private val DOCKER_IMAGE_GEARPUMP_HOME = "/opt/gearpump" - private val DOCKER_IMAGE_LOG_HOME = "/var/log/gearpump" private val HOST_GEARPUMP_HOME = "pwd".!!.trim + "/output/target/pack" - private val HOST_LOG_HOME = { - val dir = "/tmp/gearpump" - s"mkdir -p $dir".!! - s"mktemp -p $dir -d".!!.trim - } private val CLUSTER_OPTS = { masterAddrs.zipWithIndex.map { case (hostPort, index) => @@ -48,8 +42,7 @@ class BaseContainer(val host: String, command: String, Docker.createAndStartContainer(host, IMAGE_NAME, command, environ = Map("JAVA_OPTS" -> CLUSTER_OPTS), volumes = Map( - HOST_GEARPUMP_HOME -> DOCKER_IMAGE_GEARPUMP_HOME, - HOST_LOG_HOME -> DOCKER_IMAGE_LOG_HOME), + HOST_GEARPUMP_HOME -> DOCKER_IMAGE_GEARPUMP_HOME), knownHosts = masterAddrs.map(_._1).filter(_ != host).toSet, tunnelPorts = tunnelPorts) } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/835d9814/integrationtest/docker/gearpump/Dockerfile ---------------------------------------------------------------------- diff --git a/integrationtest/docker/gearpump/Dockerfile b/integrationtest/docker/gearpump/Dockerfile index 754f6de..0c5042e 100644 --- a/integrationtest/docker/gearpump/Dockerfile +++ b/integrationtest/docker/gearpump/Dockerfile @@ -18,7 +18,7 @@ FROM openjdk:8u131-jre-alpine # Install python -RUN apk add --no-cache python && \ +RUN apk add --update curl python && \ python -m ensurepip && \ rm -r /usr/lib/python*/ensurepip && \ pip install --upgrade pip setuptools && \ http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/835d9814/integrationtest/docker/gearpump/README.md ---------------------------------------------------------------------- diff --git a/integrationtest/docker/gearpump/README.md b/integrationtest/docker/gearpump/README.md index bc38ccb..26820b5 100644 --- a/integrationtest/docker/gearpump/README.md +++ b/integrationtest/docker/gearpump/README.md @@ -13,14 +13,14 @@ docker run -d \ -v $GEARPUMP_HOME:/opt/gearpump \ -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 \ -p 8090:8090 \ - grubykarol/gearpump-launcher \ + gearpump/gearpump-launcher \ master -ip master0 -port 3000 docker run -d \ --link master0 \ -v $GEARPUMP_HOME:/opt/gearpump \ -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 \ - grubykarol/gearpump-launcher \ + gearpump/gearpump-launcher \ worker docker exec master0 bin/gear info http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/835d9814/integrationtest/docker/gearpump/start.sh ---------------------------------------------------------------------- diff --git a/integrationtest/docker/gearpump/start.sh b/integrationtest/docker/gearpump/start.sh index 76d1f18..987adfd 100644 --- a/integrationtest/docker/gearpump/start.sh +++ b/integrationtest/docker/gearpump/start.sh @@ -26,14 +26,6 @@ if [ -z "$JAVA_OPTS" ]; then exit 1 fi -update_config_file() { - CONF_FILE="$SUT_HOME"/conf/gear.conf - mkdir /var/log/gearpump - sed -i 's/log\.daemon\.dir\s*=.*$/log.daemon.dir = "\/var\/log\/gearpump\/daemon"/g' $CONF_FILE - sed -i 's/log\.application\.dir\s*=.*$/log.application.dir = "\/var\/log\/gearpump\/app"/g' $CONF_FILE - sed -i 's/#\s*jarstore\.rootpath\s*=.*$/jarstore.rootpath = "\/tmp"/g' $CONF_FILE -} - set_and_export_java_opts() { JAVA_OPTS="$JAVA_OPTS $*" export JAVA_OPTS @@ -46,7 +38,6 @@ case "$COMMAND" in master|local) # Launch a container with Gearpump cluster and REST interface (in foreground) HOSTNAME=$(hostname) - update_config_file set_and_export_java_opts \ "-Dgearpump.hostname=$HOSTNAME" \ "-Dgearpump.services.host=$HOSTNAME" @@ -55,7 +46,6 @@ case "$COMMAND" in ;; worker) # Launch a container with a Gearpump worker (in foreground) - update_config_file set_and_export_java_opts \ "-Dgearpump.hostname=$(hostname -i)" nohup sh "$SUT_HOME"/bin/worker @@ -63,7 +53,6 @@ case "$COMMAND" in gear|storm) # Launch a container and execute command `gear` or `storm` # Container will be killed, when command is executed. - update_config_file set_and_export_java_opts \ "-Dgearpump.hostname=$(hostname -i)" sh "$SUT_HOME"/bin/"$COMMAND" "$@"
