Repository: incubator-gearpump Updated Branches: refs/heads/master 5498ae205 -> f7457504e
[GEARPUMP-281] Using new version of docker images for integration tests Closes #162 from karol-brejna-i/GEARPUMP-281. Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/f7457504 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/f7457504 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/f7457504 Branch: refs/heads/master Commit: f7457504e8697c9faaae263c8c72b776ab964359 Parents: 5498ae2 Author: karol brejna <[email protected]> Authored: Wed Mar 1 21:38:28 2017 +0800 Committer: huafengw <[email protected]> Committed: Wed Mar 1 21:38:59 2017 +0800 ---------------------------------------------------------------------- integrationtest/README.md | 14 +-- .../gearpump/integrationtest/Docker.scala | 5 +- .../integrationtest/kafka/KafkaCluster.scala | 2 +- .../minicluster/BaseContainer.scala | 2 +- integrationtest/docker/Dockerfile | 31 ------- integrationtest/docker/README.md | 51 +++++------ integrationtest/docker/gearpump/Dockerfile | 31 +++++++ integrationtest/docker/gearpump/README.md | 28 ++++++ integrationtest/docker/gearpump/start.sh | 94 ++++++++++++++++++++ integrationtest/docker/hadoop/README.md | 5 ++ integrationtest/docker/java/README.md | 18 ++++ integrationtest/docker/java/jdk/Dockerfile | 39 ++++++++ integrationtest/docker/java/jre/Dockerfile | 39 ++++++++ integrationtest/docker/kafka/README.md | 89 ++++++++++++++++++ integrationtest/docker/kafka/kafka/Dockerfile | 43 +++++++++ .../docker/kafka/kafka/scripts/start-kafka.sh | 67 ++++++++++++++ .../docker/kafka/kafka/supervisor/kafka.conf | 4 + .../kafka/kafka/supervisor/zookeeper.conf | 4 + .../docker/kafka/kafkaproxy/Dockerfile | 34 +++++++ .../docker/kafka/kafkaproxy/consumer.properties | 5 ++ .../docker/kafka/kafkaproxy/kafkaproxy.conf | 4 + .../docker/kafka/kafkaproxy/producer.properties | 4 + .../docker/kafka/kafkaproxy/start_proxy.sh | 21 +++++ integrationtest/docker/start.sh | 94 -------------------- 24 files changed, 564 insertions(+), 164 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/README.md ---------------------------------------------------------------------- diff --git a/integrationtest/README.md b/integrationtest/README.md index 034632b..b626c6a 100644 --- a/integrationtest/README.md +++ b/integrationtest/README.md @@ -3,10 +3,10 @@ To run the integration test, you need a Linux with Kernel version >= 3.10 and Docker 1.7 (or higher). The test framework will use several Docker images. These docker image **NEED** to be prepared **BEFOREHAND** to avoid timeout during testing: - * [The Gearpump Cluster Launcher and Storm Client](https://hub.docker.com/r/stanleyxu2005/gearpump-launcher/) - `docker pull stanleyxu2005/gearpump-launcher` - * [The standalone single node Kafka cluster with Zookeeper](https://hub.docker.com/r/spotify/kafka/) - `docker pull spotify/kafka` + * [The Gearpump Cluster Launcher and Storm Client](https://hub.docker.com/r/grubykarol/gearpump-launcher/) + `docker pull grubykarol/gearpump-launcher` + * [The standalone single node Kafka cluster with Zookeeper](https://hub.docker.com/r/grubykarol/kafka/) + `docker pull grubykarol/kafka:0.8.2.1` * [The Hadoop image](https://hub.docker.com/r/sequenceiq/hadoop-docker/) `docker pull sequenceiq/hadoop-docker:2.6.0` @@ -69,15 +69,15 @@ You can launch as many worker containers as you wish, but only one master for th export GEARPUMP_HOME=/path/to/gearpump/dist ## Start Master node -docker run -d -h master0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --name master0 stanleyxu2005/gearpump-launcher master -ip master0 -port 3000 +docker run -d -h master0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --name master0 grubykarol/gearpump-launcher master -ip master0 -port 3000 ## Start Worker0 node -docker run -d -h worker0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 stanleyxu2005/gearpump-launcher worker +docker run -d -h worker0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 grubykarol/gearpump-launcher worker ## ... ## Start Worker1 node -docker run -d -h worker1 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 stanleyxu2005/gearpump-launcher worker +docker run -d -h worker1 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 grubykarol/gearpump-launcher worker ``` http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/Docker.scala ---------------------------------------------------------------------- diff --git a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/Docker.scala b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/Docker.scala index f315ad3..d04c22a 100644 --- a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/Docker.scala +++ b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/Docker.scala @@ -155,9 +155,10 @@ object Docker { final def getNetworkGateway(container: String): String = { trace(container, s"Get gateway of container...") { - doExecute(container, "ip route").split("\\s+")(2) + Docker.inspect(container, "--format={{.NetworkSettings.Gateway}}") } } + final def killProcess(container: String, pid: Int, signal: String = "SIGKILL"): Boolean = { trace(container, s"Kill process pid: $pid") { doExecuteSilently(container, s"kill -$signal $pid") @@ -208,4 +209,4 @@ object Docker { } preview } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/kafka/KafkaCluster.scala ---------------------------------------------------------------------- diff --git a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/kafka/KafkaCluster.scala b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/kafka/KafkaCluster.scala index 15ba084..bb7c95a 100644 --- a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/kafka/KafkaCluster.scala +++ b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/kafka/KafkaCluster.scala @@ -53,7 +53,7 @@ object KafkaCluster { class KafkaCluster(val advertisedHost: String, zkChroot: String = "") { private val LOG = Logger.getLogger(getClass) - private val KAFKA_DOCKER_IMAGE = "spotify/kafka" + private val KAFKA_DOCKER_IMAGE = "grubykarol/kafka:0.8.2.1" private val KAFKA_HOST = "kafka0" private val KAFKA_HOME = "/opt/kafka_2.11-0.8.2.1/" private val ZOOKEEPER_PORT = 2181 http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala ---------------------------------------------------------------------- diff --git a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala index 73413da..6e32f63 100644 --- a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala +++ b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala @@ -28,7 +28,7 @@ class BaseContainer(val host: String, command: String, masterAddrs: List[(String, Int)], tunnelPorts: Set[Int] = Set.empty) { - private val IMAGE_NAME = "stanleyxu2005/gearpump-launcher" + private val IMAGE_NAME = "grubykarol/gearpump-launcher" private val DOCKER_IMAGE_GEARPUMP_HOME = "/opt/gearpump" private val DOCKER_IMAGE_LOG_HOME = "/var/log/gearpump" private val HOST_GEARPUMP_HOME = "pwd".!!.trim + "/output/target/pack" http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/integrationtest/docker/Dockerfile b/integrationtest/docker/Dockerfile deleted file mode 100644 index f0a436a..0000000 --- a/integrationtest/docker/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The base image contains JRE8 -FROM errordeveloper/oracle-jre - -# Add Python Support -RUN opkg-install python - -# Create SUT home, files will be mounted at runtime. -ENV SUT_HOME=/opt/gearpump -WORKDIR $SUT_HOME - -# Setup the entry point -ADD start.sh /opt/start -RUN chmod +x /opt/start - -ENTRYPOINT ["/opt/start"] http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/README.md ---------------------------------------------------------------------- diff --git a/integrationtest/docker/README.md b/integrationtest/docker/README.md index 450bec6..69d89f4 100644 --- a/integrationtest/docker/README.md +++ b/integrationtest/docker/README.md @@ -1,28 +1,23 @@ -# Gearpump Launcher Docker Image - -The image helps developer to setup/test a local [Gearpump](https://github.com/apache/incubator-gearpump.git) cluster quickly. The image is based on a minimal JRE8 environment with Python support. - -## Usage - -Here are the commands to launch a cluster. You can launch as many worker containers as you wish but only one master for the time being. -``` -export GEARPUMP_HOME=/path/to/gearpump - -docker run -d \ - -h master0 --name master0 \ - -v $GEARPUMP_HOME:/opt/gearpump \ - -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 \ - -p 8090:8090 \ - stanleyxu2005/gearpump-launcher \ - master -ip master0 -port 3000 - -docker run -d \ - --link master0 \ - -v $GEARPUMP_HOME:/opt/gearpump \ - -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 \ - stanleyxu2005/gearpump-launcher \ - worker - -docker exec master0 gear info -docker exec master0 gear app -jar /path/to/userapp.jar [mainclass] [args] -``` +This folder contains docker images definitions used in integration tests of Gearpump. + + +These include: + + * [The standalone single node Kafka cluster with Zookeeper](/kafka) + * [The Hadoop image](/hadoop) + * [The Gearpump Cluster Launcher and Storm Client](/gearpump) + * [Java 8 JRE image](/java) + + +We decided to fork spotify/kafka image, because the project does not maintain proper tagging. +For now our tests focus on Kafka 0.8.x, and the project does not support this version. + +Hadoop docker image (https://hub.docker.com/r/sequenceiq/hadoop-docker/) is well maintained. +We rely on version 2.6.0 and we feel there is no need to duplicate it. + +Gearpump Cluster Launcher helps developer to setup/test a local Gearpump cluster quickly. +The image is based on a minimal JRE8 environment with Python support. + +We used to base Gearpump Cluster Launcher on errordeveloper/oracle-jre image but it stuck on version 8u66-b17 and doesn't support tagging. +We also probably hit a Java bug (http://bugs.java.com/bugdatabase/view_bug.do?bug_id=8133205) that requires newer version of JRE. +This is experimental version of Java image that we will use until we switch to some "official" image. http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/gearpump/Dockerfile ---------------------------------------------------------------------- diff --git a/integrationtest/docker/gearpump/Dockerfile b/integrationtest/docker/gearpump/Dockerfile new file mode 100644 index 0000000..b37494f --- /dev/null +++ b/integrationtest/docker/gearpump/Dockerfile @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The base image contains JRE8 +FROM grubykarol/jre:8u121b13_server + +# Add Python Support +RUN opkg-install python + +# Create SUT home, files will be mounted at runtime. +ENV SUT_HOME=/opt/gearpump +WORKDIR $SUT_HOME + +# Setup the entry point +ADD start.sh /opt/start +RUN chmod +x /opt/start + +ENTRYPOINT ["/opt/start"] http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/gearpump/README.md ---------------------------------------------------------------------- diff --git a/integrationtest/docker/gearpump/README.md b/integrationtest/docker/gearpump/README.md new file mode 100644 index 0000000..d53876e --- /dev/null +++ b/integrationtest/docker/gearpump/README.md @@ -0,0 +1,28 @@ +# Gearpump Launcher Docker Image + +The image helps developer to setup/test a local [Gearpump](https://github.com/apache/incubator-gearpump.git) cluster quickly. The image is based on a minimal JRE8 environment with Python support. + +## Usage + +Here are the commands to launch a cluster. You can launch as many worker containers as you wish but only one master for the time being. +``` +export GEARPUMP_HOME=/path/to/gearpump + +docker run -d \ + -h master0 --name master0 \ + -v $GEARPUMP_HOME:/opt/gearpump \ + -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 \ + -p 8090:8090 \ + grubykarol/gearpump-launcher \ + master -ip master0 -port 3000 + +docker run -d \ + --link master0 \ + -v $GEARPUMP_HOME:/opt/gearpump \ + -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 \ + grubykarol/gearpump-launcher \ + worker + +docker exec master0 gear info +docker exec master0 gear app -jar /path/to/userapp.jar [mainclass] [args] +``` http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/gearpump/start.sh ---------------------------------------------------------------------- diff --git a/integrationtest/docker/gearpump/start.sh b/integrationtest/docker/gearpump/start.sh new file mode 100644 index 0000000..76d1f18 --- /dev/null +++ b/integrationtest/docker/gearpump/start.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ ! -d "$SUT_HOME/bin" ]; then + echo "FATAL: The Gearpump distribution seems to be incomplete. Please build gearpump with 'sbt clean assembly' first, so that the test driver is able to mount the distribution directory to '/opt/gearpump'." + exit 1 +fi + +if [ -z "$JAVA_OPTS" ]; then + echo "FATAL: Environment variable 'JAVA_OPTS' is NOT set." + exit 1 +fi + +update_config_file() { + CONF_FILE="$SUT_HOME"/conf/gear.conf + mkdir /var/log/gearpump + sed -i 's/log\.daemon\.dir\s*=.*$/log.daemon.dir = "\/var\/log\/gearpump\/daemon"/g' $CONF_FILE + sed -i 's/log\.application\.dir\s*=.*$/log.application.dir = "\/var\/log\/gearpump\/app"/g' $CONF_FILE + sed -i 's/#\s*jarstore\.rootpath\s*=.*$/jarstore.rootpath = "\/tmp"/g' $CONF_FILE +} + +set_and_export_java_opts() { + JAVA_OPTS="$JAVA_OPTS $*" + export JAVA_OPTS +} + +COMMAND=$1 +shift + +case "$COMMAND" in + master|local) + # Launch a container with Gearpump cluster and REST interface (in foreground) + HOSTNAME=$(hostname) + update_config_file + set_and_export_java_opts \ + "-Dgearpump.hostname=$HOSTNAME" \ + "-Dgearpump.services.host=$HOSTNAME" + nohup sh "$SUT_HOME"/bin/services & + nohup sh "$SUT_HOME"/bin/"$COMMAND" "$@" + ;; + worker) + # Launch a container with a Gearpump worker (in foreground) + update_config_file + set_and_export_java_opts \ + "-Dgearpump.hostname=$(hostname -i)" + nohup sh "$SUT_HOME"/bin/worker + ;; + gear|storm) + # Launch a container and execute command `gear` or `storm` + # Container will be killed, when command is executed. + update_config_file + set_and_export_java_opts \ + "-Dgearpump.hostname=$(hostname -i)" + sh "$SUT_HOME"/bin/"$COMMAND" "$@" + ;; + storm-drpc) + # Launch a container with a Storm DRPC daemon + # Note that this command has nothing to do with Gearpump, it only uses storm related jar libs. + LIB_HOME="$SUT_HOME"/lib + cat > "$SUT_HOME"/storm.yaml <<- EOF +drpc.servers: + - $(ip route|awk '/default/ {print $3}') +EOF + java -server -Xmx768m -cp "$LIB_HOME"/*:"$LIB_HOME"/storm/* backtype.storm.daemon.drpc + ;; + *) + cat <<- USAGE +Gearpump Commands: + master -ip [HOST] -port [PORT] + worker + gear (app|info|kill) [ARGS] + storm [ARGS] + +Storm Commands: + storm-drpc +USAGE + exit 1 + ;; +esac http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/hadoop/README.md ---------------------------------------------------------------------- diff --git a/integrationtest/docker/hadoop/README.md b/integrationtest/docker/hadoop/README.md new file mode 100644 index 0000000..ef3897f --- /dev/null +++ b/integrationtest/docker/hadoop/README.md @@ -0,0 +1,5 @@ +For hadoop testing we are using https://hub.docker.com/r/sequenceiq/hadoop-docker/. +It is one of the most used hadoop docker images on this planet. The maintainers also keeps a very decent tagging strategy. + +We are using their images AS-IS. No need for any modifications there. + http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/java/README.md ---------------------------------------------------------------------- diff --git a/integrationtest/docker/java/README.md b/integrationtest/docker/java/README.md new file mode 100644 index 0000000..6f93f20 --- /dev/null +++ b/integrationtest/docker/java/README.md @@ -0,0 +1,18 @@ +A minimalistic Oracle JDK 8 container on top of busybox. + +We used to base Gearpump Cluster Launcher on errordeveloper/oracle-jre image but it stuck on version 8u66-b17 and doesn't support tagging. +We also probably hit a Java bug (http://bugs.java.com/bugdatabase/view_bug.do?bug_id=8133205) that requires newer version of JRE. +This is experimental version of Java image that we will use until we switch to some "official" image. + +For now, the official JRE image is twice as big as the one we used: +openjdk 8-jre-alpine 108 MB +openjdk 8u121-jre 309 MB + +delitescere/java 1.8.0_92 67.8 MB +errordeveloper/oracle-jre latest 161 MB + + +By using this container or any derived containers you are accepting the terms of [Oracle Binary Code License Agreement for the Java SE Platform Products and JavaFX][1]. + + [1]: http://www.oracle.com/technetwork/java/javase/terms/license/index.html + http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/java/jdk/Dockerfile ---------------------------------------------------------------------- diff --git a/integrationtest/docker/java/jdk/Dockerfile b/integrationtest/docker/java/jdk/Dockerfile new file mode 100644 index 0000000..133d7a6 --- /dev/null +++ b/integrationtest/docker/java/jdk/Dockerfile @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM progrium/busybox +LABEL maintainer "grubykarol <[email protected]>" + +RUN opkg-install curl ca-certificates + +ENV JAVA_HOME /usr/jdk1.8.0_121 + +RUN curl \ + --silent \ + --location \ + --retry 3 \ + --cacert /etc/ssl/certs/GeoTrust_Global_CA.crt \ + --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ + "http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.tar.gz"; \ + | gunzip \ + | tar x -C /usr/ \ + && ln -s $JAVA_HOME /usr/java \ + && rm -rf $JAVA_HOME/src.zip $JAVA_HOME/javafx-src.zip $JAVA_HOME/man + +ENV PATH ${PATH}:${JAVA_HOME}/bin + +ENTRYPOINT [ "java" ] +CMD [ "-version" ] http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/java/jre/Dockerfile ---------------------------------------------------------------------- diff --git a/integrationtest/docker/java/jre/Dockerfile b/integrationtest/docker/java/jre/Dockerfile new file mode 100644 index 0000000..211a0a7 --- /dev/null +++ b/integrationtest/docker/java/jre/Dockerfile @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM progrium/busybox +LABEL maintainer "grubykarol <[email protected]>" + +RUN opkg-install curl ca-certificates + +ENV JAVA_HOME /usr/jdk1.8.0_121 + +RUN curl \ + --silent \ + --location \ + --retry 3 \ + --cacert /etc/ssl/certs/GeoTrust_Global_CA.crt \ + --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ + "http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/server-jre-8u121-linux-x64.tar.gz"; \ + | gunzip \ + | tar x -C /usr/ \ + && ln -s $JAVA_HOME /usr/java \ + && rm -rf $JAVA_HOME/man + +ENV PATH ${PATH}:${JAVA_HOME}/bin + +ENTRYPOINT [ "java" ] +CMD [ "-version" ] http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/README.md ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/README.md b/integrationtest/docker/kafka/README.md new file mode 100644 index 0000000..e75ec20 --- /dev/null +++ b/integrationtest/docker/kafka/README.md @@ -0,0 +1,89 @@ +Kafka in Docker +=== + +**This is a fork spotify/kafka image. We decided to make it, because the project does not maintain proper tagging. +For now our tests focus on Kafka 0.8.x, so here is a version that supports Kafka 0.8.x. +For latest version of kafka image go to the original project.** + +This repository provides everything you need to run Kafka in Docker. + +For convenience also contains a packaged proxy that can be used to get data from +a legacy Kafka 7 cluster into a dockerized Kafka 8. + +Why? +--- +The main hurdle of running Kafka in Docker is that it depends on Zookeeper. +Compared to other Kafka docker images, this one runs both Zookeeper and Kafka +in the same container. This means: + +* No dependency on an external Zookeeper host, or linking to another container +* Zookeeper and Kafka are configured to work together out of the box + +Run +--- + +```bash +docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=`docker-machine ip \`docker-machine active\`` --env ADVERTISED_PORT=9092 grubykarol/kafka +``` + +```bash +export KAFKA=`docker-machine ip \`docker-machine active\``:9092 +kafka-console-producer.sh --broker-list $KAFKA --topic test +``` + +```bash +export ZOOKEEPER=`docker-machine ip \`docker-machine active\``:2181 +kafka-console-consumer.sh --zookeeper $ZOOKEEPER --topic test +``` + +Running the proxy +----------------- + +Take the same parameters as the grubykarol/kafka image with some new ones: + * `CONSUMER_THREADS` - the number of threads to consume the source kafka 7 with + * `TOPICS` - whitelist of topics to mirror + * `ZK_CONNECT` - the zookeeper connect string of the source kafka 7 + * `GROUP_ID` - the group.id to use when consuming from kafka 7 + +```bash +docker run -p 2181:2181 -p 9092:9092 \ + --env ADVERTISED_HOST=`boot2docker ip` \ + --env ADVERTISED_PORT=9092 \ + --env CONSUMER_THREADS=1 \ + --env TOPICS=my-topic,some-other-topic \ + --env ZK_CONNECT=kafka7zookeeper:2181/root/path \ + --env GROUP_ID=mymirror \ + grubykarol/kafkaproxy +``` + +In the box +--- +* **grubykarol/kafka** + + The docker image with both Kafka and Zookeeper. Built from the `kafka` + directory. + +* **grubykarol/kafkaproxy** + + The docker image with Kafka, Zookeeper and a Kafka 7 proxy that can be + configured with a set of topics to mirror. + +Public Builds +--- + +https://registry.hub.docker.com/u/grubykarol/kafka/ + +https://registry.hub.docker.com/u/grubykarol/kafkaproxy/ + +Build from Source +--- + + docker build -t grubykarol/kafka kafka/ + docker build -t grubykarol/kafkaproxy kafkaproxy/ + +Todo +--- + +* Not particularily optimzed for startup time. +* Better docs + http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafka/Dockerfile ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafka/Dockerfile b/integrationtest/docker/kafka/kafka/Dockerfile new file mode 100644 index 0000000..197ddca --- /dev/null +++ b/integrationtest/docker/kafka/kafka/Dockerfile @@ -0,0 +1,43 @@ +# Kafka and Zookeeper + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM java:openjdk-8-jre + +ENV DEBIAN_FRONTEND noninteractive +ENV SCALA_VERSION 2.11 +ENV KAFKA_VERSION 0.8.2.1 +ENV KAFKA_HOME /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION" + +# Install Kafka, Zookeeper and other needed things +RUN apt-get update && \ + apt-get install -y zookeeper wget supervisor dnsutils && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + wget -q http://apache.mirrors.spacedump.net/kafka/"$KAFKA_VERSION"/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -O /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \ + tar xfz /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -C /opt && \ + rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz + +ADD scripts/start-kafka.sh /usr/bin/start-kafka.sh + +# Supervisor config +ADD supervisor/kafka.conf supervisor/zookeeper.conf /etc/supervisor/conf.d/ + +# 2181 is zookeeper, 9092 is kafka +EXPOSE 2181 9092 + +CMD ["supervisord", "-n"] http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafka/scripts/start-kafka.sh ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafka/scripts/start-kafka.sh b/integrationtest/docker/kafka/kafka/scripts/start-kafka.sh new file mode 100755 index 0000000..ff37918 --- /dev/null +++ b/integrationtest/docker/kafka/kafka/scripts/start-kafka.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +# Optional ENV variables: +# * ADVERTISED_HOST: the external ip for the container, e.g. `docker-machine ip \`docker-machine active\`` +# * ADVERTISED_PORT: the external port for Kafka, e.g. 9092 +# * ZK_CHROOT: the zookeeper chroot that's used by Kafka (without / prefix), e.g. "kafka" +# * LOG_RETENTION_HOURS: the minimum age of a log file in hours to be eligible for deletion (default is 168, for 1 week) +# * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB) +# * NUM_PARTITIONS: configure the default number of log partitions per topic + +# Configure advertised host/port if we run in helios +if [ ! -z "$HELIOS_PORT_kafka" ]; then + ADVERTISED_HOST=`echo $HELIOS_PORT_kafka | cut -d':' -f 1 | xargs -n 1 dig +short | tail -n 1` + ADVERTISED_PORT=`echo $HELIOS_PORT_kafka | cut -d':' -f 2` +fi + +# Set the external host and port +if [ ! -z "$ADVERTISED_HOST" ]; then + echo "advertised host: $ADVERTISED_HOST" + sed -r -i "s/#(advertised.host.name)=(.*)/\1=$ADVERTISED_HOST/g" $KAFKA_HOME/config/server.properties +fi +if [ ! -z "$ADVERTISED_PORT" ]; then + echo "advertised port: $ADVERTISED_PORT" + sed -r -i "s/#(advertised.port)=(.*)/\1=$ADVERTISED_PORT/g" $KAFKA_HOME/config/server.properties +fi + +# Set the zookeeper chroot +if [ ! -z "$ZK_CHROOT" ]; then + # wait for zookeeper to start up + until /usr/share/zookeeper/bin/zkServer.sh status; do + sleep 0.1 + done + + # create the chroot node + echo "create /$ZK_CHROOT \"\"" | /usr/share/zookeeper/bin/zkCli.sh || { + echo "can't create chroot in zookeeper, exit" + exit 1 + } + + # configure kafka + sed -r -i "s/(zookeeper.connect)=(.*)/\1=localhost:2181\/$ZK_CHROOT/g" $KAFKA_HOME/config/server.properties +fi + +# Allow specification of log retention policies +if [ ! -z "$LOG_RETENTION_HOURS" ]; then + echo "log retention hours: $LOG_RETENTION_HOURS" + sed -r -i "s/(log.retention.hours)=(.*)/\1=$LOG_RETENTION_HOURS/g" $KAFKA_HOME/config/server.properties +fi +if [ ! -z "$LOG_RETENTION_BYTES" ]; then + echo "log retention bytes: $LOG_RETENTION_BYTES" + sed -r -i "s/#(log.retention.bytes)=(.*)/\1=$LOG_RETENTION_BYTES/g" $KAFKA_HOME/config/server.properties +fi + +# Configure the default number of log partitions per topic +if [ ! -z "$NUM_PARTITIONS" ]; then + echo "default number of partition: $NUM_PARTITIONS" + sed -r -i "s/(num.partitions)=(.*)/\1=$NUM_PARTITIONS/g" $KAFKA_HOME/config/server.properties +fi + +# Enable/disable auto creation of topics +if [ ! -z "$AUTO_CREATE_TOPICS" ]; then + echo "auto.create.topics.enable: $AUTO_CREATE_TOPICS" + echo "auto.create.topics.enable=$AUTO_CREATE_TOPICS" >> $KAFKA_HOME/config/server.properties +fi + +# Run Kafka +$KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafka/supervisor/kafka.conf ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafka/supervisor/kafka.conf b/integrationtest/docker/kafka/kafka/supervisor/kafka.conf new file mode 100644 index 0000000..de05832 --- /dev/null +++ b/integrationtest/docker/kafka/kafka/supervisor/kafka.conf @@ -0,0 +1,4 @@ +[program:kafka] +command=/usr/bin/start-kafka.sh +autostart=true +autorestart=true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafka/supervisor/zookeeper.conf ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafka/supervisor/zookeeper.conf b/integrationtest/docker/kafka/kafka/supervisor/zookeeper.conf new file mode 100644 index 0000000..5650d95 --- /dev/null +++ b/integrationtest/docker/kafka/kafka/supervisor/zookeeper.conf @@ -0,0 +1,4 @@ +[program:zookeeper] +command=/usr/share/zookeeper/bin/zkServer.sh start-foreground +autostart=true +autorestart=true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafkaproxy/Dockerfile ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafkaproxy/Dockerfile b/integrationtest/docker/kafka/kafkaproxy/Dockerfile new file mode 100644 index 0000000..40a33e1 --- /dev/null +++ b/integrationtest/docker/kafka/kafkaproxy/Dockerfile @@ -0,0 +1,34 @@ +# Kafka, Zookeeper and Kafka 7 proxy + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM spotify/kafka + +ADD kafka-0.7.2.jar kafka-0.7.2.jar +ADD zkclient-0.1.jar zkclient-0.1.jar +ADD consumer.properties consumer.properties +ADD producer.properties producer.properties +ADD start_proxy.sh /start_proxy.sh +ADD kafkaproxy.conf /etc/supervisor/conf.d/kafkaproxy.conf + +ENV LOG_RETENTION_HOURS 1 + +ADD https://archive.apache.org/dist/kafka/0.8.1/kafka_2.8.0-0.8.1.tgz / +RUN cd / && tar xzf kafka_2.8.0-0.8.1.tgz +ENV TAIL_KAFKA_HOME /kafka_2.8.0-0.8.1 + +CMD ["supervisord", "-n"] http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafkaproxy/consumer.properties ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafkaproxy/consumer.properties b/integrationtest/docker/kafka/kafkaproxy/consumer.properties new file mode 100644 index 0000000..cbdad06 --- /dev/null +++ b/integrationtest/docker/kafka/kafkaproxy/consumer.properties @@ -0,0 +1,5 @@ +zk.connect= +zk.connectiontimeout.ms=1000000 +rebalance.retries.max=60 +groupid= +mirror.consumer.numthreads=1 http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafkaproxy/kafkaproxy.conf ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafkaproxy/kafkaproxy.conf b/integrationtest/docker/kafka/kafkaproxy/kafkaproxy.conf new file mode 100644 index 0000000..a9baf2b --- /dev/null +++ b/integrationtest/docker/kafka/kafkaproxy/kafkaproxy.conf @@ -0,0 +1,4 @@ +[program:kafka_proxy] +command=/start_proxy.sh +autostart=true +autorestart=true http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafkaproxy/producer.properties ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafkaproxy/producer.properties b/integrationtest/docker/kafka/kafkaproxy/producer.properties new file mode 100644 index 0000000..5800b30 --- /dev/null +++ b/integrationtest/docker/kafka/kafkaproxy/producer.properties @@ -0,0 +1,4 @@ +metadata.broker.list=127.0.0.1:9092 +partitioner.class=kafka.producer.ByteArrayPartitioner +producer.type=sync +compression.codec=snappy http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/kafka/kafkaproxy/start_proxy.sh ---------------------------------------------------------------------- diff --git a/integrationtest/docker/kafka/kafkaproxy/start_proxy.sh b/integrationtest/docker/kafka/kafkaproxy/start_proxy.sh new file mode 100755 index 0000000..4ccd06e --- /dev/null +++ b/integrationtest/docker/kafka/kafkaproxy/start_proxy.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +: ${KAFKA_HOME:?"Need to configure KAFKA_HOME"} +: ${CONSUMER_THREADS:?"Need to configure CONSUMER_THREADS"} +: ${TOPICS:?"Need to configure TOPICS whitelist"} +: ${ZK_CONNECT:?"Need to configure ZK_CONNECT string"} +: ${GROUP_ID:?"Need to configure GROUP_ID string"} + +sed -i "s/mirror.consumer.numthreads=.*/mirror.consumer.numthreads=$CONSUMER_THREADS/" /consumer.properties +sed -i "s/zk.connect=.*/$ZK_CONNECT/" /consumer.properties +sed -i "s/groupid=.*/$GROUP_ID/" /consumer.properties + +cd / +exec $TAIL_KAFKA_HOME/bin/kafka-run-class.sh kafka.tools.KafkaMigrationTool \ + --kafka.07.jar ../kafka-0.7.2.jar \ + --zkclient.01.jar ../zkclient-0.1.jar \ + --num.producers 16 \ + --num.streams $CONSUMER_THREADS \ + --consumer.config=../consumer.properties \ + --producer.config=../producer.properties \ + --whitelist="$TOPICS" http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7457504/integrationtest/docker/start.sh ---------------------------------------------------------------------- diff --git a/integrationtest/docker/start.sh b/integrationtest/docker/start.sh deleted file mode 100644 index 76d1f18..0000000 --- a/integrationtest/docker/start.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ ! -d "$SUT_HOME/bin" ]; then - echo "FATAL: The Gearpump distribution seems to be incomplete. Please build gearpump with 'sbt clean assembly' first, so that the test driver is able to mount the distribution directory to '/opt/gearpump'." - exit 1 -fi - -if [ -z "$JAVA_OPTS" ]; then - echo "FATAL: Environment variable 'JAVA_OPTS' is NOT set." - exit 1 -fi - -update_config_file() { - CONF_FILE="$SUT_HOME"/conf/gear.conf - mkdir /var/log/gearpump - sed -i 's/log\.daemon\.dir\s*=.*$/log.daemon.dir = "\/var\/log\/gearpump\/daemon"/g' $CONF_FILE - sed -i 's/log\.application\.dir\s*=.*$/log.application.dir = "\/var\/log\/gearpump\/app"/g' $CONF_FILE - sed -i 's/#\s*jarstore\.rootpath\s*=.*$/jarstore.rootpath = "\/tmp"/g' $CONF_FILE -} - -set_and_export_java_opts() { - JAVA_OPTS="$JAVA_OPTS $*" - export JAVA_OPTS -} - -COMMAND=$1 -shift - -case "$COMMAND" in - master|local) - # Launch a container with Gearpump cluster and REST interface (in foreground) - HOSTNAME=$(hostname) - update_config_file - set_and_export_java_opts \ - "-Dgearpump.hostname=$HOSTNAME" \ - "-Dgearpump.services.host=$HOSTNAME" - nohup sh "$SUT_HOME"/bin/services & - nohup sh "$SUT_HOME"/bin/"$COMMAND" "$@" - ;; - worker) - # Launch a container with a Gearpump worker (in foreground) - update_config_file - set_and_export_java_opts \ - "-Dgearpump.hostname=$(hostname -i)" - nohup sh "$SUT_HOME"/bin/worker - ;; - gear|storm) - # Launch a container and execute command `gear` or `storm` - # Container will be killed, when command is executed. - update_config_file - set_and_export_java_opts \ - "-Dgearpump.hostname=$(hostname -i)" - sh "$SUT_HOME"/bin/"$COMMAND" "$@" - ;; - storm-drpc) - # Launch a container with a Storm DRPC daemon - # Note that this command has nothing to do with Gearpump, it only uses storm related jar libs. - LIB_HOME="$SUT_HOME"/lib - cat > "$SUT_HOME"/storm.yaml <<- EOF -drpc.servers: - - $(ip route|awk '/default/ {print $3}') -EOF - java -server -Xmx768m -cp "$LIB_HOME"/*:"$LIB_HOME"/storm/* backtype.storm.daemon.drpc - ;; - *) - cat <<- USAGE -Gearpump Commands: - master -ip [HOST] -port [PORT] - worker - gear (app|info|kill) [ARGS] - storm [ARGS] - -Storm Commands: - storm-drpc -USAGE - exit 1 - ;; -esac
