[FLINK-9822] Add Dockerfile for StandaloneJobClusterEntryPoint image This commit adds a Dockerfile for a standalone job cluster image. The image contains the Flink distribution and a specified user code jar. The entrypoint will start the StandaloneJobClusterEntryPoint with the provided job classname.
This closes #6319. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/56e5381c Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/56e5381c Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/56e5381c Branch: refs/heads/master Commit: 56e5381cb7aba01f1d7ecfa11e4be7f505a35baf Parents: 740f2fb Author: Till Rohrmann <trohrm...@apache.org> Authored: Tue Jul 10 15:41:18 2018 +0200 Committer: Till Rohrmann <trohrm...@apache.org> Committed: Fri Jul 13 18:02:19 2018 +0200 ---------------------------------------------------------------------- flink-container/docker/Dockerfile | 50 ++++++++++ flink-container/docker/README.md | 40 ++++++++ flink-container/docker/build.sh | 116 +++++++++++++++++++++++ flink-container/docker/docker-compose.yml | 31 ++++++ flink-container/docker/docker-entrypoint.sh | 44 +++++++++ 5 files changed, 281 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/56e5381c/flink-container/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/flink-container/docker/Dockerfile b/flink-container/docker/Dockerfile new file mode 100644 index 0000000..458c214 --- /dev/null +++ b/flink-container/docker/Dockerfile @@ -0,0 +1,50 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +FROM java:8-jre-alpine + +# Install requirements +RUN apk add --no-cache bash snappy + +# Flink environment variables +ENV FLINK_INSTALL_PATH=/opt +ENV FLINK_HOME $FLINK_INSTALL_PATH/flink +ENV FLINK_LIB_DIR $FLINK_HOME/lib +ENV PATH $PATH:$FLINK_HOME/bin + +# flink-dist can point to a directory or a tarball on the local system +ARG flink_dist=NOT_SET +ARG job_jar=NOT_SET + +# Install build dependencies and flink +ADD $flink_dist $FLINK_INSTALL_PATH +ADD $job_jar $FLINK_INSTALL_PATH/job.jar + +RUN set -x && \ + ln -s $FLINK_INSTALL_PATH/flink-* $FLINK_HOME && \ + ln -s $FLINK_INSTALL_PATH/job.jar $FLINK_LIB_DIR && \ + addgroup -S flink && adduser -D -S -H -G flink -h $FLINK_HOME flink && \ + chown -R flink:flink $FLINK_INSTALL_PATH/flink-* && \ + chown -h flink:flink $FLINK_HOME + +COPY docker-entrypoint.sh / + +USER flink +EXPOSE 8081 6123 +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["--help"] http://git-wip-us.apache.org/repos/asf/flink/blob/56e5381c/flink-container/docker/README.md ---------------------------------------------------------------------- diff --git a/flink-container/docker/README.md b/flink-container/docker/README.md new file mode 100644 index 0000000..644b31c --- /dev/null +++ b/flink-container/docker/README.md @@ -0,0 +1,40 @@ +# Apache Flink job cluster deployment on docker using docker-compose + +## Installation + +Install the most recent stable version of docker +https://docs.docker.com/installation/ + +## Build + +Images are based on the official Java Alpine (OpenJDK 8) image. If you want to +build the flink image run: + + build.sh --from-local-dist --job-jar /path/to/job/jar/job.jar --image-name flink:job + +If you want to build the container for a specific version of flink/hadoop/scala +you can configure it in the respective args: + + docker build --build-arg FLINK_VERSION=1.6.0 --build-arg HADOOP_VERSION=28 --build-arg SCALA_VERSION=2.11 -t "flink:1.6.0-hadoop2.8-scala_2.11" flink + +## Deploy + +- Deploy cluster and see config/setup log output (best run in a screen session) + + docker-compose up + +- Deploy as a daemon (and return) + + docker-compose up -d + +- Scale the cluster up or down to *N* TaskManagers + + docker-compose scale taskmanager=<N> + +- Access the Job Manager container + + docker exec -it $(docker ps --filter name=flink_jobmanager --format={{.ID}}) /bin/sh + +- Kill the cluster + + docker-compose kill http://git-wip-us.apache.org/repos/asf/flink/blob/56e5381c/flink-container/docker/build.sh ---------------------------------------------------------------------- diff --git a/flink-container/docker/build.sh b/flink-container/docker/build.sh new file mode 100755 index 0000000..13a536c --- /dev/null +++ b/flink-container/docker/build.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +usage() { + cat <<HERE +Usage: + build.sh --job-jar <path-to-job-jar> --from-local-dist [--image-name <image>] + build.sh --job-jar <path-to-job-jar> --from-release --flink-version <x.x.x> --hadoop-version <x.x> --scala-version <x.xx> [--image-name <image>] + build.sh --help + + If the --image-name flag is not used the built image name will be 'flink'. +HERE + exit 1 +} + +while [[ $# -ge 1 ]] +do +key="$1" + case $key in + --job-jar) + JOB_JAR_PATH="$2" + ;; + --from-local-dist) + FROM_LOCAL="true" + ;; + --from-release) + FROM_RELEASE="true" + ;; + --image-name) + IMAGE_NAME="$2" + shift + ;; + --flink-version) + FLINK_VERSION="$2" + shift + ;; + --hadoop-version) + HADOOP_VERSION="$(echo "$2" | sed 's/\.//')" + shift + ;; + --scala-version) + SCALA_VERSION="$2" + shift + ;; + --kubernetes-certificates) + CERTIFICATES_DIR="$2" + shift + ;; + --help) + usage + ;; + *) + # unknown option + ;; + esac + shift +done + +IMAGE_NAME=${IMAGE_NAME:-flink-job} + +# TMPDIR must be contained within the working directory so it is part of the +# Docker context. (i.e. it can't be mktemp'd in /tmp) +TMPDIR=_TMP_ + +cleanup() { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir -p "${TMPDIR}" + +JOB_JAR_TARGET="${TMPDIR}/job.jar" +cp ${JOB_JAR_PATH} ${JOB_JAR_TARGET} + +if [ -n "${FROM_RELEASE}" ]; then + + [[ -n "${FLINK_VERSION}" ]] && [[ -n "${HADOOP_VERSION}" ]] && [[ -n "${SCALA_VERSION}" ]] || usage + + FLINK_DIST_FILE_NAME="flink-${FLINK_VERSION}-bin-hadoop${HADOOP_VERSION}-scala_${SCALA_VERSION}.tgz" + CURL_OUTPUT="${TMPDIR}/${FLINK_DIST_FILE_NAME}" + + echo "Downloading ${FLINK_DIST_FILE_NAME} from ${FLINK_BASE_URL}" + curl -# "https://archive.apache.org/dist/flink/flink-${FLINK_VERSION}/${FLINK_DIST_FILE_NAME}"; --output ${CURL_OUTPUT} + + FLINK_DIST="${CURL_OUTPUT}" + +elif [ -n "${FROM_LOCAL}" ]; then + + DIST_DIR="../../flink-dist/target/flink-*-bin" + FLINK_DIST="${TMPDIR}/flink.tgz" + echo "Using flink dist: ${DIST_DIR}" + tar -C ${DIST_DIR} -cvzf "${FLINK_DIST}" . + +else + + usage + +fi + +docker build --build-arg flink_dist="${FLINK_DIST}" --build-arg job_jar="${JOB_JAR_TARGET}" -t "${IMAGE_NAME}" . http://git-wip-us.apache.org/repos/asf/flink/blob/56e5381c/flink-container/docker/docker-compose.yml ---------------------------------------------------------------------- diff --git a/flink-container/docker/docker-compose.yml b/flink-container/docker/docker-compose.yml new file mode 100644 index 0000000..81e4c8c --- /dev/null +++ b/flink-container/docker/docker-compose.yml @@ -0,0 +1,31 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set the FLINK_DOCKER_IMAGE_NAME environment variable to override the image name to use + +version: "2.1" +services: + job-cluster: + image: ${FLINK_DOCKER_IMAGE_NAME:-flink-job} + ports: + - "8081:8081" + command: job-cluster --job-classname ${FLINK_JOB} -Djobmanager.rpc.address=job-cluster + + taskmanager: + image: ${FLINK_DOCKER_IMAGE_NAME:-flink-job} + command: task-manager -Djobmanager.rpc.address=job-cluster http://git-wip-us.apache.org/repos/asf/flink/blob/56e5381c/flink-container/docker/docker-entrypoint.sh ---------------------------------------------------------------------- diff --git a/flink-container/docker/docker-entrypoint.sh b/flink-container/docker/docker-entrypoint.sh new file mode 100755 index 0000000..85ba866 --- /dev/null +++ b/flink-container/docker/docker-entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +### If unspecified, the hostname of the container is taken as the JobManager address +FLINK_HOME=${FLINK_HOME:-"/opt/flink/bin"} + +JOB_CLUSTER="job-cluster" +TASK_MANAGER="task-manager" + +CMD="$1" +shift; + +if [ "${CMD}" == "--help" -o "${CMD}" == "-h" ]; then + echo "Usage: $(basename $0) (${JOB_CLUSTER}|${TASK_MANAGER})" + exit 0 +elif [ "${CMD}" == "${JOB_CLUSTER}" -o "${CMD}" == "${TASK_MANAGER}" ]; then + echo "Starting the ${CMD}" + echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml + + if [ "${CMD}" == "${TASK_MANAGER}" ]; then + exec $FLINK_HOME/bin/taskmanager.sh start-foreground "$@" + else + exec $FLINK_HOME/bin/standalone-job.sh start-foreground "$@" + fi +fi + +exec "$@"
