Repository: bigtop Updated Branches: refs/heads/master 45ea264d0 -> 51e7dccef
BIGTOP-2345 Create Flink packaging Signed-off-by: Konstantin Boudnik <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/51e7dcce Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/51e7dcce Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/51e7dcce Branch: refs/heads/master Commit: 51e7dccef3b74d53efb8a7359ebf4ef47e2208f2 Parents: 45ea264 Author: Robert Metzger <[email protected]> Authored: Mon May 2 15:59:14 2016 +0200 Committer: Konstantin Boudnik <[email protected]> Committed: Sun May 8 13:19:37 2016 -0700 ---------------------------------------------------------------------- .../src/common/flink/do-component-build | 24 +++ .../src/common/flink/flink-jobmanager.svc | 82 ++++++++++ .../src/common/flink/flink-taskmanager.svc | 82 ++++++++++ .../src/common/flink/install_flink.sh | 143 ++++++++++++++++++ .../src/common/flink/patch1-1837.diff | 81 ++++++++++ bigtop-packages/src/deb/flink/changelog | 1 + bigtop-packages/src/deb/flink/compat | 1 + bigtop-packages/src/deb/flink/control | 29 ++++ bigtop-packages/src/deb/flink/copyright | 15 ++ bigtop-packages/src/deb/flink/flink.dirs | 5 + bigtop-packages/src/deb/flink/flink.install | 7 + bigtop-packages/src/deb/flink/flink.postinst | 36 +++++ bigtop-packages/src/deb/flink/flink.preinst | 61 ++++++++ bigtop-packages/src/deb/flink/flink.prerm | 38 +++++ bigtop-packages/src/deb/flink/rules | 42 ++++++ bigtop-packages/src/deb/flink/source/format | 1 + bigtop-packages/src/rpm/flink/SPECS/flink.spec | 149 +++++++++++++++++++ bigtop.bom | 13 ++ 18 files changed, 810 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/common/flink/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/flink/do-component-build b/bigtop-packages/src/common/flink/do-component-build new file mode 100644 index 0000000..4fa9102 --- /dev/null +++ b/bigtop-packages/src/common/flink/do-component-build @@ -0,0 +1,24 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + + +#load versions +. `dirname $0`/bigtop.bom + +# Use Maven to build Flink from source +mvn install $FLINK_BUILD_OPTS -Drat.skip=true -DskipTests -Dhadoop.version=$HADOOP_VERSION "$@" http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/common/flink/flink-jobmanager.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/flink/flink-jobmanager.svc b/bigtop-packages/src/common/flink/flink-jobmanager.svc new file mode 100644 index 0000000..43825af --- /dev/null +++ b/bigtop-packages/src/common/flink/flink-jobmanager.svc @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TYPE="jobmanager" +DAEMON="flink-${TYPE}" +DESC="Flink ${TYPE}" +EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh" +WORKING_DIR="/var/lib/flink" +DAEMON_FLAGS=jobmanager +CONF_DIR="/etc/flink/conf" +SVC_USER="flink" +PIDFILE="/var/run/flink/flink-$SVC_USER-jobmanager.pid" +EXEC_MODE="cluster" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC (${DAEMON}): " + + export FLINK_LOG_DIR="/var/log/flink" + export FLINK_PID_DIR="/var/run/flink" + mkdir -p $FLINK_PID_DIR + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "${DESC} is running" + exit 0 + fi + + LOG_FILE=/var/log/flink/${DAEMON}.out + # Flink will set the pid file + su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \ + ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir $CONF_DIR --executionMode cluster \ + > $LOG_FILE 2>&1 & " + + sleep 3 + + checkstatusofproc + + RETVAL=$? + + if [ $RETVAL -eq $STATUS_RUNNING ]; then + touch $LOCKFILE + log_success_msg "Started ${DESC} (${DAEMON}): " + else + log_failure_msg "Failed to start ${DESC}. Return value: $RETVAL" + fi + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + log_success_msg "Stopping $DESC (${DAEMON}): " + killproc -p $PIDFILE java + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE + return $RETVAL +} +__EOT__ + +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/common/flink/flink-taskmanager.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/flink/flink-taskmanager.svc b/bigtop-packages/src/common/flink/flink-taskmanager.svc new file mode 100644 index 0000000..1f6dd2b --- /dev/null +++ b/bigtop-packages/src/common/flink/flink-taskmanager.svc @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TYPE="taskmanager" +DAEMON="flink-${TYPE}" +DESC="Flink ${TYPE}" +EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh" +WORKING_DIR="/var/lib/flink" +DAEMON_FLAGS="taskmanager" +SVC_USER="flink" +CONF_DIR="/etc/flink/conf" +PIDFILE="/var/run/flink/flink-$SVC_USER-taskmanager.pid" + + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC (${DAEMON}): " + + export FLINK_LOG_DIR="/var/log/flink" + export FLINK_PID_DIR="/var/run/flink" + mkdir -p $FLINK_PID_DIR + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "${DESC} is running" + exit 0 + fi + + LOG_FILE=/var/log/flink/${DAEMON}.out + + su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \ + ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir ${CONF_DIR} \ + > $LOG_FILE 2>&1 & " + + sleep 3 + + checkstatusofproc + RETVAL=$? + + if [ $RETVAL -eq $STATUS_RUNNING ]; then + touch $LOCKFILE + log_success_msg "Started ${DESC} (${DAEMON}): " + else + log_failure_msg "Failed to start ${DESC}. Return value: $RETVAL" + fi + + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + log_success_msg "Stopping $DESC (${DAEMON}): " + killproc -p $PIDFILE java + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE + return $RETVAL +} +__EOT__ + +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/common/flink/install_flink.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/flink/install_flink.sh b/bigtop-packages/src/common/flink/install_flink.sh new file mode 100644 index 0000000..f63f546 --- /dev/null +++ b/bigtop-packages/src/common/flink/install_flink.sh @@ -0,0 +1,143 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + echo " +usage: $0 <options> + Required not-so-options: + --build-dir=DIR path to flink dist.dir + --source-dir=DIR path to package shared files dir + --prefix=PREFIX path to install into + + Optional options: + --lib-dir=DIR path to install flink home [/usr/lib/flink] + --installed-lib-dir=DIR path where lib-dir will end up on target system + --bin-dir=DIR path to install bins [/usr/bin] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'lib-dir:' \ + -l 'installed-lib-dir:' \ + -l 'bin-dir:' \ + -l 'source-dir:' \ + -l 'build-dir:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --build-dir) + BUILD_DIR=$2 ; shift 2 + ;; + --source-dir) + SOURCE_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --installed-lib-dir) + INSTALLED_LIB_DIR=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX BUILD_DIR SOURCE_DIR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +# load bigtop component versions +if [ -f "$SOURCE_DIR/bigtop.bom" ]; then + . $SOURCE_DIR/bigtop.bom +fi + + +LIB_DIR=${LIB_DIR:-/usr/lib/flink} +INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/flink} +BIN_DIR=${BIN_DIR:-/usr/bin} +CONF_DIR=${CONF_DIR:-/etc/flink/conf.dist} + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/bin +install -d -m 0755 $PREFIX/$LIB_DIR/lib +install -d -m 0755 $PREFIX/$LIB_DIR/examples +install -d -m 0755 $PREFIX/$LIB_DIR/resources +install -d -m 0755 $PREFIX/$CONF_DIR +install -d -m 0755 $PREFIX/var/log/flink +install -d -m 0755 $PREFIX/var/log/flink-cli +install -d -m 0755 $PREFIX/var/run/flink + +cp -ra ${BUILD_DIR}/lib/* $PREFIX/${LIB_DIR}/lib/ +cp -a ${BUILD_DIR}/bin/* $PREFIX/${LIB_DIR}/bin/ +# delete Windows start scripts +rm -rf $PREFIX/${LIB_DIR}/bin/*.cmd +# remove log directory +rm -rf $PREFIX/${LIB_DIR}/log + +# Copy the configuration files +cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR +ln -s /etc/flink/conf $PREFIX/$LIB_DIR/conf + +cp -ra ${BUILD_DIR}/examples/* $PREFIX/${LIB_DIR}/examples/ +cp -ra ${BUILD_DIR}/resources/* $PREFIX/${LIB_DIR}/resources/ + +cp ${BUILD_DIR}/{LICENSE,NOTICE,README.txt} ${PREFIX}/${LIB_DIR}/ + +# Copy in the /usr/bin/flink wrapper +install -d -m 0755 $PREFIX/$BIN_DIR +cat > $PREFIX/$BIN_DIR/flink <<EOF +#!/bin/bash + +# Autodetect JAVA_HOME if not defined +. /usr/lib/bigtop-utils/bigtop-detect-javahome + +export HADOOP_HOME=\${HADOOP_HOME:-/usr/lib/hadoop} +export HADOOP_CONF_DIR=\${HADOOP_CONF_DIR:-/etc/hadoop/conf} +export FLINK_HOME=\${FLINK_HOME:-$INSTALLED_LIB_DIR} +export FLINK_CONF_DIR=\${FLINK_CONF_DIR:-$CONF_DIR} +export FLINK_LOG_DIR=\${FLINK_LOG_DIR:-/var/log/flink-cli} + +exec $INSTALLED_LIB_DIR/bin/flink "\$@" +EOF +chmod 755 $PREFIX/$BIN_DIR/flink http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/common/flink/patch1-1837.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/flink/patch1-1837.diff b/bigtop-packages/src/common/flink/patch1-1837.diff new file mode 100644 index 0000000..6acd35f --- /dev/null +++ b/bigtop-packages/src/common/flink/patch1-1837.diff @@ -0,0 +1,81 @@ +From 05002c01345425d6fe9814ea7f669630fa5514b3 Mon Sep 17 00:00:00 2001 +From: Stefano Baghino <[email protected]> +Date: Tue, 29 Mar 2016 19:10:46 +0200 +Subject: [PATCH 1/2] [FLINK-3678] Make Flink logs directory configurable + +* Edit config.sh +* Document the newly defined log directory configuration key +--- + docs/setup/config.md | 4 ++++ + flink-dist/src/main/flink-bin/bin/config.sh | 7 ++++++- + 2 files changed, 10 insertions(+), 1 deletion(-) + +diff --git a/docs/setup/config.md b/docs/setup/config.md +index ce2ad13..63d3f10 100644 +--- a/docs/setup/config.md ++++ b/docs/setup/config.md +@@ -275,6 +275,10 @@ For example when running Flink on YARN on an environment with a restrictive fire + + - `recovery.job.delay`: (Default 'akka.ask.timeout') Defines the delay before persisted jobs are recovered in case of a recovery situation. + ++## Environment ++ ++- `env.log.dir`: (Defaults to the `log` directory under Flink's home) Defines the directory where the Flink logs are saved. ++ + ## Background + + ### Configuring the Network Buffers +diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh +index 8faf951..43619ce 100755 +--- a/flink-dist/src/main/flink-bin/bin/config.sh ++++ b/flink-dist/src/main/flink-bin/bin/config.sh +@@ -95,6 +95,7 @@ KEY_TASKM_OFFHEAP="taskmanager.memory.off-heap" + KEY_TASKM_MEM_PRE_ALLOCATE="taskmanager.memory.preallocate" + + KEY_ENV_PID_DIR="env.pid.dir" ++KEY_ENV_LOG_DIR="env.log.dir" + KEY_ENV_LOG_MAX="env.log.max" + KEY_ENV_JAVA_HOME="env.java.home" + KEY_ENV_JAVA_OPTS="env.java.opts" +@@ -136,7 +137,7 @@ FLINK_LIB_DIR=$FLINK_ROOT_DIR/lib + FLINK_ROOT_DIR_MANGLED=`manglePath "$FLINK_ROOT_DIR"` + if [ -z "$FLINK_CONF_DIR" ]; then FLINK_CONF_DIR=$FLINK_ROOT_DIR_MANGLED/conf; fi + FLINK_BIN_DIR=$FLINK_ROOT_DIR_MANGLED/bin +-FLINK_LOG_DIR=$FLINK_ROOT_DIR_MANGLED/log ++DEFAULT_FLINK_LOG_DIR=$FLINK_ROOT_DIR_MANGLED/log + FLINK_CONF_FILE="flink-conf.yaml" + YAML_CONF=${FLINK_CONF_DIR}/${FLINK_CONF_FILE} + +@@ -211,6 +212,10 @@ if [ -z "${MAX_LOG_FILE_NUMBER}" ]; then + MAX_LOG_FILE_NUMBER=$(readFromConfig ${KEY_ENV_LOG_MAX} ${DEFAULT_ENV_LOG_MAX} "${YAML_CONF}") + fi + ++if [ -z "${FLINK_LOG_DIR}" ]; then ++ FLINK_LOG_DIR=$(readFromConfig ${KEY_ENV_LOG_DIR} "${DEFAULT_FLINK_LOG_DIR}" "${YAML_CONF}") ++fi ++ + if [ -z "${FLINK_PID_DIR}" ]; then + FLINK_PID_DIR=$(readFromConfig ${KEY_ENV_PID_DIR} "${DEFAULT_ENV_PID_DIR}" "${YAML_CONF}") + fi + +From 2ba13dd7278d6d45531ec3e599abce3322066e9d Mon Sep 17 00:00:00 2001 +From: Stefano Baghino <[email protected]> +Date: Thu, 31 Mar 2016 02:27:07 +0200 +Subject: [PATCH 2/2] Add comment about absolute path restriction. + +--- + docs/setup/config.md | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/docs/setup/config.md b/docs/setup/config.md +index 63d3f10..fdc90a2 100644 +--- a/docs/setup/config.md ++++ b/docs/setup/config.md +@@ -277,7 +277,7 @@ For example when running Flink on YARN on an environment with a restrictive fire + + ## Environment + +-- `env.log.dir`: (Defaults to the `log` directory under Flink's home) Defines the directory where the Flink logs are saved. ++- `env.log.dir`: (Defaults to the `log` directory under Flink's home) Defines the directory where the Flink logs are saved. It has to be an absolute path. + + ## Background http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/changelog ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/changelog b/bigtop-packages/src/deb/flink/changelog new file mode 100644 index 0000000..9f074a4 --- /dev/null +++ b/bigtop-packages/src/deb/flink/changelog @@ -0,0 +1 @@ +--- This is auto-generated \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/compat ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/compat b/bigtop-packages/src/deb/flink/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/bigtop-packages/src/deb/flink/compat @@ -0,0 +1 @@ +9 http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/control b/bigtop-packages/src/deb/flink/control new file mode 100644 index 0000000..7e3a2ae --- /dev/null +++ b/bigtop-packages/src/deb/flink/control @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Source: flink +Section: misc +Priority: extra +Maintainer: Bigtop <[email protected]> +Build-Depends: debhelper (>= 9) +Standards-Version: 3.9.4 +Homepage: http://flink.apache.org + +Package: flink +Architecture: all +Depends: bigtop-utils (>= 0.7) +Description: Flinkâs core is a streaming dataflow engine that provides data + distribution, communication, and fault tolerance for distributed + computations over data streams. http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/copyright ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/copyright b/bigtop-packages/src/deb/flink/copyright new file mode 100644 index 0000000..1372211 --- /dev/null +++ b/bigtop-packages/src/deb/flink/copyright @@ -0,0 +1,15 @@ +Format: http://dep.debian.net/deps/dep5 +Source: http://flink.apache.org/ +Upstream-Name: Apache Flink + +Files: * +Copyright: 2010-2011, The Apache Software Foundation +License: Apache-2.0 + +Files debian/* +Copyright: 2011, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/flink.dirs ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/flink.dirs b/bigtop-packages/src/deb/flink/flink.dirs new file mode 100644 index 0000000..b272c75 --- /dev/null +++ b/bigtop-packages/src/deb/flink/flink.dirs @@ -0,0 +1,5 @@ +/usr/lib/flink +/usr/lib/flink/lib +/usr/lib/flink/bin +/usr/lib/flink/log +/usr/bin http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/flink.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/flink.install b/bigtop-packages/src/deb/flink/flink.install new file mode 100644 index 0000000..ceea8ad --- /dev/null +++ b/bigtop-packages/src/deb/flink/flink.install @@ -0,0 +1,7 @@ +/etc/flink +/usr/lib/flink +/usr/lib/flink/lib +/usr/lib/flink/bin +/usr/bin +/var/log/flink +/var/log/flink-cli http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/flink.postinst ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/flink.postinst b/bigtop-packages/src/deb/flink/flink.postinst new file mode 100644 index 0000000..83b68af --- /dev/null +++ b/bigtop-packages/src/deb/flink/flink.postinst @@ -0,0 +1,36 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + # Install config alternatives + update-alternatives --install /etc/flink/conf flink-conf /etc/flink/conf.dist 30 + chown -R flink:flink /var/log/flink + chmod -R a+rw /var/log/flink-cli + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/flink.preinst ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/flink.preinst b/bigtop-packages/src/deb/flink/flink.preinst new file mode 100644 index 0000000..cee0a14 --- /dev/null +++ b/bigtop-packages/src/deb/flink/flink.preinst @@ -0,0 +1,61 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# preinst script for flink +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * <new-preinst> `install' +# * <new-preinst> `install' <old-version> +# * <new-preinst> `upgrade' <old-version> +# * <old-preinst> `abort-upgrade' <new-version> +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + install|upgrade) + if ! getent passwd flink >/dev/null; then + # Adding system user: flink . + adduser \ + --system \ + --group \ + --home /var/lib/flink \ + --gecos "Flink" \ + --shell /bin/false \ + flink >/dev/null + fi + ;; + + abort-upgrade) + ;; + + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/flink.prerm ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/flink.prerm b/bigtop-packages/src/deb/flink/flink.prerm new file mode 100644 index 0000000..bd4f28d --- /dev/null +++ b/bigtop-packages/src/deb/flink/flink.prerm @@ -0,0 +1,38 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + remove|upgrade|deconfigure) + update-alternatives --remove flink-conf /etc/flink/conf.dist || : + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/rules b/bigtop-packages/src/deb/flink/rules new file mode 100644 index 0000000..95284bf --- /dev/null +++ b/bigtop-packages/src/deb/flink/rules @@ -0,0 +1,42 @@ +#!/usr/bin/make -f + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +%: + dh $@ + +override_dh_auto_build: + bash debian/do-component-build + +svcs=flink-jobmanager flink-taskmanager + +$(svcs): debian/init.d.tmpl + mkdir -p debian/flink/etc/init.d/ + sh $< debian/[email protected] deb debian/flink/etc/init.d/$@ + touch $@ + +override_dh_auto_install: $(svcs) + bash -x debian/install_flink.sh \ + --build-dir=`pwd`/build-target \ + --source-dir=debian \ + --prefix=debian/tmp http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/deb/flink/source/format ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/flink/source/format b/bigtop-packages/src/deb/flink/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/bigtop-packages/src/deb/flink/source/format @@ -0,0 +1 @@ +3.0 (quilt) http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop-packages/src/rpm/flink/SPECS/flink.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/flink/SPECS/flink.spec b/bigtop-packages/src/rpm/flink/SPECS/flink.spec new file mode 100644 index 0000000..0ef42a9 --- /dev/null +++ b/bigtop-packages/src/rpm/flink/SPECS/flink.spec @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define flink_name flink +%define lib_flink /usr/lib/%{flink_name} +%define bin_flink /usr/bin +%define etc_flink /etc/%{flink_name} +%define config_flink %{etc_flink}/conf +%define man_dir %{_mandir} +%define flink_services flink-jobmanager flink-taskmanager +%define var_log_flink /var/log/%{flink_name} + + +%if %{!?suse_version:1}0 +%define doc_flink %{_docdir}/%{flink_name}-%{flink_version} +%define alternatives_cmd alternatives +%define build_flink %{_builddir}/%{flink_name}-%{flink_version}/flink-dist/target/%{flink_name}-%{flink_version}-bin/%{flink_name}-%{flink_version}/ + +%else +%define doc_flink %{_docdir}/%{flink_name}-%{flink_version} +%define alternatives_cmd update-alternatives +%endif + +Name: %{flink_name} +Version: %{flink_version} +Release: %{flink_release} +Summary: Apache Flink is an open source platform for distributed stream and batch data processing. +License: ASL 2.0 +URL: http://flink.apache.org/ +Group: Development/Libraries +Buildroot: %{_topdir}/INSTALL/%{name}-%{version} +BuildArch: noarch +Source0: flink-%{flink_base_version}.tar.gz +Source1: do-component-build +Source2: install_flink.sh +Source3: init.d.tmpl +Source4: flink-jobmanager.svc +Source5: flink-taskmanager.svc +Source6: bigtop.bom +#BIGTOP_PATCH_FILES +Requires: bigtop-utils >= 0.7 +Requires(preun): /sbin/service + +%description +Apache Flink is an open source platform for distributed stream and batch data processing. +Flinkâs core is a streaming dataflow engine that provides data distribution, communication, +and fault tolerance for distributed computations over data streams. + +Flink includes several APIs for creating applications that use the Flink engine: + * DataStream API for unbounded streams embedded in Java and Scala, and + * DataSet API for static data embedded in Java, Scala, and Python, + * Table API with a SQL-like expression language embedded in Java and Scala. + +Flink also bundles libraries for domain-specific use cases: + * Machine Learning library, and + * Gelly, a graph processing API and library. + +Some of the key features of Apache Flink includes. + * Complete Event Processing (CEP) + * Fault-tolerance via Lightweight Distributed Snapshots + * Hadoop-native YARN & HDFS implementation + +# Additions for master-worker configuration # + +%global initd_dir %{_sysconfdir}/init.d + +%if %{?suse_version:1}0 +# Required for init scripts +Requires: insserv +%global initd_dir %{_sysconfdir}/rc.d + +%else +# Required for init scripts +Requires: /lib/lsb/init-functions +%global initd_dir %{_sysconfdir}/rc.d/init.d +%endif + +############################################## + +%prep +%setup -n %{name}-%{flink_base_version} +#BIGTOP_PATCH_COMMANDS + +%build +bash $RPM_SOURCE_DIR/do-component-build + + + +# Init.d scripts +%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ + +%install +%__rm -rf $RPM_BUILD_ROOT + +sh -x %{SOURCE2} --prefix=$RPM_BUILD_ROOT --source-dir=$RPM_SOURCE_DIR --build-dir=`pwd`/build-target + + + +for service in %{flink_services} +do + # Install init script + init_file=$RPM_BUILD_ROOT/%{initd_dir}/${service} + bash %{SOURCE3} $RPM_SOURCE_DIR/${service}.svc rpm $init_file +done + + +%preun +for service in %{flink_services}; do + /sbin/service ${service} status > /dev/null 2>&1 + if [ $? -eq 0 ]; then + /sbin/service ${service} stop > /dev/null 2>&1 + fi +done + +%pre +getent group flink >/dev/null || groupadd -r flink +getent passwd flink >/dev/null || useradd -c "Flink" -s /sbin/nologin -g flink -r -d %{lib_flink} flink 2> /dev/null || : + + +%post +%{alternatives_cmd} --install %{config_flink} %{flink_name}-conf %{config_flink}.dist 30 +systemctl daemon-reload + +###### FILES ########### + +%files +%defattr(-,root,root,755) +%config(noreplace) %{config_flink}.dist + +%dir %{_sysconfdir}/%{flink_name} +%config(noreplace) %{initd_dir}/flink-jobmanager +%config(noreplace) %{initd_dir}/flink-taskmanager +#%doc %{doc_flink} +%attr(0755,flink,flink) %{var_log_flink} +%attr(0767,flink,flink) /var/log/flink-cli +%{lib_flink} +%{bin_flink}/flink http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e7dcce/bigtop.bom ---------------------------------------------------------------------- diff --git a/bigtop.bom b/bigtop.bom index 254b4c8..9e33c3a 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -292,6 +292,19 @@ bigtop { site = "${apache.APACHE_MIRROR}/${download_path}" archive = "${apache.APACHE_ARCHIVE}/${download_path}" } } + + 'flink' { + name = 'flink' + relNotes = 'Apache Flink' + version { base = '1.0.0'; pkg = base; release = 1 } + tarball { destination = "$name-${version.base}.tar.gz" + source = "$name-${version.base}-src.tgz" } + url { download_path = "/$name/$name-${version.base}" + site = "${apache.APACHE_MIRROR}/${download_path}" + archive = "${apache.APACHE_ARCHIVE}/${download_path}" } + } + + 'phoenix' { name = 'phoenix' relNotes = 'Apache Phoenix: A SQL skin over HBase'
