Repository: bigtop Updated Branches: refs/heads/master abba9b4da -> 1a9efe60f
BIGTOP-989: Add Apache Kafka to Apache Bigtop Signed-off-by: Mark Grover <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/1a9efe60 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/1a9efe60 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/1a9efe60 Branch: refs/heads/master Commit: 1a9efe60f82f9061dc42cadba5550292f77aea0f Parents: abba9b4 Author: Mani Narayan <[email protected]> Authored: Tue Feb 24 20:58:07 2015 -0800 Committer: Mark Grover <[email protected]> Committed: Thu Feb 26 13:54:08 2015 -0800 ---------------------------------------------------------------------- .../src/common/kafka/do-component-build | 41 +++++ .../src/common/kafka/install_kafka.sh | 162 ++++++++++++++++++ .../src/common/kafka/kafka-server.svc | 95 +++++++++++ bigtop-packages/src/common/kafka/kafka.default | 0 bigtop-packages/src/deb/kafka/changelog | 1 + bigtop-packages/src/deb/kafka/compat | 1 + bigtop-packages/src/deb/kafka/control | 34 ++++ bigtop-packages/src/deb/kafka/copyright | 11 ++ .../src/deb/kafka/kafka-server.postinst | 32 ++++ .../src/deb/kafka/kafka-server.postrm | 32 ++++ bigtop-packages/src/deb/kafka/kafka.postinst | 35 ++++ bigtop-packages/src/deb/kafka/kafka.preinst | 33 ++++ bigtop-packages/src/deb/kafka/kafka.prerm | 32 ++++ bigtop-packages/src/deb/kafka/rules | 38 +++++ bigtop-packages/src/deb/kafka/source/format | 1 + bigtop-packages/src/rpm/kafka/SPECS/kafka.spec | 163 +++++++++++++++++++ bigtop.mk | 14 ++ pom.xml | 6 + 18 files changed, 731 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/common/kafka/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/kafka/do-component-build b/bigtop-packages/src/common/kafka/do-component-build new file mode 100644 index 0000000..ba5a577 --- /dev/null +++ b/bigtop-packages/src/common/kafka/do-component-build @@ -0,0 +1,41 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +if [ "x$SCALA_HOME" = "x" ]; then + echo "SCALA_HOME has to be set before the build can run" + exit 2 +fi + +BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${HOME} \ + -Drepo.maven.org=$IVY_MIRROR_PROP \ + -Dreactor.repo=file://${HOME}/.m2/repository \ + -DskipTests -DrecompileMode=all" + +## this might be an issue at times +# http://maven.40175.n5.nabble.com/Not-finding-artifact-in-local-repo-td3727753.html +export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -XX:PermSize=1024m -XX:MaxPermSize=1024m" + +SCALA_VERSION=2.10.0 + +./gradlew -PscalaVersion=${SCALA_VERSION} clean releaseTarGz -x signArchives + +rm -rf build +mkdir build + +cp core/build/distributions/kafka_*.tgz build/`basename core/build/distributions/kafka* .tgz`.tar.gz + http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/common/kafka/install_kafka.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/kafka/install_kafka.sh b/bigtop-packages/src/common/kafka/install_kafka.sh new file mode 100644 index 0000000..1b044b5 --- /dev/null +++ b/bigtop-packages/src/common/kafka/install_kafka.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + echo " +usage: $0 <options> + Required not-so-options: + --build-dir=DIR path to dist.dir + --source-dir=DIR path to package shared files dir + --prefix=PREFIX path to install into + + Optional options: + --doc-dir=DIR path to install docs into [/usr/share/doc/kafka] + --lib-dir=DIR path to install Kafka home [/usr/lib/kafka] + --installed-lib-dir=DIR path where lib-dir will end up on target system + --examples-dir=DIR path to install examples [doc-dir/examples] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'doc-dir:' \ + -l 'lib-dir:' \ + -l 'installed-lib-dir:' \ + -l 'source-dir:' \ + -l 'examples-dir:' \ + -l 'build-dir:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --build-dir) + BUILD_DIR=$2 ; shift 2 + ;; + --source-dir) + SOURCE_DIR=$2 ; shift 2 + ;; + --doc-dir) + DOC_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --examples-dir) + EXAMPLES_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX BUILD_DIR SOURCE_DIR; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +MAN_DIR=${MAN_DIR:-/usr/share/man/man1} +DOC_DIR=${DOC_DIR:-/usr/share/doc/kafka} +LIB_DIR=${LIB_DIR:-/usr/lib/kafka} +EXAMPLES_DIR=${EXAMPLES_DIR:-$DOC_DIR/examples} +CONF_DIR=${CONF_DIR:-/etc/kafka/conf.dist} +KAFKA_HOME=${KAFKA_HOME:-/usr/lib/kafka} +ETC_KAFKA_DIR=${ETC_KAFKA_DIR:-/etc/kafka} +BIN_DIR=/usr/bin +INSTALLED_KAFKA_DIR=${LIB_DIR} + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$DOC_DIR +install -d -m 0755 $PREFIX/$EXAMPLES_DIR + +install -d -m 0755 $PREFIX/var/lib/kafka/ +install -d -m 0755 $PREFIX/var/log/kafka/ +install -d -m 0755 $PREFIX/var/run/kafka/ + +# Create the data directory. +install -d -m 0755 $PREFIX/var/lib/kafka/data + +# Unzip binary tarball contents to lib directory. +tar zxf ${BUILD_DIR}/build/kafka*.tar.gz -C ${PREFIX}/${LIB_DIR} --strip 1 +rm -f ${PREFIX}/${LIB_DIR}/{LICENSE,NOTICE} + +#Remove config directory. Creating symlink below. +rm -rf ${PREFIX}/${LIB_DIR}/config + +#Removing: +# - javadoc scaladoc source jars from under libs. +# - bat files from under bin +rm -rf ${PREFIX}/${LIB_DIR}/libs/kafka_*source* +rm -rf ${PREFIX}/${LIB_DIR}/libs/kafka_*javadoc* +rm -rf ${PREFIX}/${LIB_DIR}/libs/kafka_*scaladoc* +rm -rf ${PREFIX}/${LIB_DIR}/bin/windows + +chmod 755 $PREFIX/$LIB_DIR/bin/* + +# Exposing a few scripts by placing them under /usr/bin +install -d -m 0755 $PREFIX/$BIN_DIR +for file in kafka-console-consumer.sh kafka-console-producer.sh kafka-run-class.sh kafka-topics.sh +do + wrapper=$PREFIX/$BIN_DIR/$file + cat >>$wrapper <<EOF +#!/bin/bash + +# Autodetect JAVA_HOME if not defined +. /usr/lib/bigtop-utils/bigtop-detect-javahome + +exec $INSTALLED_KAFKA_DIR/bin/$file "\$@" +EOF + chmod 755 $wrapper +done + +# Copy in the configuration files +install -d -m 0755 $PREFIX/$CONF_DIR +cp -a ${BUILD_DIR}/config/{tools-log4j.properties,server.properties,log4j.properties} $PREFIX/$CONF_DIR/ +ln -s ${ETC_KAFKA_DIR}/conf $PREFIX/$LIB_DIR/config + +# Creating symlink to /var/log/kafka +ln -s /var/log/kafka ${PREFIX}/$LIB_DIR/logs + +# Removing zookeeper*.jar from under libs and dropping a symlink in place +rm -f ${PREFIX}/${LIB_DIR}/libs/zookeeper-*.jar +ln -sf /usr/lib/zookeeper/zookeeper.jar ${PREFIX}/${LIB_DIR}/libs/ + +# Copy in the defaults file +install -d -m 0755 ${PREFIX}/etc/default +cp ${SOURCE_DIR}/kafka.default ${PREFIX}/etc/default/kafka + +cp ${BUILD_DIR}/{LICENSE,NOTICE} ${PREFIX}/${LIB_DIR}/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/common/kafka/kafka-server.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/kafka/kafka-server.svc b/bigtop-packages/src/common/kafka/kafka-server.svc new file mode 100644 index 0000000..879dd43 --- /dev/null +++ b/bigtop-packages/src/common/kafka/kafka-server.svc @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DAEMON="kafka-server" +DESC="Kafka Server" +EXEC_PATH="/usr/lib/kafka/bin/kafka-server-start.sh" +SVC_USER="kafka" +WORKING_DIR="/var/lib/kafka" +DAEMON_FLAGS="" +CONF_DIR="/etc/kafka/conf" +PIDFILE="/var/run/kafka/${DAEMON}.pid" +PRIORITY=86 +DEFAULTS_DIR="/etc/default/kafka" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC (${DAEMON}): " + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "${DESC} is running" + exit 0 + fi + + LOG_FILE=/var/log/kafka/${DAEMON}.out + export LOG_DIR=`dirname $LOG_FILE` + + echo "Starting $desc (kafka-server): " + + if [ -z "$KAFKA_JAVA_OPTS" ]; then + JAVA_OPTS="-Xmx512M -server -Dlog4j.configuration=file:$CONF_DIR/log4j.properties" + fi + + su -s /bin/bash $SVC_USER -c "nohup $EXEC_PATH $CONF_DIR/server.properties > $LOG_FILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE" + sleep 3 + + checkstatusofproc + RETVAL=$? + if [ $RETVAL -eq $STATUS_RUNNING ]; then + touch $LOCKFILE + log_success_msg "Starting $DESC (${DAEMON}): " + else + log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL" + fi + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { +checkstatusofproc + if [ "$?" = "$STATUS_RUNNING" ] ; then + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -n $PID ]; then + kill -TERM $PID &>/dev/null + fi + fi + RETVAL=$? + else + RETVAL=$RETVAL_SUCCESS + fi + + if [ $RETVAL -eq $RETVAL_SUCCESS ]; then + rm -f $LOCKFILE $PIDFILE + log_success_msg "Stopped ${DESC}: " + else + log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL" + fi + return $RETVAL +} + +__EOT__ + +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/common/kafka/kafka.default ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/kafka/kafka.default b/bigtop-packages/src/common/kafka/kafka.default new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/changelog ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/changelog b/bigtop-packages/src/deb/kafka/changelog new file mode 100644 index 0000000..d4858cd --- /dev/null +++ b/bigtop-packages/src/deb/kafka/changelog @@ -0,0 +1 @@ +--- This is auto-generated http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/compat ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/compat b/bigtop-packages/src/deb/kafka/compat new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/bigtop-packages/src/deb/kafka/compat @@ -0,0 +1 @@ +7 http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/control b/bigtop-packages/src/deb/kafka/control new file mode 100644 index 0000000..2c67ea0 --- /dev/null +++ b/bigtop-packages/src/deb/kafka/control @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Source: kafka +Section: misc +Priority: extra +Maintainer: Bigtop <[email protected]> +Build-Depends: debhelper (>= 7.0.50~) +Standards-Version: 3.8.0 +Homepage: http://kafka.apache.org/ + +Package: kafka +Architecture: all +Depends: zookeeper, bigtop-utils (>= 0.7) +Description: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. + A single Kafka broker can handle hundreds of megabytes of reads and writes per second from thousands of clients. It can be elastically and transparently expanded without downtime. Data streams are partitioned and spread over a cluster of machines to allow data streams larger than the capability of any single machine and to allow clusters of co-ordinated consumers + +Package: kafka-server +Architecture: all +Depends: kafka +Description: Bundles the init script for kafka server. + This package starts the kafka server on startup http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/copyright ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/copyright b/bigtop-packages/src/deb/kafka/copyright new file mode 100644 index 0000000..35c5893 --- /dev/null +++ b/bigtop-packages/src/deb/kafka/copyright @@ -0,0 +1,11 @@ +Format: http://dep.debian.net/deps/dep7 +Source: http://kafka.apache.org/ +Upstream-Name: Apache Kafka + +Files: * +Copyright: 2008-2015, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/kafka-server.postinst ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/kafka-server.postinst b/bigtop-packages/src/deb/kafka/kafka-server.postinst new file mode 100644 index 0000000..acaf30c --- /dev/null +++ b/bigtop-packages/src/deb/kafka/kafka-server.postinst @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + update-rc.d kafka-server defaults >/dev/null || exit 1 + ;; + abort-upgrade|abort-remove|abort-deconfigure) + ;; + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/kafka-server.postrm ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/kafka-server.postrm b/bigtop-packages/src/deb/kafka/kafka-server.postrm new file mode 100644 index 0000000..ca6f39d --- /dev/null +++ b/bigtop-packages/src/deb/kafka/kafka-server.postrm @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + purge) + update-rc.d -f kafka-server remove > /dev/null || exit 1 + ;; + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/kafka.postinst ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/kafka.postinst b/bigtop-packages/src/deb/kafka/kafka.postinst new file mode 100644 index 0000000..71dc564 --- /dev/null +++ b/bigtop-packages/src/deb/kafka/kafka.postinst @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + update-alternatives --install /etc/kafka/conf kafka-conf /etc/kafka/conf.dist 30 + chown kafka:kafka -R /var/log/kafka + chown kafka:kafka -R /var/run/kafka + chown kafka:kafka -R /var/lib/kafka + ;; + abort-upgrade|abort-remove|abort-deconfigure) + ;; + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/kafka.preinst ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/kafka.preinst b/bigtop-packages/src/deb/kafka/kafka.preinst new file mode 100644 index 0000000..e20ad51 --- /dev/null +++ b/bigtop-packages/src/deb/kafka/kafka.preinst @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + install|upgrade) + getent group kafka >/dev/null || groupadd -r kafka + getent passwd kafka > /dev/null || useradd -c "kafka" -s /bin/bash -g kafka -d /var/lib/kafka kafka 2> /dev/null || : + ;; + abort-upgrade) + ;; + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/kafka.prerm ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/kafka.prerm b/bigtop-packages/src/deb/kafka/kafka.prerm new file mode 100644 index 0000000..6d64f02 --- /dev/null +++ b/bigtop-packages/src/deb/kafka/kafka.prerm @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + remove|upgrade|deconfigure) + update-alternatives --remove kafka-conf /etc/kafka/conf.dist || : + ;; + failed-upgrade) + ;; + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/rules b/bigtop-packages/src/deb/kafka/rules new file mode 100644 index 0000000..b5e590b --- /dev/null +++ b/bigtop-packages/src/deb/kafka/rules @@ -0,0 +1,38 @@ +#!/usr/bin/make -f +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +%: + dh $@ + +override_dh_auto_build: + env FULL_VERSION=${KAFKA_BASE_VERSION} bash debian/do-component-build + +override_dh_auto_install: + env FULL_VERSION=${KAFKA_BASE_VERSION} bash -x debian/install_kafka.sh \ + --build-dir=./ \ + --source-dir=debian \ + --prefix=debian/kafka + mkdir -p debian/kafka-server/etc/init.d/ + bash debian/init.d.tmpl debian/kafka-server.svc deb debian/kafka-server/etc/init.d/kafka-server http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/deb/kafka/source/format ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kafka/source/format b/bigtop-packages/src/deb/kafka/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/bigtop-packages/src/deb/kafka/source/format @@ -0,0 +1 @@ +3.0 (quilt) http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop-packages/src/rpm/kafka/SPECS/kafka.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/kafka/SPECS/kafka.spec b/bigtop-packages/src/rpm/kafka/SPECS/kafka.spec new file mode 100644 index 0000000..58aa132 --- /dev/null +++ b/bigtop-packages/src/rpm/kafka/SPECS/kafka.spec @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define kafka_name kafka +%define lib_kafka /usr/lib/%{kafka_name} +%define var_lib_kafka /var/lib/%{kafka_name} +%define var_run_kafka /var/run/%{kafka_name} +%define var_log_kafka /var/log/%{kafka_name} +%define bin_kafka /usr/lib/%{kafka_name}/bin +%define etc_kafka /etc/%{kafka_name} +%define config_kafka %{etc_kafka}/conf +%define bin /usr/bin +%define man_dir /usr/share/man + +%if %{!?suse_version:1}0 +%define doc_kafka %{_docdir}/%{kafka_name}-%{kafka_version} +%define alternatives_cmd alternatives +%global initd_dir %{_sysconfdir}/rc.d/init.d + +%else + +# Only tested on openSUSE 11.4. le'ts update it for previous release when confirmed +%if 0%{suse_version} > 1130 +%define suse_check \# Define an empty suse_check for compatibility with older sles +%endif + +%define alternatives_cmd update-alternatives +%define doc_kafka %{_docdir}/%{kafka_name}-%{kafka_version} +%global initd_dir %{_sysconfdir}/rc.d + +%define __os_install_post \ + %{suse_check} ; \ + /usr/lib/rpm/brp-compress ; \ + %{nil} + +%endif + + +# disable repacking jars +%define __os_install_post %{nil} + +Name: kafka +Version: %{kafka_version} +Release: %{kafka_release} +Summary: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. +URL: http://kafka.apache.org/ +Group: Development/Libraries +BuildArch: noarch +Buildroot: %(mktemp -ud %{_tmppath}/%{kafka_name}-%{version}-%{release}-XXXXXX) +License: ASL 2.0 +Source0: %{kafka_name}-%{kafka_base_version}.tar.gz +Source1: do-component-build +Source2: install_%{kafka_name}.sh +Source3: kafka-server.svc +Source4: init.d.tmpl +Source6: kafka.default +Requires: zookeeper +Requires: bigtop-utils >= 0.7 +Requires(preun): /sbin/service + +%description +Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. +A single Kafka broker can handle hundreds of megabytes of reads and writes per second +from thousands of clients. It can be elastically and transparently expanded without downtime. +Data streams are partitioned and spread over a cluster of machines to allow data streams +larger than the capability of any single machine and to allow clusters of co-ordinated consumers + +%package server +Summary: Server for kafka +Group: System/Daemons +Requires: kafka = %{version}-%{release} + +%description server +Bundles the init script for kafka server. + +%prep +%setup -n %{kafka_name}-%{kafka_base_version}-src + +%build +bash $RPM_SOURCE_DIR/do-component-build + +%install +%__rm -rf $RPM_BUILD_ROOT +%__install -d -m 0755 $RPM_BUILD_ROOT/etc/default/ +%__install -m 0644 %{SOURCE6} $RPM_BUILD_ROOT/etc/default/%{kafka_name} + +bash $RPM_SOURCE_DIR/install_kafka.sh \ + --build-dir=`pwd` \ + --source-dir=$RPM_SOURCE_DIR \ + --prefix=$RPM_BUILD_ROOT \ + --doc-dir=%{doc_kafka} \ + +# Generate the init script +init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{kafka_name}-server +bash $RPM_SOURCE_DIR/init.d.tmpl $RPM_SOURCE_DIR/%{kafka_name}-server.svc rpm $init_file + + +####################### +#### Kafka core section #### +####################### +%pre +getent group kafka >/dev/null || groupadd -r kafka +getent passwd kafka >/dev/null || useradd -c "Kafka" -s /sbin/nologin -g kafka -r -d %{var_lib_kafka} kafka 2> /dev/null || : + +%post +%{alternatives_cmd} --install %{config_kafka} %{kafka_name}-conf %{config_kafka}.dist 30 + +%preun +if [ "$1" = 0 ]; then + %{alternatives_cmd} --remove %{kafka_name}-conf %{config_kafka}.dist || : +fi + +/sbin/service %{kafka_name}-server status > /dev/null 2>&1 +if [ $? -eq 0 ]; then + /sbin/service %{kafka_name}-server stop > /dev/null 2>&1 +fi + +####################### +#### Kafka-server section #### +####################### +%post server +chkconfig --add %{kafka_name}-server + +%preun server +/sbin/service %{kafka_name}-server status > /dev/null 2>&1 +if [ $? -eq 0 ] ; then + service kafka-server stop > /dev/null 2>&1 + chkconfig --del %{kafka_name}-server +fi + +%postun server +if [ $1 -ge 1 ]; then + service %{kafka_name}-server condrestart >/dev/null 2>&1 +fi + +####################### +#### FILES SECTION #### +####################### +%files server +%attr(0755,root,root) %{initd_dir}/%{kafka_name}-server + +%files +%defattr(-,root,root,755) +%{bin} +%config(noreplace) %{config_kafka}.dist +%config(noreplace) /etc/default/kafka +%attr(0755,kafka,kafka) %{lib_kafka} +%attr(0755,kafka,kafka) %docdir %{doc_kafka} +%attr(0755,kafka,kafka) %{var_lib_kafka} +%attr(0755,kafka,kafka) %{var_run_kafka} +%attr(0755,kafka,kafka) %{var_log_kafka} http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index 5789e4c..bec0f82 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -332,3 +332,17 @@ TACHYON_TARBALL_SRC=v$(TACHYON_BASE_VERSION).tar.gz TACHYON_SITE=https://github.com/amplab/tachyon/archive TACHYON_ARCHIVE=$(TACHYON_SITE) $(eval $(call PACKAGE,tachyon,TACHYON)) + +# Kafka +KAFKA_NAME=kafka +KAFKA_RELNOTES_NAME=Apache Kafka +KAFKA_PKG_NAME=kafka +KAFKA_BASE_VERSION=0.8.1.1 +KAFKA_PKG_VERSION=$(KAFKA_BASE_VERSION) +KAFKA_RELEASE_VERSION=1 +KAFKA_TARBALL_DST=kafka-$(KAFKA_BASE_VERSION).tar.gz +KAFKA_TARBALL_SRC=kafka-$(KAFKA_BASE_VERSION)-src.tgz +KAFKA_DOWNLOAD_PATH=/kafka/$(KAFKA_BASE_VERSION) +KAFKA_SITE=$(APACHE_MIRROR)$(KAFKA_DOWNLOAD_PATH) +KAFKA_ARCHIVE=$(APACHE_ARCHIVE)$(KAFKA_DOWNLOAD_PATH) +$(eval $(call PACKAGE,kafka,KAFKA)) http://git-wip-us.apache.org/repos/asf/bigtop/blob/1a9efe60/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index bc0d1d5..1e8df77 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ <giraph.version>1.0.0</giraph.version> <solr.version>4.6.0</solr.version> <spark.version>0.9.1</spark.version> + <kafka.version>0.8.1.1</kafka.version> <phoenix.version>4.2.2</phoenix.version> <spark-smoke.version>${project.version}</spark-smoke.version> @@ -244,6 +245,11 @@ <artifactId>spark-core_2.10</artifactId> <version>${spark.version}</version> </dependency> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka</artifactId> + <version>${kafka.version}</version> + </dependency> <dependency> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-core</artifactId>
