This is an automated email from the ASF dual-hosted git repository.
oflebbe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 4a4c6a2 BIGTOP-3293. Add hadoop-kms package.
4a4c6a2 is described below
commit 4a4c6a2bbf79d1f04cabd327b846feac228f6a66
Author: Masatake Iwasaki <[email protected]>
AuthorDate: Tue Jan 28 15:04:56 2020 +0000
BIGTOP-3293. Add hadoop-kms package.
fixed kms misconfiguration.
added deb resources for hadoop-kms.
fixed invalid path in tomcat-deployment script.
added temporary workaround for init script not working under systemctl
redirect.
runuser must be used instead of su to make init script work under systemd
control.
Signed-off-by: Olaf Flebbe <[email protected]>
---
bigtop-packages/src/common/hadoop/hadoop-kms.svc | 94 ++++++++++++++++++++++
.../src/common/hadoop/install_hadoop.sh | 57 ++++++++++---
.../src/common/hadoop/kms-tomcat-deployment.sh | 37 +++++++++
bigtop-packages/src/common/hadoop/kms.default | 24 ++++++
bigtop-packages/src/deb/hadoop/control | 6 ++
bigtop-packages/src/deb/hadoop/hadoop-kms.dirs | 3 +
bigtop-packages/src/deb/hadoop/hadoop-kms.install | 5 ++
bigtop-packages/src/deb/hadoop/hadoop-kms.postinst | 40 +++++++++
bigtop-packages/src/deb/hadoop/hadoop-kms.preinst | 61 ++++++++++++++
bigtop-packages/src/deb/hadoop/hadoop-kms.prerm | 58 +++++++++++++
bigtop-packages/src/deb/hadoop/rules | 4 +-
bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec | 65 +++++++++++++--
12 files changed, 437 insertions(+), 17 deletions(-)
diff --git a/bigtop-packages/src/common/hadoop/hadoop-kms.svc
b/bigtop-packages/src/common/hadoop/hadoop-kms.svc
new file mode 100644
index 0000000..cf44a48
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/hadoop-kms.svc
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+HADOOP_NAME="kms"
+DAEMON="hadoop-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop-kms/sbin/kms.sh"
+SVC_USER="$HADOOP_NAME"
+WORKING_DIR="/var/run/hadoop-kms"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/$DAEMON/conf"
+PIDFILE="/var/run/$DAEMON/hadoop-$SVC_USER-$HADOOP_NAME.pid"
+
+CHKCONFIG=${CHKCONFIG:-"2345 90 10"}
+CHKCONFIG="${CHKCONFIG%% *} 90 10"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+ [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+ [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
+
+ export KMS_USER="$SVC_USER"
+ export KMS_CONFIG="$CONF_DIR"
+ export KMS_LOG=${KMS_LOG:-"/var/log/hadoop-kms/"}
+ export KMS_TEMP="$(dirname $PIDFILE)"
+ export KMS_SLEEP_TIME="$SLEEP_TIME"
+ export
CATALINA_BASE=${CATALINA_BASE:-"/var/lib/hadoop-kms/tomcat-deployment"}
+ export CATALINA_PID="$PIDFILE"
+ export CATALINA_TMPDIR="$KMS_TEMP"
+
+ . /usr/lib/hadoop-kms/tomcat-deployment.sh
+
+ runuser -s /bin/bash -c "${EXEC_PATH} start $DAEMON_FLAGS" $KMS_USER
+
+ for second in {5..0}
+ do
+ checkstatusofproc
+ RETVAL=$?
+ if [ "$RETVAL" -eq $RETVAL_SUCCESS ] ; then
+ break
+ fi
+ sleep 1
+ done
+
+ if [ $RETVAL -eq $STATUS_RUNNING ]; then
+ touch $LOCKFILE
+ log_success_msg "Started ${DESC} (${DAEMON}): "
+ else
+ log_failure_msg "Failed to start ${DESC}. Return value: $RETVAL"
+ fi
+
+ return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+
+ # FIXME: workaround for BIGTOP-537
+ checkstatusofproc
+ if [ "$?" = "$STATUS_RUNNING" ] ; then
+ su -s /bin/bash $SVC_USER -c "${EXEC_PATH} stop $SLEEP_TIME -force"
+ RETVAL=$?
+ else
+ RETVAL=$RETVAL_SUCCESS
+ fi
+
+ if [ $RETVAL -eq $RETVAL_SUCCESS ]; then
+ log_success_msg "Stopped ${DESC}: "
+ rm -f $LOCKFILE $PIDFILE
+ else
+ log_failure_msg "Failure to stop ${DESC}. Return value: $RETVAL"
+ fi
+}
+__EOT__
+
+}
diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh
b/bigtop-packages/src/common/hadoop/install_hadoop.sh
index 1926862..3c82d1d 100755
--- a/bigtop-packages/src/common/hadoop/install_hadoop.sh
+++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh
@@ -41,6 +41,7 @@ OPTS=$(getopt \
-l 'installed-lib-dir:' \
-l 'hadoop-dir:' \
-l 'httpfs-dir:' \
+ -l 'kms-dir:' \
-l 'hdfs-dir:' \
-l 'yarn-dir:' \
-l 'mapreduce-dir:' \
@@ -50,6 +51,7 @@ OPTS=$(getopt \
-l 'system-libexec-dir:' \
-l 'hadoop-etc-dir:' \
-l 'httpfs-etc-dir:' \
+ -l 'kms-etc-dir:' \
-l 'doc-dir:' \
-l 'man-dir:' \
-l 'example-dir:' \
@@ -72,6 +74,9 @@ while true ; do
--httpfs-dir)
HTTPFS_DIR=$2 ; shift 2
;;
+ --kms-dir)
+ KMS_DIR=$2 ; shift 2
+ ;;
--hadoop-dir)
HADOOP_DIR=$2 ; shift 2
;;
@@ -111,6 +116,9 @@ while true ; do
--httpfs-etc-dir)
HTTPFS_ETC_DIR=$2 ; shift 2
;;
+ --kms-etc-dir)
+ KMS_ETC_DIR=$2 ; shift 2
+ ;;
--installed-lib-dir)
INSTALLED_LIB_DIR=$2 ; shift 2
;;
@@ -144,6 +152,7 @@ YARN_DIR=${YARN_DIR:-$PREFIX/usr/lib/hadoop-yarn}
MAPREDUCE_DIR=${MAPREDUCE_DIR:-$PREFIX/usr/lib/hadoop-mapreduce}
CLIENT_DIR=${CLIENT_DIR:-$PREFIX/usr/lib/hadoop/client}
HTTPFS_DIR=${HTTPFS_DIR:-$PREFIX/usr/lib/hadoop-httpfs}
+KMS_DIR=${KMS_DIR:-$PREFIX/usr/lib/hadoop-kms}
SYSTEM_LIB_DIR=${SYSTEM_LIB_DIR:-/usr/lib}
BIN_DIR=${BIN_DIR:-$PREFIX/usr/bin}
DOC_DIR=${DOC_DIR:-$PREFIX/usr/share/doc/hadoop}
@@ -153,6 +162,7 @@
SYSTEM_LIBEXEC_DIR=${SYSTEM_LIBEXEC_DIR:-$PREFIX/usr/libexec}
EXAMPLE_DIR=${EXAMPLE_DIR:-$DOC_DIR/examples}
HADOOP_ETC_DIR=${HADOOP_ETC_DIR:-$PREFIX/etc/hadoop}
HTTPFS_ETC_DIR=${HTTPFS_ETC_DIR:-$PREFIX/etc/hadoop-httpfs}
+KMS_ETC_DIR=${KMS_ETC_DIR:-$PREFIX/etc/hadoop-kms}
BASH_COMPLETION_DIR=${BASH_COMPLETION_DIR:-$PREFIX/etc/bash_completion.d}
INSTALLED_HADOOP_DIR=${INSTALLED_HADOOP_DIR:-/usr/lib/hadoop}
@@ -331,18 +341,18 @@ install -d -m 0755 $HTTPFS_ETC_DIR/conf.empty
install -m 0755 ${DISTRO_DIR}/httpfs-tomcat-deployment.sh
${HTTPFS_DIR}/tomcat-deployment.sh
-HTTP_DIRECTORY=$HTTPFS_ETC_DIR/tomcat-conf.dist
-HTTPS_DIRECTORY=$HTTPFS_ETC_DIR/tomcat-conf.https
+HTTPFS_HTTP_DIRECTORY=$HTTPFS_ETC_DIR/tomcat-conf.dist
+HTTPFS_HTTPS_DIRECTORY=$HTTPFS_ETC_DIR/tomcat-conf.https
-install -d -m 0755 ${HTTP_DIRECTORY}
-cp -r ${BUILD_DIR}/share/hadoop/httpfs/tomcat/conf ${HTTP_DIRECTORY}
-chmod 644 ${HTTP_DIRECTORY}/conf/*
-install -d -m 0755 ${HTTP_DIRECTORY}/WEB-INF
-mv ${HTTPFS_DIR}/webapps/webhdfs/WEB-INF/*.xml ${HTTP_DIRECTORY}/WEB-INF/
+install -d -m 0755 ${HTTPFS_HTTP_DIRECTORY}
+cp -r ${BUILD_DIR}/share/hadoop/httpfs/tomcat/conf ${HTTPFS_HTTP_DIRECTORY}
+chmod 644 ${HTTPFS_HTTP_DIRECTORY}/conf/*
+install -d -m 0755 ${HTTPFS_HTTP_DIRECTORY}/WEB-INF
+mv ${HTTPFS_DIR}/webapps/webhdfs/WEB-INF/*.xml
${HTTPFS_HTTP_DIRECTORY}/WEB-INF/
-cp -r ${HTTP_DIRECTORY} ${HTTPS_DIRECTORY}
-mv ${HTTPS_DIRECTORY}/conf/ssl-server.xml ${HTTPS_DIRECTORY}/conf/server.xml
-rm ${HTTP_DIRECTORY}/conf/ssl-server.xml
+cp -r ${HTTPFS_HTTP_DIRECTORY} ${HTTPFS_HTTPS_DIRECTORY}
+mv ${HTTPFS_HTTPS_DIRECTORY}/conf/ssl-server.xml
${HTTPFS_HTTPS_DIRECTORY}/conf/server.xml
+rm ${HTTPFS_HTTP_DIRECTORY}/conf/ssl-server.xml
mv $HADOOP_ETC_DIR/conf.empty/httpfs* $HTTPFS_ETC_DIR/conf.empty
sed -i -e '/<\/configuration>/i\
@@ -351,6 +361,31 @@ sed -i -e '/<\/configuration>/i\
<value>/etc/hadoop/conf</value>\
</property>' $HTTPFS_ETC_DIR/conf.empty/httpfs-site.xml
+# KMS
+install -d -m 0755 ${KMS_DIR}/sbin
+cp ${BUILD_DIR}/sbin/kms.sh ${KMS_DIR}/sbin/
+cp -r ${BUILD_DIR}/share/hadoop/kms/tomcat/webapps ${KMS_DIR}/webapps
+install -d -m 0755 ${PREFIX}/var/lib/hadoop-kms
+install -d -m 0755 $KMS_ETC_DIR/conf.empty
+
+install -m 0755 ${DISTRO_DIR}/kms-tomcat-deployment.sh
${KMS_DIR}/tomcat-deployment.sh
+
+KMS_HTTP_DIRECTORY=$KMS_ETC_DIR/tomcat-conf.dist
+KMS_HTTPS_DIRECTORY=$KMS_ETC_DIR/tomcat-conf.https
+
+install -d -m 0755 ${KMS_HTTP_DIRECTORY}
+cp -r ${BUILD_DIR}/share/hadoop/kms/tomcat/conf ${KMS_HTTP_DIRECTORY}
+chmod 644 ${KMS_HTTP_DIRECTORY}/conf/*
+install -d -m 0755 ${KMS_HTTP_DIRECTORY}/WEB-INF
+cp ${KMS_DIR}/webapps/kms/WEB-INF/*.xml ${KMS_HTTP_DIRECTORY}/WEB-INF/
+
+cp -r ${KMS_HTTP_DIRECTORY} ${KMS_HTTPS_DIRECTORY}
+mv ${KMS_HTTPS_DIRECTORY}/conf/ssl-server.xml
${KMS_HTTPS_DIRECTORY}/conf/server.xml
+rm ${KMS_HTTP_DIRECTORY}/conf/ssl-server.xml
+
+mv $HADOOP_ETC_DIR/conf.empty/kms* $KMS_ETC_DIR/conf.empty
+cp $HADOOP_ETC_DIR/conf.empty/core-site.xml $KMS_ETC_DIR/conf.empty
+
# Make the pseudo-distributed config
for conf in conf.pseudo ; do
install -d -m 0755 $HADOOP_ETC_DIR/$conf
@@ -379,7 +414,7 @@ install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-yarn
install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-mapreduce
# Remove all source and create version-less symlinks to offer integration
point with other projects
-for DIR in ${HADOOP_DIR} ${HDFS_DIR} ${YARN_DIR} ${MAPREDUCE_DIR}
${HTTPFS_DIR} ; do
+for DIR in ${HADOOP_DIR} ${HDFS_DIR} ${YARN_DIR} ${MAPREDUCE_DIR}
${HTTPFS_DIR} ${KMS_DIR}; do
(cd $DIR &&
rm -fv *-sources.jar
rm -fv lib/hadoop-*.jar
diff --git a/bigtop-packages/src/common/hadoop/kms-tomcat-deployment.sh
b/bigtop-packages/src/common/hadoop/kms-tomcat-deployment.sh
new file mode 100644
index 0000000..3872fd0
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/kms-tomcat-deployment.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script must be sourced so that it can set CATALINA_BASE for the parent
process
+
+TOMCAT_CONF=${TOMCAT_CONF:-`readlink -e /etc/hadoop-kms/tomcat-conf`}
+TOMCAT_DEPLOYMENT=${TOMCAT_DEPLOYMENT:-/var/lib/hadoop-kms/tomcat-deployment}
+KMS_HOME=${KMS_HOME:-/usr/lib/hadoop-kms}
+
+rm -rf ${TOMCAT_DEPLOYMENT}
+mkdir ${TOMCAT_DEPLOYMENT}
+cp -r ${TOMCAT_CONF}/conf ${TOMCAT_DEPLOYMENT}/
+cp -r ${KMS_HOME}/webapps ${TOMCAT_DEPLOYMENT}/
+cp -r ${TOMCAT_CONF}/WEB-INF/* ${TOMCAT_DEPLOYMENT}/webapps/kms/WEB-INF/
+
+if [ -n "${BIGTOP_CLASSPATH}" ] ; then
+ sed -i -e "s#^\(common.loader=.*\)\$#\1,${BIGTOP_CLASSPATH/:/,}#"
${TOMCAT_DEPLOYMENT}/conf/catalina.properties
+fi
+
+chown -R kms:kms ${TOMCAT_DEPLOYMENT}
+chmod -R 755 ${TOMCAT_DEPLOYMENT}
+
+export CATALINA_BASE=${TOMCAT_DEPLOYMENT}
diff --git a/bigtop-packages/src/common/hadoop/kms.default
b/bigtop-packages/src/common/hadoop/kms.default
new file mode 100644
index 0000000..ac537f8
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/kms.default
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+export KMS_USER=kms
+export KMS_CONFIG=/etc/hadoop-kms/conf
+export KMS_LOG=/var/log/hadoop-kms/
+export KMS_TEMP=/var/run/hadoop-kms/
+export KMS_CATALINA_HOME=/usr/lib/bigtop-tomcat
+export CATALINA_PID=/var/run/hadoop-kms/hadoop-kms-kms.pid
+export CATALINA_BASE=/var/lib/hadoop-kms/tomcat-deployment
+export CATALINA_TMPDIR=/var/run/hadoop-kms/
+# KMS_HTTP_PORT
+# KMS_ADMIN_PORT
diff --git a/bigtop-packages/src/deb/hadoop/control
b/bigtop-packages/src/deb/hadoop/control
index 9ca5870..789076f 100644
--- a/bigtop-packages/src/deb/hadoop/control
+++ b/bigtop-packages/src/deb/hadoop/control
@@ -200,6 +200,12 @@ Description: HTTPFS for Hadoop
The server providing HTTP REST API support for the complete
FileSystem/FileContext
interface in HDFS.
+Package: hadoop-kms
+Architecture: any
+Depends: hadoop (= ${binary:Version}), bigtop-tomcat
+Description: KMS for Hadoop
+ The server providing cryptographic key management based on Hadoop
KeyProvider API.
+
Package: hadoop-client
Architecture: any
Depends: hadoop (= ${binary:Version}), hadoop-hdfs (= ${binary:Version}),
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-kms.dirs
b/bigtop-packages/src/deb/hadoop/hadoop-kms.dirs
new file mode 100644
index 0000000..9abd9ee
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/hadoop-kms.dirs
@@ -0,0 +1,3 @@
+/etc/hadoop-kms
+/usr/lib/hadoop-kms
+/var/log/hadoop-kms
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-kms.install
b/bigtop-packages/src/deb/hadoop/hadoop-kms.install
new file mode 100644
index 0000000..d221e07
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/hadoop-kms.install
@@ -0,0 +1,5 @@
+/etc/default/hadoop-kms
+/etc/hadoop-kms
+/usr/lib/hadoop/libexec/kms-config.sh
+/usr/lib/hadoop-kms
+/var/lib/hadoop-kms
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-kms.postinst
b/bigtop-packages/src/deb/hadoop/hadoop-kms.postinst
new file mode 100644
index 0000000..fa300e1
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/hadoop-kms.postinst
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# postinst script for hadoop
+
+set -e
+
+case "$1" in
+ configure)
+ mkdir -p /var/log/hadoop-kms /var/run/hadoop-kms || :
+ chown kms:kms /var/lib/hadoop-kms /var/log/hadoop-kms
/var/run/hadoop-kms
+ update-alternatives --install /etc/hadoop-kms/conf hadoop-kms-conf
/etc/hadoop-kms/conf.empty 10
+ update-alternatives --install /etc/hadoop-kms/tomcat-conf
hadoop-kms-tomcat-conf /etc/hadoop-kms/tomcat-conf.dist 10
+ update-alternatives --install /etc/hadoop-kms/tomcat-conf
hadoop-kms-tomcat-conf /etc/hadoop-kms/tomcat-conf.https 5
+ ;;
+
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-kms.preinst
b/bigtop-packages/src/deb/hadoop/hadoop-kms.preinst
new file mode 100644
index 0000000..3218897
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/hadoop-kms.preinst
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# preinst script for hadoop-kms
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <new-preinst> `install'
+# * <new-preinst> `install' <old-version>
+# * <new-preinst> `upgrade' <old-version>
+# * <old-preinst> `abort-upgrade' <new-version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ install|upgrade)
+ getent group kms >/dev/null || groupadd -r kms
+ if ! getent passwd kms >/dev/null; then
+ adduser \
+ --system \
+ --ingroup kms \
+ --home /var/lib/hadoop-kms \
+ --gecos "Hadoop KMS" \
+ --shell /bin/bash \
+ kms >/dev/null 2>/dev/null || :
+ fi
+ ;;
+
+ abort-upgrade)
+ ;;
+
+ *)
+ echo "preinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-kms.prerm
b/bigtop-packages/src/deb/hadoop/hadoop-kms.prerm
new file mode 100644
index 0000000..8ff9afd
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/hadoop-kms.prerm
@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# prerm script for hadoop-kms
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <prerm> `remove'
+# * <old-prerm> `upgrade' <new-version>
+# * <new-prerm> `failed-upgrade' <old-version>
+# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
+# * <deconfigured's-prerm> `deconfigure' `in-favour'
+# <package-being-installed> <version> `removing'
+# <conflicting-package> <version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ remove|upgrade|deconfigure)
+ update-alternatives --remove hadoop-kms-conf /etc/hadoop-kms/conf.empty
|| :
+ update-alternatives --remove-all hadoop-kms-tomcat-conf || :
+ ;;
+
+ failed-upgrade)
+ ;;
+
+ *)
+ echo "prerm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
+
+
diff --git a/bigtop-packages/src/deb/hadoop/rules
b/bigtop-packages/src/deb/hadoop/rules
index e74c7f5..c587fcc 100755
--- a/bigtop-packages/src/deb/hadoop/rules
+++ b/bigtop-packages/src/deb/hadoop/rules
@@ -41,7 +41,7 @@ override_dh_auto_build:
hadoop_svcs=hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc
hdfs-journalnode \
yarn-resourcemanager yarn-nodemanager yarn-proxyserver
yarn-timelineserver\
- mapreduce-historyserver httpfs
+ mapreduce-historyserver httpfs kms
$(hadoop_svcs): debian/init.d.tmpl
bash $< debian/[email protected] deb debian/[email protected]
@@ -59,6 +59,8 @@ override_dh_auto_install:
--build-dir=${PWD}/build \
--httpfs-dir=debian/tmp/usr/lib/hadoop-httpfs \
--httpfs-etc-dir=debian/tmp/etc/hadoop-httpfs \
+ --kms-dir=debian/tmp/usr/lib/hadoop-kms \
+ --kms-etc-dir=debian/tmp/etc/hadoop-kms \
--system-lib-dir=debian/tmp/usr/lib/ \
--system-libexec-dir=debian/tmp/usr/lib/hadoop/libexec/ \
--system-include-dir=debian/tmp/usr/include \
diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
index 1223dea..f36becb 100644
--- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
+++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
@@ -25,13 +25,17 @@
%define etc_hadoop /etc/%{name}
%define etc_yarn /etc/yarn
%define etc_httpfs /etc/%{name}-httpfs
+%define etc_kms /etc/%{name}-kms
%define config_hadoop %{etc_hadoop}/conf
%define config_yarn %{etc_yarn}/conf
%define config_httpfs %{etc_httpfs}/conf
+%define config_kms %{etc_kms}/conf
%define tomcat_deployment_httpfs %{etc_httpfs}/tomcat-conf
+%define tomcat_deployment_kms %{etc_kms}/tomcat-conf
%define lib_hadoop_dirname /usr/lib
%define lib_hadoop %{lib_hadoop_dirname}/%{name}
%define lib_httpfs %{lib_hadoop_dirname}/%{name}-httpfs
+%define lib_kms %{lib_hadoop_dirname}/%{name}-kms
%define lib_hdfs %{lib_hadoop_dirname}/%{name}-hdfs
%define lib_yarn %{lib_hadoop_dirname}/%{name}-yarn
%define lib_mapreduce %{lib_hadoop_dirname}/%{name}-mapreduce
@@ -40,12 +44,14 @@
%define log_yarn %{log_hadoop_dirname}/%{name}-yarn
%define log_hdfs %{log_hadoop_dirname}/%{name}-hdfs
%define log_httpfs %{log_hadoop_dirname}/%{name}-httpfs
+%define log_kms %{log_hadoop_dirname}/%{name}-kms
%define log_mapreduce %{log_hadoop_dirname}/%{name}-mapreduce
%define run_hadoop_dirname /var/run
%define run_hadoop %{run_hadoop_dirname}/hadoop
%define run_yarn %{run_hadoop_dirname}/%{name}-yarn
%define run_hdfs %{run_hadoop_dirname}/%{name}-hdfs
%define run_httpfs %{run_hadoop_dirname}/%{name}-httpfs
+%define run_kms %{run_hadoop_dirname}/%{name}-kms
%define run_mapreduce %{run_hadoop_dirname}/%{name}-mapreduce
%define state_hadoop_dirname /var/lib
%define state_hadoop %{state_hadoop_dirname}/hadoop
@@ -53,14 +59,16 @@
%define state_hdfs %{state_hadoop_dirname}/%{name}-hdfs
%define state_mapreduce %{state_hadoop_dirname}/%{name}-mapreduce
%define state_httpfs %{state_hadoop_dirname}/%{name}-httpfs
+%define state_kms %{state_hadoop_dirname}/%{name}-kms
%define bin_hadoop %{_bindir}
%define man_hadoop %{_mandir}
%define doc_hadoop %{_docdir}/%{name}-%{hadoop_version}
%define httpfs_services httpfs
+%define kms_services kms
%define mapreduce_services mapreduce-historyserver
%define hdfs_services hdfs-namenode hdfs-secondarynamenode hdfs-datanode
hdfs-zkfc hdfs-journalnode
%define yarn_services yarn-resourcemanager yarn-nodemanager yarn-proxyserver
yarn-timelineserver
-%define hadoop_services %{hdfs_services} %{mapreduce_services}
%{yarn_services} %{httpfs_services}
+%define hadoop_services %{hdfs_services} %{mapreduce_services}
%{yarn_services} %{httpfs_services} %{kms_services}
# Hadoop outputs built binaries into %{hadoop_build}
%define hadoop_build_path build
%define static_images_dir src/webapps/static/images
@@ -169,6 +177,9 @@ Source26: yarn.1
Source27: hdfs.1
Source28: mapred.1
Source29: hadoop-yarn-timelineserver.svc
+Source30: hadoop-kms.svc
+Source31: kms.default
+Source32: kms-tomcat-deployment.sh
#BIGTOP_PATCH_FILES
Buildroot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id} -u -n)
BuildRequires: fuse-devel, fuse, cmake
@@ -334,6 +345,15 @@ Requires(pre): %{name}-hdfs = %{version}-%{release}
The server providing HTTP REST API support for the complete
FileSystem/FileContext
interface in HDFS.
+%package kms
+Summary: KMS for Hadoop
+Group: System/Daemons
+Requires: %{name}-client = %{version}-%{release}, bigtop-tomcat
+Requires(pre): %{name} = %{version}-%{release}
+
+%description kms
+Cryptographic Key Management Server based on Hadoop KeyProvider API.
+
%package yarn-resourcemanager
Summary: YARN Resource Manager
Group: System/Daemons
@@ -490,11 +510,13 @@ env HADOOP_VERSION=%{hadoop_base_version} bash %{SOURCE2}
\
--distro-dir=$RPM_SOURCE_DIR \
--build-dir=$PWD/build \
--httpfs-dir=$RPM_BUILD_ROOT%{lib_httpfs} \
+ --kms-dir=$RPM_BUILD_ROOT%{lib_kms} \
--system-include-dir=$RPM_BUILD_ROOT%{_includedir} \
--system-lib-dir=$RPM_BUILD_ROOT%{_libdir} \
--system-libexec-dir=$RPM_BUILD_ROOT/%{lib_hadoop}/libexec \
--hadoop-etc-dir=$RPM_BUILD_ROOT%{etc_hadoop} \
--httpfs-etc-dir=$RPM_BUILD_ROOT%{etc_httpfs} \
+ --kms-etc-dir=$RPM_BUILD_ROOT%{etc_kms} \
--prefix=$RPM_BUILD_ROOT \
--doc-dir=$RPM_BUILD_ROOT%{doc_hadoop} \
--example-dir=$RPM_BUILD_ROOT%{doc_hadoop}/examples \
@@ -544,11 +566,13 @@ done
%__install -d -m 0755 $RPM_BUILD_ROOT/%{log_hdfs}
%__install -d -m 0755 $RPM_BUILD_ROOT/%{log_mapreduce}
%__install -d -m 0755 $RPM_BUILD_ROOT/%{log_httpfs}
+%__install -d -m 0755 $RPM_BUILD_ROOT/%{log_kms}
# /var/run/*
%__install -d -m 0755 $RPM_BUILD_ROOT/%{run_yarn}
%__install -d -m 0755 $RPM_BUILD_ROOT/%{run_hdfs}
%__install -d -m 0755 $RPM_BUILD_ROOT/%{run_mapreduce}
%__install -d -m 0755 $RPM_BUILD_ROOT/%{run_httpfs}
+%__install -d -m 0755 $RPM_BUILD_ROOT/%{run_kms}
%pre
getent group hadoop >/dev/null || groupadd -r hadoop
@@ -561,6 +585,10 @@ getent passwd hdfs >/dev/null || /usr/sbin/useradd
--comment "Hadoop HDFS" --she
getent group httpfs >/dev/null || groupadd -r httpfs
getent passwd httpfs >/dev/null || /usr/sbin/useradd --comment "Hadoop HTTPFS"
--shell /bin/bash -M -r -g httpfs -G httpfs --home %{run_httpfs} httpfs
+%pre kms
+getent group kms >/dev/null || groupadd -r kms
+getent passwd kms >/dev/null || /usr/sbin/useradd --comment "Hadoop KMS"
--shell /bin/bash -M -r -g kms -G kms --home %{state_kms} kms
+
%pre yarn
getent group yarn >/dev/null || groupadd -r yarn
getent passwd yarn >/dev/null || /usr/sbin/useradd --comment "Hadoop Yarn"
--shell /bin/bash -M -r -g yarn -G hadoop --home %{state_yarn} yarn
@@ -579,6 +607,13 @@ getent passwd mapred >/dev/null || /usr/sbin/useradd
--comment "Hadoop MapReduce
chkconfig --add %{name}-httpfs
+%post kms
+%{alternatives_cmd} --install %{config_kms} %{name}-kms-conf
%{etc_kms}/conf.empty 10
+%{alternatives_cmd} --install %{tomcat_deployment_kms} %{name}-kms-tomcat-conf
%{etc_kms}/tomcat-conf.dist 10
+%{alternatives_cmd} --install %{tomcat_deployment_kms} %{name}-kms-tomcat-conf
%{etc_kms}/tomcat-conf.https 5
+
+chkconfig --add %{name}-kms
+
%preun
if [ "$1" = 0 ]; then
%{alternatives_cmd} --remove %{name}-conf %{etc_hadoop}/conf.empty || :
@@ -598,6 +633,19 @@ if [ $1 -ge 1 ]; then
service %{name}-httpfs condrestart >/dev/null 2>&1
fi
+%preun kms
+if [ $1 = 0 ]; then
+ service %{name}-kms stop > /dev/null 2>&1
+ chkconfig --del %{name}-kms
+ %{alternatives_cmd} --remove %{name}-kms-conf %{etc_kms}/conf.empty || :
+ %{alternatives_cmd} --remove %{name}-kms-tomcat-conf
%{etc_kms}/tomcat-conf.dist || :
+ %{alternatives_cmd} --remove %{name}-kms-tomcat-conf
%{etc_kms}/tomcat-conf.https || :
+fi
+
+%postun kms
+if [ $1 -ge 1 ]; then
+ service %{name}-kms condrestart >/dev/null 2>&1
+fi
%files yarn
%defattr(-,root,root)
@@ -658,10 +706,6 @@ fi
%config(noreplace) %{etc_hadoop}/conf.empty/configuration.xsl
%config(noreplace) %{etc_hadoop}/conf.empty/hadoop-env.sh
%config(noreplace) %{etc_hadoop}/conf.empty/hadoop-policy.xml
-%config(noreplace) %{etc_hadoop}/conf.empty/kms-acls.xml
-%config(noreplace) %{etc_hadoop}/conf.empty/kms-env.sh
-%config(noreplace) %{etc_hadoop}/conf.empty/kms-log4j.properties
-%config(noreplace) %{etc_hadoop}/conf.empty/kms-site.xml
%config(noreplace) /etc/default/hadoop
/etc/bash_completion.d/hadoop
%{lib_hadoop}/*.jar
@@ -696,6 +740,17 @@ fi
%attr(0775,httpfs,httpfs) %{log_httpfs}
%attr(0775,httpfs,httpfs) %{state_httpfs}
+%files kms
+%defattr(-,root,root)
+%config(noreplace) %{etc_kms}
+%config(noreplace) /etc/default/%{name}-kms
+%{lib_hadoop}/libexec/kms-config.sh
+%{initd_dir}/%{name}-kms
+%{lib_kms}
+%attr(0775,kms,kms) %{run_kms}
+%attr(0775,kms,kms) %{log_kms}
+%attr(0775,kms,kms) %{state_kms}
+
# Service file management RPMs
%define service_macro() \
%files %1 \
