This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new b6d8cad METRON-2074 Script to handle TGT renewal with Storm and
Kerberos enabled (mmiklavc) closes apache/metron#1382
b6d8cad is described below
commit b6d8cade6aec7fe6290d4fd5660ff205de06bd64
Author: mmiklavc <[email protected]>
AuthorDate: Wed Apr 17 12:09:14 2019 -0600
METRON-2074 Script to handle TGT renewal with Storm and Kerberos enabled
(mmiklavc) closes apache/metron#1382
---
metron-deployment/Kerberos-manual-setup.md | 45 +++++++++++
.../packaging/docker/rpm-docker/SPECS/metron.spec | 1 +
.../metron-common/src/main/scripts/tgt_renew.py | 90 ++++++++++++++++++++++
3 files changed, 136 insertions(+)
diff --git a/metron-deployment/Kerberos-manual-setup.md
b/metron-deployment/Kerberos-manual-setup.md
index ecb0cb7..d50da34 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -31,6 +31,7 @@ This document provides instructions for kerberizing Metron's
Vagrant-based devel
* [Push Data](#push-data)
* [More Information](#more-information)
* [Elasticseach X-Pack](#X-Pack)
+* [TGT Ticket Renew](#tgt-ticket-renew)
Setup
-----
@@ -636,3 +637,47 @@ The random access indexer topology fails with the
following exception. This exc
#### Solution
This can occur when an HDFS Client is not installed on the Storm worker nodes.
This might occur on any Storm worker node where an HDFS Client is not
installed. Installing the HDFS Client on all Storm worker nodes should resolve
the problem.
+
+## TGT Ticket Renew
+
+Apache Storm doesn't handle automatic TGT ticket renewal for their running
topologies. Instead, it is left up to the operations team deploying the Storm
topologies
+in a Kerberized environment to manage this themselves. We've included a Python
script that can be setup with a cron process to automatically manage the renewal
+process for you. The script should be run on an interval that is shorter than
the renew_lifetime configured for your TGT.
+
+### Setup Instructions
+
+Run the following on a node with a Storm and Metron client installed. We need
python 2.7 via virtualenv for this to work correctly.
+
+```
+# run yum commands as root
+for item in epel-release centos-release-scl "@Development tools" python27
python27-scldevel python27-python-virtualenv libselinux-python; do yum install
-y $item; done
+sudo yum install -y gcc krb5-devel python-devel
+sudo yum install -y libffi libffi-devel
+sudo yum install -y python-cffi
+sudo yum install -y openssl-devel
+# setup python with metron user
+su - metron
+export PYTHON27_HOME=/opt/rh/python27/root
+export LD_LIBRARY_PATH="/opt/rh/python27/root/usr/lib64"
+mkdir project_dir
+cd project_dir
+${PYTHON27_HOME}/usr/bin/virtualenv venv
+source venv/bin/activate
+pip install --upgrade setuptools==18.5
+pip install requests-kerberos
+```
+
+The script `$METRON_HOME/bin/tgt_renew.py` takes two arguments:
+
+* arg1 = host:port for Storm UI server
+* arg2 = topology owner - typically "metron" for a kerberized cluster with
metron topologies
+
+Execute it like the following example:
+
+```
+# run as the metron user
+su - metron
+python $METRON_HOME/bin/tgt_renew.py node1:8744 metron
+```
+
+
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index fbffe28..dc40967 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -136,6 +136,7 @@ This package installs the Metron common files %{metron_home}
%{metron_home}/bin/zk_load_configs.sh
%{metron_home}/bin/stellar
%{metron_home}/bin/cluster_info.py
+%{metron_home}/bin/tgt_renew.py
%{metron_home}/config/zookeeper/global.json
%attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar
diff --git a/metron-platform/metron-common/src/main/scripts/tgt_renew.py
b/metron-platform/metron-common/src/main/scripts/tgt_renew.py
new file mode 100755
index 0000000..b0a8d0f
--- /dev/null
+++ b/metron-platform/metron-common/src/main/scripts/tgt_renew.py
@@ -0,0 +1,90 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from requests_kerberos import HTTPKerberosAuth
+from contextlib import closing
+import json
+import requests
+import subprocess
+import sys
+
+INDENT_SIZE = 2
+
+class ShellHandler(object):
+
+ def __init__(self):
+ pass
+
+ # returns full stdout of process call
+ def call(self, command):
+ try:
+ return subprocess.call(command, shell=True)
+ except OSError as e:
+ print >> sys.stderr, "Execution failed:", e
+
+ # partly hijacked from Python 2.7+ check_output for use in 2.6
+ def ret_output(self, cmd):
+ process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+ output, unused_err = process.communicate()
+ retcode = process.poll()
+ if retcode:
+ raise subprocess.CalledProcessError(retcode, cmd, output=output)
+ return output
+
+def get_topologies(storm_ui_host, requested_owner):
+ print "Connecting to Storm via " + storm_ui_host
+ response = requests.get(storm_ui_host, auth=HTTPKerberosAuth())
+ topology_ids = []
+ if response.status_code == 200:
+ print "Connected to server ok, parsing results"
+ for feature, value in response.json().iteritems():
+ if feature == 'topologies':
+ for topology in value:
+ t_id = ''
+ t_owner = ''
+ for k, v in topology.iteritems():
+ if k == 'name':
+ t_id = v
+ if k == 'owner':
+ t_owner = v
+ if t_owner == requested_owner:
+ topology_ids.append(t_id)
+ else:
+ print "Request for topologies failed with status code: " +
str(response.status_code)
+ print "Finished. Found {0} topologies for user
'{1}'".format(len(topology_ids), requested_owner)
+ return topology_ids
+
+def upload_credentials(topologies):
+ if len(topologies) == 0:
+ print "No topologies running for specified username/owner. Exiting"
+ return
+ storm_cmd_template = "/usr/hdp/current/storm-client/bin/storm -c
topology.auto-credentials=\'[\"org.apache.storm.security.auth.kerberos.AutoTGT\"]\'
upload-credentials $topology_name"
+ for t in topologies:
+ cmd = storm_cmd_template.replace("$topology_name", t)
+ print "Running renew-credentials command: " + cmd
+ ShellHandler().call(cmd)
+
+if len(sys.argv) != 3:
+ raise ValueError("Need host info 'host:port' for Storm and topology owner
name")
+
+host_info = sys.argv[1]
+requested_owner = sys.argv[2]
+storm_ui_host = 'http://{0}/api/v1/topology/summary'.format(host_info)
+topologies = get_topologies(storm_ui_host, requested_owner)
+upload_credentials(topologies)
+
