METRON-799 The MPack should function in a kerberized cluster (dlyle via justinleet) closes apache/incubator-metron#518
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/55062fb7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/55062fb7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/55062fb7 Branch: refs/heads/Metron_0.4.0 Commit: 55062fb79a85722c909906874e0772f51b18953d Parents: 187ef37 Author: David Lyle <[email protected]> Authored: Thu Apr 13 11:23:40 2017 -0400 Committer: leet <[email protected]> Committed: Thu Apr 20 16:03:05 2017 -0400 ---------------------------------------------------------------------- .../METRON/CURRENT/configuration/metron-env.xml | 19 ++++---- .../METRON/CURRENT/kerberos.json | 22 ++++++++- .../common-services/METRON/CURRENT/metainfo.xml | 14 ++++++ .../package/scripts/enrichment_commands.py | 12 ----- .../package/scripts/enrichment_master.py | 25 +++++++++- .../CURRENT/package/scripts/metron_client.py | 49 ++++++++++++++++++++ .../CURRENT/package/scripts/metron_security.py | 9 ++++ .../CURRENT/package/scripts/metron_service.py | 23 +++++++-- .../package/scripts/params/status_params.py | 5 +- .../METRON/CURRENT/service_advisor.py | 5 -- 10 files changed, 145 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 334e4f3..d09614b 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -128,21 +128,20 @@ <name>metron_principal_name</name> <display-name>Metron Principal Name</display-name> <description>Metron principal name</description> - </property> - <property> - <name>metron_keytab</name> - <display-name>Metron Keytab</display-name> - <description>Metron keytab path</description> - </property> - <property> - <name>metron_service_principal_name</name> - <display-name>Metron Service Principal Name</display-name> - <description>Metron service principal name</description> + <property-type>KERBEROS_PRINCIPAL</property-type> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> </property> <property> <name>metron_service_keytab</name> <display-name>Metron Service User Keytab</display-name> <description>Metron Service user keytab path</description> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> </property> <property> <name>global-json</name> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json index e1a847b..cba1847 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/kerberos.json @@ -21,11 +21,29 @@ "name": "${cluster-env/user_group}", "access": "r" }, - "configuration": "metron-env/metron_keytab" + "configuration": "metron-env/metron_service_keytab" } } + ], + "configurations": [ + { + "storm-site": { + "nimbus.credential.renewers.classes": "['org.apache.storm.security.auth.kerberos.AutoTGT']", + "supervisor.run.worker.as.user": "true" + } + } + ], + "components": [ + { + "name": "METRON_PARSERS" + }, + { + "name": "METRON_ENRICHMENT_MASTER" + }, + { + "name": "METRON_INDEXING" + } ] } ] } - http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index bbe7187..25237f5 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -174,6 +174,20 @@ </customCommand> </customCommands> </component> + + <component> + <name>METRON_CLIENT</name> + <displayName>METRON Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <commandScript> + <script>scripts/metron_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + </component> + </components> <themes> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index aa53391..4fda07b 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -21,8 +21,6 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File import metron_service -import metron_security - # Wrap major operations and functionality in this class class EnrichmentCommands: @@ -143,11 +141,6 @@ class EnrichmentCommands: -s {1} \ -z {2}""" Logger.info('Starting ' + self.__enrichment_topology) - if self.__params.security_enabled: - metron_security.kinit(self.__params.kinit_path_local, - self.__params.metron_keytab_path, - self.__params.metron_principal_name, - execute_user=self.__params.metron_user) Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum), user=self.__params.metron_user) @@ -156,11 +149,6 @@ class EnrichmentCommands: def stop_enrichment_topology(self): Logger.info('Stopping ' + self.__enrichment_topology) stop_cmd = 'storm kill ' + self.__enrichment_topology - if self.__params.security_enabled: - metron_security.kinit(self.__params.kinit_path_local, - self.__params.metron_keytab_path, - self.__params.metron_principal_name, - execute_user=self.__params.metron_user) Execute(stop_cmd, user=self.__params.metron_user) Logger.info('Done stopping enrichment topologies') http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index e8cb8d5..4ab81d4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -24,14 +24,13 @@ from resource_management.core.logger import Logger from enrichment_commands import EnrichmentCommands from metron_security import storm_security_setup import metron_service - +import metron_security class Enrichment(Script): def install(self, env): from params import params env.set_params(params) self.install_packages(env) - self.configure(env) def configure(self, env, upgrade_type=None, config_dir=None): from params import params @@ -52,6 +51,13 @@ class Enrichment(Script): env.set_params(params) self.configure(env) commands = EnrichmentCommands(params) + + if params.security_enabled: + metron_security.kinit(params.kinit_path_local, + params.metron_keytab_path, + params.metron_principal_name, + execute_user=params.metron_user) + metron_service.load_global_config(params) if not commands.is_kafka_configured(): @@ -69,8 +75,16 @@ class Enrichment(Script): def stop(self, env, upgrade_type=None): from params import params + env.set_params(params) commands = EnrichmentCommands(params) + + if params.security_enabled: + metron_security.kinit(params.kinit_path_local, + params.metron_keytab_path, + params.metron_principal_name, + execute_user=params.metron_user) + commands.stop_enrichment_topology() def status(self, env): @@ -78,6 +92,13 @@ class Enrichment(Script): env.set_params(status_params) commands = EnrichmentCommands(status_params) + if status_params.security_enabled: + metron_security.kinit(status_params.kinit_path_local, + status_params.metron_keytab_path, + status_params.metron_principal_name, + execute_user=status_params.metron_user) + + if not commands.is_topology_active(env): raise ComponentIsNotRunning() http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_client.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_client.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_client.py new file mode 100644 index 0000000..65831c9 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_client.py @@ -0,0 +1,49 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script +from resource_management.core.exceptions import ClientComponentHasNoStatus +from resource_management.core.resources.system import Directory +from metron_security import storm_security_setup + +class MetronClient(Script): + + def install(self, env): + from params import params + env.set_params(params) + self.configure(env) + + def configure(self, env): + from params import params + env.set_params(params) + storm_security_setup(params) + + def start(self, env, upgrade_type=None): + from params import params + env.set_params(params) + + def stop(self, env, upgrade_type=None): + from params import params + env.set_params(params) + + def status(self, env): + raise ClientComponentHasNoStatus() + +if __name__ == "__main__": + MetronClient().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py index 057339a..4f04daf 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py @@ -30,6 +30,15 @@ def storm_security_setup(params): # Ambari's Directory doesn't do tilde expansion. metron_storm_dir_tilde = '~' + params.metron_user + '/.storm' metron_storm_dir = os.path.expanduser(metron_storm_dir_tilde) + + + Directory(params.metron_home, + mode=0755, + owner=params.metron_user, + group=params.metron_group, + create_parents=True + ) + Directory(metron_storm_dir, mode=0755, owner=params.metron_user, http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index b213947..f2cb88d 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@ -22,6 +22,7 @@ from resource_management.core.resources.system import Directory, File from resource_management.core.resources.system import Execute from resource_management.core.source import InlineTemplate from resource_management.libraries.functions import format as ambari_format +from resource_management.libraries.functions.get_user_call_output import get_user_call_output from metron_security import kinit def init_config(): @@ -40,8 +41,7 @@ def get_running_topologies(params): # Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets. # This is becuase we need to run a command with a return as the metron user. Sigh negotiate = '--negotiate -u : ' if params.security_enabled else '' - sudo = ambari_format('sudo -u {metron_user} ') if params.security_enabled else '' - cmd = ambari_format(sudo + 'curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary') + cmd = ambari_format('curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary') if params.security_enabled: kinit(params.kinit_path_local, @@ -50,8 +50,23 @@ def get_running_topologies(params): execute_user=params.metron_user) Logger.info('Running cmd: ' + cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (stdout, stderr) = proc.communicate() + return_code, stdout, sdterr = get_user_call_output(cmd, user=params.metron_user) + + try: + stormjson = json.loads(stdout) + except ValueError, e: + Logger.info('Stdout: ' + str(stdout)) + Logger.info('Stderr: ' + str(stderr)) + Logger.exception(str(e)) + return {} + + topologiesDict = {} + + for topology in stormjson['topologies']: + topologiesDict[topology['name']] = topology['status'] + + Logger.info("Topologies: " + str(topologiesDict)) + return topologiesDict try: stormjson = json.loads(stdout) http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 65e1eaf..154b3de 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -73,6 +73,5 @@ tmp_dir = Script.get_tmp_dir() metron_user = config['configurations']['metron-env']['metron_user'] -if security_enabled: - metron_principal_name = config['configurations']['metron-env']['metron_principal_name'] - metron_keytab_path = config['configurations']['metron-env']['metron_keytab'] +metron_principal_name = config['configurations']['metron-env']['metron_principal_name'] +metron_keytab_path = config['configurations']['metron-env']['metron_service_keytab'] http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55062fb7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py index c16e3e6..23aebb4 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py @@ -156,10 +156,5 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor storm_site_desired_values = { "topology.classpath" : "/etc/hbase/conf:/etc/hadoop/conf" } - if is_secured: - storm_site_desired_values.update({ - "nimbus.credential.renewers.classes":"['org.apache.storm.security.auth.kerberos.AutoTGT']", - "supervisor.run.worker.as.user":"true" - }) return storm_site_desired_values
