METRON-1738: Pcap directories should have correct permissions (merrimanr via mmiklavc) closes apache/metron#1166
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/076a6a19 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/076a6a19 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/076a6a19 Branch: refs/heads/master Commit: 076a6a197df629d3b04e0757320e7681ea2fb3d9 Parents: 6b70571 Author: merrimanr <merrim...@gmail.com> Authored: Wed Aug 15 16:02:47 2018 -0600 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Wed Aug 15 16:02:47 2018 -0600 ---------------------------------------------------------------------- .../package/scripts/params/params_linux.py | 1 + .../package/scripts/params/status_params.py | 1 + .../CURRENT/package/scripts/rest_commands.py | 23 +++++++++++++++----- .../CURRENT/package/scripts/rest_master.py | 5 +++++ 4 files changed, 24 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 115a54c..9be09f1 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -393,6 +393,7 @@ pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size'] pcap_configured_flag_file = status_params.pcap_configured_flag_file +pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file # MapReduce metron_user_hdfs_dir = '/user/' + metron_user http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 0a9fdd0..99f5ec0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -120,6 +120,7 @@ metron_keytab_path = config['configurations']['metron-env']['metron_service_keyt # Pcap pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured' +pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured' # MapReduce metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index 463dca1..d44f478 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -38,6 +38,7 @@ class RestCommands: __hbase_configured = False __hbase_acl_configured = False __pcap_configured = False + __pcap_perm_configured = False __metron_user_hdfs_dir_configured = False def __init__(self, params): @@ -49,6 +50,7 @@ class RestCommands: self.__hbase_configured = os.path.isfile(self.__params.rest_hbase_configured_flag_file) self.__hbase_acl_configured = os.path.isfile(self.__params.rest_hbase_acl_configured_flag_file) self.__pcap_configured = os.path.isfile(self.__params.pcap_configured_flag_file) + self.__pcap_perm_configured = os.path.isfile(self.__params.pcap_perm_configured_flag_file) self.__metron_user_hdfs_dir_configured = os.path.isfile(self.__params.metron_user_hdfs_dir_configured_flag_file) Directory(params.metron_rest_pid_dir, mode=0755, @@ -81,6 +83,9 @@ class RestCommands: def is_pcap_configured(self): return self.__pcap_configured + def is_pcap_perm_configured(self): + return self.__pcap_perm_configured + def is_metron_user_hdfs_dir_configured(self): return self.__metron_user_hdfs_dir_configured @@ -99,6 +104,9 @@ class RestCommands: def set_pcap_configured(self): metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file, "Setting Pcap configured to True") + def set_pcap_perm_configured(self): + metron_service.set_configured(self.__params.metron_user, self.__params.pcap_perm_configured_flag_file, "Setting Pcap perm configured to True") + def set_metron_user_hdfs_dir_configured(self): metron_service.set_configured(self.__params.metron_user, self.__params.metron_user_hdfs_dir_configured_flag_file, "Setting Metron user HDFS directory configured to True") @@ -118,26 +126,29 @@ class RestCommands: def init_pcap(self): Logger.info("Creating HDFS locations for Pcap") + # Non Kerberized Metron runs under 'storm', requiring write under the 'hadoop' group. + # Kerberized Metron runs under it's own user. + ownership = 0755 if self.__params.security_enabled else 0775 self.__params.HdfsResource(self.__params.pcap_base_path, type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.metron_group, - mode=0755, + group=self.__params.hadoop_group, + mode=ownership, ) self.__params.HdfsResource(self.__params.pcap_base_interim_result_path, type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.metron_group, - mode=0755, + group=self.__params.hadoop_group, + mode=ownership, ) self.__params.HdfsResource(self.__params.pcap_final_output_path, type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.metron_group, - mode=0755, + group=self.__params.hadoop_group, + mode=ownership, ) def create_metron_user_hdfs_dir(self): http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py index c842214..791ca77 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py @@ -60,6 +60,11 @@ class RestMaster(Script): if params.security_enabled and not commands.is_kafka_acl_configured(): commands.init_kafka_acls() commands.set_kafka_acl_configured() + if params.security_enabled and not commands.is_pcap_perm_configured(): + # If we Kerberize the cluster, we need to call this again, to remove write perms from hadoop group + # If we start off Kerberized, it just does the same thing twice. + commands.init_pcap() + commands.set_pcap_perm_configured() def start(self, env, upgrade_type=None): from params import params