AMBARI-17462. Install Log search client package with Ranger Service (Part-2) (Mugdha Varadkar via gautam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ef29683c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ef29683c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ef29683c Branch: refs/heads/trunk Commit: ef29683ce2cd49c0fc06baf4cba83ee39baf7997 Parents: 9e43840 Author: Gautam Borad <gau...@apache.org> Authored: Thu Jun 30 08:09:44 2016 +0530 Committer: Gautam Borad <gau...@apache.org> Committed: Thu Jun 30 10:52:56 2016 +0530 ---------------------------------------------------------------------- .../RANGER/0.4.0/package/scripts/params.py | 12 +- .../0.4.0/package/scripts/ranger_admin.py | 5 + .../0.4.0/package/scripts/setup_ranger_xml.py | 4 - .../stacks/2.5/RANGER/test_ranger_admin.py | 158 ++++++++++--------- 4 files changed, 93 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ef29683c/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py index e708843..ba8d3b3 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py @@ -269,15 +269,17 @@ logsearch_solr_hosts = default("/clusterHostInfo/logsearch_solr_hosts", []) has_logsearch = len(logsearch_solr_hosts) > 0 is_solrCloud_enabled = default('/configurations/ranger-env/is_solrCloud_enabled', False) solr_znode = '/ranger_audits' -if is_solrCloud_enabled: - solr_znode = config['configurations']['ranger-admin-site']['ranger.audit.solr.zookeepers'] +if stack_supports_logsearch_client and is_solrCloud_enabled: + solr_znode = default('/configurations/ranger-admin-site/ranger.audit.solr.zookeepers', 'NONE') if solr_znode != '' and solr_znode.upper() != 'NONE': - solr_znode = solr_znode.split('/')[1] - solr_znode = format('/{solr_znode}') + solr_znode = solr_znode.split('/') + if len(solr_znode) > 1 and len(solr_znode) == 2: + solr_znode = solr_znode[1] + solr_znode = format('/{solr_znode}') if has_logsearch: solr_znode = config['configurations']['logsearch-solr-env']['logsearch_solr_znode'] solr_user = default('/configurations/logsearch-solr-env/logsearch_solr_user', unix_user) -custom_log4j = True if has_logsearch else False +custom_log4j = has_logsearch # get comma separated list of zookeeper hosts zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) http://git-wip-us.apache.org/repos/asf/ambari/blob/ef29683c/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py index 6367281..c6e5b70 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py @@ -30,6 +30,7 @@ from resource_management.core.logger import Logger from resource_management.core import shell from ranger_service import ranger_service from setup_ranger_xml import setup_ranger_audit_solr +from resource_management.libraries.functions import solr_cloud_util import upgrade import os, errno @@ -82,6 +83,10 @@ class RangerAdmin(Script): env.set_params(params) self.configure(env, upgrade_type=upgrade_type) + if params.stack_supports_logsearch_client and params.is_solrCloud_enabled: + solr_cloud_util.setup_solr_client(params.config, user = params.solr_user, custom_log4j = params.custom_log4j) + setup_ranger_audit_solr() + ranger_service('ranger_admin') http://git-wip-us.apache.org/repos/asf/ambari/blob/ef29683c/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index 23bb764..e35ea5f 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -191,10 +191,6 @@ def setup_ranger_admin(upgrade_type=None): create_core_site_xml(ranger_conf) - if params.stack_supports_logsearch_client and params.is_solrCloud_enabled: - solr_cloud_util.setup_solr_client(params.config, user = params.solr_user, custom_log4j = params.custom_log4j) - setup_ranger_audit_solr() - def setup_ranger_db(stack_version=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/ef29683c/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py b/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py index 9b2ef3f..10c40f4 100644 --- a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py +++ b/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py @@ -50,6 +50,44 @@ class TestRangerAdmin(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_default() + + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client', + owner = 'logsearch-solr', + group = 'hadoop', + create_parents = True, + mode = 0755, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client', + group = 'hadoop', + cd_access = 'a', + create_parents = True, + mode = 0755, + owner = 'logsearch-solr', + recursive_ownership = True, + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh', + content = StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'), + owner = 'logsearch-solr', + group = 'hadoop', + mode = 0755, + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties', + owner = 'logsearch-solr', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-client-log4j']['content']), + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log', + content = '', + owner = 'logsearch-solr', + group = 'hadoop', + mode = 0664, + ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') + self.assertResourceCalled('Execute', '/usr/bin/ranger-admin-start', environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, not_if = 'ps -ef | grep proc_rangeradmin | grep -v grep', @@ -99,6 +137,48 @@ class TestRangerAdmin(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_secured() + + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client', + owner = 'logsearch-solr', + group = 'hadoop', + create_parents = True, + mode = 0755, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client', + group = 'hadoop', + cd_access = 'a', + create_parents = True, + mode = 0755, + owner = 'logsearch-solr', + recursive_ownership = True, + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh', + content = StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'), + owner = 'logsearch-solr', + group = 'hadoop', + mode = 0755, + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties', + owner = 'logsearch-solr', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-client-log4j']['content']), + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log', + content = '', + owner = 'logsearch-solr', + group = 'hadoop', + mode = 0664, + ) + self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger_solr_jass.conf', + content = Template('ranger_solr_jass_conf.j2'), + owner = 'ranger', + ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') + self.assertResourceCalled('Execute', '/usr/bin/ranger-admin-start', environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, not_if = 'ps -ef | grep proc_rangeradmin | grep -v grep', @@ -249,42 +329,6 @@ class TestRangerAdmin(RMFTestCase): configuration_attributes = self.getConfig()['configuration_attributes']['core-site'], mode = 0644 ) - self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client', - owner = 'logsearch-solr', - group = 'hadoop', - create_parents = True, - mode = 0755, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client', - group = 'hadoop', - cd_access = 'a', - create_parents = True, - mode = 0755, - owner = 'logsearch-solr', - recursive_ownership = True, - ) - self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh', - content = StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'), - owner = 'logsearch-solr', - group = 'hadoop', - mode = 0755, - ) - self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties', - owner = 'logsearch-solr', - content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-client-log4j']['content']), - group = 'hadoop', - mode = 0644, - ) - self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log', - content = '', - owner = 'logsearch-solr', - group = 'hadoop', - mode = 0664, - ) - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') def assert_configure_secured(self): @@ -427,44 +471,4 @@ class TestRangerAdmin(RMFTestCase): configurations = self.getConfig()['configurations']['core-site'], configuration_attributes = self.getConfig()['configuration_attributes']['core-site'], mode = 0644 - ) - self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client', - owner = 'logsearch-solr', - group = 'hadoop', - create_parents = True, - mode = 0755, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client', - group = 'hadoop', - cd_access = 'a', - create_parents = True, - mode = 0755, - owner = 'logsearch-solr', - recursive_ownership = True, - ) - self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh', - content = StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'), - owner = 'logsearch-solr', - group = 'hadoop', - mode = 0755, - ) - self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties', - owner = 'logsearch-solr', - content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-client-log4j']['content']), - group = 'hadoop', - mode = 0644, - ) - self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log', - content = '', - owner = 'logsearch-solr', - group = 'hadoop', - mode = 0664, - ) - self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger_solr_jass.conf', - content = Template('ranger_solr_jass_conf.j2'), - owner = 'ranger', - ) - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') \ No newline at end of file + ) \ No newline at end of file
