AMBARI-17499. Editable solrconfig.xml for Logsearch (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2e50b0ba Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2e50b0ba Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2e50b0ba Branch: refs/heads/branch-2.4 Commit: 2e50b0ba5a41361e71af0794031b9ca2851bcf3a Parents: 4429829 Author: oleewere <[email protected]> Authored: Wed Jul 6 15:38:20 2016 +0200 Committer: oleewere <[email protected]> Committed: Wed Jul 6 16:26:58 2016 +0200 ---------------------------------------------------------------------- .../libraries/functions/solr_cloud_util.py | 32 ++++++++++++++--- .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 22 ++++++------ .../LOGSEARCH/0.5.0/metainfo.xml | 4 +++ .../0.5.0/package/scripts/setup_logsearch.py | 19 +++++----- .../properties/solr-client-log4j.properties.j2 | 2 +- .../0.4.0/package/scripts/setup_ranger_xml.py | 6 +--- .../stacks/2.3/ATLAS/test_metadata_server.py | 16 +++++++-- .../stacks/2.4/LOGSEARCH/test_logsearch.py | 38 ++++++++++++++++---- .../stacks/2.5/ATLAS/test_atlas_server.py | 16 +++++++-- .../stacks/2.5/RANGER/test_ranger_admin.py | 14 ++++++-- .../src/test/python/stacks/utils/RMFTestCase.py | 11 +++++- 11 files changed, 134 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py index 038f64d..4ecddf2 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py @@ -16,6 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +import random from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.core.resources.system import Directory, Execute, File @@ -36,26 +37,47 @@ def __append_flags_if_exists(command, flagsDict): return command -def upload_configuration_to_zk(zookeeper_quorum, solr_znode, config_set, config_set_dir, tmp_config_set_dir, - java64_home, user, retry = 5, interval = 10): +def upload_configuration_to_zk(zookeeper_quorum, solr_znode, config_set, config_set_dir, tmp_dir, + java64_home, user, retry = 5, interval = 10, solrconfig_content = None): """ Upload configuration set to zookeeper with solrCloudCli.sh At first, it tries to download configuration set if exists into a temporary location, then upload that one to zookeeper. (if the configuration changed there, in that case the user wont redefine it) - If the configuration set does not exits in zookeeper then upload it based on the config_set_dir parameter. + If the configuration set does not exist in zookeeper then upload it based on the config_set_dir parameter. """ + random_num = random.random() + tmp_config_set_dir = format('{tmp_dir}/solr_config_{config_set}_{random_num}') solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home) Execute(format('{solr_cli_prefix} --download-config --config-dir {tmp_config_set_dir} --config-set {config_set} --retry {retry} --interval {interval}'), only_if=as_user(format("{solr_cli_prefix} --check-config --config-set {config_set} --retry {retry} --interval {interval}"), user), user=user ) + if solrconfig_content is not None: + File(format("{tmp_config_set_dir}/solrconfig.xml"), + content=solrconfig_content, + owner=user, + only_if=format("test -d {tmp_config_set_dir}") + ) + + Execute(format( + '{solr_cli_prefix} --upload-config --config-dir {tmp_config_set_dir} --config-set {config_set} --retry {retry} --interval {interval}'), + user=user, + only_if=format("test -d {tmp_config_set_dir}") + ) + Execute(format( '{solr_cli_prefix} --upload-config --config-dir {config_set_dir} --config-set {config_set} --retry {retry} --interval {interval}'), - not_if=format("test -d {tmp_config_set_dir}"), - user=user + user=user, + not_if=format("test -d {tmp_config_set_dir}") ) + Directory(tmp_config_set_dir, + action="delete", + owner=user, + create_parents=True + ) + def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java64_home, user, shards = 1, replication_factor = 1, max_shards = 1, retry = 5, interval = 10, router_name = None, router_field = None, jaas_file = None, key_store_location = None, http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index e45560e..d75e87d 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -17,10 +17,11 @@ See the License for the specific language governing permissions and limitations under the License. """ -import random -from resource_management import Directory, Fail, Logger, File, \ - InlineTemplate, PropertiesFile, StaticFile, XmlConfig -from resource_management.libraries.functions import format + +from resource_management.core.resources.system import Directory, File +from resource_management.core.source import StaticFile, InlineTemplate +from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.libraries.functions.format import format from resource_management.libraries.resources.template_config import TemplateConfig from resource_management.libraries.functions import solr_cloud_util @@ -50,7 +51,8 @@ def metadata(type='server'): cd_access='a', owner=params.metadata_user, group=params.user_group, - create_parents = True + create_parents = True, + recursive_ownership=True ) Directory(params.log_dir, mode=0755, @@ -106,9 +108,7 @@ def metadata(type='server'): if type == 'server' and params.search_backend_solr and params.has_logsearch_solr: solr_cloud_util.setup_solr_client(params.config) - random_num = random.random() - - upload_conf_set('basic_configs', random_num) + upload_conf_set('basic_configs') create_collection('vertex_index', 'basic_configs') create_collection('edge_index', 'basic_configs') @@ -118,18 +118,18 @@ def metadata(type='server'): TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) -def upload_conf_set(config_set, random_num): +def upload_conf_set(config_set): import params - tmp_config_set_folder = format('{tmp_dir}/solr_config_{config_set}_{random_num}') solr_cloud_util.upload_configuration_to_zk( zookeeper_quorum=params.zookeeper_quorum, solr_znode=params.logsearch_solr_znode, config_set_dir=format("{conf_dir}/solr"), config_set=config_set, - tmp_config_set_dir=tmp_config_set_folder, + tmp_dir=params.tmp_dir, java64_home=params.java64_home, user=params.metadata_user, + solrconfig_content=InlineTemplate(params.metadata_solrconfig_content), retry=30, interval=5) def create_collection(collection, config_set): http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml index ffa9e13..69da4f0 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -70,6 +70,8 @@ <config-type>logsearch-log4j</config-type> <config-type>logsearch-admin-json</config-type> <config-type>logsearch-solr-env</config-type> + <config-type>logsearch-service_logs-solrconfig.xml</config-type> + <config-type>logsearch-audit_logs-solrconfig.xml</config-type> </configuration-dependencies> </component> @@ -112,6 +114,8 @@ <config-type>logsearch-solr-env</config-type> <config-type>logsearch-solr-xml</config-type> <config-type>logsearch-solr-log4j</config-type> + <config-type>logsearch-service_logs-solrconfig.xml</config-type> + <config-type>logsearch-audit_logs-solrconfig.xml</config-type> </configuration-dependencies> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py index 57d6e8b..d656012 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -17,7 +17,6 @@ limitations under the License. """ -import random from resource_management.libraries.functions import solr_cloud_util from resource_management.core.resources.system import Directory, Execute, File from resource_management.libraries.functions.format import format @@ -99,29 +98,31 @@ def setup_logsearch(): owner=params.logsearch_user ) - random_num = random.random() + upload_conf_set(format('{logsearch_solr_collection_service_logs}'), + solrconfig_content=InlineTemplate(params.logsearch_service_logs_solrconfig_content) + ) - upload_conf_set(format('{logsearch_solr_collection_service_logs}'), random_num) + upload_conf_set('history') - upload_conf_set('history', random_num) - - upload_conf_set(format('{logsearch_solr_collection_audit_logs}'), random_num) + upload_conf_set(format('{logsearch_solr_collection_audit_logs}'), + solrconfig_content=InlineTemplate(params.logsearch_audit_logs_solrconfig_content) + ) Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")), sudo=True ) -def upload_conf_set(config_set, random_num): +def upload_conf_set(config_set, solrconfig_content = None): import params - tmp_config_set_folder = format('{tmp_dir}/solr_config_{config_set}_{random_num}') solr_cloud_util.upload_configuration_to_zk( zookeeper_quorum=params.zookeeper_quorum, solr_znode=params.logsearch_solr_znode, config_set_dir=format("{logsearch_server_conf}/solr_configsets/{config_set}/conf"), config_set=config_set, - tmp_config_set_dir=tmp_config_set_folder, + tmp_dir=params.tmp_dir, java64_home=params.java64_home, user=params.logsearch_solr_user, + solrconfig_content= solrconfig_content, retry=30, interval=5) http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/solr-client-log4j.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/solr-client-log4j.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/solr-client-log4j.properties.j2 index 67806d7..1d04a7c 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/solr-client-log4j.properties.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/solr-client-log4j.properties.j2 @@ -19,7 +19,7 @@ log4j.rootLogger=INFO,file,stdout,stderr log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.File={{logsearch_solr_client_log}} +log4j.appender.file.File={{logsearch_solr_client_log|default('/var/log/ambari-logsearch-solr-client/solr-client.log')}} log4j.appender.file.MaxFileSize=80MB log4j.appender.file.MaxBackupIndex=60 log4j.appender.file.layout=org.apache.log4j.PatternLayout http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index 6b04075..5e675e6 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -19,7 +19,6 @@ limitations under the License. """ import os import re -import random from resource_management.core.logger import Logger from resource_management.core.resources.system import File, Directory, Execute, Link from resource_management.core.source import DownloadSource, InlineTemplate, Template @@ -559,9 +558,6 @@ def create_core_site_xml(conf_dir): def setup_ranger_audit_solr(): import params - random_num = random.random() - tmp_config_set_folder = format('{tmp_dir}/ranger_config_{ranger_solr_config_set}_{random_num}') - if params.security_enabled and params.stack_supports_ranger_kerberos: File(format("{solr_jaas_file}"), content=Template("ranger_solr_jass_conf.j2"), @@ -573,7 +569,7 @@ def setup_ranger_audit_solr(): solr_znode = params.solr_znode, config_set = params.ranger_solr_config_set, config_set_dir = params.ranger_solr_conf, - tmp_config_set_dir = tmp_config_set_folder, + tmp_dir = params.tmp_dir, java64_home = params.java_home, user = params.unix_user, retry=30, interval=5) http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index 1109e51..0da6e4d 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -53,7 +53,8 @@ class TestMetadataServer(RMFTestCase): group='hadoop', create_parents = True, cd_access='a', - mode=0755 + mode=0755, + recursive_ownership = True ) # Log dir self.assertResourceCalled('Directory', '/var/log/atlas', @@ -152,7 +153,18 @@ class TestMetadataServer(RMFTestCase): content = '' ) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --download-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/atlas/conf/solr --config-set basic_configs --retry 30 --interval 5') + self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_basic_configs_0.[0-9]*', + content=InlineTemplate(self.getConfig()['configurations']['atlas-solrconfig']['content']), + owner='atlas', + only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5', + only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/atlas/conf/solr --config-set basic_configs --retry 30 --interval 5', + not_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*') + self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_basic_configs_0.[0-9]*', + action=['delete'], + owner='atlas', + create_parents=True) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --create-collection --collection vertex_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --create-collection --collection edge_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py index cd45027..b53943f 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -18,10 +18,7 @@ See the License for the specific language governing permissions and limitations under the License. ''' -from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile -from resource_management.core.exceptions import ComponentIsNotRunning -from mock.mock import MagicMock, patch -from resource_management.libraries.script.config_dictionary import UnknownConfiguration +from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate class TestLogSearch(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" @@ -111,14 +108,41 @@ class TestLogSearch(RMFTestCase): ) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --download-config --config-dir /tmp/solr_config_hadoop_logs_0.[0-9]* --config-set hadoop_logs --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf --config-set hadoop_logs --retry 30 --interval 5') + self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_hadoop_logs_0.[0-9]*', + content=InlineTemplate(self.getConfig()['configurations']['logsearch-service_logs-solrconfig']['content']), + owner='solr', + only_if='test -d /tmp/solr_config_hadoop_logs_0.[0-9]*') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /tmp/solr_config_hadoop_logs_0.[0-9]* --config-set hadoop_logs --retry 30 --interval 5', + only_if='test -d /tmp/solr_config_hadoop_logs_0.[0-9]*') + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf --config-set hadoop_logs --retry 30 --interval 5', + not_if='test -d /tmp/solr_config_hadoop_logs_0.[0-9]*') + self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_hadoop_logs_0.[0-9]*', + action=['delete'], + owner='solr', + create_parents=True) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --download-config --config-dir /tmp/solr_config_history_0.[0-9]* --config-set history --retry 30 --interval 5') self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/ambari-logsearch-portal/conf/solr_configsets/history/conf --config-set history --retry 30 --interval 5') + self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_history_0.[0-9]*', + action=['delete'], + owner='solr', + create_parents=True) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --download-config --config-dir /tmp/solr_config_audit_logs_0.[0-9]* --config-set audit_logs --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf --config-set audit_logs --retry 30 --interval 5') - + self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_audit_logs_0.[0-9]*', + content=InlineTemplate(self.getConfig()['configurations']['logsearch-audit_logs-solrconfig']['content']), + owner='solr', + only_if='test -d /tmp/solr_config_audit_logs_0.[0-9]*') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /tmp/solr_config_audit_logs_0.[0-9]* --config-set audit_logs --retry 30 --interval 5', + only_if='test -d /tmp/solr_config_audit_logs_0.[0-9]*') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf --config-set audit_logs --retry 30 --interval 5', + not_if='test -d /tmp/solr_config_audit_logs_0.[0-9]*') + self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_audit_logs_0.[0-9]*', + action=['delete'], + owner='solr', + create_parents=True) self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/etc/ambari-logsearch-portal/conf/solr_configsets'), sudo=True ) http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py index 65bd1d2..1120743 100644 --- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py +++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py @@ -49,7 +49,8 @@ class TestAtlasServer(RMFTestCase): group='hadoop', create_parents = True, cd_access='a', - mode=0755 + mode=0755, + recursive_ownership = True ) # Log dir self.assertResourceCalled('Directory', '/var/log/atlas', @@ -151,7 +152,18 @@ class TestAtlasServer(RMFTestCase): ) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --download-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/atlas/conf/solr --config-set basic_configs --retry 30 --interval 5') + self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_basic_configs_0.[0-9]*', + content=InlineTemplate(self.getConfig()['configurations']['atlas-solrconfig']['content']), + owner='atlas', + only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5', + only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --upload-config --config-dir /etc/atlas/conf/solr --config-set basic_configs --retry 30 --interval 5', + not_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*') + self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_basic_configs_0.[0-9]*', + action=['delete'], + owner='atlas', + create_parents=True) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --create-collection --collection vertex_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --create-collection --collection edge_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py b/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py index 10c40f4..247b978 100644 --- a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py +++ b/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py @@ -84,8 +84,12 @@ class TestRangerAdmin(RMFTestCase): group = 'hadoop', mode = 0664, ) - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/solr_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5') self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5') + self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_ranger_audits_0.[0-9]*', + action=['delete'], + owner='ranger', + create_parents=True) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') self.assertResourceCalled('Execute', '/usr/bin/ranger-admin-start', @@ -175,9 +179,13 @@ class TestRangerAdmin(RMFTestCase): content = Template('ranger_solr_jass_conf.j2'), owner = 'ranger', ) - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/solr_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5') self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5') - self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') + self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_ranger_audits_0.[0-9]*', + action=['delete'], + owner='ranger', + create_parents=True) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10') self.assertResourceCalled('Execute', '/usr/bin/ranger-admin-start', environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, http://git-wip-us.apache.org/repos/asf/ambari/blob/2e50b0ba/ambari-server/src/test/python/stacks/utils/RMFTestCase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py index 53a854c..9240d38 100644 --- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py +++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py @@ -290,12 +290,21 @@ class RMFTestCase(TestCase): for key in set(resource.arguments.keys()) | set(kwargs.keys()): resource_value = resource.arguments.get(key, '') actual_value = kwargs.get(key, '') - self.assertRegexpMatches(resource_value, actual_value, msg="Key " + key + " doesn't match") + if self.isstring(resource_value): + self.assertRegexpMatches(resource_value, actual_value, + msg="Key '%s': '%s' does not match with '%s'" % (key, resource_value, actual_value)) + else: # check only the type of a custom object + self.assertEquals(resource_value.__class__.__name__, actual_value.__class__.__name__) def assertRegexpMatches(self, value, pattern, msg=None): if not re.match(pattern, value): raise AssertionError, msg or 'pattern %s does not match %s' % (pattern, value) + def isstring(self, s): + if (sys.version_info[0] == 3): + return isinstance(s, str) + return isinstance(s, basestring) + def assertNoMoreResources(self): self.assertEquals(len(RMFTestCase.env.resource_list), 0, "There were other resources executed!")
