AMBARI-18279. Use the PropertiesFile python command to create property files instead of jinja templates (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/57ef91f9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/57ef91f9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/57ef91f9 Branch: refs/heads/logsearch-ga Commit: 57ef91f9762a4e201dbd6c85027be75dceed4b77 Parents: cd08bde Author: Miklos Gergely <[email protected]> Authored: Wed Sep 7 22:17:05 2016 +0200 Committer: oleewere <[email protected]> Committed: Tue Sep 27 22:56:29 2016 +0200 ---------------------------------------------------------------------- .../configuration/logsearch-properties.xml | 6 +- .../LOGSEARCH/0.5.0/package/scripts/params.py | 125 +++++++++---------- .../0.5.0/package/scripts/setup_logfeeder.py | 10 +- .../0.5.0/package/scripts/setup_logsearch.py | 7 +- .../package/templates/logfeeder.properties.j2 | 32 ----- .../package/templates/logsearch.properties.j2 | 72 ----------- .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 18 ++- .../stacks/2.4/LOGSEARCH/test_logsearch.py | 30 ++++- .../test/python/stacks/2.4/configs/default.json | 13 +- ambari-web/app/data/HDP2/site_properties.js | 6 +- 10 files changed, 120 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml index 56ff2d1..e521c8e 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml @@ -133,7 +133,7 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>logsearch.external.auth.enabled</name> + <name>logsearch.auth.external_auth.enable</name> <value>true</value> <display-name>External authentication</display-name> <description>Enable external authentication</description> @@ -143,14 +143,14 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>logsearch.external.auth.host_url</name> + <name>logsearch.auth.external_auth.host_url</name> <value>{ambari_server_auth_host_url}</value> <display-name>External authentication url - host and port</display-name> <description>The host and port of the external server used to authenticate</description> <on-ambari-upgrade add="true"/> </property> <property> - <name>logsearch.external.auth.login_url</name> + <name>logsearch.auth.external_auth.login_url</name> <value>/api/v1/users/$USERNAME/privileges?fields=*</value> <display-name>External authentication url - path and query</display-name> <description>The path and query of the external server used to authenticate</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 5c3954f..8b0ce75 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -75,9 +75,9 @@ if 'metrics_collector_hosts' in config['clusterHostInfo']: else: metrics_collector_hosts = '' -logsearch_solr_metrics_collector_hosts = format(config['configurations']['logsearch-properties']['logsearch.solr.metrics.collector.hosts']) - +##################################### # Infra Solr configs +##################################### infra_solr_znode = default('/configurations/infra-solr-env/infra_solr_znode', '/infra-solr') infra_solr_instance_count = len(config['clusterHostInfo']['infra_solr_hosts']) infra_solr_ssl_enabled = default('configurations/infra-solr-env/infra_solr_ssl_enabled', False) @@ -108,57 +108,17 @@ if security_enabled: ##################################### logsearch_dir = '/usr/lib/ambari-logsearch-portal' -logsearch_collection_service_logs_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.service.logs.numshards'] -logsearch_collection_audit_logs_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.audit.logs.numshards'] - -if logsearch_collection_service_logs_numshards_config > 0: - logsearch_collection_service_logs_numshards = str(logsearch_collection_service_logs_numshards_config) -else: - logsearch_collection_service_logs_numshards = format(str(infra_solr_instance_count)) - -if logsearch_collection_audit_logs_numshards_config > 0: - logsearch_collection_audit_logs_numshards = str(logsearch_collection_audit_logs_numshards_config) -else: - logsearch_collection_audit_logs_numshards = format(str(infra_solr_instance_count)) - -logsearch_collection_service_logs_replication_factor = str(config['configurations']['logsearch-properties']['logsearch.collection.service.logs.replication.factor']) -logsearch_collection_audit_logs_replication_factor = str(config['configurations']['logsearch-properties']['logsearch.collection.audit.logs.replication.factor']) - -logsearch_solr_collection_service_logs = default('/configurations/logsearch-properties/logsearch.solr.collection.service.logs', 'hadoop_logs') -logsearch_solr_collection_audit_logs = default('/configurations/logsearch-properties/logsearch.solr.collection.audit.logs','audit_logs') - logsearch_service_logs_max_retention = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_max_retention'] logsearch_service_logs_merge_factor = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_merge_factor'] -logsearch_service_logs_fields = config['configurations']['logsearch-properties']['logsearch.service.logs.fields'] -logsearch_service_logs_split_interval_mins = config['configurations']['logsearch-properties']['logsearch.service.logs.split.interval.mins'] logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention'] logsearch_audit_logs_merge_factor = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_merge_factor'] -logsearch_audit_logs_split_interval_mins = config['configurations']['logsearch-properties']['logsearch.audit.logs.split.interval.mins'] -logsearch_logfeeder_include_default_level = default('/configurations/logsearch-properties/logsearch.logfeeder.include.default.level', 'fatal,error,warn') - -logsearch_solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node', zookeeper_quorum) -logsearch_solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum', infra_solr_znode) +logsearch_solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node', infra_solr_znode) +logsearch_solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum', zookeeper_quorum) logsearch_solr_audit_logs_zk_node = format(logsearch_solr_audit_logs_zk_node) logsearch_solr_audit_logs_zk_quorum = format(logsearch_solr_audit_logs_zk_quorum) -# create custom properties - remove defaults -logsearch_custom_properties = dict(config['configurations']['logsearch-properties']) -logsearch_custom_properties.pop("logsearch.service.logs.fields", None) -logsearch_custom_properties.pop("logsearch.audit.logs.split.interval.mins", None) -logsearch_custom_properties.pop("logsearch.collection.service.logs.replication.factor", None) -logsearch_custom_properties.pop("logsearch.solr.collection.service.logs", None) -logsearch_custom_properties.pop("logsearch.solr.metrics.collector.hosts", None) -logsearch_custom_properties.pop("logsearch.solr.collection.audit.logs", None) -logsearch_custom_properties.pop("logsearch.logfeeder.include.default.level", None) -logsearch_custom_properties.pop("logsearch.collection.audit.logs.replication.factor", None) -logsearch_custom_properties.pop("logsearch.collection.service.logs.numshards", None) -logsearch_custom_properties.pop("logsearch.service.logs.split.interval.mins", None) -logsearch_custom_properties.pop("logsearch.collection.audit.logs.numshards", None) -logsearch_custom_properties.pop("logsearch.external.auth.enabled", None) -logsearch_custom_properties.pop("logsearch.external.auth.host_url", None) -logsearch_custom_properties.pop("logsearch.external.auth.login_url", None) # logsearch-env configs logsearch_user = config['configurations']['logsearch-env']['logsearch_user'] @@ -215,9 +175,7 @@ hdfs_user = default('configurations/hadoop-env/hdfs_user', 'hdfs') mapred_user = default('configurations/mapred-env/mapred_user', 'mapred') yarn_user = default('configurations/yarn-env/yarn_user', 'yarn') -##################################### # Logsearch auth configs -##################################### logsearch_admin_credential_file = 'logsearch-admin.json' logsearch_admin_username = default('/configurations/logsearch-admin-json/logsearch_admin_username', "admin") @@ -236,9 +194,42 @@ if 'ambari_server_host' in config['clusterHostInfo']: else: ambari_server_auth_host_url = '' -logsearch_auth_external_enabled = str(config['configurations']['logsearch-properties']['logsearch.external.auth.enabled']).lower() -logsearch_auth_external_host_url = format(config['configurations']['logsearch-properties']['logsearch.external.auth.host_url']) -logsearch_auth_external_login_url = config['configurations']['logsearch-properties']['logsearch.external.auth.login_url'] +# Logsearch propreties + +logsearch_properties = dict(config['configurations']['logsearch-properties']) + +logsearch_properties['logsearch.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode +logsearch_properties['logsearch.solr.audit.logs.zk_connect_string'] = logsearch_solr_audit_logs_zk_quorum + logsearch_solr_audit_logs_zk_node + +logsearch_properties['logsearch.solr.collection.history'] = 'history' +logsearch_properties['logsearch.solr.history.config.name'] = 'history' +logsearch_properties['logsearch.collection.history.replication.factor'] = '1' + +logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearch_properties['logsearch.solr.metrics.collector.hosts']) + +logsearch_properties['logsearch.solr.jmx.port'] = infra_solr_jmx_port + +logsearch_properties['logsearch.login.credentials.file'] = logsearch_admin_credential_file +logsearch_properties['logsearch.auth.file.enable'] = 'true' +logsearch_properties['logsearch.auth.ldap.enable'] = 'false' +logsearch_properties['logsearch.auth.simple.enable'] = 'false' +logsearch_properties['logsearch.roles.allowed'] = 'AMBARI.ADMINISTRATOR' +logsearch_properties['logsearch.auth.external_auth.host_url'] = format(logsearch_properties['logsearch.auth.external_auth.host_url']) + +logsearch_properties['logsearch.protocol'] = logsearch_ui_protocol + +if security_enabled: + logfeeder_properties['logsearch.solr.kerberos.enable'] = 'true' + logfeeder_properties['logsearch.solr.jaas.file'] = logsearch_jaas_file + + +logsearch_solr_collection_service_logs = logsearch_properties['logsearch.solr.collection.service.logs'] +logsearch_service_logs_split_interval_mins = logsearch_properties['logsearch.service.logs.split.interval.mins'] +logsearch_collection_service_logs_numshards = logsearch_properties['logsearch.collection.service.logs.numshards'] + +logsearch_solr_collection_audit_logs = logsearch_properties['logsearch.solr.collection.audit.logs'] +logsearch_audit_logs_split_interval_mins = logsearch_properties['logsearch.audit.logs.split.interval.mins'] +logsearch_collection_audit_logs_numshards = logsearch_properties['logsearch.collection.audit.logs.numshards'] ##################################### # Logfeeder configs @@ -268,29 +259,31 @@ logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfe logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password'] logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type'] -logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.checkpoint.folder', - '/etc/ambari-logsearch-logfeeder/conf/checkpoints') +logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'infra', + 'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', + 'zeppelin', 'zookeeper'] -logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable', True)).lower() -logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval', 5) +logfeeder_config_file_names = \ + ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] -logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'infra', 'kafka', - 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper'] +default_config_files = ','.join(logfeeder_config_file_names) -logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in - logfeeder_supported_services] +logfeeder_properties = dict(config['configurations']['logfeeder-properties']) -default_config_files = ','.join(logfeeder_config_file_names) +logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts']) +logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files']) +logfeeder_properties['logfeeder.solr.core.config.name'] = 'history' +logfeeder_properties['logfeeder.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode -logfeeder_config_files = format(config['configurations']['logfeeder-properties']['logfeeder.config.files']) -logfeeder_metrics_collector_hosts = format(config['configurations']['logfeeder-properties']['logfeeder.metrics.collector.hosts']) +if security_enabled: + logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true' + logfeeder_properties['logfeeder.solr.jaas.file'] = logfeeder_jaas_file -logfeeder_custom_properties = dict(config['configurations']['logfeeder-properties']) -logfeeder_custom_properties.pop('logfeeder.config.files', None) -logfeeder_custom_properties.pop('logfeeder.checkpoint.folder', None) -logfeeder_custom_properties.pop('logfeeder.metrics.collector.hosts', None) -logfeeder_custom_properties.pop('logfeeder.log.filter.enable', None) -logfeeder_custom_properties.pop('logfeeder.solr.config.interval', None) +logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder'] + +##################################### +# Smoke command +##################################### logsearch_server_hosts = config['clusterHostInfo']['logsearch_server_hosts'] logsearch_server_host = "" http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py index d3851d3..688a9b0 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py @@ -20,12 +20,12 @@ limitations under the License. from resource_management.core.resources.system import Directory, File from resource_management.libraries.functions.format import format from resource_management.core.source import InlineTemplate, Template +from resource_management.libraries.resources.properties_file import PropertiesFile def setup_logfeeder(): import params - Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir, - params.logfeeder_checkpoint_folder], + Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir, params.logfeeder_checkpoint_folder], mode=0755, cd_access='a', create_parents=True @@ -43,9 +43,9 @@ def setup_logfeeder(): content='' ) - File(format("{logsearch_logfeeder_conf}/logfeeder.properties"), - content=Template("logfeeder.properties.j2"), - ) + PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"), + properties = params.logfeeder_properties + ) File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"), content=InlineTemplate(params.logfeeder_env_content), http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py index aa89609..8d84093 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -23,6 +23,7 @@ from resource_management.core.resources.system import Directory, Execute, File from resource_management.libraries.functions.format import format from resource_management.libraries.functions.decorator import retry from resource_management.core.source import InlineTemplate, Template +from resource_management.libraries.resources.properties_file import PropertiesFile def setup_logsearch(): @@ -51,10 +52,8 @@ def setup_logsearch(): content='' ) - File(format("{logsearch_server_conf}/logsearch.properties"), - content=Template("logsearch.properties.j2"), - owner=params.logsearch_user, - group=params.user_group + PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"), + properties=params.logsearch_properties ) File(format("{logsearch_server_conf}/HadoopServiceConfig.json"), http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 deleted file mode 100644 index 86f0be4..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -logfeeder.checkpoint.folder={{logfeeder_checkpoint_folder}} -logfeeder.metrics.collector.hosts={{logfeeder_metrics_collector_hosts}} -logfeeder.config.files={{logfeeder_config_files}} -logfeeder.log.filter.enable={{logfeeder_log_filter_enable}} -logfeeder.solr.config.interval={{logfeeder_solr_config_interval}} -logfeeder.solr.core.config.name=history -logfeeder.solr.zk_connect_string={{zookeeper_quorum}}{{infra_solr_znode}} - -# Custom properties -{% for key, value in logfeeder_custom_properties.items() %} -{{key}}={{value}} -{% endfor %} - -{% if security_enabled -%} -logfeeder.solr.kerberos.enable=true -logfeeder.solr.jaas.file={{logfeeder_jaas_file}} -{% endif %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 deleted file mode 100644 index 1bcee2d..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -logsearch.solr.zk_connect_string={{zookeeper_quorum}}{{infra_solr_znode}} - -# Service Logs -logsearch.solr.collection.service.logs={{logsearch_solr_collection_service_logs}} - -logsearch.service.logs.split.interval.mins={{logsearch_service_logs_split_interval_mins}} -logsearch.collection.service.logs.numshards={{logsearch_collection_service_logs_numshards}} -logsearch.collection.service.logs.replication.factor={{logsearch_collection_service_logs_replication_factor}} - -logsearch.service.logs.fields={{logsearch_service_logs_fields}} - -# Audit logs -logsearch.solr.audit.logs.zk_connect_string={{logsearch_solr_audit_logs_zk_quorum}}{{logsearch_solr_audit_logs_zk_node}} -logsearch.solr.collection.audit.logs={{logsearch_solr_collection_audit_logs}} -logsearch.solr.audit.logs.url= - -logsearch.audit.logs.split.interval.mins={{logsearch_audit_logs_split_interval_mins}} -logsearch.collection.audit.logs.numshards={{logsearch_collection_audit_logs_numshards}} -logsearch.collection.audit.logs.replication.factor={{logsearch_collection_audit_logs_replication_factor}} - -# History logs -logsearch.solr.collection.history=history -logsearch.solr.history.config.name=history -logsearch.collection.history.replication.factor=1 - -# Metrics -logsearch.solr.metrics.collector.hosts={{logsearch_solr_metrics_collector_hosts}} -logsearch.solr.jmx.port={{infra_solr_jmx_port}} - -# Logfeeder Settings - -logsearch.logfeeder.include.default.level={{logsearch_logfeeder_include_default_level}} - -# logsearch-admin.json -logsearch.auth.file.enable=true -logsearch.login.credentials.file={{logsearch_admin_credential_file}} - -logsearch.auth.ldap.enable=false -logsearch.auth.simple.enable=false - -logsearch.auth.external_auth.enable={{logsearch_auth_external_enabled}} -logsearch.auth.external_auth.host_url={{logsearch_auth_external_host_url}} -logsearch.auth.external_auth.login_url={{logsearch_auth_external_login_url}} -logsearch.roles.allowed=AMBARI.ADMINISTRATOR - -logsearch.protocol={{logsearch_ui_protocol}} - -# Custom properties -{% for key, value in logsearch_custom_properties.items() %} -{{key}}={{value}} -{% endfor %} - -{% if security_enabled -%} -logsearch.solr.kerberos.enable=true -logsearch.solr.jaas.file={{logsearch_jaas_file}} -{% endif %} - http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index f4dbd63..7f9acba 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -60,8 +60,13 @@ class TestLogFeeder(RMFTestCase): mode=0644, content='' ) - self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties', - content=Template('logfeeder.properties.j2') + self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties', + properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints', + 'logfeeder.config.files': 'global.config.json,output.config.json,input.config-accumulo.json,input.config-ambari.json,input.config-ams.json,input.config-atlas.json,input.config-falcon.json,input.config-flume.json,input.config-hbase.json,input.config-hdfs.json,input.config-hive.json,input.config-hst.json,input.config-infra.json,input.config-kafka.json,input.config-knox.json,input.config-logsearch.json,input.config-nifi.json,input.config-oozie.json,input.config-ranger.json,input.config-spark.json,input.config-spark2.json,input.config-storm.json,input.config-yarn.json,input.config-zeppelin.json,input.config-zookeeper.json', + 'logfeeder.metrics.collector.hosts': '', + 'logfeeder.solr.core.config.name': 'history', + 'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr' + } ) self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh', mode=0755, @@ -75,11 +80,12 @@ class TestLogFeeder(RMFTestCase): encoding='utf-8' ) - logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'infra', 'kafka', - 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper'] + logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', + 'infra', 'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', + 'yarn', 'zeppelin', 'zookeeper'] - logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag - in logfeeder_supported_services] + logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + \ + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] for file_name in logfeeder_config_file_names: self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name, http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py index 5a2e6e8..0378a94 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -70,10 +70,32 @@ class TestLogSearch(RMFTestCase): mode = 0644, content = '' ) - self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch.properties', - owner = 'logsearch', - group='hadoop', - content = Template('logsearch.properties.j2') + self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-portal/conf/logsearch.properties', + properties = {'logsearch.audit.logs.split.interval.mins': '1', + 'logsearch.auth.external_auth.enabled': 'false', + 'logsearch.auth.external_auth.host_url': 'http://c6401.ambari.apache.org:8080', + 'logsearch.auth.external_auth.login_url': '/api/v1/users/$USERNAME/privileges?fields=*', + 'logsearch.auth.file.enable': 'true', + 'logsearch.auth.ldap.enable': 'false', + 'logsearch.auth.simple.enable': 'false', + 'logsearch.collection.audit.logs.numshards': '10', + 'logsearch.collection.audit.logs.replication.factor': '1', + 'logsearch.collection.history.replication.factor': '1', + 'logsearch.collection.service.logs.numshards': '10', + 'logsearch.collection.service.logs.replication.factor': '1', + 'logsearch.login.credentials.file': 'logsearch-admin.json', + 'logsearch.protocol': 'http', + 'logsearch.roles.allowed': 'AMBARI.ADMINISTRATOR', + 'logsearch.service.logs.split.interval.mins': '1', + 'logsearch.solr.audit.logs.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr', + 'logsearch.solr.collection.audit.logs': 'audit_logs', + 'logsearch.solr.collection.history': 'history', + 'logsearch.solr.collection.service.logs': 'hadoop_logs', + 'logsearch.solr.history.config.name': 'history', + 'logsearch.solr.metrics.collector.hosts': '', + 'logsearch.solr.jmx.port': '1', + 'logsearch.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr' + } ) self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/HadoopServiceConfig.json', owner = 'logsearch', http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index a016ce0..8ac6fa7 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -250,6 +250,7 @@ "infra_solr_datadir": "/opt/ambari_infra_solr/data", "infra_solr_log_dir": "/var/log/ambari-infra-solr", "infra_solr_client_log_dir": "/var/log/ambari-infra-solr-client", + "infra_solr_jmx_port" : "1", "content": "# By default the script will use JAVA_HOME to determine which java\n# to use, but you can set a specific path for Solr to use without\n# affecting other Java applications on your server/workstation.\nSOLR_JAVA_HOME={{java64_home}}\n\n# Increase Java Min/Max Heap as needed to support your indexing / query needs\nSOLR_JAVA_MEM=\"-Xms{{solr_min_mem}} -Xmx{{solr_max_mem}}\"\n\n# Enable verbose GC logging\nGC_LOG_OPTS=\"-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \\\n-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime\"\n\n# These GC settings have shown to work well for a number of common Solr workloads\nGC_TUNE=\"-XX:NewRatio=3 \\\n-XX:SurvivorRatio=4 \\\n-XX:TargetSurvivorRatio=90 \\\n-XX:MaxTenuringThreshold=8 \\\n-XX:+UseConcMarkSweepGC \\\n-XX:+UseParNewGC \\\n-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \\\n-XX:+CMSScavengeBeforeRemark \\\n-XX:PretenureSizeThreshold=64m \\\n-XX:+UseCMSInitiatingOc cupancyOnly \\\n-XX:CMSInitiatingOccupancyFraction=50 \\\n-XX:CMSMaxAbortablePrecleanTime=6000 \\\n-XX:+CMSParallelRemarkEnabled \\\n-XX:+ParallelRefProcEnabled\"\n\n# Set the ZooKeeper connection string if using an external ZooKeeper ensemble\n# e.g. host1:2181,host2:2181/chroot\n# Leave empty if not using SolrCloud\nZK_HOST=\"{{zookeeper_quorum}}{{solr_znode}}\"\n\n# Set the ZooKeeper client timeout (for SolrCloud mode)\nZK_CLIENT_TIMEOUT=\"60000\"\n\n# By default the start script uses \"localhost\"; override the hostname here\n# for production SolrCloud environments to control the hostname exposed to cluster state\n#SOLR_HOST=\"192.168.1.1\"\n\n# By default the start script uses UTC; override the timezone if needed\n#SOLR_TIMEZONE=\"UTC\"\n\n# Set to true to activate the JMX RMI connector to allow remote JMX client applications\n# to monitor the JVM hosting Solr; set to \"false\" to disable that behavior\n# (false is recommended in production environments)\nENABLE_REMOTE_JMX_OPTS =\"false\"\n\n# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here\n# RMI_PORT=18983\n\n# Anything you add to the SOLR_OPTS variable will be included in the java\n# start command line as-is, in ADDITION to other options. If you specify the\n# -a option on start script, those options will be appended as well. Examples:\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.clustering.enabled=true\"\n\n# Location where the bin/solr script will save PID files for running instances\n# If not set, the script will create PID files in $SOLR_TIP/bin\nSOLR_PID_DIR={{solr_piddir}}\n\n# Path to a directory where Solr creates index files, the specified directory\n# must contain a solr.xml; by default, Solr will use server/solr\nSOLR_HOME={{infra_solr_datadir}}\n\n# Solr provides a default Log4J configuration properties file in server/resources\n# however, you may want to cu stomize the log settings and file appender location\n# so you can point the script to use a different log4j.properties file\nLOG4J_PROPS={{logsearch_solr_datadir}}/resources/log4j.properties\n\n# Location where Solr should write logs to; should agree with the file appender\n# settings in server/resources/log4j.properties\nSOLR_LOGS_DIR={{solr_log_dir}}\n\n# Sets the port Solr binds to, default is 8983\nSOLR_PORT={{solr_port}}\n\n# Be sure to update the paths to the correct keystore for your environment\n{% if logsearch_solr_ssl_enabled %}\nSOLR_SSL_KEY_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_KEY_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_TRUST_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_TRUST_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_NEED_CLIENT_AUTH=false\nSOLR_SSL_WANT_CLIENT_AUTH=false\n{% endif %}\n\n# Uncomment to set a specific SSL port (-Djetty.ssl.port=N); if not set\n# and you are using SSL, then the start script will use SOLR_PORT for the SSL port\n#SOLR_SSL_PORT=" }, "infra-solr-xml": { @@ -280,17 +281,21 @@ "content": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n<!DOCTYPE log4j:configuration SYSTEM \"log4j.dtd\">\n<log4j:configuration xmlns:log4j=\"http://jakarta. apache.org/log4j/\">\n <appender name=\"console\" class=\"org.apache.log4j.ConsoleAppender\">\n <param name=\"Target\" value=\"System.out\" />\n <layout class=\"org.apache.log4j.PatternLayout\">\n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\" />\n </layout>\n </appender>\n\n <appender name=\"rolling_file\" class=\"org.apache.log4j.RollingFileAppender\"> \n <param name=\"file\" value=\"{{logsearch_log_dir}}/logsearch.log\" /> \n <param name=\"append\" value=\"true\" /> \n <param name=\"maxFileSize\" value=\"10MB\" /> \n <param name=\"maxBackupIndex\" value=\"10\" /> \n <layout class=\"org.apache.log4j.PatternLayout\"> \n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\"/> \n </layout> \n </appender> \n\n <appender name=\"performance_analyzer\" class=\"org.apache.log4j.RollingFileAppender\" >\n <param name=\"file\" value=\"{{logsearch_log_dir}}/logsearch-performance.log\" />\n <param name=\"Threshold\" value=\"info\" />\n <param name=\"append\" value=\"true\" />\n <param name=\"maxFileSize\" value=\"10MB\" /> \n <param name=\"maxBackupIndex\" value=\"10\" /> \n <layout class=\"org.apache.log4j.PatternLayout\">\n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\" />\n </layout>\n </appender>\n \n <logger name=\"org.apache.ambari.logsearch.perfomance\" additivity=\"false\">\n <appender-ref ref=\"performance_analyzer\" />\n </logger>\n\n <category name=\"org.apache.ambari.logsearch\" additivity=\"false\">\n <priority value=\"info\" />\n <appender-ref ref=\"rolling_file\" />\n </category>\n\n <root>\n <priority value=\"warn\" />\n <appender-ref ref=\"rolling_file\" />\n </root>\ n</log4j:configuration>" }, "logsearch-properties": { + "logsearch.solr.collection.service.logs" : "hadoop_logs", + "logsearch.solr.collection.audit.logs" : "audit_logs", + "logsearch.service.logs.split.interval.mins": "1", + "logsearch.audit.logs.split.interval.mins" : "1", "logsearch.collection.service.logs.numshards": "10", "logsearch.collection.service.logs.replication.factor": "1", "logsearch.collection.audit.logs.numshards": "10", "logsearch.collection.audit.logs.replication.factor": "1", - "logsearch.app.max.memory": "1g", "logsearch.solr.metrics.collector.hosts" : "{metrics_collector_hosts}", - "logsearch.external.auth.enabled" : "false", - "logsearch.external.auth.host_url" : "{ambari_server_auth_host_url}", - "logsearch.external.auth.login_url" : "/api/v1/users/$USERNAME/privileges?fields=*" + "logsearch.auth.external_auth.enabled" : "false", + "logsearch.auth.external_auth.host_url" : "{ambari_server_auth_host_url}", + "logsearch.auth.external_auth.login_url" : "/api/v1/users/$USERNAME/privileges?fields=*" }, "logfeeder-properties": { + "logfeeder.checkpoint.folder" : "/etc/ambari-logsearch-logfeeder/conf/checkpoints", "logfeeder.metrics.collector.hosts" : "{metrics_collector_hosts}", "logfeeder.config.files" : "{default_config_files}" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/57ef91f9/ambari-web/app/data/HDP2/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js index 1fad1d6..9311e75 100644 --- a/ambari-web/app/data/HDP2/site_properties.js +++ b/ambari-web/app/data/HDP2/site_properties.js @@ -2215,21 +2215,21 @@ var hdp2properties = [ "index": 6 }, { - "name": "logsearch.external.auth.enabled", + "name": "logsearch.auth.external_auth.enable", "serviceName": "LOGSEARCH", "filename": "logsearch-properties.xml", "category": "Advanced logsearch-properties", "index": 7 }, { - "name": "logsearch.external.auth.host_url", + "name": "logsearch.auth.external_auth.host_url", "serviceName": "LOGSEARCH", "filename": "logsearch-properties.xml", "category": "Advanced logsearch-properties", "index": 8 }, { - "name": "logsearch.external.auth.login_url", + "name": "logsearch.auth.external_auth.login_url", "serviceName": "LOGSEARCH", "filename": "logsearch-properties.xml", "category": "Advanced logsearch-properties",
