AMBARI-19012 Abillity to use external Solr for Log Search instead of AMBARI_INFRA_SOLR
Change-Id: I9501d854005e52153ed0bc38ba51d6b9113120c8 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a85000b8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a85000b8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a85000b8 Branch: refs/heads/branch-feature-AMBARI-18901 Commit: a85000b8a1e9012f95b14894a44baf861792d4bf Parents: aedf2c0 Author: Miklos Gergely <[email protected]> Authored: Thu Dec 8 10:20:51 2016 +0100 Committer: Miklos Gergely <[email protected]> Committed: Thu Dec 8 10:20:51 2016 +0100 ---------------------------------------------------------------------- .../libraries/functions/package_conditions.py | 2 +- .../0.5.0/configuration/logfeeder-env.xml | 14 + .../0.5.0/configuration/logsearch-env.xml | 94 ++++++- .../LOGSEARCH/0.5.0/metainfo.xml | 34 +-- .../LOGSEARCH/0.5.0/package/scripts/params.py | 79 +++--- .../0.5.0/package/scripts/setup_logfeeder.py | 2 +- .../0.5.0/package/scripts/setup_logsearch.py | 10 +- .../0.5.0/properties/logfeeder-env.sh.j2 | 2 +- .../0.5.0/properties/logsearch-env.sh.j2 | 2 +- .../0.5.0/properties/output.config.json.j2 | 6 +- .../LOGSEARCH/0.5.0/themes/theme.json | 253 +++++++++++++++++++ .../stacks/HDP/2.2/services/stack_advisor.py | 55 ++-- .../test/python/stacks/2.4/configs/default.json | 2 +- ambari-web/app/data/HDP2/site_properties.js | 36 +-- 14 files changed, 472 insertions(+), 119 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py index 8257022..5a16061 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -70,7 +70,7 @@ def should_install_infra_solr(): def should_install_infra_solr_client(): config = Script.get_config() - return _has_applicable_local_component(config, ['INFRA_SOLR_CLIENT', 'ATLAS_SERVER', 'RANGER_ADMIN']) + return _has_applicable_local_component(config, ['INFRA_SOLR_CLIENT', 'ATLAS_SERVER', 'RANGER_ADMIN', 'LOGSEARCH_SERVER']) def should_install_logsearch_portal(): config = Script.get_config() http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml index ee885e3..aba638a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml @@ -119,6 +119,20 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>logfeeder_external_solr_kerberos_keytab</name> + <value></value> + <display-name>Logfeeder External Solr keytab</display-name> + <description>The path to the Kerberos Keytab file containing service principal of Logfeeder e.g. /etc/security/keytabs/logfeeder.service.keytab</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>logfeeder_external_solr_kerberos_principal</name> + <value></value> + <display-name>Logfeeder External Solr principal</display-name> + <description>The service principal for Logfeeder which will be used to access SolrClient e.g. logfeeder/_HOST@REALM</description> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>logfeeder_kerberos_keytab</name> <value>/etc/security/keytabs/logfeeder.service.keytab</value> <display-name>Logfeeder Solr keytab</display-name> http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml index c5b9b4e..2f13710 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml @@ -99,20 +99,100 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>logsearch_solr_audit_logs_zk_node</name> - <value>{infra_solr_znode}</value> - <display-name>Solr Audit Logs Znode</display-name> - <description>Only needed if using custom solr cloud. E.g. /audit_logs</description> + <name>logsearch_use_external_solr</name> + <value>false</value> + <display-name>Use External Solr</display-name> + <description>Use External Solr to store logs</description> + <value-attributes> + <type>value-list</type> + <overridable>false</overridable> + <entries> + <entry> + <value>true</value> + <label>ON</label> + </entry> + <entry> + <value>false</value> + <label>OFF</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>logsearch_external_solr_zk_znode</name> + <value></value> + <display-name>External Solr Logs Znode</display-name> + <description>Only needed if using custom solr cloud. E.g. /logsearch_solr</description> <on-ambari-upgrade add="true"/> </property> <property> - <name>logsearch_solr_audit_logs_zk_quorum</name> - <value>{zookeeper_quorum}</value> - <display-name>Solr Audit Logs ZK Quorum</display-name> + <name>logsearch_external_solr_zk_quorum</name> + <value></value> + <display-name>External Solr Logs ZK Quorum</display-name> <description>Only needed if using custom solr cloud. E.g. zk1:2181,zk2:2182</description> <on-ambari-upgrade add="true"/> </property> <property> + <name>logsearch_external_solr_ssl_enabled</name> + <value>false</value> + <display-name>Log Search External Solr SSL Enabled</display-name> + <description>Use SSL to connect to the external solr</description> + <value-attributes> + <type>value-list</type> + <overridable>false</overridable> + <entries> + <entry> + <value>true</value> + <label>ON</label> + </entry> + <entry> + <value>false</value> + <label>OFF</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>logsearch_external_solr_kerberos_enabled</name> + <value>false</value> + <display-name>Log Search External Solr Kerberos Enabled</display-name> + <description>Use Kerberos to connect to the external solr</description> + <value-attributes> + <type>value-list</type> + <overridable>false</overridable> + <entries> + <entry> + <value>true</value> + <label>ON</label> + </entry> + <entry> + <value>false</value> + <label>OFF</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>logsearch_external_solr_kerberos_keytab</name> + <value></value> + <display-name>Log Search External Solr keytab</display-name> + <description>The path to the Kerberos Keytab file containing service principal of Log Search e.g. /etc/security/keytabs/logsearch.service.keytab</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>logsearch_external_solr_kerberos_principal</name> + <value></value> + <display-name>Log Search External Solr principal</display-name> + <description>The service principal for Log Search which will be used to access SolrClient e.g. logsearch/_HOST@REALM</description> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>logsearch_truststore_location</name> <value>/etc/security/serverKeys/logsearch.trustStore.jks</value> <display-name>Log Search trust store location</display-name> http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml index ed74bdf..6921a8e 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -55,15 +55,6 @@ <enabled>true</enabled> </auto-deploy> </dependency> - - <dependency> - <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> - <scope>cluster</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - </dependencies> <configuration-dependencies> <config-type>infra-solr-env</config-type> @@ -95,20 +86,9 @@ </log> </logs> - <dependencies> - - <dependency> - <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> - <scope>cluster</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - - </dependencies> - <configuration-dependencies> <config-type>infra-solr-env</config-type> + <config-type>logsearch-env</config-type> <config-type>logfeeder-properties</config-type> <config-type>logfeeder-env</config-type> <config-type>logfeeder-grok</config-type> @@ -135,6 +115,10 @@ <skipUpgrade>true</skipUpgrade> <condition>should_install_logsearch_portal</condition> </package> + <package> + <name>ambari-infra-solr-client</name> + <condition>should_install_infra_solr_client</condition> + </package> </packages> </osSpecific> <osSpecific> @@ -149,6 +133,10 @@ <skipUpgrade>true</skipUpgrade> <condition>should_install_logsearch_portal</condition> </package> + <package> + <name>ambari-infra-solr-client</name> + <condition>should_install_infra_solr_client</condition> + </package> </packages> </osSpecific> </osSpecifics> @@ -159,10 +147,6 @@ <timeout>300</timeout> </commandScript> - <requiredServices> - <service>AMBARI_INFRA</service> - </requiredServices> - <quickLinksConfigurations> <quickLinksConfiguration> <fileName>quicklinks.json</fileName> http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index d086f9f..c20d92d 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -38,7 +38,6 @@ def get_port_from_url(address): config = Script.get_config() tmp_dir = Script.get_tmp_dir() -stack_version = default("/commandParams/version", None) sudo = AMBARI_SUDO_BINARY security_enabled = status_params.security_enabled @@ -56,16 +55,10 @@ logfeeder_pid_dir = status_params.logfeeder_pid_dir logfeeder_pid_file = status_params.logfeeder_pid_file user_group = config['configurations']['cluster-env']['user_group'] -fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_groups"] # shared configs java64_home = config['hostLevelParams']['java_home'] -zookeeper_hosts_list = config['clusterHostInfo']['zookeeper_hosts'] -zookeeper_hosts_list.sort() -# get comma separated list of zookeeper hosts from clusterHostInfo -zookeeper_hosts = ",".join(zookeeper_hosts_list) cluster_name = str(config['clusterName']) -availableServices = config['availableServices'] configurations = config['configurations'] # need reference inside logfeeder jinja templates logserch_meta_configs = get_logsearch_meta_configs(configurations) @@ -84,30 +77,31 @@ else: ##################################### # Infra Solr configs ##################################### -infra_solr_znode = default('/configurations/infra-solr-env/infra_solr_znode', '/infra-solr') -infra_solr_instance_count = len(config['clusterHostInfo']['infra_solr_hosts']) -infra_solr_ssl_enabled = default('configurations/infra-solr-env/infra_solr_ssl_enabled', False) -infra_solr_jmx_port = config['configurations']['infra-solr-env']['infra_solr_jmx_port'] - -zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) -index = 0 -zookeeper_quorum = "" -for host in config['clusterHostInfo']['zookeeper_hosts']: - zookeeper_quorum += host + ":" + str(zookeeper_port) - index += 1 - if index < len(config['clusterHostInfo']['zookeeper_hosts']): - zookeeper_quorum += "," +infra_solr_znode = '/infra-solr' +infra_solr_ssl_enabled = False +infra_solr_jmx_port = '' +if 'infra-solr-env' in config['configurations']: + infra_solr_znode = default('/configurations/infra-solr-env/infra_solr_znode', '/infra-solr') + infra_solr_ssl_enabled = default('configurations/infra-solr-env/infra_solr_ssl_enabled', False) + infra_solr_jmx_port = config['configurations']['infra-solr-env']['infra_solr_jmx_port'] if security_enabled: kinit_path_local = status_params.kinit_path_local _hostname_lowercase = config['hostname'].lower() logsearch_jaas_file = logsearch_server_conf + '/logsearch_jaas.conf' logfeeder_jaas_file = logsearch_logfeeder_conf + '/logfeeder_jaas.conf' - logsearch_kerberos_keytab = config['configurations']['logsearch-env']['logsearch_kerberos_keytab'] - logsearch_kerberos_principal = config['configurations']['logsearch-env']['logsearch_kerberos_principal'].replace('_HOST',_hostname_lowercase) - logfeeder_kerberos_keytab = config['configurations']['logfeeder-env']['logfeeder_kerberos_keytab'] - logfeeder_kerberos_principal = config['configurations']['logfeeder-env']['logfeeder_kerberos_principal'].replace('_HOST',_hostname_lowercase) + use_external_solr_with_kerberos = default('configurations/logsearch-env/logsearch_external_solr_kerberos_enabled', False) + if use_external_solr_with_kerberos: + logsearch_kerberos_keytab = config['configurations']['logsearch-env']['logsearch_external_solr_kerberos_keytab'] + logsearch_kerberos_principal = config['configurations']['logsearch-env']['logsearch_external_solr_kerberos_principal'].replace('_HOST',_hostname_lowercase) + logfeeder_kerberos_keytab = config['configurations']['logfeeder-env']['logfeeder_external_solr_kerberos_keytab'] + logfeeder_kerberos_principal = config['configurations']['logfeeder-env']['logfeeder_external_solr_kerberos_principal'].replace('_HOST',_hostname_lowercase) + else: + logsearch_kerberos_keytab = config['configurations']['logsearch-env']['logsearch_kerberos_keytab'] + logsearch_kerberos_principal = config['configurations']['logsearch-env']['logsearch_kerberos_principal'].replace('_HOST',_hostname_lowercase) + logfeeder_kerberos_keytab = config['configurations']['logfeeder-env']['logfeeder_kerberos_keytab'] + logfeeder_kerberos_principal = config['configurations']['logfeeder-env']['logfeeder_kerberos_principal'].replace('_HOST',_hostname_lowercase) ##################################### # Logsearch configs @@ -120,10 +114,29 @@ logsearch_service_logs_merge_factor = config['configurations']['logsearch-servic logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention'] logsearch_audit_logs_merge_factor = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_merge_factor'] -logsearch_solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node', infra_solr_znode) -logsearch_solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum', zookeeper_quorum) -logsearch_solr_audit_logs_zk_node = format(logsearch_solr_audit_logs_zk_node) -logsearch_solr_audit_logs_zk_quorum = format(logsearch_solr_audit_logs_zk_quorum) +logsearch_use_external_solr = default('/configurations/logsearch-env/logsearch_use_external_solr', False) + +if logsearch_use_external_solr: + logsearch_solr_zk_znode = config['configurations']['logsearch-env']['logsearch_external_solr_zk_znode'] + logsearch_solr_zk_quorum = config['configurations']['logsearch-env']['logsearch_external_solr_zk_quorum'] + logsearch_solr_ssl_enabled = default('configurations/logsearch-env/logsearch_external_solr_ssl_enabled', False) + logsearch_solr_kerberos_enabled = security_enabled and default('configurations/logsearch-env/logsearch_external_solr_kerberos_enabled', False) +else: + logsearch_solr_zk_znode = infra_solr_znode + + logsearch_solr_zk_quorum = "" + zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) + if 'zookeeper_hosts' in config['clusterHostInfo']: + for host in config['clusterHostInfo']['zookeeper_hosts']: + if logsearch_solr_zk_quorum: + logsearch_solr_zk_quorum += ',' + logsearch_solr_zk_quorum += host + ":" + str(zookeeper_port) + + logsearch_solr_ssl_enabled = infra_solr_ssl_enabled + logsearch_solr_kerberos_enabled = security_enabled + +zookeeper_quorum = logsearch_solr_zk_quorum + # logsearch-env configs @@ -179,8 +192,8 @@ logsearch_properties = {} # default values -logsearch_properties['logsearch.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode -logsearch_properties['logsearch.solr.audit.logs.zk_connect_string'] = logsearch_solr_audit_logs_zk_quorum + logsearch_solr_audit_logs_zk_node +logsearch_properties['logsearch.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode +logsearch_properties['logsearch.solr.audit.logs.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode logsearch_properties['logsearch.solr.collection.history'] = 'history' logsearch_properties['logsearch.solr.history.config.name'] = 'history' @@ -210,7 +223,7 @@ del logsearch_properties['logsearch.solr.audit.logs.use.ranger'] logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearch_properties['logsearch.solr.metrics.collector.hosts']) logsearch_properties['logsearch.auth.external_auth.host_url'] = format(logsearch_properties['logsearch.auth.external_auth.host_url']) -if security_enabled: +if logsearch_solr_kerberos_enabled: logsearch_properties['logsearch.solr.kerberos.enable'] = 'true' logsearch_properties['logsearch.solr.jaas.file'] = logsearch_jaas_file @@ -293,9 +306,9 @@ logfeeder_properties = dict(logfeeder_properties.items() + dict(config['configur logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts']) logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files']) -logfeeder_properties['logfeeder.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode +logfeeder_properties['logfeeder.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode -if security_enabled: +if logsearch_solr_kerberos_enabled: if 'logfeeder.solr.kerberos.enable' not in logfeeder_properties: logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true' if 'logfeeder.solr.jaas.file' not in logfeeder_properties: http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py index 9582334..14f8d20 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py @@ -92,7 +92,7 @@ def setup_logfeeder(): ) - if params.security_enabled: + if params.logsearch_solr_kerberos_enabled: File(format("{logfeeder_jaas_file}"), content=Template("logfeeder_jaas.conf.j2") ) http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py index 351126b..874b90b 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -118,10 +118,10 @@ def setup_logsearch(): def upload_conf_set(config_set, solrconfig_content = None): import params - jaas_file = params.logsearch_jaas_file if params.security_enabled else None + jaas_file = params.logsearch_jaas_file if params.logsearch_solr_kerberos_enabled else None solr_cloud_util.upload_configuration_to_zk( - zookeeper_quorum=params.zookeeper_quorum, - solr_znode=params.infra_solr_znode, + zookeeper_quorum=params.logsearch_solr_zk_quorum, + solr_znode=params.logsearch_solr_zk_znode, config_set_dir=format("{logsearch_server_conf}/solr_configsets/{config_set}/conf"), config_set=config_set, tmp_dir=params.tmp_dir, @@ -134,7 +134,7 @@ def upload_conf_set(config_set, solrconfig_content = None): def check_znode(): import params solr_cloud_util.check_znode( - zookeeper_quorum=params.zookeeper_quorum, - solr_znode=params.infra_solr_znode, + zookeeper_quorum=params.logsearch_solr_zk_quorum, + solr_znode=params.logsearch_solr_zk_znode, java64_home=params.java64_home, retry=30, interval=5) http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 index 2818708..6795dab 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 @@ -32,7 +32,7 @@ if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then export LOGFEEDER_JAVA_MEM=-Xmx{{logfeeder_max_mem}} fi -{% if infra_solr_ssl_enabled %} +{% if logsearch_solr_ssl_enabled %} export LOGFEEDER_SSL="true" export LOGFEEDER_KEYSTORE_LOCATION={{logfeeder_keystore_location}} export LOGFEEDER_KEYSTORE_PASSWORD={{logfeeder_keystore_password}} http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 index eb7306c..a179983 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 @@ -38,7 +38,7 @@ export LOGSEARCH_DEBUG={{logsearch_debug_enabled}} export LOGSEARCH_DEBUG_PORT={{logsearch_debug_port}} -{% if infra_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl %} +{% if logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl %} export LOGSEARCH_SSL="true" export LOGSEARCH_KEYSTORE_LOCATION={{logsearch_keystore_location}} export LOGSEARCH_KEYSTORE_PASSWORD={{logsearch_keystore_password}} http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 index 062d636..214e5ba 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 @@ -21,7 +21,7 @@ "comment":"Output to solr for service logs", "is_enabled":"{{solr_service_logs_enable}}", "destination":"solr", - "zk_connect_string":"{{zookeeper_quorum}}{{infra_solr_znode}}", + "zk_connect_string":"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}", "collection":"{{logsearch_solr_collection_service_logs}}", "number_of_shards": "{{logsearch_collection_service_logs_numshards}}", "splits_interval_mins": "{{logsearch_service_logs_split_interval_mins}}", @@ -40,7 +40,7 @@ "comment":"Output to solr for audit records", "is_enabled":"{{solr_audit_logs_enable}}", "destination":"solr", - "zk_connect_string":"{{zookeeper_quorum}}{{infra_solr_znode}}", + "zk_connect_string":"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}", "collection":"{{logsearch_solr_collection_audit_logs}}", "number_of_shards": "{{logsearch_collection_audit_logs_numshards}}", "splits_interval_mins": "{{logsearch_audit_logs_split_interval_mins}}", @@ -57,5 +57,5 @@ } ] - + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json index 0193689..2858825 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json @@ -57,6 +57,25 @@ "column-span": "1" } ] + }, + { + "name": "section-logsearch-solr-connection", + "display-name": "Solr Connection", + "row-index": "4", + "column-index": "1", + "row-span": "3", + "column-span": "1", + "section-columns": "1", + "section-rows": "4", + "subsections": [ + { + "name": "subsection-logsearch-solr-connection-col1", + "row-index": "4", + "column-index": "0", + "row-span": "3", + "column-span": "1" + } + ] } ] } @@ -138,6 +157,186 @@ } } ] + }, + { + "config": "logsearch-env/logsearch_use_external_solr", + "subsection-name": "subsection-logsearch-solr-connection-col1" + }, + { + "config": "logsearch-env/logsearch_external_solr_zk_znode", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-env/logsearch_use_external_solr" + ], + "if": "${logsearch-env/logsearch_use_external_solr}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-env/logsearch_external_solr_zk_quorum", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-env/logsearch_use_external_solr" + ], + "if": "${logsearch-env/logsearch_use_external_solr}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-env/logsearch_external_solr_ssl_enabled", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-env/logsearch_use_external_solr" + ], + "if": "${logsearch-env/logsearch_use_external_solr}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_enabled", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-env/logsearch_use_external_solr" + ], + "if": "${logsearch-env/logsearch_use_external_solr}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_keytab", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-env/logsearch_external_solr_kerberos_enabled" + ], + "if": "${logsearch-env/logsearch_external_solr_kerberos_enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_principal", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-env/logsearch_external_solr_kerberos_enabled" + ], + "if": "${logsearch-env/logsearch_external_solr_kerberos_enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logfeeder-env/logfeeder_external_solr_kerberos_keytab", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-env/logsearch_external_solr_kerberos_enabled" + ], + "if": "${logsearch-env/logsearch_external_solr_kerberos_enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logfeeder-env/logfeeder_external_solr_kerberos_principal", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-env/logsearch_external_solr_kerberos_enabled" + ], + "if": "${logsearch-env/logsearch_external_solr_kerberos_enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] } ] }, @@ -220,6 +419,60 @@ "widget": { "type": "directories" } + }, + { + "config": "logsearch-env/logsearch_use_external_solr", + "widget": { + "type": "toggle" + } + }, + { + "config": "logsearch-env/logsearch_external_solr_zk_znode", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-env/logsearch_external_solr_zk_quorum", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-env/logsearch_external_solr_ssl_enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_keytab", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_principal", + "widget": { + "type": "text-field" + } + }, + { + "config": "logfeeder-env/logfeeder_external_solr_kerberos_keytab", + "widget": { + "type": "text-field" + } + }, + { + "config": "logfeeder-env/logfeeder_external_solr_kerberos_principal", + "widget": { + "type": "text-field" + } } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 4f0a9d0..4802193 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -1015,27 +1015,50 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def recommendLogsearchConfigurations(self, configurations, clusterData, services, hosts): putLogsearchProperty = self.putProperty(configurations, "logsearch-properties", services) + putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties") + putLogsearchEnvProperty = self.putProperty(configurations, "logsearch-env", services) + putLogsearchEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-env") + putLogfeederEnvAttribute = self.putPropertyAttribute(configurations, "logfeeder-env") + infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR") - if infraSolrHosts is not None and len(infraSolrHosts) > 0 \ - and "logsearch-properties" in services["configurations"]: + if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]: + replicationReccomendFloat = math.log(len(infraSolrHosts), 5) + recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat)) + recommendedMinShards = len(infraSolrHosts) recommendedShards = 2 * len(infraSolrHosts) recommendedMaxShards = 3 * len(infraSolrHosts) - # recommend number of shard - putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties") - putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards) - putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards) - putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards) - - putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards) - putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards) - putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards) - # recommend replication factor - replicationReccomendFloat = math.log(len(infraSolrHosts), 5) - recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat)) - putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor) - putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor) + else: + recommendedReplicationFactor = 2 + + recommendedMinShards = 1 + recommendedShards = 1 + recommendedMaxShards = 100 + + putLogsearchEnvProperty('logsearch_use_external_solr', 'true') + putLogsearchEnvAttribute('logsearch_use_external_solr', 'visible', 'false') + + # recommend number of shard + putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards) + putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards) + putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards) + + putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards) + putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards) + putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards) + # recommend replication factor + putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor) + putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor) + + kerberos_authentication_enabled = self.isSecurityEnabled(services) + if not kerberos_authentication_enabled: + putLogsearchEnvProperty('logsearch_external_solr_kerberos_enabled', 'false') + putLogsearchEnvAttribute('logsearch_external_solr_kerberos_enabled', 'visible', 'false') + putLogsearchEnvAttribute('logsearch_external_solr_kerberos_keytab', 'visible', 'false') + putLogsearchEnvAttribute('logsearch_external_solr_kerberos_principal', 'visible', 'false') + putLogfeederEnvAttribute('logfeeder_external_solr_kerberos_keytab', 'visible', 'false') + putLogfeederEnvAttribute('logfeeder_external_solr_kerberos_principal', 'visible', 'false') def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
