Repository: ambari Updated Branches: refs/heads/branch-2.5 8ca19525a -> 50e36c7f8
AMBARI-19330 LogSearch upgrade support 2.4 -> 2.5 Change-Id: If6d7a7f94c2e858cbd7d73ad0c9440aeec89c0ab Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/50e36c7f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/50e36c7f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/50e36c7f Branch: refs/heads/branch-2.5 Commit: 50e36c7f80b12b03106ab8277956d7999eb9b8da Parents: 8ca1952 Author: Miklos Gergely <[email protected]> Authored: Fri Jan 6 00:39:08 2017 +0100 Committer: Miklos Gergely <[email protected]> Committed: Fri Jan 6 00:39:08 2017 +0100 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 80 ++++++++++++ .../configuration/logfeeder-ambari-config.xml | 2 +- .../logfeeder-custom-logsearch-conf.xml | 6 +- .../0.5.0/configuration/logfeeder-env.xml | 38 +++--- .../0.5.0/configuration/logfeeder-log4j.xml | 20 +-- .../configuration/logfeeder-output-config.xml | 4 +- .../configuration/logfeeder-properties.xml | 8 +- .../configuration/logfeeder-system_log-env.xml | 2 + .../logsearch-audit_logs-solrconfig.xml | 2 +- .../0.5.0/configuration/logsearch-log4j.xml | 42 +++---- .../0.5.0/properties/logsearch-log4j.xml.j2 | 18 +-- .../server/upgrade/UpgradeCatalog250Test.java | 125 +++++++++++++++++-- ambari-web/app/data/HDP2/site_properties.js | 88 ++++++++++++- 13 files changed, 355 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index f6fa118..3701e2f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -158,6 +159,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { updateHiveLlapConfigs(); updateTablesForZeppelinViewRemoval(); updateAtlasConfigs(); + updateLogSearchConfigs(); addManageServiceAutoStartPermissions(); } @@ -594,6 +596,84 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { } /** + * Updates Log Search configs. + * + * @throws AmbariException + */ + protected void updateLogSearchConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Config logSearchProperties = cluster.getDesiredConfigByType("logsearch-properties"); + if (logSearchProperties != null) { + Map<String, String> newProperties = new HashMap<>(); + newProperties.put("logsearch.auth.external_auth.enabled", logSearchProperties.getProperties().get("logsearch.external.auth.enabled")); + newProperties.put("logsearch.auth.external_auth.host_url", logSearchProperties.getProperties().get("logsearch.external.auth.host_url")); + newProperties.put("logsearch.auth.external_auth.login_url", logSearchProperties.getProperties().get("logsearch.external.auth.login_url")); + + Set<String> removeProperties = new HashSet<>(); + removeProperties.add("logsearch.external.auth.enabled"); + removeProperties.add("logsearch.external.auth.host_url"); + removeProperties.add("logsearch.external.auth.login_url"); + + updateConfigurationPropertiesForCluster(cluster, "logsearch-properties", newProperties, removeProperties, true, true); + } + + Config logfeederEnvProperties = cluster.getDesiredConfigByType("logfeeder-env"); + if (logfeederEnvProperties != null) { + String content = logfeederEnvProperties.getProperties().get("content"); + if (content.contains("infra_solr_ssl_enabled")) { + content = content.replace("infra_solr_ssl_enabled", "logsearch_solr_ssl_enabled"); + updateConfigurationPropertiesForCluster(cluster, "logfeeder-env", Collections.singletonMap("content", content), true, true); + } + } + + Config logsearchEnvProperties = cluster.getDesiredConfigByType("logsearch-env"); + if (logsearchEnvProperties != null) { + Map<String, String> newProperties = new HashMap<>(); + String content = logsearchEnvProperties.getProperties().get("content"); + if (content.contains("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'")) { + content = content.replace("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'", + "infra_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl"); + } + if (content.contains("infra_solr_ssl_enabled")) { + content = content.replace("infra_solr_ssl_enabled", "logsearch_solr_ssl_enabled"); + } + if (!content.equals(logsearchEnvProperties.getProperties().get("content"))) { + newProperties.put("content", content); + } + + Set<String> removeProperties = new HashSet<>(); + removeProperties.add("logsearch_solr_audit_logs_use_ranger"); + removeProperties.add("logsearch_solr_audit_logs_zk_node"); + removeProperties.add("logsearch_solr_audit_logs_zk_quorum"); + + updateConfigurationPropertiesForCluster(cluster, "logsearch-env", newProperties, removeProperties, true, true); + } + + Config logsearchLog4jProperties = cluster.getDesiredConfigByType("logsearch-log4j"); + if (logsearchLog4jProperties != null) { + String content = logsearchLog4jProperties.getProperties().get("content"); + if (content.contains("{{logsearch_log_dir}}/logsearch.err")) { + content = content.replace("{{logsearch_log_dir}}/logsearch.err", "{{logsearch_log_dir}}/logsearch.log"); + } + if (content.contains("<priority value=\"warn\"/>")) { + content = content.replace("<priority value=\"warn\"/>", "<priority value=\"info\"/>"); + } + if (!content.equals(logsearchLog4jProperties.getProperties().get("content"))) { + updateConfigurationPropertiesForCluster(cluster, "logsearch-log4j", Collections.singletonMap("content", content), true, true); + } + } + } + } + } + } + + /** * Add permissions for managing service auto-start. * <p> * <ul> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml index 64d7946..19bbd4a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml @@ -22,7 +22,7 @@ <configuration supports_final="false" supports_adding_forbidden="true"> <property> <name>content</name> - <display-name>Ambari LogFeeder config</display-name> + <display-name>Ambari Log Feeder config</display-name> <description>Grok filters and input file locations for ambari related log files</description> <value/> <property-type>VALUE_FROM_PROPERTY_FILE</property-type> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml index 8358266..30a93a5 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml @@ -23,7 +23,7 @@ <property> <name>service_component_mappings</name> <display-name>Service Component mapping</display-name> - <description>Logsearch service component logid mapping list (e.g.: SERVICE1=S1_COMPONENT1:logid1,logid2;S1_COMPONENT2:logid3|SERVICE2=...)</description> + <description>Log Search service component logid mapping list (e.g.: SERVICE1=S1_COMPONENT1:logid1,logid2;S1_COMPONENT2:logid3|SERVICE2=...)</description> <value></value> <value-attributes> <empty-value-valid>true</empty-value-valid> @@ -32,8 +32,8 @@ </property> <property> <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <display-name>Log Feeder Config</display-name> + <description>Metadata jinja template for Log Feeder which contains grok patterns for reading service specific logs.</description> <value> </value> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml index bc65e3c..fb641a8 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml @@ -24,8 +24,8 @@ <property> <name>logfeeder_pid_dir</name> <value>/var/run/ambari-logsearch-logfeeder</value> - <description>logfeeder Process ID Directory</description> - <display-name>Logfeeder pid dir</display-name> + <description>Log Feeder Process ID Directory</description> + <display-name>Log Feeder pid dir</display-name> <value-attributes> <type>directory</type> </value-attributes> @@ -34,8 +34,8 @@ <property> <name>logfeeder_log_dir</name> <value>/var/log/ambari-logsearch-logfeeder</value> - <description>Log dir for logfeeder</description> - <display-name>Logfeeder log dir</display-name> + <description>Log dir for Log Feeder</description> + <display-name>Log Feeder log dir</display-name> <value-attributes> <type>directory</type> </value-attributes> @@ -45,7 +45,7 @@ <name>logfeeder_max_mem</name> <value>512m</value> <description>Max Memory for Logfeeder</description> - <display-name>Logfeeder max memory</display-name> + <display-name>Log Feeder max memory</display-name> <on-ambari-upgrade add="true"/> </property> <property> @@ -71,21 +71,21 @@ <property> <name>logfeeder_truststore_location</name> <value>/etc/security/serverKeys/logsearch.trustStore.jks</value> - <display-name>Logfeeder trust store location</display-name> + <display-name>Log Feeder trust store location</display-name> <description>Location of the trust store file.</description> <on-ambari-upgrade add="true"/> </property> <property> <name>logfeeder_truststore_type</name> <value>jks</value> - <display-name>Logfeeder trust store type</display-name> + <display-name>Log Feeder trust store type</display-name> <description>Type of the trust store file.</description> <on-ambari-upgrade add="true"/> </property> <property> <name>logfeeder_truststore_password</name> <value>bigdata</value> - <display-name>Logfeeder trust store password</display-name> + <display-name>Log Feeder trust store password</display-name> <property-type>PASSWORD</property-type> <description>Password to open the trust store file.</description> <value-attributes> @@ -96,21 +96,21 @@ <property> <name>logfeeder_keystore_location</name> <value>/etc/security/serverKeys/logsearch.keyStore.jks</value> - <display-name>Logfeeder key store location</display-name> + <display-name>Log Feeder key store location</display-name> <description>Location of the key store file.</description> <on-ambari-upgrade add="true"/> </property> <property> <name>logfeeder_keystore_type</name> <value>jks</value> - <display-name>Logfeeder store type</display-name> + <display-name>Log Feeder store type</display-name> <description>Type of the key store file.</description> <on-ambari-upgrade add="true"/> </property> <property> <name>logfeeder_keystore_password</name> <value>bigdata</value> - <display-name>Logfeeder key store password</display-name> + <display-name>Log Feeder key store password</display-name> <property-type>PASSWORD</property-type> <description>Password to open the key store file.</description> <value-attributes> @@ -121,29 +121,29 @@ <property> <name>logfeeder_external_solr_kerberos_keytab</name> <value></value> - <display-name>Logfeeder External Solr keytab</display-name> - <description>The path to the Kerberos Keytab file containing service principal of Logfeeder e.g. /etc/security/keytabs/logfeeder.service.keytab</description> + <display-name>Log Feeder External Solr keytab</display-name> + <description>The path to the Kerberos Keytab file containing service principal of Log Feeder e.g. /etc/security/keytabs/logfeeder.service.keytab</description> <on-ambari-upgrade add="true"/> </property> <property> <name>logfeeder_external_solr_kerberos_principal</name> <value></value> - <display-name>Logfeeder External Solr principal</display-name> - <description>The service principal for Logfeeder which will be used to access SolrClient e.g. logfeeder/_HOST@REALM</description> + <display-name>Log Feeder External Solr principal</display-name> + <description>The service principal for Log Feeder which will be used to access SolrClient e.g. logfeeder/_HOST@REALM</description> <on-ambari-upgrade add="true"/> </property> <property> <name>logfeeder_kerberos_keytab</name> <value>/etc/security/keytabs/logfeeder.service.keytab</value> - <display-name>Logfeeder Solr keytab</display-name> - <description>The path to the Kerberos Keytab file containing service principal of the Logfeeder.</description> + <display-name>Log Feeder Solr keytab</display-name> + <description>The path to the Kerberos Keytab file containing service principal of the Log Feeder.</description> <on-ambari-upgrade add="true"/> </property> <property> <name>logfeeder_kerberos_principal</name> <value>logfeeder</value> - <display-name>Logfeeder Solr principal</display-name> - <description>The service principal for Logfeeder which will be used to access Log Search Solr.</description> + <display-name>Log Feeder Solr principal</display-name> + <description>The service principal for Log Feeder which will be used to access Log Search Solr.</description> <on-ambari-upgrade add="true"/> </property> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml index 9371530..1843e75 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml @@ -25,38 +25,38 @@ <name>logfeeder_log_maxfilesize</name> <value>10</value> <description>The maximum size of backup file before the log is rotated</description> - <display-name>Logfeeder Log: backup file size</display-name> + <display-name>Log Feeder Log: backup file size</display-name> <value-attributes> <unit>MB</unit> </value-attributes> <on-ambari-upgrade add="false"/> - </property> - <property> + </property> + <property> <name>logfeeder_log_maxbackupindex</name> <value>10</value> <description>The number of backup files</description> - <display-name>Logfeeder Log: # of backup files</display-name> + <display-name>Log Feeder Log: # of backup files</display-name> <value-attributes> <type>int</type> <minimum>0</minimum> </value-attributes> <on-ambari-upgrade add="false"/> </property> - <property> + <property> <name>logfeeder_json_log_maxfilesize</name> <value>10</value> <description>The maximum size of backup file before the log is rotated</description> - <display-name>Logfeeder Json Log: backup file size</display-name> + <display-name>Log Feeder Json Log: backup file size</display-name> <value-attributes> <unit>MB</unit> </value-attributes> <on-ambari-upgrade add="false"/> - </property> - <property> + </property> + <property> <name>logfeeder_json_log_maxbackupindex</name> <value>10</value> <description>The number of backup files</description> - <display-name>Logfeeder Json Log: # of backup files</display-name> + <display-name>Log Feeder Json Log: # of backup files</display-name> <value-attributes> <type>int</type> <minimum>0</minimum> @@ -66,7 +66,7 @@ <property> <name>content</name> <display-name>logfeeder-log4j template</display-name> - <description>This is the jinja template for log4j.xml file for logfeeder</description> + <description>This is the jinja template for log4j.xml file for Log Feeder</description> <value/> <property-type>VALUE_FROM_PROPERTY_FILE</property-type> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml index 85654c8..652918a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml @@ -22,8 +22,8 @@ <configuration supports_final="false" supports_adding_forbidden="true"> <property> <name>content</name> - <display-name>LogFeeder output config</display-name> - <description>Output configuration for LogFeeder (data shipping)</description> + <display-name>Log Feeder output config</display-name> + <description>Output configuration for Log Feeder (data shipping)</description> <value/> <property-type>VALUE_FROM_PROPERTY_FILE</property-type> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml index e7cd476..2acb57e 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml @@ -24,14 +24,14 @@ <name>logfeeder.config.files</name> <value>{default_config_files}</value> <description>Comma separated config files in grok format</description> - <display-name>Logfeeder config files</display-name> + <display-name>Log Feeder config files</display-name> <on-ambari-upgrade add="true"/> </property> <property> <name>logfeeder.checkpoint.folder</name> <value>/etc/ambari-logsearch-logfeeder/conf/checkpoints</value> - <description>Checkpoint folder for logfeeder</description> - <display-name>Logfeeder checkpoint dir</display-name> + <description>Checkpoint folder for Log Feeder</description> + <display-name>Log Feeder checkpoint dir</display-name> <value-attributes> <type>directory</type> </value-attributes> @@ -40,7 +40,7 @@ <property> <name>logfeeder.metrics.collector.hosts</name> <value>{metrics_collector_hosts}</value> - <description>Metrics collector hosts for pushing metrics by Logfeeder</description> + <description>Metrics collector hosts for pushing metrics by Log Feeder</description> <display-name>Metrics collector hosts</display-name> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml index b8dd4b9..0461409 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml @@ -49,6 +49,8 @@ <property> <name>logfeeder_system_log_enabled</name> + <display-name>Enable system logs</display-name> + <description>Enable Log Feeders to load system logs to the Log Search solr</description> <value>false</value> <value-attributes> <type>boolean</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-audit_logs-solrconfig.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-audit_logs-solrconfig.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-audit_logs-solrconfig.xml index 6272dd1..5a07b69 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-audit_logs-solrconfig.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-audit_logs-solrconfig.xml @@ -49,7 +49,7 @@ <!-- solrconfig.xml --> <property> <name>content</name> - <display-name>solrconfig template</display-name> + <display-name>Solrconfig template</display-name> <description>This is the jinja template for solrconfig.xml file for audit logs</description> <value/> <property-type>VALUE_FROM_PROPERTY_FILE</property-type> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-log4j.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-log4j.xml index 66b9bba..729eba6 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-log4j.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-log4j.xml @@ -22,84 +22,84 @@ <!-- This is a special config file for properties used to monitor status of the service --> <configuration supports_adding_forbidden="true"> <!-- log4j.xml --> - <property> + <property> <name>logsearch_log_maxfilesize</name> <value>10</value> <description>The maximum size of backup file before the log is rotated</description> - <display-name>Logsearch Log: backup file size</display-name> + <display-name>Log Search Log: backup file size</display-name> <value-attributes> <unit>MB</unit> </value-attributes> <on-ambari-upgrade add="false"/> - </property> - <property> + </property> + <property> <name>logsearch_log_maxbackupindex</name> <value>10</value> <description>The number of backup files</description> - <display-name>Logsearch Log: # of backup files</display-name> + <display-name>Log Search Log: # of backup files</display-name> <value-attributes> <type>int</type> <minimum>0</minimum> </value-attributes> <on-ambari-upgrade add="false"/> </property> - <property> + <property> <name>logsearch_json_log_maxfilesize</name> <value>10</value> <description>The maximum size of backup file before the log is rotated</description> - <display-name>Logsearch Json Log: backup file size</display-name> + <display-name>Log Search Json Log: backup file size</display-name> <value-attributes> <unit>MB</unit> </value-attributes> <on-ambari-upgrade add="false"/> - </property> - <property> + </property> + <property> <name>logsearch_json_log_maxbackupindex</name> <value>10</value> <description>The number of backup files</description> - <display-name>Logsearch Json Log: # of backup files</display-name> + <display-name>Log Search Json Log: # of backup files</display-name> <value-attributes> <type>int</type> <minimum>0</minimum> </value-attributes> <on-ambari-upgrade add="false"/> </property> - <property> + <property> <name>logsearch_audit_log_maxfilesize</name> <value>10</value> <description>The maximum size of backup file before the log is rotated</description> - <display-name>Logsearch Audit Log: backup file size</display-name> + <display-name>Log Search Audit Log: backup file size</display-name> <value-attributes> <unit>MB</unit> </value-attributes> <on-ambari-upgrade add="false"/> - </property> - <property> + </property> + <property> <name>logsearch_audit_log_maxbackupindex</name> <value>10</value> <description>The number of backup files</description> - <display-name>Logsearch Audit Log: # of backup files</display-name> + <display-name>Log Search Audit Log: # of backup files</display-name> <value-attributes> <type>int</type> <minimum>0</minimum> </value-attributes> <on-ambari-upgrade add="false"/> </property> - <property> + <property> <name>logsearch_perf_log_maxfilesize</name> <value>10</value> <description>The maximum size of backup file before the log is rotated</description> - <display-name>Logsearch Performance Log: backup file size</display-name> + <display-name>Log Search Performance Log: backup file size</display-name> <value-attributes> <unit>MB</unit> </value-attributes> <on-ambari-upgrade add="false"/> - </property> - <property> + </property> + <property> <name>logsearch_perf_log_maxbackupindex</name> <value>10</value> <description>The number of backup files</description> - <display-name>Logsearch Performance Log: # of backup files</display-name> + <display-name>Log Search Performance Log: # of backup files</display-name> <value-attributes> <type>int</type> <minimum>0</minimum> @@ -109,7 +109,7 @@ <property> <name>content</name> <display-name>logsearch-log4j template</display-name> - <description>This is the jinja template for log4j.xml file for logsearch server</description> + <description>This is the jinja template for log4j.xml file for Log Search server</description> <value/> <property-type>VALUE_FROM_PROPERTY_FILE</property-type> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-log4j.xml.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-log4j.xml.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-log4j.xml.j2 index cc5addf..63f9f0a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-log4j.xml.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-log4j.xml.j2 @@ -25,13 +25,13 @@ limitations under the License. </appender> <appender name="rolling_file" class="org.apache.log4j.RollingFileAppender"> - <param name="file" value="{{logsearch_log_dir}}/logsearch.log" /> - <param name="Threshold" value="info" /> - <param name="append" value="true" /> - <param name="maxFileSize" value="{{logsearch_log_maxfilesize}}MB" /> - <param name="maxBackupIndex" value="{{logsearch_log_maxbackupindex}}" /> + <param name="file" value="{{logsearch_log_dir}}/logsearch.log"/> + <param name="Threshold" value="info"/> + <param name="append" value="true"/> + <param name="maxFileSize" value="{{logsearch_log_maxfilesize}}MB"/> + <param name="maxBackupIndex" value="{{logsearch_log_maxbackupindex}}"/> <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/> </layout> </appender> @@ -40,7 +40,7 @@ limitations under the License. <param name="append" value="true"/> <param name="maxFileSize" value="{{logsearch_json_log_maxfilesize}}MB"/> <param name="maxBackupIndex" value="{{logsearch_json_log_maxbackupindex}}"/> - <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion" /> + <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion"/> </appender> <appender name="audit_rolling_file_json" class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender"> @@ -48,7 +48,7 @@ limitations under the License. <param name="append" value="true"/> <param name="maxFileSize" value="{{logsearch_audit_log_maxfilesize}}MB"/> <param name="maxBackupIndex" value="{{logsearch_audit_log_maxbackupindex}}"/> - <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion" /> + <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion"/> </appender> <appender name="performance_analyzer_json" class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender"> @@ -57,7 +57,7 @@ limitations under the License. <param name="append" value="true"/> <param name="maxFileSize" value="{{logsearch_perf_log_maxfilesize}}MB"/> <param name="maxBackupIndex" value="{{logsearch_perf_log_maxbackupindex}}"/> - <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion" /> + <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion"/> </appender> <logger name="org.apache.ambari.logsearch.audit" additivity="true"> http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index 370588a..c89522f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.upgrade; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.gson.Gson; import com.google.inject.Binder; @@ -274,6 +275,7 @@ public class UpgradeCatalog250Test { Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); Method updateHIVEInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHIVEInteractiveConfigs"); Method updateTEZInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateTEZInteractiveConfigs"); + Method updateLogSearchConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateLogSearchConfigs"); Method addManageServiceAutoStartPermissions = UpgradeCatalog250.class.getDeclaredMethod("addManageServiceAutoStartPermissions"); UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class) @@ -285,6 +287,7 @@ public class UpgradeCatalog250Test { .addMockedMethod(updateHiveLlapConfigs) .addMockedMethod(updateTablesForZeppelinViewRemoval) .addMockedMethod(updateAtlasConfigs) + .addMockedMethod(updateLogSearchConfigs) .addMockedMethod(addNewConfigurationsFromXml) .addMockedMethod(addManageServiceAutoStartPermissions) .createMock(); @@ -317,6 +320,9 @@ public class UpgradeCatalog250Test { upgradeCatalog250.updateAtlasConfigs(); expectLastCall().once(); + upgradeCatalog250.updateLogSearchConfigs(); + expectLastCall().once(); + upgradeCatalog250.addManageServiceAutoStartPermissions(); expectLastCall().once(); @@ -328,6 +334,7 @@ public class UpgradeCatalog250Test { } @Test + @SuppressWarnings("unchecked") public void testAmsEnvUpdateConfigs() throws Exception { Map<String, String> oldPropertiesAmsEnv = new HashMap<String, String>() { @@ -369,7 +376,7 @@ public class UpgradeCatalog250Test { replay(clusters, mockAmsEnv, cluster); - AmbariManagementControllerImpl controller = (AmbariManagementControllerImpl)createMockBuilder(AmbariManagementControllerImpl.class) + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) .addMockedMethod("createConfiguration") .addMockedMethod("getClusters", new Class[] { }) .addMockedMethod("createConfig") @@ -377,7 +384,7 @@ public class UpgradeCatalog250Test { .createNiceMock(); Injector injector2 = easyMockSupport.createNiceMock(Injector.class); - Capture<Map> propertiesCapture = EasyMock.newCapture(); + Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture(); expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); @@ -419,6 +426,7 @@ public class UpgradeCatalog250Test { } + @SuppressWarnings("unchecked") private void testAmsHbaseRootDir(Map<String, String> oldProperties, Map<String, String> newProperties) throws AmbariException { Map<String, String> amsSite = new HashMap<String, String>() { { @@ -442,7 +450,7 @@ public class UpgradeCatalog250Test { replay(clusters, mockAmsHbaseSite, mockAmsSite, cluster); - AmbariManagementControllerImpl controller = (AmbariManagementControllerImpl)createMockBuilder(AmbariManagementControllerImpl.class) + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) .addMockedMethod("createConfiguration") .addMockedMethod("getClusters", new Class[] { }) .addMockedMethod("createConfig") @@ -450,7 +458,7 @@ public class UpgradeCatalog250Test { .createNiceMock(); Injector injector2 = easyMockSupport.createNiceMock(Injector.class); - Capture<Map> propertiesCapture = EasyMock.newCapture(); + Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture(); expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); @@ -466,6 +474,7 @@ public class UpgradeCatalog250Test { } @Test + @SuppressWarnings("unchecked") public void testKafkaUpdateConfigs() throws Exception{ Map<String, String> oldProperties = new HashMap<String, String>() { @@ -491,7 +500,7 @@ public class UpgradeCatalog250Test { replay(clusters, mockKafkaBroker, cluster); - AmbariManagementControllerImpl controller = (AmbariManagementControllerImpl)createMockBuilder(AmbariManagementControllerImpl.class) + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) .addMockedMethod("createConfiguration") .addMockedMethod("getClusters", new Class[] { }) .addMockedMethod("createConfig") @@ -499,7 +508,7 @@ public class UpgradeCatalog250Test { .createNiceMock(); Injector injector2 = easyMockSupport.createNiceMock(Injector.class); - Capture<Map> propertiesCapture = EasyMock.newCapture(); + Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture(); expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); @@ -515,6 +524,105 @@ public class UpgradeCatalog250Test { } @Test + @SuppressWarnings("unchecked") + public void testLogSearchUpdateConfigs() throws Exception { + reset(clusters, cluster); + expect(clusters.getClusters()).andReturn(ImmutableMap.of("normal", cluster)).once(); + + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + Injector injector2 = easyMockSupport.createNiceMock(Injector.class); + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) + .addMockedMethod("createConfiguration") + .addMockedMethod("getClusters", new Class[] {}) + .addMockedMethod("createConfig") + .withConstructor(actionManager, clusters, injector) + .createNiceMock(); + + expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + + Map<String, String> oldLogSearchProperties = ImmutableMap.of( + "logsearch.external.auth.enabled", "true", + "logsearch.external.auth.host_url", "host_url", + "logsearch.external.auth.login_url", "login_url"); + + Map<String, String> expectedLogSearchProperties = ImmutableMap.of( + "logsearch.auth.external_auth.enabled", "true", + "logsearch.auth.external_auth.host_url", "host_url", + "logsearch.auth.external_auth.login_url", "login_url"); + + Config mockLogSearchProperties = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logsearch-properties")).andReturn(mockLogSearchProperties).atLeastOnce(); + expect(mockLogSearchProperties.getProperties()).andReturn(oldLogSearchProperties).anyTimes(); + Capture<Map<String, String>> logSearchPropertiesCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(logSearchPropertiesCapture), anyString(), + anyObject(Map.class))).andReturn(config).once(); + + Map<String, String> oldLogFeederEnv = ImmutableMap.of( + "content", "infra_solr_ssl_enabled"); + + Map<String, String> expectedLogFeederEnv = ImmutableMap.of( + "content", "logsearch_solr_ssl_enabled"); + + Config mockLogFeederEnv = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logfeeder-env")).andReturn(mockLogFeederEnv).atLeastOnce(); + expect(mockLogFeederEnv.getProperties()).andReturn(oldLogFeederEnv).anyTimes(); + Capture<Map<String, String>> logFeederEnvCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(logFeederEnvCapture), anyString(), + anyObject(Map.class))).andReturn(config).once(); + + Map<String, String> oldLogSearchEnv = ImmutableMap.of( + "logsearch_solr_audit_logs_use_ranger", "false", + "logsearch_solr_audit_logs_zk_node", "zk_node", + "logsearch_solr_audit_logs_zk_quorum", "zk_quorum", + "content", "infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'"); + + Map<String, String> expectedLogSearchEnv = ImmutableMap.of( + "content", "logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl"); + + Config mockLogSearchEnv = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logsearch-env")).andReturn(mockLogSearchEnv).atLeastOnce(); + expect(mockLogSearchEnv.getProperties()).andReturn(oldLogSearchEnv).anyTimes(); + Capture<Map<String, String>> logSearchEnvCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(logSearchEnvCapture), anyString(), + anyObject(Map.class))).andReturn(config).once(); + + Map<String, String> oldLogSearchLog4j = ImmutableMap.of( + "content", "{{logsearch_log_dir}}/logsearch.err\n" + + "<priority value=\"warn\"/>"); + + Map<String, String> expectedLogSearchLog4j = ImmutableMap.of( + "content", "{{logsearch_log_dir}}/logsearch.log\n" + + "<priority value=\"info\"/>"); + + Config mockLogSearchLog4j = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logsearch-log4j")).andReturn(mockLogSearchLog4j).atLeastOnce(); + expect(mockLogSearchLog4j.getProperties()).andReturn(oldLogSearchLog4j).anyTimes(); + Capture<Map<String, String>> logSearchLog4jCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(logSearchLog4jCapture), anyString(), + anyObject(Map.class))).andReturn(config).once(); + + replay(clusters, cluster); + replay(controller, injector2); + replay(mockLogSearchProperties, mockLogFeederEnv, mockLogSearchEnv, mockLogSearchLog4j); + new UpgradeCatalog250(injector2).updateLogSearchConfigs(); + easyMockSupport.verifyAll(); + + Map<String, String> updatedLogSearchProperties = logSearchPropertiesCapture.getValue(); + assertTrue(Maps.difference(expectedLogSearchProperties, updatedLogSearchProperties).areEqual()); + + Map<String, String> updatedLogFeederEnv = logFeederEnvCapture.getValue(); + assertTrue(Maps.difference(expectedLogFeederEnv, updatedLogFeederEnv).areEqual()); + + Map<String, String> updatedLogSearchEnv = logSearchEnvCapture.getValue(); + assertTrue(Maps.difference(expectedLogSearchEnv, updatedLogSearchEnv).areEqual()); + + Map<String, String> updatedLogSearchLog4j = logSearchLog4jCapture.getValue(); + assertTrue(Maps.difference(expectedLogSearchLog4j, updatedLogSearchLog4j).areEqual()); + } + + @Test public void testUpdateAtlasConfigs() throws Exception { Map<String, String> oldHiveProperties = new HashMap<String, String>(); @@ -543,6 +651,7 @@ public class UpgradeCatalog250Test { testUpdateAtlasHookConfig(oldSqoopProperties, newSqoopProperties, "sqoop-env"); } + @SuppressWarnings("unchecked") public void testUpdateAtlasHookConfig(Map<String, String> oldProperties, Map<String, String> newProperties, String configType) throws Exception { Map<String, Service> installedServices = new HashMap<String, Service>() { @@ -570,7 +679,7 @@ public class UpgradeCatalog250Test { replay(clusters, mockAtlasConfig, cluster); - AmbariManagementControllerImpl controller = (AmbariManagementControllerImpl)createMockBuilder(AmbariManagementControllerImpl.class) + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) .addMockedMethod("createConfiguration") .addMockedMethod("getClusters", new Class[] { }) .addMockedMethod("createConfig") @@ -578,7 +687,7 @@ public class UpgradeCatalog250Test { .createNiceMock(); Injector injector2 = easyMockSupport.createNiceMock(Injector.class); - Capture<Map> propertiesCapture = EasyMock.newCapture(); + Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture(); expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); http://git-wip-us.apache.org/repos/asf/ambari/blob/50e36c7f/ambari-web/app/data/HDP2/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js index abe900f..6c2d5e1 100644 --- a/ambari-web/app/data/HDP2/site_properties.js +++ b/ambari-web/app/data/HDP2/site_properties.js @@ -1979,12 +1979,40 @@ var hdp2properties = [ }, /*logfeeder-log4j*/ { - "name": "content", + "name": "logfeeder_log_maxfilesize", "serviceName": "LOGSEARCH", "filename": "logfeeder-log4j.xml", "category": "Advanced logfeeder-log4j", "index": 1 }, + { + "name": "logfeeder_log_maxbackupindex", + "serviceName": "LOGSEARCH", + "filename": "logfeeder-log4j.xml", + "category": "Advanced logfeeder-log4j", + "index": 2 + }, + { + "name": "logfeeder_json_log_maxfilesize", + "serviceName": "LOGSEARCH", + "filename": "logfeeder-log4j.xml", + "category": "Advanced logfeeder-log4j", + "index": 3 + }, + { + "name": "logfeeder_json_log_maxbackupindex", + "serviceName": "LOGSEARCH", + "filename": "logfeeder-log4j.xml", + "category": "Advanced logfeeder-log4j", + "index": 4 + }, + { + "name": "content", + "serviceName": "LOGSEARCH", + "filename": "logfeeder-log4j.xml", + "category": "Advanced logfeeder-log4j", + "index": 5 + }, /*logsearch-admin-json*/ { "name": "logsearch_admin_username", @@ -2144,12 +2172,68 @@ var hdp2properties = [ }, /*logsearch-log4j*/ { - "name": "content", + "name": "logsearch_log_maxfilesize", "serviceName": "LOGSEARCH", "filename": "logsearch-log4j.xml", "category": "Advanced logsearch-log4j", "index": 1 }, + { + "name": "logsearch_log_maxbackupindex", + "serviceName": "LOGSEARCH", + "filename": "logsearch-log4j.xml", + "category": "Advanced logsearch-log4j", + "index": 2 + }, + { + "name": "logsearch_json_log_maxfilesize", + "serviceName": "LOGSEARCH", + "filename": "logsearch-log4j.xml", + "category": "Advanced logsearch-log4j", + "index": 3 + }, + { + "name": "logsearch_json_log_maxbackupindex", + "serviceName": "LOGSEARCH", + "filename": "logsearch-log4j.xml", + "category": "Advanced logsearch-log4j", + "index": 4 + }, + { + "name": "logsearch_audit_log_maxfilesize", + "serviceName": "LOGSEARCH", + "filename": "logsearch-log4j.xml", + "category": "Advanced logsearch-log4j", + "index": 5 + }, + { + "name": "logsearch_audit_log_maxbackupindex", + "serviceName": "LOGSEARCH", + "filename": "logsearch-log4j.xml", + "category": "Advanced logsearch-log4j", + "index": 6 + }, + { + "name": "logsearch_perf_log_maxfilesize", + "serviceName": "LOGSEARCH", + "filename": "logsearch-log4j.xml", + "category": "Advanced logsearch-log4j", + "index": 7 + }, + { + "name": "logsearch_perf_log_maxbackupindex", + "serviceName": "LOGSEARCH", + "filename": "logsearch-log4j.xml", + "category": "Advanced logsearch-log4j", + "index": 8 + }, + { + "name": "content", + "serviceName": "LOGSEARCH", + "filename": "logsearch-log4j.xml", + "category": "Advanced logsearch-log4j", + "index": 9 + }, /*logsearch-service_logs-solrconfig*/ { "name": "logsearch_service_logs_max_retention",
