Repository: ambari Updated Branches: refs/heads/trunk 0458ec851 -> ca09c208e
AMBARI-19345 Storm REST metrics are not available when Storm UI is running in HTTPS (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca09c208 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca09c208 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca09c208 Branch: refs/heads/trunk Commit: ca09c208ea602e1f7b612d4e40b42b63861231a6 Parents: 0458ec8 Author: Dmytro Sen <[email protected]> Authored: Thu Feb 9 18:40:15 2017 +0200 Committer: Dmytro Sen <[email protected]> Committed: Thu Feb 9 18:40:15 2017 +0200 ---------------------------------------------------------------------- .../HDP/2.1.1/services/STORM/metrics.json | 10 +- .../metrics/RestMetricsPropertyProvider.java | 106 +++++++++----- .../server/upgrade/UpgradeCatalog250.java | 47 ++++++ .../common-services/STORM/0.10.0/metrics.json | 8 +- .../common-services/STORM/0.9.1/alerts.json | 30 +--- .../common-services/STORM/0.9.1/metrics.json | 8 +- .../STORM/0.9.1/quicklinks/quicklinks.json | 25 +++- .../common-services/STORM/0.9.3/metrics.json | 8 +- .../RestMetricsPropertyProviderTest.java | 4 +- .../server/upgrade/UpgradeCatalog250Test.java | 146 +++++++++++++++---- .../HDP/2.1.1/services/STORM/metrics.json | 10 +- 11 files changed, 292 insertions(+), 110 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json index 7c149bb..247e59e 100644 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json +++ b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json @@ -7,7 +7,9 @@ "default_port": "8745", "port_config_type": "storm-site", "port_property_name": "storm.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { @@ -62,7 +64,9 @@ "default_port": "8745", "port_config_type": "storm-site", "port_property_name": "storm.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { @@ -111,4 +115,4 @@ } ] } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java index 8c23fe0..7a6021f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java @@ -102,16 +102,17 @@ public class RestMetricsPropertyProvider extends ThreadPoolEnabledPropertyProvid private static final String DEFAULT_PORT_PROPERTY = "default_port"; private static final String PORT_CONFIG_TYPE_PROPERTY = "port_config_type"; private static final String PORT_PROPERTY_NAME_PROPERTY = "port_property_name"; + private static final String HTTPS_PORT_PROPERTY_NAME_PROPERTY = "https_port_property_name"; /** * Protocol to use when connecting */ private static final String PROTOCOL_OVERRIDE_PROPERTY = "protocol"; + private static final String HTTPS_PROTOCOL_PROPERTY = "https_property_name"; private static final String HTTP_PROTOCOL = "http"; private static final String HTTPS_PROTOCOL = "https"; private static final String DEFAULT_PROTOCOL = HTTP_PROTOCOL; - /** * String that separates JSON URL from path inside JSON in metrics path */ @@ -208,7 +209,7 @@ public class RestMetricsPropertyProvider extends ThreadPoolEnabledPropertyProvid // If there are no metrics defined for the given component then there is nothing to do. return resource; } - String protocol = resolveProtocol(); + String protocol = null; String port = "-1"; String hostname = null; try { @@ -221,7 +222,8 @@ public class RestMetricsPropertyProvider extends ThreadPoolEnabledPropertyProvid LOG.warn(msg); return resource; } - port = resolvePort(cluster, hostname, resourceComponentName, metricsProperties); + protocol = resolveProtocol(cluster, hostname); + port = resolvePort(cluster, hostname, resourceComponentName, metricsProperties, protocol); } catch (Exception e) { rethrowSystemException(e); } @@ -283,52 +285,24 @@ public class RestMetricsPropertyProvider extends ThreadPoolEnabledPropertyProvid // ----- helper methods ---------------------------------------------------- /** - * Uses port_config_type, port_property_name, default_port parameters from + * If protocol is equal to HTTPS_PROTOCOL than returns HTTPS_PORT_PROPERTY_NAME_PROPERTY value from PORT_CONFIG_TYPE_PROPERTY + * else uses port_config_type, port_property_name, default_port parameters from * metricsProperties to find out right port value for service * * @return determines REST port for service */ protected String resolvePort(Cluster cluster, String hostname, String componentName, - Map<String, String> metricsProperties) + Map<String, String> metricsProperties, String protocol) throws AmbariException { String portConfigType = null; + String portPropertyNameInMetricsProperties = protocol.equalsIgnoreCase(HTTPS_PROTOCOL) ? HTTPS_PORT_PROPERTY_NAME_PROPERTY : PORT_PROPERTY_NAME_PROPERTY; String portPropertyName = null; if (metricsProperties.containsKey(PORT_CONFIG_TYPE_PROPERTY) && - metricsProperties.containsKey(PORT_PROPERTY_NAME_PROPERTY)) { + metricsProperties.containsKey(portPropertyNameInMetricsProperties)) { portConfigType = metricsProperties.get(PORT_CONFIG_TYPE_PROPERTY); - portPropertyName = metricsProperties.get(PORT_PROPERTY_NAME_PROPERTY); - } - String portStr = null; - if (portConfigType != null && portPropertyName != null) { - try { - Map<String, Map<String, String>> configTags = - amc.findConfigurationTagsWithOverrides(cluster, hostname); - if (configTags.containsKey(portConfigType)) { - Map<String, Map<String, String>> properties = amc.getConfigHelper().getEffectiveConfigProperties(cluster, - Collections.singletonMap(portConfigType, configTags.get(portConfigType))); - Map<String, String> config = properties.get(portConfigType); - if (config != null && config.containsKey(portPropertyName)) { - portStr = config.get(portPropertyName); - } - } - } catch (AmbariException e) { - String message = String.format("Can not extract configs for " + - "component = %s, hostname = %s, config type = %s, property name = %s", componentName, - hostname, portConfigType, portPropertyName); - LOG.warn(message, e); - } - if (portStr == null) { - String message = String.format( - "Can not extract REST port for " + - "component %s from configurations. " + - "Config tag = %s, config key name = %s, " + - "hostname = %s. Probably metrics.json file for " + - "service is misspelled. Trying default port", - componentName, portConfigType, - portPropertyName, hostname); - LOG.debug(message); - } + portPropertyName = metricsProperties.get(portPropertyNameInMetricsProperties); } + String portStr = getPropertyValueByNameAndConfigType(portPropertyName, portConfigType, cluster, hostname); if (portStr == null && metricsProperties.containsKey(DEFAULT_PORT_PROPERTY)) { if (metricsProperties.containsKey(DEFAULT_PORT_PROPERTY)) { portStr = metricsProperties.get(DEFAULT_PORT_PROPERTY); @@ -345,13 +319,65 @@ public class RestMetricsPropertyProvider extends ThreadPoolEnabledPropertyProvid return portStr; } + /** + * Tries to get propertyName property from configType config for specified cluster and hostname + * @param propertyName + * @param configType + * @param cluster + * @param hostname + * @return + */ + private String getPropertyValueByNameAndConfigType(String propertyName, String configType, Cluster cluster, String hostname){ + String result = null; + if (configType != null && propertyName != null) { + try { + Map<String, Map<String, String>> configTags = + amc.findConfigurationTagsWithOverrides(cluster, hostname); + if (configTags.containsKey(configType)) { + Map<String, Map<String, String>> properties = amc.getConfigHelper().getEffectiveConfigProperties(cluster, + Collections.singletonMap(configType, configTags.get(configType))); + Map<String, String> config = properties.get(configType); + if (config != null && config.containsKey(propertyName)) { + result = config.get(propertyName); + } + } + } catch (AmbariException e) { + String message = String.format("Can not extract configs for " + + "component = %s, hostname = %s, config type = %s, property name = %s", componentName, + hostname, configType, propertyName); + LOG.warn(message, e); + } + if (result == null) { + String message = String.format( + "Can not extract property for " + + "component %s from configurations. " + + "Config tag = %s, config key name = %s, " + + "hostname = %s. Probably metrics.json file for " + + "service is misspelled.", + componentName, configType, + propertyName, hostname); + LOG.debug(message); + } + } + return result; + } /** - * Extracts protocol type from metrics properties. If no protocol is defined, + * if HTTPS_PROTOCOL_PROPERTY is present in metrics properties then checks if it is present in PORT_CONFIG_TYPE_PROPERTY and returns "https" if it is. + * + * Otherwise extracts protocol type from metrics properties. If no protocol is defined, * uses default protocol. */ - private String resolveProtocol() { + private String resolveProtocol(Cluster cluster, String hostname) { String protocol = DEFAULT_PROTOCOL; + if (metricsProperties.containsKey(PORT_CONFIG_TYPE_PROPERTY) && metricsProperties.containsKey(HTTPS_PROTOCOL_PROPERTY)) { + String configType = metricsProperties.get(PORT_CONFIG_TYPE_PROPERTY); + String propertyName = metricsProperties.get(HTTPS_PROTOCOL_PROPERTY); + String value = getPropertyValueByNameAndConfigType(propertyName, configType, cluster, hostname); + if (value != null) { + return HTTPS_PROTOCOL; + } + } if (metricsProperties.containsKey(PROTOCOL_OVERRIDE_PROPERTY)) { protocol = metricsProperties.get(PROTOCOL_OVERRIDE_PROPERTY).toLowerCase(); if (!protocol.equals(HTTP_PROTOCOL) && !protocol.equals(HTTPS_PROTOCOL)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 8f0d218..b617dc4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -37,7 +37,9 @@ import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor; import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; +import org.apache.ambari.server.orm.dao.AlertDefinitionDAO; import org.apache.ambari.server.orm.dao.DaoUtils; +import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; @@ -46,6 +48,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.jdbc.support.JdbcUtils; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.google.inject.Inject; import com.google.inject.Injector; @@ -158,6 +162,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { protected void executeDMLUpdates() throws AmbariException, SQLException { addNewConfigurationsFromXml(); updateAMSConfigs(); + updateStormAlerts(); updateHadoopEnvConfigs(); updateKafkaConfigs(); updateHIVEInteractiveConfigs(); @@ -172,6 +177,48 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { addManageServiceAutoStartPermissions(); } + protected void updateStormAlerts() { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class); + Clusters clusters = ambariManagementController.getClusters(); + + Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); + for (final Cluster cluster : clusterMap.values()) { + long clusterID = cluster.getClusterId(); + LOG.info("Updating storm alert definitions on cluster : " + cluster.getClusterName()); + + final AlertDefinitionEntity stormServerProcessDefinitionEntity = alertDefinitionDAO.findByName( + clusterID, "storm_server_process"); + + final AlertDefinitionEntity stormWebAlert = alertDefinitionDAO.findByName( + clusterID, "storm_webui"); + + if (stormServerProcessDefinitionEntity != null) { + LOG.info("Removing alert definition : " + stormServerProcessDefinitionEntity.toString()); + alertDefinitionDAO.remove(stormServerProcessDefinitionEntity); + } + + if (stormWebAlert != null) { + LOG.info("Updating alert definition : " + stormWebAlert.getDefinitionName()); + String source = stormWebAlert.getSource(); + JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); + LOG.debug("Source before update : " + sourceJson); + + JsonObject uriJson = sourceJson.get("uri").getAsJsonObject(); + uriJson.remove("https"); + uriJson.remove("https_property"); + uriJson.remove("https_property_value"); + uriJson.addProperty("https", "{{storm-site/ui.https.port}}"); + uriJson.addProperty("https_property", "{{storm-site/ui.https.keystore.type}}"); + uriJson.addProperty("https_property_value", "jks"); + + LOG.debug("Source after update : " + sourceJson); + stormWebAlert.setSource(sourceJson.toString()); + alertDefinitionDAO.merge(stormWebAlert); + } + } + } + protected void updateHostVersionTable() throws SQLException { LOG.info("Updating the {} table", HOST_VERSION_TABLE); http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json b/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json index b2b0404..2c27d58 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json @@ -7,7 +7,9 @@ "default_port": "8744", "port_config_type": "storm-site", "port_property_name": "ui.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { @@ -63,7 +65,9 @@ "default_port": "8744", "port_config_type": "storm-site", "port_property_name": "ui.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json index 842538d..acd9d85 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json @@ -30,31 +30,6 @@ ], "STORM_UI_SERVER": [ { - "name": "storm_server_process", - "label": "Storm Server Process", - "interval": 1, - "scope": "ANY", - "enabled": true, - "source": { - "type": "PORT", - "uri": "{{storm-site/ui.port}}", - "default_port": 8744, - "reporting": { - "ok": { - "text": "TCP OK - {0:.3f}s response on port {1}" - }, - "warning": { - "text": "TCP OK - {0:.3f}s response on port {1}", - "value": 1.5 - }, - "critical": { - "text": "Connection failed: {0} to {1}:{2}", - "value": 5.0 - } - } - } - }, - { "name": "storm_webui", "label": "Storm Web UI", "interval": 1, @@ -64,9 +39,12 @@ "type": "WEB", "uri": { "http": "{{storm-site/ui.port}}", + "https" : "{{storm-site/ui.https.port}}", "kerberos_keytab": "{{storm-env/storm_ui_keytab}}", "kerberos_principal": "{{storm-env/storm_ui_principal_name}}", - "connection_timeout": 5.0 + "connection_timeout": 5.0, + "https_property": "{{storm-site/ui.https.keystore.type}}", + "https_property_value": "jks" }, "reporting": { "ok": { http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/main/resources/common-services/STORM/0.9.1/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/metrics.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1/metrics.json index 43ed608..e7f1d93 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/metrics.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/metrics.json @@ -7,7 +7,9 @@ "default_port": "8745", "port_config_type": "storm-site", "port_property_name": "storm.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { @@ -62,7 +64,9 @@ "default_port": "8745", "port_config_type": "storm-site", "port_property_name": "storm.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/main/resources/common-services/STORM/0.9.1/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1/quicklinks/quicklinks.json index 492f0a0..d45f337 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/quicklinks/quicklinks.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/quicklinks/quicklinks.json @@ -4,7 +4,24 @@ "configuration": { "protocol": { - "type":"HTTP_ONLY" + "type":"https", + "checks":[ + { + "property":"ui.https.keystore.path", + "desired":"EXIST", + "site":"storm-site" + }, + { + "property":"ui.https.key.password", + "desired":"EXIST", + "site":"storm-site" + }, + { + "property":"ui.https.port", + "desired":"EXIST", + "site":"storm-site" + } + ] }, "links": [ @@ -17,12 +34,12 @@ "port":{ "http_property": "ui.port", "http_default_port": "8744", - "https_property": "ui.port", - "https_default_port": "8744", + "https_property": "ui.https.port", + "https_default_port": "8740", "regex": "^(\\d+)$", "site": "storm-site" } } ] } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json b/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json index d9a11b2..5daaeb1 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json @@ -7,7 +7,9 @@ "default_port": "8744", "port_config_type": "storm-site", "port_property_name": "ui.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { @@ -62,7 +64,9 @@ "default_port": "8744", "port_config_type": "storm-site", "port_property_name": "ui.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java index 8455484..05b9835 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java @@ -256,7 +256,7 @@ public class RestMetricsPropertyProviderTest { Map<String, String> customMetricsProperties = new HashMap<>(metricsProperties); customMetricsProperties.put("port_property_name", "wrong_property"); String resolvedPort = restMetricsPropertyProvider.resolvePort(c1, "domu-12-31-39-0e-34-e1.compute-1.internal", - "STORM_REST_API", customMetricsProperties); + "STORM_REST_API", customMetricsProperties, "http"); Assert.assertEquals(DEFAULT_STORM_UI_PORT, resolvedPort); // a port property exists (8745). Should return it, not a default_port (8746) @@ -264,7 +264,7 @@ public class RestMetricsPropertyProviderTest { // custom default customMetricsProperties.put("default_port", "8746"); resolvedPort = restMetricsPropertyProvider.resolvePort(c1, "domu-12-31-39-0e-34-e1.compute-1.internal", - "STORM_REST_API", customMetricsProperties); + "STORM_REST_API", customMetricsProperties, "http"); Assert.assertEquals(DEFAULT_STORM_UI_PORT, resolvedPort); } http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index 4dffea8..7084769 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -47,6 +47,12 @@ import java.util.Set; import javax.persistence.EntityManager; +import com.google.inject.AbstractModule; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.Provider; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.ActionManager; import org.apache.ambari.server.configuration.Configuration; @@ -55,9 +61,11 @@ import org.apache.ambari.server.controller.AmbariManagementControllerImpl; import org.apache.ambari.server.controller.KerberosHelper; import org.apache.ambari.server.controller.MaintenanceStateHelper; import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.orm.dao.AlertDefinitionDAO; import org.apache.ambari.server.orm.dao.PermissionDAO; import org.apache.ambari.server.orm.dao.ResourceTypeDAO; import org.apache.ambari.server.orm.dao.RoleAuthorizationDAO; +import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; import org.apache.ambari.server.orm.entities.PermissionEntity; import org.apache.ambari.server.orm.entities.ResourceTypeEntity; import org.apache.ambari.server.orm.entities.RoleAuthorizationEntity; @@ -81,11 +89,6 @@ import org.junit.runner.RunWith; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.gson.Gson; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Provider; import junit.framework.AssertionFailedError; @@ -270,6 +273,92 @@ public class UpgradeCatalog250Test { Assert.assertEquals(false, capturedCredentialStoreEnabledColValues.isNullable()); } + @Test + public void testUpdateAlerts_StormUIWebAlert() { + EasyMockSupport easyMockSupport = new EasyMockSupport(); + final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class); + final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class); + final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class); + final AlertDefinitionDAO mockAlertDefinitionDAO = easyMockSupport.createNiceMock(AlertDefinitionDAO.class); + final AlertDefinitionEntity stormWebUIAlertMock = easyMockSupport.createNiceMock(AlertDefinitionEntity.class); + + final Injector mockInjector = Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(AmbariManagementController.class).toInstance(mockAmbariManagementController); + bind(Clusters.class).toInstance(mockClusters); + bind(EntityManager.class).toInstance(entityManager); + bind(AlertDefinitionDAO.class).toInstance(mockAlertDefinitionDAO); + bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); + bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + } + }); + + long clusterId = 1; + + expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once(); + expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", mockClusterExpected); + }}).atLeastOnce(); + expect(mockClusterExpected.getClusterId()).andReturn(clusterId).anyTimes(); + expect(mockAlertDefinitionDAO.findByName(eq(clusterId), eq("storm_webui"))) + .andReturn(stormWebUIAlertMock).atLeastOnce(); + expect(stormWebUIAlertMock.getSource()).andReturn("{\"uri\": {\n" + + " \"http\": \"{{storm-site/ui.port}}\",\n" + + " \"kerberos_keytab\": \"{{storm-env/storm_ui_keytab}}\",\n" + + " \"kerberos_principal\": \"{{storm-env/storm_ui_principal_name}}\",\n" + + " \"connection_timeout\": 5.0\n" + + " } }"); + + stormWebUIAlertMock.setSource("{\"uri\":{\"http\":\"{{storm-site/ui.port}}\",\"kerberos_keytab\":\"{{storm-env/storm_ui_keytab}}\",\"kerberos_principal\":\"{{storm-env/storm_ui_principal_name}}\",\"connection_timeout\":5.0,\"https\":\"{{storm-site/ui.https.port}}\",\"https_property\":\"{{storm-site/ui.https.keystore.type}}\",\"https_property_value\":\"jks\"}}"); + + expectLastCall().once(); + + easyMockSupport.replayAll(); + mockInjector.getInstance(UpgradeCatalog250.class).updateStormAlerts(); + easyMockSupport.verifyAll(); + } + + @Test + public void testUpdateAlerts_StormUIPortAlert() { + EasyMockSupport easyMockSupport = new EasyMockSupport(); + final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class); + final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class); + final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class); + final AlertDefinitionDAO mockAlertDefinitionDAO = easyMockSupport.createNiceMock(AlertDefinitionDAO.class); + final AlertDefinitionEntity stormUIPortAlertMock = easyMockSupport.createNiceMock(AlertDefinitionEntity.class); + + final Injector mockInjector = Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(AmbariManagementController.class).toInstance(mockAmbariManagementController); + bind(Clusters.class).toInstance(mockClusters); + bind(EntityManager.class).toInstance(entityManager); + bind(AlertDefinitionDAO.class).toInstance(mockAlertDefinitionDAO); + bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); + bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + } + }); + long clusterId = 1; + + expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once(); + expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", mockClusterExpected); + }}).atLeastOnce(); + expect(mockClusterExpected.getClusterId()).andReturn(clusterId).anyTimes(); + expect(mockAlertDefinitionDAO.findByName(eq(clusterId), eq("storm_server_process"))) + .andReturn(stormUIPortAlertMock).atLeastOnce(); + + mockAlertDefinitionDAO.remove(stormUIPortAlertMock); + expectLastCall().once(); + + easyMockSupport.replayAll(); + + mockInjector.getInstance(UpgradeCatalog250.class).updateStormAlerts(); + easyMockSupport.verifyAll(); + } + + @Test public void testExecuteDMLUpdates() throws Exception { Method updateAmsConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAMSConfigs"); @@ -286,6 +375,7 @@ public class UpgradeCatalog250Test { Method updateAmbariInfraConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAmbariInfraConfigs"); Method updateRangerUrlConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateRangerUrlConfigs"); Method updateYarnSite = UpgradeCatalog250.class.getDeclaredMethod("updateYarnSite"); + Method updateAlerts = UpgradeCatalog250.class.getDeclaredMethod("updateStormAlerts"); UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class) .addMockedMethod(updateAmsConfigs) @@ -302,6 +392,7 @@ public class UpgradeCatalog250Test { .addMockedMethod(updateAmbariInfraConfigs) .addMockedMethod(updateRangerUrlConfigs) .addMockedMethod(updateYarnSite) + .addMockedMethod(updateAlerts) .createMock(); upgradeCatalog250.updateAMSConfigs(); @@ -346,6 +437,9 @@ public class UpgradeCatalog250Test { upgradeCatalog250.updateYarnSite(); expectLastCall().once(); + upgradeCatalog250.updateStormAlerts(); + expectLastCall().once(); + replay(upgradeCatalog250); upgradeCatalog250.executeDMLUpdates(); @@ -991,9 +1085,9 @@ public class UpgradeCatalog250Test { public void testLogSearchUpdateConfigs() throws Exception { reset(clusters, cluster); expect(clusters.getClusters()).andReturn(ImmutableMap.of("normal", cluster)).once(); - + EasyMockSupport easyMockSupport = new EasyMockSupport(); - + Injector injector2 = easyMockSupport.createNiceMock(Injector.class); AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) .addMockedMethod("createConfiguration") @@ -1004,17 +1098,17 @@ public class UpgradeCatalog250Test { expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); - + Map<String, String> oldLogSearchProperties = ImmutableMap.of( "logsearch.external.auth.enabled", "true", "logsearch.external.auth.host_url", "host_url", "logsearch.external.auth.login_url", "login_url"); - + Map<String, String> expectedLogSearchProperties = ImmutableMap.of( "logsearch.auth.external_auth.enabled", "true", "logsearch.auth.external_auth.host_url", "host_url", "logsearch.auth.external_auth.login_url", "login_url"); - + Config mockLogSearchProperties = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("logsearch-properties")).andReturn(mockLogSearchProperties).atLeastOnce(); expect(mockLogSearchProperties.getProperties()).andReturn(oldLogSearchProperties).anyTimes(); @@ -1024,10 +1118,10 @@ public class UpgradeCatalog250Test { Map<String, String> oldLogFeederEnv = ImmutableMap.of( "content", "infra_solr_ssl_enabled"); - + Map<String, String> expectedLogFeederEnv = ImmutableMap.of( "content", "logfeeder_use_ssl"); - + Config mockLogFeederEnv = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("logfeeder-env")).andReturn(mockLogFeederEnv).atLeastOnce(); expect(mockLogFeederEnv.getProperties()).andReturn(oldLogFeederEnv).anyTimes(); @@ -1040,10 +1134,10 @@ public class UpgradeCatalog250Test { "logsearch_solr_audit_logs_zk_node", "zk_node", "logsearch_solr_audit_logs_zk_quorum", "zk_quorum", "content", "infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'"); - + Map<String, String> expectedLogSearchEnv = ImmutableMap.of( "content", "logsearch_use_ssl"); - + Config mockLogSearchEnv = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("logsearch-env")).andReturn(mockLogSearchEnv).atLeastOnce(); expect(mockLogSearchEnv.getProperties()).andReturn(oldLogSearchEnv).anyTimes(); @@ -1071,7 +1165,7 @@ public class UpgradeCatalog250Test { " <param name=\"maxBackupIndex\" value=\"14\" />\n" + " <layout class=\"org.apache.ambari.logsearch.appender.LogsearchConversion\" />\n" + " </appender>"); - + Map<String, String> expectedLogFeederLog4j = ImmutableMap.of( "content", " <appender name=\"rolling_file\" class=\"org.apache.log4j.RollingFileAppender\">\n" + @@ -1096,7 +1190,7 @@ public class UpgradeCatalog250Test { "logfeeder_log_maxbackupindex", "12", "logfeeder_json_log_maxfilesize", "13", "logfeeder_json_log_maxbackupindex", "14"); - + Config mockLogFeederLog4j = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("logfeeder-log4j")).andReturn(mockLogFeederLog4j).atLeastOnce(); expect(mockLogFeederLog4j.getProperties()).andReturn(oldLogFeederLog4j).anyTimes(); @@ -1154,7 +1248,7 @@ public class UpgradeCatalog250Test { " <priority value=\"warn\"/>\n" + " <appender-ref ref=\"rolling_file_json\"/>\n" + " </category>"); - + Map<String, String> expectedLogSearchLog4j = new HashMap<>(); expectedLogSearchLog4j.put("content", " <appender name=\"rolling_file\" class=\"org.apache.log4j.RollingFileAppender\">\n" + @@ -1214,7 +1308,7 @@ public class UpgradeCatalog250Test { expectedLogSearchLog4j.put("logsearch_audit_log_maxbackupindex", "16"); expectedLogSearchLog4j.put("logsearch_perf_log_maxfilesize", "17"); expectedLogSearchLog4j.put("logsearch_perf_log_maxbackupindex", "18"); - + Config mockLogSearchLog4j = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("logsearch-log4j")).andReturn(mockLogSearchLog4j).atLeastOnce(); expect(mockLogSearchLog4j.getProperties()).andReturn(oldLogSearchLog4j).anyTimes(); @@ -1248,9 +1342,9 @@ public class UpgradeCatalog250Test { public void testAmbariInfraUpdateConfigs() throws Exception { reset(clusters, cluster); expect(clusters.getClusters()).andReturn(ImmutableMap.of("normal", cluster)).once(); - + EasyMockSupport easyMockSupport = new EasyMockSupport(); - + Injector injector2 = easyMockSupport.createNiceMock(Injector.class); AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) .addMockedMethod("createConfiguration") @@ -1267,13 +1361,13 @@ public class UpgradeCatalog250Test { "SOLR_SSL_TRUST_STORE_PASSWORD={{infra_solr_keystore_password}}\n" + "SOLR_KERB_NAME_RULES={{infra_solr_kerberos_name_rules}}\n" + "SOLR_AUTHENTICATION_OPTS=\" -DauthenticationPlugin=org.apache.solr.security.KerberosPlugin -Djava.security.auth.login.config=$SOLR_JAAS_FILE -Dsolr.kerberos.principal=${SOLR_KERB_PRINCIPAL} -Dsolr.kerberos.keytab=${SOLR_KERB_KEYTAB} -Dsolr.kerberos.cookie.domain=${SOLR_HOST} -Dsolr.kerberos.name.rules=${SOLR_KERB_NAME_RULES}\""); - + Map<String, String> expectedInfraSolrEnv = ImmutableMap.of( "content", "SOLR_SSL_TRUST_STORE={{infra_solr_truststore_location}}\n" + "SOLR_SSL_TRUST_STORE_PASSWORD={{infra_solr_truststore_password}}\n" + "SOLR_KERB_NAME_RULES=\"{{infra_solr_kerberos_name_rules}}\"\n" + "SOLR_AUTHENTICATION_OPTS=\" -DauthenticationPlugin=org.apache.solr.security.KerberosPlugin -Djava.security.auth.login.config=$SOLR_JAAS_FILE -Dsolr.kerberos.principal=${SOLR_KERB_PRINCIPAL} -Dsolr.kerberos.keytab=${SOLR_KERB_KEYTAB} -Dsolr.kerberos.cookie.domain=${SOLR_HOST}\""); - + Config mockInfraSolrEnv = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("infra-solr-env")).andReturn(mockInfraSolrEnv).atLeastOnce(); expect(mockInfraSolrEnv.getProperties()).andReturn(oldInfraSolrEnv).anyTimes(); @@ -1284,13 +1378,13 @@ public class UpgradeCatalog250Test { Map<String, String> oldInfraSolrLog4j = ImmutableMap.of( "content", "log4j.appender.file.MaxFileSize=15MB\n" + "log4j.appender.file.MaxBackupIndex=5\n"); - + Map<String, String> expectedInfraSolrLog4j = ImmutableMap.of( "content", "log4j.appender.file.MaxFileSize={{infra_log_maxfilesize}}MB\n" + "log4j.appender.file.MaxBackupIndex={{infra_log_maxbackupindex}}\n", "infra_log_maxfilesize", "15", "infra_log_maxbackupindex", "5"); - + Config mockInfraSolrLog4j = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("infra-solr-log4j")).andReturn(mockInfraSolrLog4j).atLeastOnce(); expect(mockInfraSolrLog4j.getProperties()).andReturn(oldInfraSolrLog4j).anyTimes(); @@ -1302,14 +1396,14 @@ public class UpgradeCatalog250Test { "content", "log4j.appender.file.File\u003d{{infra_client_log|default(\u0027/var/log/ambari-infra-solr-client/solr-client.log\u0027)}}\n" + "log4j.appender.file.MaxFileSize=55MB\n" + "log4j.appender.file.MaxBackupIndex=10\n"); - + Map<String, String> expectedInfraSolrClientLog4j = ImmutableMap.of( "content", "log4j.appender.file.File\u003d{{solr_client_log|default(\u0027/var/log/ambari-infra-solr-client/solr-client.log\u0027)}}\n" + "log4j.appender.file.MaxFileSize={{solr_client_log_maxfilesize}}MB\n" + "log4j.appender.file.MaxBackupIndex={{solr_client_log_maxbackupindex}}\n", "infra_client_log_maxfilesize", "55", "infra_client_log_maxbackupindex", "10"); - + Config mockInfraSolrClientLog4j = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("infra-solr-client-log4j")).andReturn(mockInfraSolrClientLog4j).atLeastOnce(); expect(mockInfraSolrClientLog4j.getProperties()).andReturn(oldInfraSolrClientLog4j).anyTimes(); http://git-wip-us.apache.org/repos/asf/ambari/blob/ca09c208/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json index 7c149bb..247e59e 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json @@ -7,7 +7,9 @@ "default_port": "8745", "port_config_type": "storm-site", "port_property_name": "storm.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { @@ -62,7 +64,9 @@ "default_port": "8745", "port_config_type": "storm-site", "port_property_name": "storm.port", - "protocol": "http" + "protocol": "http", + "https_port_property_name" : "ui.https.port", + "https_property_name" : "ui.https.keystore.type" }, "metrics": { "default": { @@ -111,4 +115,4 @@ } ] } -} \ No newline at end of file +}
