AMBARI-21893 : NameNode Heap Usage (Daily) metric alert status flips to UNKNOWN intermittently when AMS HTTPS is enabled. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7fa7a6c1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7fa7a6c1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7fa7a6c1 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 7fa7a6c17607341e8a6888f3dd8b938dce7425ff Parents: bb4645f Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Tue Sep 12 17:01:51 2017 -0700 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Tue Sep 12 17:01:51 2017 -0700 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog260.java | 30 +++++++++ .../0.1.0/configuration/ams-ssl-client.xml | 9 --- .../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 10 +-- .../0.1.0/package/scripts/params.py | 10 ++- .../server/upgrade/UpgradeCatalog260Test.java | 70 +++++++++++++++++++- 5 files changed, 110 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java index d05f39a..2de85fc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java @@ -21,6 +21,7 @@ import static org.apache.ambari.server.view.ViewContextImpl.CORE_SITE; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -29,6 +30,7 @@ import javax.persistence.EntityManager; import javax.persistence.Query; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor; import org.apache.ambari.server.orm.dao.ArtifactDAO; import org.apache.ambari.server.orm.entities.ArtifactEntity; @@ -122,6 +124,9 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { public static final String HOST_COMPONENT_DESIRED_STATE = "hostcomponentdesiredstate"; public static final String HOST_COMPONENT_STATE = "hostcomponentstate"; + public static final String AMS_SSL_CLIENT = "ams-ssl-client"; + public static final String METRIC_TRUSTSTORE_ALIAS = "ssl.client.truststore.alias"; + /** * Logger. */ @@ -395,6 +400,7 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { removeSupersetFromDruid(); ensureZeppelinProxyUserConfigs(); updateKerberosDescriptorArtifacts(); + updateAmsConfigs(); } public int getCurrentVersionID() throws AmbariException, SQLException { @@ -535,4 +541,28 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { } } } + + protected void updateAmsConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + + + Config amsSslClient = cluster.getDesiredConfigByType(AMS_SSL_CLIENT); + if (amsSslClient != null) { + Map<String, String> amsSslClientProperties = amsSslClient.getProperties(); + + if (amsSslClientProperties.containsKey(METRIC_TRUSTSTORE_ALIAS)) { + LOG.info("Removing " + METRIC_TRUSTSTORE_ALIAS + " from " + AMS_SSL_CLIENT); + removeConfigurationPropertiesFromCluster(cluster, AMS_SSL_CLIENT, Collections.singleton(METRIC_TRUSTSTORE_ALIAS)); + } + + } + } + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml index cac39de..d75bba2 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml @@ -39,13 +39,4 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> - <property> - <name>ssl.client.truststore.alias</name> - <value></value> - <description>Alias used to create certificate for AMS. (Default is hostname)</description> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py index 9a31ade..51e0756 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py @@ -515,10 +515,12 @@ def export_ca_certs(dir_path): truststore_p12 = os.path.join(tmpdir,'truststore.p12') if (params.metric_truststore_type.lower() == 'jks'): - # Convert truststore from JKS to PKCS12 - cmd = format("{sudo} {java64_home}/bin/keytool -importkeystore -srckeystore {metric_truststore_path} -destkeystore {truststore_p12} -srcalias {metric_truststore_alias} -deststoretype PKCS12 -srcstorepass {metric_truststore_password} -deststorepass {metric_truststore_password}") - Execute(cmd, - ) + if not params.metric_truststore_alias: + for alias in params.metric_truststore_alias_list: + # Convert truststore from JKS to PKCS12 + cmd = format("{sudo} {java64_home}/bin/keytool -importkeystore -srckeystore {metric_truststore_path} -destkeystore {truststore_p12} -srcalias " + alias + " -deststoretype PKCS12 -srcstorepass {metric_truststore_password} -deststorepass {metric_truststore_password}") + Execute(cmd, + ) truststore = truststore_p12 # Export all CA certificates from the truststore to the conf directory http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index 756da26..0242b73 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -100,11 +100,15 @@ else: metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") -metric_truststore_alias = default("/configurations/ams-ssl-client/ssl.client.truststore.alias", None) -if not metric_truststore_alias: - metric_truststore_alias = metric_collector_host metric_truststore_ca_certs='ca.pem' +metric_truststore_alias_list = [] +for host in ams_collector_hosts.split(","): + metric_truststore_alias = default("/configurations/ams-ssl-client/{host}.ssl.client.truststore.alias", None) + if not metric_truststore_alias: + metric_truststore_alias = host + metric_truststore_alias_list.append(metric_truststore_alias) + agent_cache_dir = config['hostLevelParams']['agentCacheDir'] service_package_folder = config['commandParams']['service_package_folder'] stack_name = default("/hostLevelParams/stack_name", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java index 33c29bc..b70f37b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java @@ -23,6 +23,7 @@ import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; @@ -30,6 +31,7 @@ import static org.easymock.EasyMock.newCapture; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertTrue; import java.io.File; import java.net.URL; @@ -40,16 +42,19 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import javax.persistence.EntityManager; +import com.google.common.collect.Maps; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.ActionManager; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.AmbariManagementControllerImpl; import org.apache.ambari.server.controller.KerberosHelper; import org.apache.ambari.server.controller.MaintenanceStateHelper; import org.apache.ambari.server.orm.DBAccessor; @@ -67,7 +72,9 @@ import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; import org.apache.ambari.server.state.stack.OsFamily; import org.easymock.Capture; +import org.easymock.EasyMock; import org.easymock.EasyMockRunner; +import org.easymock.EasyMockSupport; import org.easymock.Mock; import org.easymock.MockType; import org.junit.After; @@ -480,8 +487,8 @@ public class UpgradeCatalog260Test { } public void verifyGetCurrentVersionID(Capture<String[]> scdcaptureKey, Capture<String[]> scdcaptureValue) { - Assert.assertTrue(Arrays.equals(scdcaptureKey.getValue(), new String[]{UpgradeCatalog260.STATE_COLUMN})); - Assert.assertTrue(Arrays.equals(scdcaptureValue.getValue(), new String[]{UpgradeCatalog260.CURRENT})); + assertTrue(Arrays.equals(scdcaptureKey.getValue(), new String[]{UpgradeCatalog260.STATE_COLUMN})); + assertTrue(Arrays.equals(scdcaptureValue.getValue(), new String[]{UpgradeCatalog260.CURRENT})); } public void expectUpdateServiceComponentDesiredStateTable(Capture<DBColumnInfo> scdstadd1, Capture<DBColumnInfo> scdstalter1, Capture<DBColumnInfo> scdstadd2, Capture<DBColumnInfo> scdstalter2) throws SQLException { @@ -625,7 +632,7 @@ public class UpgradeCatalog260Test { verify(clusters, cluster, zeppelinEnvConf, coreSiteConf, coreSiteConfNew, controller); - Assert.assertTrue(captureCoreSiteConfProperties.hasCaptured()); + assertTrue(captureCoreSiteConfProperties.hasCaptured()); Assert.assertEquals("existing_value", captureCoreSiteConfProperties.getValue().get("hadoop.proxyuser.zeppelin_user.hosts")); Assert.assertEquals("*", captureCoreSiteConfProperties.getValue().get("hadoop.proxyuser.zeppelin_user.groups")); } @@ -673,4 +680,61 @@ public class UpgradeCatalog260Test { Assert.assertNull(kerberosDescriptorUpdated.getService("RANGER_KMS").getComponent("RANGER_KMS_SERVER").getIdentity("/smokeuser")); } + + @Test + public void testUpdateAmsConfigs() throws Exception{ + + Map<String, String> oldProperties = new HashMap<String, String>() { + { + put("ssl.client.truststore.location", "/some/location"); + put("ssl.client.truststore.alias", "test_alias"); + } + }; + Map<String, String> newProperties = new HashMap<String, String>() { + { + put("ssl.client.truststore.location", "/some/location"); + } + }; + + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + Clusters clusters = easyMockSupport.createNiceMock(Clusters.class); + final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class); + Config mockAmsSslClient = easyMockSupport.createNiceMock(Config.class); + + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).once(); + expect(cluster.getDesiredConfigByType("ams-ssl-client")).andReturn(mockAmsSslClient).atLeastOnce(); + expect(mockAmsSslClient.getProperties()).andReturn(oldProperties).anyTimes(); + + Injector injector = easyMockSupport.createNiceMock(Injector.class); + expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes(); + expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes(); + + replay(injector, clusters, mockAmsSslClient, cluster); + + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) + .addMockedMethod("createConfiguration") + .addMockedMethod("getClusters", new Class[] { }) + .addMockedMethod("createConfig") + .withConstructor(createNiceMock(ActionManager.class), clusters, injector) + .createNiceMock(); + + Injector injector2 = easyMockSupport.createNiceMock(Injector.class); + Capture<Map> propertiesCapture = EasyMock.newCapture(); + + expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(propertiesCapture), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + + replay(controller, injector2); + new UpgradeCatalog260(injector2).updateAmsConfigs(); + easyMockSupport.verifyAll(); + + Map<String, String> updatedProperties = propertiesCapture.getValue(); + assertTrue(Maps.difference(newProperties, updatedProperties).areEqual()); + } + }