AMBARI-19561. After ambari only upgrade Property:"yarn.nodemanager.linux-container-executor.cgroups.mount-path" become required (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b7d8f5e9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b7d8f5e9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b7d8f5e9 Branch: refs/heads/branch-dev-patch-upgrade Commit: b7d8f5e9767e94516a6990ac09bbbfb6ddece2fd Parents: 112cea4 Author: Lisnichenko Dmitro <[email protected]> Authored: Tue Jan 17 12:06:41 2017 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Tue Jan 17 12:06:41 2017 +0200 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 39 +++++++ .../YARN/2.1.0.2.0/kerberos.json | 1 - .../YARN/3.0.0.3.0/kerberos.json | 1 - .../stacks/HDP/2.2/services/YARN/kerberos.json | 1 - .../HDP/2.3.ECS/services/YARN/kerberos.json | 1 - .../stacks/HDP/2.3/services/YARN/kerberos.json | 1 - .../stacks/HDP/2.5/services/YARN/kerberos.json | 1 - .../stacks/PERF/1.0/services/YARN/kerberos.json | 1 - .../server/upgrade/UpgradeCatalog250Test.java | 106 +++++++++++++++++++ .../2.2/configs/pig-service-check-secure.json | 3 +- .../test_kerberos_descriptor_2_1_3.json | 1 - .../data/stacks/HDP-2.1/service_components.json | 1 - 12 files changed, 146 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 3d84968..29e1f17 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -64,6 +64,10 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { private static final String HBASE_ROOTDIR = "hbase.rootdir"; private static final String HADOOP_ENV = "hadoop-env"; private static final String KAFKA_BROKER = "kafka-broker"; + private static final String YARN_SITE_CONFIG = "yarn-site"; + private static final String YARN_ENV_CONFIG = "yarn-env"; + private static final String YARN_LCE_CGROUPS_MOUNT_PATH = "yarn.nodemanager.linux-container-executor.cgroups.mount-path"; + private static final String YARN_CGROUPS_ENABLED = "yarn_cgroups_enabled"; private static final String KAFKA_TIMELINE_METRICS_HOST = "kafka.timeline.metrics.host"; public static final String COMPONENT_TABLE = "servicecomponentdesiredstate"; @@ -161,6 +165,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { updateAtlasConfigs(); updateLogSearchConfigs(); updateAmbariInfraConfigs(); + updateYarnSite(); addManageServiceAutoStartPermissions(); } @@ -179,6 +184,40 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { dbAccessor.addUniqueConstraint(GROUPS_TABLE, "UNQ_groups_0", "group_name", "group_type"); } + /** + * Updates {@code yarn-site} in the following ways: + * + * Remove {@code YARN_LCE_CGROUPS_MOUNT_PATH} if {@code YARN_CGROUPS_ENABLED} is {@code false} and + * {@code YARN_LCE_CGROUPS_MOUNT_PATH} is empty string + * + * @throws AmbariException + */ + protected void updateYarnSite() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); + + for (final Cluster cluster : clusterMap.values()) { + Config yarnEnvConfig = cluster.getDesiredConfigByType(YARN_ENV_CONFIG); + Config yarnSiteConfig = cluster.getDesiredConfigByType(YARN_SITE_CONFIG); + + if (yarnEnvConfig != null && yarnSiteConfig != null) { + String cgroupEnabled = yarnEnvConfig.getProperties().get(YARN_CGROUPS_ENABLED); + String mountPath = yarnSiteConfig.getProperties().get(YARN_LCE_CGROUPS_MOUNT_PATH); + + if (StringUtils.isEmpty(mountPath) && cgroupEnabled != null + && cgroupEnabled.trim().equalsIgnoreCase("false")){ + + removeConfigurationPropertiesFromCluster(cluster, YARN_SITE_CONFIG, new HashSet<String>(){{ + add(YARN_LCE_CGROUPS_MOUNT_PATH); + }}); + + } + } + + } + } + protected void updateHiveLlapConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); Clusters clusters = ambariManagementController.getClusters(); http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json index c307800..6b61c13 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json @@ -31,7 +31,6 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda", "hadoop.registry.secure" : "true", "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm" http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json index af920f1..d334887 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json @@ -32,7 +32,6 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda", "hadoop.registry.secure" : "true", "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm" http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json index 3a183cc..ad30b76 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json @@ -31,7 +31,6 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure", "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda", "hadoop.registry.secure" : "true", http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json index e11ce84..7977941 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json @@ -34,7 +34,6 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda", "hadoop.registry.secure" : "true", "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm" http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json index 1a6cf5b..73addb1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json @@ -32,7 +32,6 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda", "hadoop.registry.secure" : "true", "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm" http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json index af920f1..d334887 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json @@ -32,7 +32,6 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda", "hadoop.registry.secure" : "true", "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm" http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json index f33c07d..7e74237 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json @@ -32,7 +32,6 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure" } }, http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index 093de4b..1830f24 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -22,6 +22,7 @@ import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; @@ -39,8 +40,10 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.persistence.EntityManager; @@ -84,6 +87,8 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provider; +import junit.framework.AssertionFailedError; + /** * {@link UpgradeCatalog250} unit tests. */ @@ -339,6 +344,107 @@ public class UpgradeCatalog250Test { } @Test + public void testUpdateYarnSite() throws Exception{ + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + final String propertyToRemove = "yarn.nodemanager.linux-container-executor.cgroups.mount-path"; + final AmbariManagementController ambariManagementController = createNiceMock(AmbariManagementController.class); + Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class); + Config mockYarnSite = easyMockSupport.createNiceMock(Config.class); + + HashMap<String, String> yarnEnv = new HashMap<String, String>(){{ + put("yarn_cgroups_enabled", "false"); + }}; + + HashMap<String, String> yarnSite = new HashMap<String, String>() {{ + put(propertyToRemove, ""); + }}; + + reset(clusters, cluster, injector); + + expect(injector.getInstance(AmbariManagementController.class)).andReturn(ambariManagementController).atLeastOnce(); + expect(ambariManagementController.getClusters()).andReturn(clusters).atLeastOnce(); + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).once(); + expect(cluster.getDesiredConfigByType("yarn-env")).andReturn(mockYarnEnv).atLeastOnce(); + expect(mockYarnEnv.getProperties()).andReturn(yarnEnv).anyTimes(); + expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(mockYarnSite).atLeastOnce(); + expect(mockYarnSite.getProperties()).andReturn(yarnSite).anyTimes(); + + replay(clusters, cluster, injector, ambariManagementController, mockYarnEnv, mockYarnSite); + + UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class) + .addMockedMethod("removeConfigurationPropertiesFromCluster") + .withConstructor(injector) + .createNiceMock(); + + Capture<HashSet<String>> removeConfigName = EasyMock.newCapture(); + + upgradeCatalog250.removeConfigurationPropertiesFromCluster(anyObject(Cluster.class), eq("yarn-site"), capture(removeConfigName)); + EasyMock.expectLastCall(); + + replay(upgradeCatalog250); + + upgradeCatalog250.updateYarnSite(); + + easyMockSupport.verifyAll(); + + Set<String> updatedProperties = removeConfigName.getValue(); + assertTrue(updatedProperties.contains(propertyToRemove)); + + reset(injector); + } + + @Test + public void testUpdateYarnSiteWithEnabledCGroups() throws Exception{ + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + final String propertyToRemove = "yarn.nodemanager.linux-container-executor.cgroups.mount-path"; + final AmbariManagementController ambariManagementController = createNiceMock(AmbariManagementController.class); + Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class); + Config mockYarnSite = easyMockSupport.createNiceMock(Config.class); + + HashMap<String, String> yarnEnv = new HashMap<String, String>(){{ + put("yarn_cgroups_enabled", "true"); + }}; + + HashMap<String, String> yarnSite = new HashMap<String, String>() {{ + put(propertyToRemove, ""); + }}; + + reset(clusters, cluster, injector); + + expect(injector.getInstance(AmbariManagementController.class)).andReturn(ambariManagementController).atLeastOnce(); + expect(ambariManagementController.getClusters()).andReturn(clusters).atLeastOnce(); + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).once(); + expect(cluster.getDesiredConfigByType("yarn-env")).andReturn(mockYarnEnv).atLeastOnce(); + expect(mockYarnEnv.getProperties()).andReturn(yarnEnv).anyTimes(); + expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(mockYarnSite).atLeastOnce(); + expect(mockYarnSite.getProperties()).andReturn(yarnSite).anyTimes(); + + replay(clusters, cluster, injector, ambariManagementController, mockYarnEnv, mockYarnSite); + + UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class) + .addMockedMethod("removeConfigurationPropertiesFromCluster") + .withConstructor(injector) + .createNiceMock(); + + Capture<HashSet<String>> removeConfigName = EasyMock.newCapture(); + + upgradeCatalog250.removeConfigurationPropertiesFromCluster(anyObject(Cluster.class), eq("yarn-site"), capture(removeConfigName)); + EasyMock.expectLastCall().andThrow(new AssertionFailedError()).anyTimes(); + + replay(upgradeCatalog250); + + upgradeCatalog250.updateYarnSite(); + + reset(injector); + } + + @Test public void testAmsEnvUpdateConfigs() throws Exception{ Map<String, String> oldPropertiesAmsEnv = new HashMap<String, String>() { http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json b/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json index 0d0c6f5..f14eb52 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json @@ -470,8 +470,7 @@ "yarn.resourcemanager.recovery.enabled": "true", "yarn.timeline-service.http-authentication.cookie.domain": "", "yarn.resourcemanager.zk-retry-interval-ms": "1000", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", - "yarn.admin.acl": "", + "yarn.admin.acl": "", "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels", "yarn.client.nodemanager-connect.retry-interval-ms": "10000", "yarn.resourcemanager.admin.address": "c6402.ambari.apache.org:8141", http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json b/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json index a9e0bcd..bcc5359 100644 --- a/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json +++ b/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json @@ -802,7 +802,6 @@ "yarn.timeline-service.http-authentication.kerberos.name.rules": "", "yarn.timeline-service.http-authentication.cookie.path": "", "yarn.timeline-service.http-authentication.type": "kerberos", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", "yarn.acl.enable": "true", "yarn.timeline-service.http-authentication.signer.secret.provider": "", http://git-wip-us.apache.org/repos/asf/ambari/blob/b7d8f5e9/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json ---------------------------------------------------------------------- diff --git a/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json b/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json index d853f24..147c1c0 100644 --- a/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json +++ b/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json @@ -2898,7 +2898,6 @@ "yarn.timeline-service.http-authentication.kerberos.name.rules" : "", "yarn.timeline-service.http-authentication.cookie.path" : "", "yarn.timeline-service.http-authentication.type" : "kerberos", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path" : "", "yarn.resourcemanager.proxy-user-privileges.enabled" : "true", "yarn.acl.enable" : "true", "yarn.timeline-service.http-authentication.signer.secret.provider" : "",
