AMBARI-14689. Increase the default value for MAX_COUNTERS (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa6b943d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa6b943d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa6b943d Branch: refs/heads/branch-dev-patch-upgrade Commit: fa6b943de81b799ebb2d63d98f965c2515ca57ec Parents: b9226d3 Author: Andrew Onishuk <[email protected]> Authored: Fri Jan 15 16:54:49 2016 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Fri Jan 15 16:54:49 2016 +0200 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog221.java | 37 ++++++++++++ .../stacks/HDP/2.2/upgrades/config-upgrade.xml | 14 +++++ .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 4 ++ .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 6 +- .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 1 + .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 2 + .../2.3/services/TEZ/configuration/tez-site.xml | 12 ++++ .../server/upgrade/UpgradeCatalog221Test.java | 60 ++++++++++++++++++++ 8 files changed, 135 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java index 914c547..4eebb66 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java @@ -34,6 +34,8 @@ import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.utils.VersionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +67,7 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { private static final String SECURITY_ENABLED = "security_enabled"; private static final String TOPOLOGY_HOST_INFO_TABLE = "topology_host_info"; private static final String TOPOLOGY_HOST_INFO_RACK_INFO_COLUMN = "rack_info"; + private static final String TEZ_SITE = "tez-site"; @Inject DaoUtils daoUtils; @@ -90,6 +93,9 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { private static final String RANGER_KMS_JDBC_DRIVER = "ranger.ks.jpa.jdbc.driver"; private static final String RANGER_KMS_PROPERTIES = "kms-properties"; + private static final String TEZ_COUNTERS_MAX = "tez.counters.max"; + private static final String TEZ_COUNTERS_MAX_GROUPS = "tez.counters.max.groups"; + // ----- Constructors ------------------------------------------------------ @@ -155,6 +161,7 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { addNewConfigurationsFromXml(); updateAlerts(); updateOozieConfigs(); + updateTezConfigs(); updateRangerKmsDbksConfigs(); } @@ -363,6 +370,36 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { } } + protected void updateTezConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { + Config tezSiteProps = cluster.getDesiredConfigByType(TEZ_SITE); + if (tezSiteProps != null) { + + // Update tez.counters.max and tez.counters.max.groups configurations + String tezCountersMaxProperty = tezSiteProps.getProperties().get(TEZ_COUNTERS_MAX); + String tezCountersMaxGroupesProperty = tezSiteProps.getProperties().get(TEZ_COUNTERS_MAX_GROUPS); + + StackId stackId = cluster.getCurrentStackVersion(); + boolean isStackNotLess23 = (stackId != null && stackId.getStackName().equals("HDP") && + VersionUtils.compareVersions(stackId.getStackVersion(), "2.3") >= 0); + + if (isStackNotLess23) { + Map<String, String> updates = new HashMap<String, String>(); + if (tezCountersMaxProperty != null && tezCountersMaxProperty.equals("2000")) { + updates.put(TEZ_COUNTERS_MAX, "10000"); + } + if (tezCountersMaxGroupesProperty != null && tezCountersMaxGroupesProperty.equals("1000")) { + updates.put(TEZ_COUNTERS_MAX_GROUPS, "3000"); + } + if (!updates.isEmpty()) { + updateConfigurationPropertiesForCluster(cluster, TEZ_SITE, updates, true, false); + } + } + } + } + } + protected void updateRangerKmsDbksConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml index c2844fa..ffc7ffb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml @@ -107,6 +107,20 @@ <type>tez-site</type> <set key="tez.lib.uris" value="/hdp/apps/${hdp.version}/tez/tez.tar.gz"/> </definition> + + <definition xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"> + <condition type="tez-site" key="tez.counters.max" value="2000"> + <type>tez-site</type> + <key>tez.counters.max</key> + <value>10000</value> + </condition> + <condition type="tez-site" key="tez.counters.max.groups" value="1000"> + <type>tez-site</type> + <key>tez.counters.max.groups</key> + <value>3000</value> + </condition> + </definition> + </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml index 5dfa2bd..816fe13 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml @@ -412,6 +412,10 @@ </execute-stage> <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez"> + <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/> + </execute-stage> + + <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez"> <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_tez_lib_uris_property"/> </execute-stage> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml index 00b9e96..5166c4b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml @@ -446,6 +446,10 @@ </execute-stage> <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez"> + <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/> + </execute-stage> + + <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez"> <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/> </execute-stage> @@ -1358,4 +1362,4 @@ </service> </processing> -</upgrade> \ No newline at end of file +</upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index fd64c8e..375da12 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -626,6 +626,7 @@ <component name="TEZ_CLIENT"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/> + <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/> <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_tez_lib_uris_property"/> <task xsi:type="configure" id="hdp_2_3_0_0_tez_keep_ats_v1"/> </pre-upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml index 7637830..ea3d6a6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml @@ -634,6 +634,8 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/> + <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/> + <!-- Apply HDP 2.4 since same as that in HDP 2.3 --> <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml index a80da09..b2db250 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml @@ -103,4 +103,16 @@ <value>org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService</value> </property> + <property> + <name>tez.counters.max</name> + <value>10000</value> + <description>The number of allowed counters for the executing DAG</description> + </property> + + <property> + <name>tez.counters.max.groups</name> + <value>3000</value> + <description>The number of allowed counter groups for the executing DAG</description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java index c14149f..5a14bc9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java @@ -44,6 +44,9 @@ import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.RepositoryInfo; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.stack.OsFamily; import org.easymock.Capture; import org.easymock.EasyMock; @@ -55,6 +58,7 @@ import org.junit.Test; import javax.persistence.EntityManager; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -138,12 +142,14 @@ public class UpgradeCatalog221Test { Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); Method updateAlerts = UpgradeCatalog221.class.getDeclaredMethod("updateAlerts"); Method updateOozieConfigs = UpgradeCatalog221.class.getDeclaredMethod("updateOozieConfigs"); + Method updateTezConfigs = UpgradeCatalog221.class.getDeclaredMethod("updateTezConfigs"); Method updateRangerKmsDbksConfigs = UpgradeCatalog221.class.getDeclaredMethod("updateRangerKmsDbksConfigs"); UpgradeCatalog221 upgradeCatalog221 = createMockBuilder(UpgradeCatalog221.class) .addMockedMethod(addNewConfigurationsFromXml) .addMockedMethod(updateAlerts) .addMockedMethod(updateOozieConfigs) + .addMockedMethod(updateTezConfigs) .addMockedMethod(updateRangerKmsDbksConfigs) .createMock(); @@ -153,6 +159,8 @@ public class UpgradeCatalog221Test { expectLastCall().once(); upgradeCatalog221.updateOozieConfigs(); expectLastCall().once(); + upgradeCatalog221.updateTezConfigs(); + expectLastCall().once(); upgradeCatalog221.updateRangerKmsDbksConfigs(); expectLastCall().once(); @@ -246,6 +254,58 @@ public class UpgradeCatalog221Test { } @Test + public void testUpdateTezConfigs() throws Exception { + EasyMockSupport easyMockSupport = new EasyMockSupport(); + final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class); + final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class); + final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class); + + final Config tezSiteConf = easyMockSupport.createNiceMock(Config.class); + final Map<String, String> propertiesTezSite = new HashMap<String, String>() {{ + put("tez.counters.max", "2000"); + put("tez.counters.max.groups", "1000"); + }}; + + StackId stackId = new StackId("HDP","2.3"); + + final Injector mockInjector = Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(AmbariManagementController.class).toInstance(mockAmbariManagementController); + bind(Clusters.class).toInstance(mockClusters); + bind(EntityManager.class).toInstance(entityManager); + + bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); + bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + } + }); + + expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once(); + expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", mockClusterExpected); + }}).atLeastOnce(); + + + expect(mockClusterExpected.getDesiredConfigByType("tez-site")).andReturn(tezSiteConf).atLeastOnce(); + expect(tezSiteConf.getProperties()).andReturn(propertiesTezSite).once(); + expect(mockClusterExpected.getCurrentStackVersion()).andReturn(stackId).once(); + + UpgradeCatalog221 upgradeCatalog221 = createMockBuilder(UpgradeCatalog221.class) + .withConstructor(Injector.class) + .withArgs(mockInjector) + .addMockedMethod("updateConfigurationPropertiesForCluster", Cluster.class, String.class, + Map.class, boolean.class, boolean.class) + .createMock(); + Map<String, String> updates = new HashMap<String, String>(); + updates.put("tez.counters.max", "10000"); + updates.put("tez.counters.max.groups", "3000"); + upgradeCatalog221.updateConfigurationPropertiesForCluster(mockClusterExpected, "tez-site", + updates, true, false); + expectLastCall().once(); + + } + + @Test public void testUpdateRangerKmsDbksConfigs() throws Exception { EasyMockSupport easyMockSupport = new EasyMockSupport(); final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
