YARN-5833. Add validation to ensure default ports are unique in Configuration. (Konstantinos Karanasos via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29e3b341 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29e3b341 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29e3b341 Branch: refs/heads/YARN-4752 Commit: 29e3b3417c16c83dc8e753f94d7ca9957dddbedd Parents: 3f93ac0 Author: Subru Krishnan <[email protected]> Authored: Tue Nov 8 14:38:18 2016 -0800 Committer: Subru Krishnan <[email protected]> Committed: Tue Nov 8 14:38:18 2016 -0800 ---------------------------------------------------------------------- .../conf/TestConfigurationFieldsBase.java | 47 ++++++++++++++++++++ .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../yarn/conf/TestYarnConfigurationFields.java | 11 +++++ 3 files changed, 59 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/29e3b341/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java index eab0161..9007c20 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java @@ -18,6 +18,7 @@ package org.apache.hadoop.conf; +import org.apache.commons.lang.StringUtils; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -147,6 +148,14 @@ public abstract class TestConfigurationFieldsBase { private Set<String> xmlFieldsMissingInConfiguration = null; /** + * A set of strings used to check for collision of default values. + * For each of the set's strings, the default values containing that string + * in their name should not coincide. + */ + @SuppressWarnings("checkstyle:visibilitymodifier") + protected Set<String> filtersForDefaultValueCollisionCheck = new HashSet<>(); + + /** * Member variable for debugging base class operation */ protected boolean configDebug = false; @@ -719,4 +728,42 @@ public abstract class TestConfigurationFieldsBase { System.out.println("====="); System.out.println(); } + + /** + * For each specified string, get the default parameter values whose names + * contain the string. Then check whether any of these default values collide. + * This is, for example, useful to make sure there is no collision of default + * ports across different services. + */ + @Test + public void testDefaultValueCollision() { + for (String filter : filtersForDefaultValueCollisionCheck) { + System.out.println("Checking if any of the default values whose name " + + "contains string \"" + filter + "\" collide."); + + // Map from filtered default value to name of the corresponding parameter. + Map<String, String> filteredValues = new HashMap<>(); + + int valuesChecked = 0; + for (Map.Entry<String, String> ent : + configurationDefaultVariables.entrySet()) { + // Apply the name filter to the default parameters. + if (ent.getKey().contains(filter)) { + // Check only for numerical values. + if (StringUtils.isNumeric(ent.getValue())) { + String crtValue = + filteredValues.putIfAbsent(ent.getValue(), ent.getKey()); + assertTrue("Parameters " + ent.getKey() + " and " + crtValue + + " are using the same default value!", crtValue == null); + } + valuesChecked++; + } + } + + System.out.println( + "Checked " + valuesChecked + " default values for collision."); + } + + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/29e3b341/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index c16e1ea..1fd25a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1744,7 +1744,7 @@ public class YarnConfiguration extends Configuration { public static final String AMRM_PROXY_ADDRESS = NM_PREFIX + "amrmproxy.address"; - public static final int DEFAULT_AMRM_PROXY_PORT = 8048; + public static final int DEFAULT_AMRM_PROXY_PORT = 8049; public static final String DEFAULT_AMRM_PROXY_ADDRESS = "0.0.0.0:" + DEFAULT_AMRM_PROXY_PORT; http://git-wip-us.apache.org/repos/asf/hadoop/blob/29e3b341/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 0c40fa9..db9364a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -147,5 +147,16 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { // Currently defined in RegistryConstants/core-site.xml xmlPrefixToSkipCompare.add("hadoop.registry"); + + // Add the filters used for checking for collision of default values. + initDefaultValueCollisionCheck(); + } + + /** + * Add filters used to perform the check of default values collision by + * {@link TestConfigurationFieldsBase#filtersForDefaultValueCollisionCheck}. + */ + private void initDefaultValueCollisionCheck() { + filtersForDefaultValueCollisionCheck.add("_PORT"); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
