Author: harsh Date: Wed Aug 14 15:50:48 2013 New Revision: 1513939 URL: http://svn.apache.org/r1513939 Log: HADOOP-9855. Backport HADOOP-6578 to branch-1. Contributed by James Kinley. (harsh)
Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/conf/TestConfiguration.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1513939&r1=1513938&r2=1513939&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Wed Aug 14 15:50:48 2013 @@ -34,6 +34,9 @@ Release 1.3.0 - unreleased HDFS-4963. Improve multihoming support in namenode. (Arpit Agarwal via cnauroth) + HADOOP-9855. Backport HADOOP-6578 to branch-1. + (James Kinley via harsh) + BUG FIXES MAPREDUCE-5047. keep.failed.task.files=true causes job failure on Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java?rev=1513939&r1=1513938&r2=1513939&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java (original) +++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java Wed Aug 14 15:50:48 2013 @@ -398,6 +398,29 @@ public class Configuration implements It } /** + * Get the value of the <code>name</code> property as a trimmed <code>String</code>, + * <code>null</code> if no such property exists. + * If the key is deprecated, it returns the value of + * the first key which replaces the deprecated key and is not null + * + * Values are processed for <a href="#VariableExpansion">variable expansion</a> + * before being returned. + * + * @param name the property name. + * @return the value of the <code>name</code> or its replacing property, + * or null if no such property exists. + */ + public String getTrimmed(String name) { + String value = get(name); + + if (null == value) { + return null; + } else { + return value.trim(); + } + } + + /** * Get the value of the <code>name</code> property, without doing * <a href="#VariableExpansion">variable expansion</a>. * @@ -472,7 +495,7 @@ public class Configuration implements It * or <code>defaultValue</code>. */ public int getInt(String name, int defaultValue) { - String valueString = get(name); + String valueString = getTrimmed(name); if (valueString == null) return defaultValue; try { @@ -508,7 +531,7 @@ public class Configuration implements It * or <code>defaultValue</code>. */ public long getLong(String name, long defaultValue) { - String valueString = get(name); + String valueString = getTrimmed(name); if (valueString == null) return defaultValue; try { @@ -561,7 +584,7 @@ public class Configuration implements It * or <code>defaultValue</code>. */ public float getFloat(String name, float defaultValue) { - String valueString = get(name); + String valueString = getTrimmed(name); if (valueString == null) return defaultValue; try { @@ -591,7 +614,7 @@ public class Configuration implements It * or <code>defaultValue</code>. */ public boolean getBoolean(String name, boolean defaultValue) { - String valueString = get(name); + String valueString = getTrimmed(name); if ("true".equals(valueString)) return true; else if ("false".equals(valueString)) @@ -740,6 +763,20 @@ public class Configuration implements It return new IntegerRanges(get(name, defaultValue)); } + /** + * Get the comma delimited values of the <code>name</code> property as + * an array of <code>String</code>s, trimmed of the leading and trailing whitespace. + * If no such property is specified then an empty array is returned. + * + * @param name property name. + * @return property value as an array of trimmed <code>String</code>s, + * or empty array. + */ + public String[] getTrimmedStrings(String name) { + String valueString = get(name); + return StringUtils.getTrimmedStrings(valueString); + } + /** * Get the comma delimited values of the <code>name</code> property as * a collection of <code>String</code>s. @@ -823,7 +860,7 @@ public class Configuration implements It * or <code>defaultValue</code>. */ public Class<?>[] getClasses(String name, Class<?> ... defaultValue) { - String[] classnames = getStrings(name); + String[] classnames = getTrimmedStrings(name); if (classnames == null) return defaultValue; try { @@ -848,7 +885,7 @@ public class Configuration implements It * or <code>defaultValue</code>. */ public Class<?> getClass(String name, Class<?> defaultValue) { - String valueString = get(name); + String valueString = getTrimmed(name); if (valueString == null) return defaultValue; try { Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java?rev=1513939&r1=1513938&r2=1513939&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java (original) +++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java Wed Aug 14 15:50:48 2013 @@ -319,6 +319,20 @@ public class StringUtils { return values; } + /** + * Splits a comma separated value <code>String</code>, trimming leading and trailing whitespace on each value. + * @param str a comma separated <String> with values + * @return an array of <code>String</code> values + */ + public static String[] getTrimmedStrings(String str){ + if (null == str || str.trim().isEmpty()) { + return emptyStringArray; + } + + return str.trim().split("\\s*,\\s*"); + } + + final public static String[] emptyStringArray = {}; final public static char COMMA = ','; final public static String COMMA_STR = ","; final public static char ESCAPE_CHAR = '\\'; Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/conf/TestConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/conf/TestConfiguration.java?rev=1513939&r1=1513938&r2=1513939&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/conf/TestConfiguration.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/conf/TestConfiguration.java Wed Aug 14 15:50:48 2013 @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Random; import junit.framework.TestCase; +import static org.junit.Assert.assertArrayEquals; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; @@ -385,6 +386,7 @@ public class TestConfiguration extends T appendProperty("test.int1", "20"); appendProperty("test.int2", "020"); appendProperty("test.int3", "-20"); + appendProperty("test.int4", " -20 "); endConfig(); Path fileResource = new Path(CONFIG); conf.addResource(fileResource); @@ -394,8 +396,80 @@ public class TestConfiguration extends T assertEquals(20, conf.getLong("test.int2", 0)); assertEquals(-20, conf.getInt("test.int3", 0)); assertEquals(-20, conf.getLong("test.int3", 0)); + assertEquals(-20, conf.getInt("test.int4", 0)); + assertEquals(-20, conf.getLong("test.int4", 0)); } - + + public void testBooleanValues() throws IOException { + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.bool1", "true"); + appendProperty("test.bool2", "false"); + appendProperty("test.bool3", " true "); + appendProperty("test.bool4", " false "); + appendProperty("test.bool5", "foo"); + endConfig(); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + assertEquals(true, conf.getBoolean("test.bool1", false)); + assertEquals(false, conf.getBoolean("test.bool2", true)); + assertEquals(true, conf.getBoolean("test.bool3", false)); + assertEquals(false, conf.getBoolean("test.bool4", true)); + assertEquals(true, conf.getBoolean("test.bool5", true)); + } + + public void testFloatValues() throws IOException { + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.float1", "3.1415"); + appendProperty("test.float2", "003.1415"); + appendProperty("test.float3", "-3.1415"); + appendProperty("test.float4", " -3.1415 "); + endConfig(); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + assertEquals(3.1415f, conf.getFloat("test.float1", 0.0f)); + assertEquals(3.1415f, conf.getFloat("test.float2", 0.0f)); + assertEquals(-3.1415f, conf.getFloat("test.float3", 0.0f)); + assertEquals(-3.1415f, conf.getFloat("test.float4", 0.0f)); + } + + public void testGetClass() throws IOException { + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.class1", "java.lang.Integer"); + appendProperty("test.class2", " java.lang.Integer "); + endConfig(); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + assertEquals("java.lang.Integer", conf.getClass("test.class1", null).getCanonicalName()); + assertEquals("java.lang.Integer", conf.getClass("test.class2", null).getCanonicalName()); + } + + public void testGetClasses() throws IOException { + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.classes1", "java.lang.Integer,java.lang.String"); + appendProperty("test.classes2", " java.lang.Integer , java.lang.String "); + endConfig(); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + String[] expectedNames = { "java.lang.Integer", "java.lang.String" }; + Class<?>[] defaultClasses = {}; + Class<?>[] classes1 = conf.getClasses("test.classes1", defaultClasses); + Class<?>[] classes2 = conf.getClasses("test.classes2", defaultClasses); + assertArrayEquals(expectedNames, extractClassNames(classes1)); + assertArrayEquals(expectedNames, extractClassNames(classes2)); + } + + private static String[] extractClassNames(Class<?>[] classes) { + String[] classNames = new String[classes.length]; + for (int i = 0; i < classNames.length; i++) { + classNames[i] = classes[i].getCanonicalName(); + } + return classNames; + } + enum Dingo { FOO, BAR }; enum Yak { RAB, FOO }; public void testEnum() throws IOException {