Author: kasha
Date: Thu Jun 12 21:26:39 2014
New Revision: 1602297
URL: http://svn.apache.org/r1602297
Log:
YARN-2155. FairScheduler: Incorrect threshold check for preemption. (Wei Yan
via kasha)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritable.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1602297&r1=1602296&r2=1602297&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Jun 12 21:26:39 2014
@@ -200,6 +200,9 @@ Release 2.5.0 - UNRELEASED
HADOOP-10622. Shell.runCommand can deadlock (Gera Shegalov via jlowe)
+ HADOOP-10686. Writables are not always configured.
+ (Abraham Elmahrek via kasha)
+
BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
HADOOP-10520. Extended attributes definition and FileSystem APIs for
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java?rev=1602297&r1=1602296&r2=1602297&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
Thu Jun 12 21:26:39 2014
@@ -256,7 +256,7 @@ public class MapFile {
} else {
keyClass=
(Class<? extends WritableComparable>) keyClassOption.getValue();
- this.comparator = WritableComparator.get(keyClass);
+ this.comparator = WritableComparator.get(keyClass, conf);
}
this.lastKey = comparator.newKey();
FileSystem fs = dirName.getFileSystem(conf);
@@ -428,12 +428,13 @@ public class MapFile {
this.data = createDataFileReader(dataFile, conf, options);
this.firstPosition = data.getPosition();
- if (comparator == null)
- this.comparator =
- WritableComparator.get(data.getKeyClass().
- asSubclass(WritableComparable.class));
- else
+ if (comparator == null) {
+ Class<? extends WritableComparable> cls;
+ cls = data.getKeyClass().asSubclass(WritableComparable.class);
+ this.comparator = WritableComparator.get(cls, conf);
+ } else {
this.comparator = comparator;
+ }
// open the index
SequenceFile.Reader.Option[] indexOptions =
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java?rev=1602297&r1=1602296&r2=1602297&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
Thu Jun 12 21:26:39 2014
@@ -2676,7 +2676,7 @@ public class SequenceFile {
/** Sort and merge files containing the named classes. */
public Sorter(FileSystem fs, Class<? extends WritableComparable> keyClass,
Class valClass, Configuration conf) {
- this(fs, WritableComparator.get(keyClass), keyClass, valClass, conf);
+ this(fs, WritableComparator.get(keyClass, conf), keyClass, valClass,
conf);
}
/** Sort and merge using an arbitrary {@link RawComparator}. */
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java?rev=1602297&r1=1602296&r2=1602297&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java
Thu Jun 12 21:26:39 2014
@@ -52,7 +52,7 @@ public class SetFile extends MapFile {
Class<? extends WritableComparable> keyClass,
SequenceFile.CompressionType compress)
throws IOException {
- this(conf, fs, dirName, WritableComparator.get(keyClass), compress);
+ this(conf, fs, dirName, WritableComparator.get(keyClass, conf),
compress);
}
/** Create a set naming the element comparator and compression type. */
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java?rev=1602297&r1=1602296&r2=1602297&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
Thu Jun 12 21:26:39 2014
@@ -24,6 +24,8 @@ import java.util.concurrent.ConcurrentHa
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
/** A Comparator for {@link WritableComparable}s.
@@ -37,13 +39,21 @@ import org.apache.hadoop.util.Reflection
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class WritableComparator implements RawComparator {
+public class WritableComparator implements RawComparator, Configurable {
private static final ConcurrentHashMap<Class, WritableComparator>
comparators
= new ConcurrentHashMap<Class, WritableComparator>(); // registry
- /** Get a comparator for a {@link WritableComparable} implementation. */
+ private Configuration conf;
+
+ /** For backwards compatibility. **/
public static WritableComparator get(Class<? extends WritableComparable> c) {
+ return get(c, null);
+ }
+
+ /** Get a comparator for a {@link WritableComparable} implementation. */
+ public static WritableComparator get(
+ Class<? extends WritableComparable> c, Configuration conf) {
WritableComparator comparator = comparators.get(c);
if (comparator == null) {
// force the static initializers to run
@@ -52,12 +62,24 @@ public class WritableComparator implemen
comparator = comparators.get(c);
// if not, use the generic one
if (comparator == null) {
- comparator = new WritableComparator(c, true);
+ comparator = new WritableComparator(c, conf, true);
}
}
+ // Newly passed Configuration objects should be used.
+ ReflectionUtils.setConf(comparator, conf);
return comparator;
}
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
/**
* Force initialization of the static members.
* As of Java 5, referencing a class doesn't force it to initialize. Since
@@ -91,12 +113,19 @@ public class WritableComparator implemen
/** Construct for a {@link WritableComparable} implementation. */
protected WritableComparator(Class<? extends WritableComparable> keyClass) {
- this(keyClass, false);
+ this(keyClass, null, false);
}
protected WritableComparator(Class<? extends WritableComparable> keyClass,
boolean createInstances) {
+ this(keyClass, null, createInstances);
+ }
+
+ protected WritableComparator(Class<? extends WritableComparable> keyClass,
+ Configuration conf,
+ boolean createInstances) {
this.keyClass = keyClass;
+ this.conf = (conf != null) ? conf : new Configuration();
if (createInstances) {
key1 = newKey();
key2 = newKey();
@@ -112,7 +141,7 @@ public class WritableComparator implemen
/** Construct a new {@link WritableComparable} instance. */
public WritableComparable newKey() {
- return ReflectionUtils.newInstance(keyClass, null);
+ return ReflectionUtils.newInstance(keyClass, conf);
}
/** Optimization hook. Override this to make SequenceFile.Sorter's scream.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritable.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritable.java?rev=1602297&r1=1602296&r2=1602297&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritable.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritable.java
Thu Jun 12 21:26:39 2014
@@ -23,6 +23,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Random;
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
@@ -30,6 +31,11 @@ import junit.framework.TestCase;
/** Unit tests for Writable. */
public class TestWritable extends TestCase {
+private static final String TEST_CONFIG_PARAM = "frob.test";
+private static final String TEST_CONFIG_VALUE = "test";
+private static final String TEST_WRITABLE_CONFIG_PARAM = "test.writable";
+private static final String TEST_WRITABLE_CONFIG_VALUE = TEST_CONFIG_VALUE;
+
public TestWritable(String name) { super(name); }
/** Example class used in test cases below. */
@@ -64,6 +70,25 @@ public class TestWritable extends TestCa
}
}
+ public static class SimpleWritableComparable extends SimpleWritable
+ implements WritableComparable<SimpleWritableComparable>, Configurable {
+ private Configuration conf;
+
+ public SimpleWritableComparable() {}
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ public int compareTo(SimpleWritableComparable o) {
+ return this.state - o.state;
+ }
+ }
+
/** Test 1: Check that SimpleWritable. */
public void testSimpleWritable() throws Exception {
testWritable(new SimpleWritable());
@@ -121,9 +146,34 @@ public class TestWritable extends TestCa
@Override public int compareTo(Frob o) { return 0; }
}
- /** Test that comparator is defined. */
+ /** Test that comparator is defined and configured. */
public static void testGetComparator() throws Exception {
- assert(WritableComparator.get(Frob.class) instanceof FrobComparator);
+ Configuration conf = new Configuration();
+
+ // Without conf.
+ WritableComparator frobComparator = WritableComparator.get(Frob.class);
+ assert(frobComparator instanceof FrobComparator);
+ assertNotNull(frobComparator.getConf());
+ assertNull(frobComparator.getConf().get(TEST_CONFIG_PARAM));
+
+ // With conf.
+ conf.set(TEST_CONFIG_PARAM, TEST_CONFIG_VALUE);
+ frobComparator = WritableComparator.get(Frob.class, conf);
+ assert(frobComparator instanceof FrobComparator);
+ assertNotNull(frobComparator.getConf());
+ assertEquals(conf.get(TEST_CONFIG_PARAM), TEST_CONFIG_VALUE);
+
+ // Without conf. should reuse configuration.
+ frobComparator = WritableComparator.get(Frob.class);
+ assert(frobComparator instanceof FrobComparator);
+ assertNotNull(frobComparator.getConf());
+ assertEquals(conf.get(TEST_CONFIG_PARAM), TEST_CONFIG_VALUE);
+
+ // New conf. should use new configuration.
+ frobComparator = WritableComparator.get(Frob.class, new Configuration());
+ assert(frobComparator instanceof FrobComparator);
+ assertNotNull(frobComparator.getConf());
+ assertNull(frobComparator.getConf().get(TEST_CONFIG_PARAM));
}
/**
@@ -153,4 +203,17 @@ public class TestWritable extends TestCa
.compare(writable1, writable3) == 0);
}
+ /**
+ * Test that Writable's are configured by Comparator.
+ */
+ public void testConfigurableWritableComparator() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(TEST_WRITABLE_CONFIG_PARAM, TEST_WRITABLE_CONFIG_VALUE);
+
+ WritableComparator wc =
WritableComparator.get(SimpleWritableComparable.class, conf);
+ SimpleWritableComparable key = ((SimpleWritableComparable)wc.newKey());
+ assertNotNull(wc.getConf());
+ assertNotNull(key.getConf());
+ assertEquals(key.getConf().get(TEST_WRITABLE_CONFIG_PARAM),
TEST_WRITABLE_CONFIG_VALUE);
+ }
}
