Author: omalley
Date: Fri Mar 4 03:26:34 2011
New Revision: 1076966
URL: http://svn.apache.org/viewvc?rev=1076966&view=rev
Log:
commit 4640e24bd88304f22968e1433ad5804a12192506
Author: Yahoo\! <[email protected]>
Date: Tue Aug 11 10:07:23 2009 -0700
Walk back the 0.20 branch to the correct point
Revert "MAPREDUCE-735. Fixes a problem in the KeyFieldHelper to do with the
end index for some inputs. Contributed by Amar Kamat."
This reverts commit 66a3d58e6f3ca322bac445739021e765009e677b.
Removed:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldHelper.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java?rev=1076966&r1=1076965&r2=1076966&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.java
Fri Mar 4 03:26:34 2011
@@ -67,7 +67,7 @@ public class KeyFieldBasedPartitioner<K2
List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs();
if (allKeySpecs.size() == 0) {
- return getPartition(key.toString().hashCode(), numReduceTasks);
+ return (key.toString().hashCode() & Integer.MAX_VALUE) % numReduceTasks;
}
try {
@@ -96,7 +96,7 @@ public class KeyFieldBasedPartitioner<K2
currentHash = hashCode(keyBytes, startChar, endChar,
currentHash);
}
- return getPartition(currentHash, numReduceTasks);
+ return (currentHash & Integer.MAX_VALUE) % numReduceTasks;
}
protected int hashCode(byte[] b, int start, int end, int currentHash) {
@@ -106,7 +106,4 @@ public class KeyFieldBasedPartitioner<K2
return currentHash;
}
- protected int getPartition(int hash, int numReduceTasks) {
- return (hash & Integer.MAX_VALUE) % numReduceTasks;
- }
}
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java?rev=1076966&r1=1076965&r2=1076966&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/lib/KeyFieldHelper.java
Fri Mar 4 03:26:34 2011
@@ -47,13 +47,6 @@ class KeyFieldHelper {
int endChar = 0;
boolean numeric;
boolean reverse;
- @Override
- public String toString() {
- return "-k"
- + beginFieldIdx + "." + beginChar + ","
- + endFieldIdx + "." + endChar
- + (numeric ? "n" : "") + (reverse ? "r" : "");
- }
}
private List<KeyDescription> allKeySpecs = new ArrayList<KeyDescription>();
@@ -137,7 +130,7 @@ class KeyFieldHelper {
if (k.endFieldIdx == 0) {
//there is no end field specified for this keyspec. So the remaining
//part of the key is considered in its entirety.
- return end - 1;
+ return end;
}
if (lengthIndices[0] >= k.endFieldIdx) {
int position = 0;
@@ -151,9 +144,9 @@ class KeyFieldHelper {
if (position + k.endChar <= (end - start)) {
return start + position + k.endChar - 1;
}
- return end - 1;
+ return end;
}
- return end - 1;
+ return end;
}
public void parseOption(String option) {
if (option == null || option.equals("")) {
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java?rev=1076966&r1=1076965&r2=1076966&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
Fri Mar 4 03:26:34 2011
@@ -133,8 +133,6 @@ public class TestKeyFieldBasedComparator
configure("-k10,10",2);
localTestWithoutMRJob("-k9,9", 1);
-
- localTestWithoutMRJob("-k9n", 1);
}
byte[] line1_bytes = line1.getBytes();
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java?rev=1076966&r1=1076965&r2=1076966&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
Fri Mar 4 03:26:34 2011
@@ -29,98 +29,12 @@ public class TestKeyFieldBasedPartitione
* Test is key-field-based partitioned works with empty key.
*/
public void testEmptyKey() throws Exception {
- int numReducers = 10;
KeyFieldBasedPartitioner<Text, Text> kfbp =
new KeyFieldBasedPartitioner<Text, Text>();
JobConf conf = new JobConf();
conf.setInt("num.key.fields.for.partition", 10);
kfbp.configure(conf);
assertEquals("Empty key should map to 0th partition",
- 0, kfbp.getPartition(new Text(), new Text(), numReducers));
-
- // check if the hashcode is correct when no keyspec is specified
- kfbp = new KeyFieldBasedPartitioner<Text, Text>();
- conf = new JobConf();
- kfbp.configure(conf);
- String input = "abc\tdef\txyz";
- int hashCode = input.hashCode();
- int expectedPartition = kfbp.getPartition(hashCode, numReducers);
- assertEquals("Partitioner doesnt work as expected", expectedPartition,
- kfbp.getPartition(new Text(input), new Text(), numReducers));
-
- // check if the hashcode is correct with specified keyspec
- kfbp = new KeyFieldBasedPartitioner<Text, Text>();
- conf = new JobConf();
- conf.set("mapred.text.key.partitioner.options", "-k2,2");
- kfbp.configure(conf);
- String expectedOutput = "def";
- byte[] eBytes = expectedOutput.getBytes();
- hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
- expectedPartition = kfbp.getPartition(hashCode, numReducers);
- assertEquals("Partitioner doesnt work as expected", expectedPartition,
- kfbp.getPartition(new Text(input), new Text(), numReducers));
-
- // test with invalid end index in keyspecs
- kfbp = new KeyFieldBasedPartitioner<Text, Text>();
- conf = new JobConf();
- conf.set("mapred.text.key.partitioner.options", "-k2,5");
- kfbp.configure(conf);
- expectedOutput = "def\txyz";
- eBytes = expectedOutput.getBytes();
- hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
- expectedPartition = kfbp.getPartition(hashCode, numReducers);
- assertEquals("Partitioner doesnt work as expected", expectedPartition,
- kfbp.getPartition(new Text(input), new Text(), numReducers));
-
- // test with 0 end index in keyspecs
- kfbp = new KeyFieldBasedPartitioner<Text, Text>();
- conf = new JobConf();
- conf.set("mapred.text.key.partitioner.options", "-k2");
- kfbp.configure(conf);
- expectedOutput = "def\txyz";
- eBytes = expectedOutput.getBytes();
- hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
- expectedPartition = kfbp.getPartition(hashCode, numReducers);
- assertEquals("Partitioner doesnt work as expected", expectedPartition,
- kfbp.getPartition(new Text(input), new Text(), numReducers));
-
- // test with invalid keyspecs
- kfbp = new KeyFieldBasedPartitioner<Text, Text>();
- conf = new JobConf();
- conf.set("mapred.text.key.partitioner.options", "-k10");
- kfbp.configure(conf);
- assertEquals("Partitioner doesnt work as expected", 0,
- kfbp.getPartition(new Text(input), new Text(), numReducers));
-
- // test with multiple keyspecs
- kfbp = new KeyFieldBasedPartitioner<Text, Text>();
- conf = new JobConf();
- conf.set("mapred.text.key.partitioner.options", "-k2,2 -k4,4");
- kfbp.configure(conf);
- input = "abc\tdef\tpqr\txyz";
- expectedOutput = "def";
- eBytes = expectedOutput.getBytes();
- hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
- expectedOutput = "xyz";
- eBytes = expectedOutput.getBytes();
- hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, hashCode);
- expectedPartition = kfbp.getPartition(hashCode, numReducers);
- assertEquals("Partitioner doesnt work as expected", expectedPartition,
- kfbp.getPartition(new Text(input), new Text(), numReducers));
-
- // test with invalid start index in keyspecs
- kfbp = new KeyFieldBasedPartitioner<Text, Text>();
- conf = new JobConf();
- conf.set("mapred.text.key.partitioner.options", "-k2,2 -k30,21 -k4,4 -k5");
- kfbp.configure(conf);
- expectedOutput = "def";
- eBytes = expectedOutput.getBytes();
- hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0);
- expectedOutput = "xyz";
- eBytes = expectedOutput.getBytes();
- hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, hashCode);
- expectedPartition = kfbp.getPartition(hashCode, numReducers);
- assertEquals("Partitioner doesnt work as expected", expectedPartition,
- kfbp.getPartition(new Text(input), new Text(), numReducers));
+ 0, kfbp.getPartition(new Text(), new Text(), 10));
}
-}
+}
\ No newline at end of file
