Repository: tez Updated Branches: refs/heads/master 80f95b1d6 -> e3ee7a63b
TEZ-3723. TezIndexRecord#hasData() returns true for empty index record in the Unordered case (kshukla) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e3ee7a63 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e3ee7a63 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e3ee7a63 Branch: refs/heads/master Commit: e3ee7a63b0289868db535008ed3c4a5d2fc6c4e3 Parents: 80f95b1 Author: Kuhu Shukla <[email protected]> Authored: Tue May 16 16:25:33 2017 -0500 Committer: Kuhu Shukla <[email protected]> Committed: Tue May 16 16:25:33 2017 -0500 ---------------------------------------------------------------------- .../tez/runtime/library/common/sort/impl/TezIndexRecord.java | 2 +- .../tez/runtime/library/common/shuffle/TestShuffleUtils.java | 5 ++--- .../common/writers/TestUnorderedPartitionedKVWriter.java | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/e3ee7a63/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java index 2f04a4d..872cd10 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java @@ -51,6 +51,6 @@ public class TezIndexRecord { public boolean hasData() { //TEZ-941 - Avoid writing out empty partitions //EOF_MARKER + Header bytes - return !(rawLength == (IFile.HEADER.length + 2)); + return !(rawLength <= (IFile.HEADER.length + 2)); } } http://git-wip-us.apache.org/repos/asf/tez/blob/e3ee7a63/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java index b1ce716..da09324 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java @@ -43,8 +43,8 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.BitSet; +import java.util.concurrent.ThreadLocalRandom; import java.util.List; -import java.util.Random; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; @@ -137,9 +137,8 @@ public class TestShuffleUtils { TezSpillRecord spillRecord = new TezSpillRecord(numPartitions); long startOffset = 0; long partLen = 200; //compressed - Random rnd = new Random(); for(int i=0;i<numPartitions;i++) { - long rawLen = rnd.nextLong(); + long rawLen = ThreadLocalRandom.current().nextLong(100, 200); if (i % 2 == 0 || allEmptyPartitions) { rawLen = 6; //indicates empty partition } http://git-wip-us.apache.org/repos/asf/tez/blob/e3ee7a63/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java index 5cda126..6ea1562 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java @@ -933,10 +933,11 @@ public class TestUnorderedPartitionedKVWriter { IntWritable keyDeser = new IntWritable(); LongWritable valDeser = new LongWritable(); for (int i = 0; i < numOutputs; i++) { + TezIndexRecord indexRecord = spillRecord.getIndex(i); if (skippedPartitions != null && skippedPartitions.contains(i)) { + assertFalse("The Index Record for partition " + i + " should not have any data", indexRecord.hasData()); continue; } - TezIndexRecord indexRecord = spillRecord.getIndex(i); FSDataInputStream inStream = FileSystem.getLocal(conf).open(outputFilePath); inStream.seek(indexRecord.getStartOffset()); IFile.Reader reader = new IFile.Reader(inStream, indexRecord.getPartLength(), codec, null,
