Repository: tez Updated Branches: refs/heads/master fccb6f127 -> 34f9bdaa3
TEZ-2849. Implement Specific Workaround for JDK-8026049 & JDK-8073093 (gopalv via rbalamohan) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/34f9bdaa Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/34f9bdaa Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/34f9bdaa Branch: refs/heads/master Commit: 34f9bdaa3afd2ccc555522b9b3edc037c777e9a1 Parents: fccb6f1 Author: Rajesh Balamohan <[email protected]> Authored: Wed Nov 4 05:44:50 2015 +0530 Committer: Rajesh Balamohan <[email protected]> Committed: Wed Nov 4 05:44:50 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../common/comparator/TezBytesComparator.java | 2 +- .../common/sort/impl/PipelinedSorter.java | 26 +++++++++++--------- 3 files changed, 17 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/34f9bdaa/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 8234e70..d96fcae 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-2679. Admin forms of launch env settings ALL CHANGES: + TEZ-2849. Implement Specific Workaround for JDK-8026049 & JDK-8073093. TEZ-2828. Fix typo in "Shuffle assigned " log statement in shuffle.orderedgrouped.Shuffle. TEZ-2909. Tez UI: Application link in All DAGs table is disable when applicationhistory is unavailable TEZ-808. Handle task attempts that are not making progress http://git-wip-us.apache.org/repos/asf/tez/blob/34f9bdaa/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java index 09b2b03..81df0cf 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java @@ -24,7 +24,7 @@ import org.apache.hadoop.io.WritableComparator; @Public @Unstable -public class TezBytesComparator extends WritableComparator implements +public final class TezBytesComparator extends WritableComparator implements ProxyComparator<BytesWritable> { public TezBytesComparator() { http://git-wip-us.apache.org/repos/asf/tez/blob/34f9bdaa/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java index f512a5d..2d53a2e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java @@ -848,13 +848,14 @@ public class PipelinedSorter extends ExternalSorter { } } - private final class SortSpan implements IndexedSortable { + private final class SortSpan implements IndexedSortable { final IntBuffer kvmeta; + final byte[] rawkvmeta; + final int kvmetabase; final ByteBuffer kvbuffer; final DataOutputStream out; final RawComparator comparator; - final int imeta[] = new int[NMETA]; - final int jmeta[] = new int[NMETA]; + final byte[] imeta = new byte[METASIZE]; private int index = 0; private long eq = 0; @@ -878,8 +879,10 @@ public class PipelinedSorter extends ExternalSorter { kvbuffer = reserved.slice(); reserved.flip(); reserved.limit(metasize); - kvmeta = reserved - .slice() + ByteBuffer kvmetabuffer = reserved.slice(); + rawkvmeta = kvmetabuffer.array(); + kvmetabase = kvmetabuffer.arrayOffset(); + kvmeta = kvmetabuffer .order(ByteOrder.nativeOrder()) .asIntBuffer(); out = new DataOutputStream( @@ -894,7 +897,7 @@ public class PipelinedSorter extends ExternalSorter { } LOG.info(outputContext.getDestinationVertexName() + ": " + "done sorting span=" + index + ", length=" + length() + ", " + "time=" + (System.currentTimeMillis() - start)); - return new SpanIterator(this); + return new SpanIterator((SortSpan)this); } int offsetFor(int i) { @@ -905,13 +908,14 @@ public class PipelinedSorter extends ExternalSorter { final int kvi = offsetFor(mi); final int kvj = offsetFor(mj); - kvmeta.position(kvi); kvmeta.get(imeta); - kvmeta.position(kvj); kvmeta.get(jmeta); - kvmeta.position(kvj); kvmeta.put(imeta); - kvmeta.position(kvi); kvmeta.put(jmeta); + final int kvioff = kvmetabase + (kvi << 2); + final int kvjoff = kvmetabase + (kvj << 2); + System.arraycopy(rawkvmeta, kvioff, imeta, 0, METASIZE); + System.arraycopy(rawkvmeta, kvjoff, rawkvmeta, kvioff, METASIZE); + System.arraycopy(imeta, 0, rawkvmeta, kvjoff, METASIZE); } - private int compareKeys(final int kvi, final int kvj) { + protected int compareKeys(final int kvi, final int kvj) { final int istart = kvmeta.get(kvi + KEYSTART); final int jstart = kvmeta.get(kvj + KEYSTART); final int ilen = kvmeta.get(kvi + VALSTART) - istart;
