[tez] branch master updated: TEZ-4216 : RLE check in MergeManager::finalMerge could be disabled (Rajesh Balamohan via Ashutosh Chauhan)

[hashutosh]

This is an automated email from the ASF dual-hosted git repository.

hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git


The following commit(s) were added to refs/heads/master by this push:
     new 3986f5b  TEZ-4216 : RLE check in MergeManager::finalMerge could be 
disabled (Rajesh Balamohan via Ashutosh Chauhan)
3986f5b is described below

commit 3986f5bc2c2e34ad160f77afd7478b26935fffa5
Author: Rajesh Balamohan <rbalamo...@apache.org>
AuthorDate: Mon Aug 24 14:11:52 2020 -0700

    TEZ-4216 : RLE check in MergeManager::finalMerge could be disabled (Rajesh 
Balamohan via Ashutosh Chauhan)
    
    Signed-off-by: Ashutosh Chauhan <hashut...@apache.org>
---
 .../common/shuffle/orderedgrouped/MergeManager.java       |  8 ++++----
 .../tez/runtime/library/common/sort/impl/TezMerger.java   | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git 
a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
 
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
index 70f9e55..dd23b3b 100644
--- 
a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
+++ 
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
@@ -1316,10 +1316,10 @@ public class MergeManager implements 
FetchedInputAllocatorOrderedGrouped {
       LOG.info(finalMergeLog.toString());
     }
     // This is doing nothing but creating an iterator over the segments.
-    return TezMerger.merge(job, fs, keyClass, valueClass,
-                 finalSegments, finalSegments.size(), tmpDir,
-                 comparator, progressable, spilledRecordsCounter, null,
-                 additionalBytesRead, null);
+    return TezMerger.merge(job, fs, keyClass, valueClass, codec,
+        finalSegments, finalSegments.size(), tmpDir,
+        comparator, progressable, spilledRecordsCounter, null,
+        additionalBytesRead, null);
   }
 
 
diff --git 
a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
 
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
index 3e04e74..726810b 100644
--- 
a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
+++ 
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
@@ -128,6 +128,21 @@ public class TezMerger {
                                                bytesReadCounter, mergePhase);
   }
 
+  public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
+      Class keyClass, Class valueClass, CompressionCodec codec,
+      List<Segment> segments,
+      int mergeFactor, Path tmpDir,
+      RawComparator comparator, Progressable reporter,
+      TezCounter readsCounter,
+      TezCounter writesCounter,
+      TezCounter bytesReadCounter,
+      Progress mergePhase) throws IOException, InterruptedException {
+    return new MergeQueue(conf, fs, segments, comparator, reporter,
+        false, codec, false, false)
+        .merge(keyClass, valueClass, mergeFactor, tmpDir,
+            readsCounter, writesCounter, bytesReadCounter, mergePhase);
+  }
+
   public static <K extends Object, V extends Object>
   TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
       Class keyClass, Class valueClass,