Author: rohini
Date: Tue Jan 17 01:00:33 2023
New Revision: 1906719

URL: http://svn.apache.org/viewvc?rev=1906719&view=rev
Log:
PIG-5439: pig.exec.reducers.max does not take effect for skewed join 
(vnarayanan7 via rohini)

Modified:
    pig/trunk/CHANGES.txt
    
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java
    
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java

Modified: pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1906719&r1=1906718&r2=1906719&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 17 01:00:33 2023
@@ -120,6 +120,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5439: pig.exec.reducers.max does not take effect for skewed join 
(vnarayanan7 via rohini)
+
 PIG-5431: Date datatype is different between Hive 1.x and Hive 3.x (rohini)
 
 PIG-5433: Fix test failures with TestHBaseStorage and htrace dependency 
(rohini)

Modified: 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java
URL: 
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java?rev=1906719&r1=1906718&r2=1906719&view=diff
==============================================================================
--- 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java
 (original)
+++ 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java
 Tue Jan 17 01:00:33 2023
@@ -84,7 +84,8 @@ public class FindQuantilesTez extends Fi
 
             long estimatedInputSize = 
(long)((double)sampleSize/mySamples.size() * totalInputRows);
             estimatedNumReducers = 
(int)Math.ceil((double)estimatedInputSize/bytesPerTask);
-            estimatedNumReducers = Math.min(estimatedNumReducers, 
InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
+            int maxReducers = 
PigMapReduce.sJobConfInternal.get().getInt(InputSizeReducerEstimator.MAX_REDUCER_COUNT_PARAM,
 InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
+            estimatedNumReducers = Math.min(estimatedNumReducers, maxReducers);
             if (estimatedNumReducers==0) {
                 estimatedNumReducers = 1;
             }

Modified: 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java
URL: 
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java?rev=1906719&r1=1906718&r2=1906719&view=diff
==============================================================================
--- 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java
 (original)
+++ 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java
 Tue Jan 17 01:00:33 2023
@@ -70,7 +70,8 @@ public class PartitionSkewedKeysTez exte
                     InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER);
 
             estimatedNumReducers = 
(int)Math.ceil((double)estimatedInputSize/bytesPerTask);
-            estimatedNumReducers = Math.min(estimatedNumReducers, 
InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
+            int maxReducers = 
PigMapReduce.sJobConfInternal.get().getInt(InputSizeReducerEstimator.MAX_REDUCER_COUNT_PARAM,
 InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
+            estimatedNumReducers = Math.min(estimatedNumReducers, maxReducers);
 
             LOG.info("Estimating parallelism: estimatedInputSize is " + 
estimatedInputSize + ". bytesPerTask is " + bytesPerTask + ". 
estimatedNumReducers is " + estimatedNumReducers + ".");
 


Reply via email to