Author: rohini Date: Tue Jan 17 01:00:33 2023 New Revision: 1906719 URL: http://svn.apache.org/viewvc?rev=1906719&view=rev Log: PIG-5439: pig.exec.reducers.max does not take effect for skewed join (vnarayanan7 via rohini)
Modified: pig/trunk/CHANGES.txt pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java Modified: pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1906719&r1=1906718&r2=1906719&view=diff ============================================================================== --- pig/trunk/CHANGES.txt (original) +++ pig/trunk/CHANGES.txt Tue Jan 17 01:00:33 2023 @@ -120,6 +120,8 @@ OPTIMIZATIONS BUG FIXES +PIG-5439: pig.exec.reducers.max does not take effect for skewed join (vnarayanan7 via rohini) + PIG-5431: Date datatype is different between Hive 1.x and Hive 3.x (rohini) PIG-5433: Fix test failures with TestHBaseStorage and htrace dependency (rohini) Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java?rev=1906719&r1=1906718&r2=1906719&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java (original) +++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java Tue Jan 17 01:00:33 2023 @@ -84,7 +84,8 @@ public class FindQuantilesTez extends Fi long estimatedInputSize = (long)((double)sampleSize/mySamples.size() * totalInputRows); estimatedNumReducers = (int)Math.ceil((double)estimatedInputSize/bytesPerTask); - estimatedNumReducers = Math.min(estimatedNumReducers, InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM); + int maxReducers = PigMapReduce.sJobConfInternal.get().getInt(InputSizeReducerEstimator.MAX_REDUCER_COUNT_PARAM, InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM); + estimatedNumReducers = Math.min(estimatedNumReducers, maxReducers); if (estimatedNumReducers==0) { estimatedNumReducers = 1; } Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java?rev=1906719&r1=1906718&r2=1906719&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java (original) +++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java Tue Jan 17 01:00:33 2023 @@ -70,7 +70,8 @@ public class PartitionSkewedKeysTez exte InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER); estimatedNumReducers = (int)Math.ceil((double)estimatedInputSize/bytesPerTask); - estimatedNumReducers = Math.min(estimatedNumReducers, InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM); + int maxReducers = PigMapReduce.sJobConfInternal.get().getInt(InputSizeReducerEstimator.MAX_REDUCER_COUNT_PARAM, InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM); + estimatedNumReducers = Math.min(estimatedNumReducers, maxReducers); LOG.info("Estimating parallelism: estimatedInputSize is " + estimatedInputSize + ". bytesPerTask is " + bytesPerTask + ". estimatedNumReducers is " + estimatedNumReducers + ".");