HIVE-16742 : cap the number of reducers for LLAP at the configured value (Sergey Shelukhin, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f81031ca Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f81031ca Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f81031ca Branch: refs/heads/hive-14535 Commit: f81031cae2728350feeb6f438666dd4e2b45d09b Parents: f66820c Author: sergey <[email protected]> Authored: Tue May 23 17:30:18 2017 -0700 Committer: sergey <[email protected]> Committed: Tue May 23 17:41:18 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f81031ca/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index c81131e..38e4b94 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -182,7 +182,9 @@ public class LlapDecider implements PhysicalPlanResolver { } // We only increase the targets here. if (reduceWork.isAutoReduceParallelism()) { - int newMin = Math.max(reduceWork.getMinReduceTasks(), targetCount); + // Do not exceed the configured max reducers. + int newMin = Math.min(conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS), + Math.max(reduceWork.getMinReduceTasks(), targetCount)); if (newMin < reduceWork.getMaxReduceTasks()) { reduceWork.setMinReduceTasks(newMin); reduceWork.getEdgePropRef().setAutoReduce(conf, true, newMin,
