DRILL-5669: Add a configurable option for minimum memory allocation to buffered 
ops

closes #879


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6b0e8378
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6b0e8378
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6b0e8378

Branch: refs/heads/master
Commit: 6b0e8378cc198c926f5b1cf95a07fd4c9fc092de
Parents: 35a7ad0
Author: Boaz Ben-Zvi <boazben-zvi@BBenZvi-E754-MBP13.local>
Authored: Mon Jul 17 14:21:29 2017 -0700
Committer: Arina Ielchiieva <arina.yelchiy...@gmail.com>
Committed: Fri Jul 21 16:32:26 2017 +0300

----------------------------------------------------------------------
 .../src/main/java/org/apache/drill/exec/ExecConstants.java  | 9 +++++++++
 .../drill/exec/server/options/SystemOptionManager.java      | 1 +
 .../apache/drill/exec/util/MemoryAllocationUtilities.java   | 8 ++++++--
 3 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java 
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 5b82d1f..97cb321 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -343,6 +343,15 @@ public interface ExecConstants {
       MAX_QUERY_MEMORY_PER_NODE_KEY, 1024 * 1024, Long.MAX_VALUE, 2 * 1024 * 
1024 * 1024L);
 
   /**
+   * Minimum memory alocated to each buffered operator instance.
+   * <p/>
+   * DEFAULT: 40 MB
+   */
+  String MIN_MEMORY_PER_BUFFERED_OP_KEY = 
"planner.memory.min_memory_per_buffered_op";
+  LongValidator MIN_MEMORY_PER_BUFFERED_OP = new RangeLongValidator(
+      MIN_MEMORY_PER_BUFFERED_OP_KEY, 1024 * 1024, Long.MAX_VALUE, 40 * 1024 * 
1024L);
+
+  /**
    * Extra query memory per node for non-blocking operators.
    * NOTE: This option is currently used only for memory estimation.
    * <p/>

http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index c2a4d65..3392a21 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -144,6 +144,7 @@ public class SystemOptionManager extends BaseOptionManager 
implements OptionMana
       ExecConstants.EARLY_LIMIT0_OPT,
       ExecConstants.ENABLE_MEMORY_ESTIMATION,
       ExecConstants.MAX_QUERY_MEMORY_PER_NODE,
+      ExecConstants.MIN_MEMORY_PER_BUFFERED_OP,
       ExecConstants.NON_BLOCKING_OPERATORS_MEMORY,
       ExecConstants.HASH_JOIN_TABLE_FACTOR,
       ExecConstants.HASH_AGG_TABLE_FACTOR,

http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
----------------------------------------------------------------------
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
index 79b49e4..4580222 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
@@ -66,11 +66,15 @@ public class MemoryAllocationUtilities {
       final long maxOperatorAlloc = maxAllocPerNode / (bufferedOpList.size() * 
maxWidthPerNode);
       logger.debug("Max buffered operator alloc: {}", maxOperatorAlloc);
 
+      // User configurable option to allow forcing minimum memory.
+      // Ensure that the buffered ops receive the minimum memory needed to 
make progress.
+      // Without this, the math might work out to allocate too little memory.
+      final long opMinMem = 
queryContext.getOptions().getOption(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY).num_val;
+
       for(final PhysicalOperator op : bufferedOpList) {
-        // Ensure that the sort receives the minimum memory needed to make 
progress.
-        // Without this, the math might work out to allocate too little memory.
 
         long alloc = Math.max(maxOperatorAlloc, op.getInitialAllocation());
+        alloc = Math.max(alloc, opMinMem);
         op.setMaxAllocation(alloc);
       }
     }

Reply via email to