Repository: drill Updated Branches: refs/heads/master 07dc46c11 -> bcc01c05a
DRILL-3045: Make planning time memory limit configurable Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dfa47da6 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dfa47da6 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dfa47da6 Branch: refs/heads/master Commit: dfa47da669dc2455389151c4e4071d405030c7a9 Parents: 07dc46c Author: Mehant Baid <meha...@gmail.com> Authored: Fri Aug 28 11:23:54 2015 -0700 Committer: Mehant Baid <meha...@gmail.com> Committed: Sun Aug 30 21:43:13 2015 -0700 ---------------------------------------------------------------------- .../org/apache/drill/exec/ops/QueryContext.java | 7 ++----- .../exec/planner/physical/PlannerSettings.java | 18 ++++++++++++++++++ .../exec/server/options/SystemOptionManager.java | 1 + 3 files changed, 21 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java index c3cd4e6..1cd67ac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java @@ -57,9 +57,6 @@ import org.apache.drill.exec.util.Utilities; public class QueryContext implements AutoCloseable, OptimizerRulesContext { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class); - private static final int INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES = 1024 * 1024; - private static final int MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 256 * 1024 * 1024; - private final DrillbitContext drillbitContext; private final UserSession session; private final OptionManager queryOptions; @@ -94,8 +91,8 @@ public class QueryContext implements AutoCloseable, OptimizerRulesContext { contextInformation = new ContextInformation(session.getCredentials(), queryContextInfo); try { - allocator = drillbitContext.getAllocator().getChildAllocator(null, INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, - MAX_OFF_HEAP_ALLOCATION_IN_BYTES, false); + allocator = drillbitContext.getAllocator().getChildAllocator(null, plannerSettings.getInitialPlanningMemorySize(), + plannerSettings.getPlanningMemoryLimit(), false); } catch (OutOfMemoryException e) { throw new DrillRuntimeException("Error creating off-heap allocator for planning context.",e); } http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java index 3355018..0a05d08 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java @@ -19,6 +19,7 @@ package org.apache.drill.exec.planner.physical; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValidator; import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator; @@ -38,6 +39,13 @@ public class PlannerSettings implements Context{ public static final int MAX_BROADCAST_THRESHOLD = Integer.MAX_VALUE; public static final int DEFAULT_IDENTIFIER_MAX_LENGTH = 1024; + // initial off heap memory allocation (1M) + private static final long INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES = 1024 * 1024; + // default off heap memory for planning (256M) + private static final long DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 256 * 1024 * 1024; + // max off heap memory for planning (16G) + private static final long MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 16l * 1024 * 1024 * 1024; + public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding", true); public static final OptionValidator EXCHANGE = new BooleanValidator("planner.disable_exchanges", false); public static final OptionValidator HASHAGG = new BooleanValidator("planner.enable_hashagg", true); @@ -65,6 +73,8 @@ public class PlannerSettings implements Context{ public static final String ENABLE_DECIMAL_DATA_TYPE_KEY = "planner.enable_decimal_data_type"; public static final OptionValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY, false); public static final OptionValidator HEP_JOIN_OPT = new BooleanValidator("planner.enable_hep_join_opt", true); + public static final OptionValidator PLANNER_MEMORY_LIMIT = new RangeLongValidator("planner.memory_limit", + INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES, DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES); public static final OptionValidator IDENTIFIER_MAX_LENGTH = new RangeLongValidator("planner.identifier_max_length", 128 /* A minimum length is needed because option names are identifiers themselves */, @@ -188,6 +198,14 @@ public class PlannerSettings implements Context{ return options.getOption(IDENTIFIER_MAX_LENGTH.getOptionName()).num_val; } + public long getPlanningMemoryLimit() { + return options.getOption(PLANNER_MEMORY_LIMIT.getOptionName()).num_val; + } + + public static long getInitialPlanningMemorySize() { + return INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES; + } + @Override public <T> T unwrap(Class<T> clazz) { if(clazz == PlannerSettings.class){ http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 2d41740..12db4d3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -73,6 +73,7 @@ public class SystemOptionManager extends BaseOptionManager { PlannerSettings.PARTITION_SENDER_SET_THREADS, PlannerSettings.ENABLE_DECIMAL_DATA_TYPE, PlannerSettings.HEP_JOIN_OPT, + PlannerSettings.PLANNER_MEMORY_LIMIT, ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION, ExecConstants.OUTPUT_FORMAT_VALIDATOR, ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
