DRILL-1163: Estimate memory and re-planning query + system options UI.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ba027c7f Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ba027c7f Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ba027c7f Branch: refs/heads/master Commit: ba027c7f555782a0369aeffaeee1dd0bd60072df Parents: bb21263 Author: Sudheesh Katkam <skat...@maprtech.com> Authored: Tue Aug 12 22:37:44 2014 +0530 Committer: Aditya Kishore <adi...@maprtech.com> Committed: Mon Aug 18 14:56:51 2014 +0530 ---------------------------------------------------------------------- .../org/apache/drill/exec/ExecConstants.java | 33 +++++++ .../drill/exec/compile/QueryClassLoader.java | 25 +++-- .../apache/drill/exec/ops/FragmentContext.java | 26 +++--- .../org/apache/drill/exec/ops/QueryContext.java | 16 +++- .../apache/drill/exec/opt/BasicOptimizer.java | 15 ++- .../drill/exec/planner/cost/DrillCostBase.java | 33 +++++-- .../exec/planner/physical/HashAggPrel.java | 42 +++++---- .../exec/planner/physical/HashAggPrule.java | 10 +- .../exec/planner/physical/HashJoinPrel.java | 27 ++++-- .../exec/planner/physical/HashJoinPrule.java | 13 +-- .../exec/planner/physical/PlannerSettings.java | 8 ++ .../drill/exec/planner/physical/SortPrel.java | 21 +++-- .../visitor/MemoryEstimationVisitor.java | 66 +++++++++++++ .../planner/sql/handlers/DefaultSqlHandler.java | 33 +++++-- .../planner/sql/handlers/SetOptionHandler.java | 32 ++++--- .../apache/drill/exec/rpc/user/UserSession.java | 14 ++- .../server/options/FragmentOptionManager.java | 82 +++++++++++++++++ .../server/options/FragmentOptionsManager.java | 85 ----------------- .../exec/server/options/OptionManager.java | 4 +- .../drill/exec/server/options/OptionValue.java | 33 ++++++- .../exec/server/options/QueryOptionManager.java | 97 ++++++++++++++++++++ .../server/options/SessionOptionManager.java | 43 +++++---- .../server/options/SystemOptionManager.java | 36 ++++---- .../exec/server/options/TypeValidators.java | 30 +++++- .../drill/exec/server/rest/DrillRoot.java | 62 ++++++++++++- .../drill/exec/server/rest/StatusResources.java | 96 ++++++++++++++++++- .../apache/drill/exec/work/foreman/Foreman.java | 2 +- .../src/main/resources/rest/generic.ftl | 6 +- .../java-exec/src/main/resources/rest/index.ftl | 13 ++- .../src/main/resources/rest/options.ftl | 47 ++++++++++ .../src/main/resources/rest/status.ftl | 1 + .../java/org/apache/drill/PlanningBase.java | 29 +++--- 32 files changed, 802 insertions(+), 278 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index f2a84ee..f018b78 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator; import org.apache.drill.exec.server.options.TypeValidators.DoubleValidator; import org.apache.drill.exec.server.options.TypeValidators.LongValidator; import org.apache.drill.exec.server.options.TypeValidators.PositiveLongValidator; +import org.apache.drill.exec.server.options.TypeValidators.PowerOfTwoLongValidator; import org.apache.drill.exec.server.options.TypeValidators.StringValidator; public interface ExecConstants { @@ -118,6 +119,38 @@ public interface ExecConstants { public static final String AFFINITY_FACTOR_KEY = "planner.affinity_factor"; public static final OptionValidator AFFINITY_FACTOR = new DoubleValidator(AFFINITY_FACTOR_KEY, 1.2d); + public static final String ENABLE_MEMORY_ESTIMATION_KEY = "planner.memory.enable_memory_estimation"; + public static final OptionValidator ENABLE_MEMORY_ESTIMATION = new BooleanValidator(ENABLE_MEMORY_ESTIMATION_KEY, false); + + /** + * Maximum query memory per node (in MB). Re-plan with cheaper operators if memory estimation exceeds this limit. + * <p/> + * DEFAULT: 2048 MB + */ + public static final String MAX_QUERY_MEMORY_PER_NODE_KEY = "planner.memory.max_query_memory_per_node"; + public static final OptionValidator MAX_QUERY_MEMORY_PER_NODE = new PowerOfTwoLongValidator( + MAX_QUERY_MEMORY_PER_NODE_KEY, Runtime.getRuntime().maxMemory(), 1 << 11); + + /** + * Extra query memory per node for non-blocking operators. + * NOTE: This option is currently used only for memory estimation. + * <p/> + * DEFAULT: 64 MB + * MAXIMUM: 2048 MB + */ + public static final String NON_BLOCKING_OPERATORS_MEMORY_KEY = "planner.memory.non_blocking_operators_memory"; + public static final OptionValidator NON_BLOCKING_OPERATORS_MEMORY = new PowerOfTwoLongValidator( + NON_BLOCKING_OPERATORS_MEMORY_KEY, 1 << 11, 1 << 6); + + public static final String HASH_JOIN_TABLE_FACTOR_KEY = "planner.memory.hash_join_table_factor"; + public static final OptionValidator HASH_JOIN_TABLE_FACTOR = new DoubleValidator(HASH_JOIN_TABLE_FACTOR_KEY, 1.1d); + + public static final String HASH_AGG_TABLE_FACTOR_KEY = "planner.memory.hash_agg_table_factor"; + public static final OptionValidator HASH_AGG_TABLE_FACTOR = new DoubleValidator(HASH_AGG_TABLE_FACTOR_KEY, 1.1d); + + public static final String AVERAGE_FIELD_WIDTH_KEY = "planner.memory.average_field_width"; + public static final OptionValidator AVERAGE_FIELD_WIDTH = new PositiveLongValidator(AVERAGE_FIELD_WIDTH_KEY, Long.MAX_VALUE, 8); + public static final String ENABLE_QUEUE_KEY = "exec.queue.enable"; public static final OptionValidator ENABLE_QUEUE = new BooleanValidator(ENABLE_QUEUE_KEY, false); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java index 8de14ec..2dcb52b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java @@ -17,13 +17,7 @@ */ package org.apache.drill.exec.compile; -import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.Arrays; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicLong; - +import com.google.common.collect.MapMaker; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.ExpressionParsingException; import org.apache.drill.exec.compile.ClassTransformer.ClassNames; @@ -35,9 +29,13 @@ import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator; import org.apache.drill.exec.server.options.TypeValidators.LongValidator; import org.apache.drill.exec.server.options.TypeValidators.StringValidator; import org.codehaus.commons.compiler.CompileException; -import org.eigenbase.sql.SqlLiteral; -import com.google.common.collect.MapMaker; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; public class QueryClassLoader extends URLClassLoader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryClassLoader.class); @@ -45,15 +43,14 @@ public class QueryClassLoader extends URLClassLoader { public static final String JAVA_COMPILER_OPTION = "exec.java_compiler"; public static final StringValidator JAVA_COMPILER_VALIDATOR = new StringValidator(JAVA_COMPILER_OPTION, CompilerPolicy.DEFAULT.toString()) { @Override - public OptionValue validate(SqlLiteral value) throws ExpressionParsingException { - OptionValue ov = super.validate(value); + public void validate(OptionValue v) throws ExpressionParsingException { + super.validate(v); try { - CompilerPolicy.valueOf(ov.string_val.toUpperCase()); + CompilerPolicy.valueOf(v.string_val.toUpperCase()); } catch (IllegalArgumentException e) { throw new ExpressionParsingException(String.format("Invalid value '%s' specified for option '%s'. Valid values are %s.", - ov.string_val, getOptionName(), Arrays.toString(CompilerPolicy.values()))); + v.string_val, getOptionName(), Arrays.toString(CompilerPolicy.values()))); } - return ov; } }; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java index 0b8b88a..8a670fb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java @@ -17,14 +17,10 @@ */ package org.apache.drill.exec.ops; -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import java.util.Map; - +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.jdbc.SimpleOptiqSchema; - import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.compile.ClassTransformer; @@ -42,13 +38,15 @@ import org.apache.drill.exec.rpc.control.ControlTunnel; import org.apache.drill.exec.rpc.data.DataTunnel; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.server.options.FragmentOptionsManager; +import org.apache.drill.exec.server.options.FragmentOptionManager; import org.apache.drill.exec.server.options.OptionList; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.work.batch.IncomingBuffers; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; /** * Contextual objects required for execution of a particular fragment. @@ -71,7 +69,7 @@ public class FragmentContext implements Closeable { private IncomingBuffers buffers; private final long queryStartTime; private final int rootFragmentTimeZone; - private final OptionManager sessionOptions; + private final OptionManager fragmentOptions; private volatile Throwable failureCause; private volatile boolean failed = false; @@ -96,16 +94,16 @@ public class FragmentContext implements Closeable { }else{ list = dbContext.getConfig().getMapper().readValue(fragment.getOptionsJson(), OptionList.class); } - this.sessionOptions = new FragmentOptionsManager(context.getOptionManager(), list); + this.fragmentOptions = new FragmentOptionManager(context.getOptionManager(), list); }catch(Exception e){ throw new ExecutionSetupException("Failure while reading plan options.", e); } this.allocator = context.getAllocator().getChildAllocator(fragment.getHandle(), fragment.getMemInitial(), fragment.getMemMax()); - this.loader = new QueryClassLoader(dbContext.getConfig(), sessionOptions); + this.loader = new QueryClassLoader(dbContext.getConfig(), fragmentOptions); } - public OptionManager getOptions(){ - return sessionOptions; + public OptionManager getOptions() { + return fragmentOptions; } public void setBuffers(IncomingBuffers buffers) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java index 78e98fb..50d753f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java @@ -17,11 +17,8 @@ */ package org.apache.drill.exec.ops; -import java.util.Collection; - import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.jdbc.SimpleOptiqSchema; - import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.cache.DistributedCache; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; @@ -35,9 +32,12 @@ import org.apache.drill.exec.rpc.data.DataConnectionCreator; import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.server.options.QueryOptionManager; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.sys.PStoreProvider; +import java.util.Collection; + public class QueryContext{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class); @@ -45,6 +45,7 @@ public class QueryContext{ private final DrillbitContext drillbitContext; private final WorkEventBus workBus; private UserSession session; + private OptionManager queryOptions; public final Multitimer<QuerySetup> timer; private final PlannerSettings plannerSettings; private final DrillOperatorTable table; @@ -56,7 +57,8 @@ public class QueryContext{ this.workBus = drllbitContext.getWorkBus(); this.session = session; this.timer = new Multitimer<>(QuerySetup.class); - this.plannerSettings = new PlannerSettings(session.getOptions()); + this.queryOptions = new QueryOptionManager(session.getOptions()); + this.plannerSettings = new PlannerSettings(queryOptions); this.plannerSettings.setNumEndPoints(this.getActiveEndpoints().size()); this.table = new DrillOperatorTable(getFunctionRegistry()); } @@ -89,7 +91,11 @@ public class QueryContext{ return rootSchema; } - public OptionManager getOptions(){ + public OptionManager getOptions() { + return queryOptions; + } + + public OptionManager getSessionOptions() { return session.getOptions(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java index 24d9cfe..1f473c5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java @@ -17,12 +17,7 @@ */ package org.apache.drill.exec.opt; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - +import com.google.common.collect.Lists; import org.apache.drill.common.JSONOptions; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.ExecutionSetupException; @@ -60,7 +55,11 @@ import org.apache.drill.exec.store.StoragePlugin; import org.eigenbase.rel.RelFieldCollation.Direction; import org.eigenbase.rel.RelFieldCollation.NullDirection; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; public class BasicOptimizer extends Optimizer{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicOptimizer.class); @@ -125,7 +124,7 @@ public class BasicOptimizer extends Optimizer{ .type(PlanProperties.PlanType.APACHE_DRILL_PHYSICAL) .version(plan.getProperties().version) .generator(plan.getProperties().generator) - .options(new JSONOptions(context.getOptions().getSessionOptionList())).build(); + .options(new JSONOptions(context.getOptions().getOptionList())).build(); PhysicalPlan p = new PhysicalPlan(props, physOps); return p; //return new PhysicalPlan(props, physOps); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillCostBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillCostBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillCostBase.java index 412c218..50e4fb6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillCostBase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillCostBase.java @@ -112,12 +112,18 @@ public class DrillCostBase implements DrillRelOptCost { final double cpu; final double io; final double network; - + final double memory; + public DrillCostBase(double rowCount, double cpu, double io, double network) { + this(rowCount, cpu, io, network, 0); + } + + public DrillCostBase(double rowCount, double cpu, double io, double network, double memory) { this.rowCount = rowCount; this.cpu = cpu; this.io = io; this.network = network; + this.memory = memory; } @Override @@ -140,6 +146,10 @@ public class DrillCostBase implements DrillRelOptCost { return network; } + public double getMemory() { + return memory; + } + @Override public int hashCode() { return Util.hashCode(rowCount) + Util.hashCode(cpu) + Util.hashCode(io) + Util.hashCode(network); @@ -211,8 +221,9 @@ public class DrillCostBase implements DrillRelOptCost { return new DrillCostBase( this.rowCount + that.rowCount, this.cpu + that.cpu, - this.io + that.io, - this.network + that.network); + this.io + that.io, + this.network + that.network, + this.memory + that.memory); } @Override @@ -224,8 +235,9 @@ public class DrillCostBase implements DrillRelOptCost { return new DrillCostBase( this.rowCount - that.rowCount, this.cpu - that.cpu, - this.io - that.io, - this.network - that.network); + this.io - that.io, + this.network - that.network, + this.memory - that.memory); } @Override @@ -279,16 +291,21 @@ public class DrillCostBase implements DrillRelOptCost { } public String toString() { - return "{" + rowCount + " rows, " + cpu + " cpu, " + io + " io, " + network + " network}"; + return "{" + rowCount + " rows, " + cpu + " cpu, " + io + " io, " + network + " network, " + memory + " memory}"; } public static class DrillCostFactory implements DrillRelOptCostFactory { + + public RelOptCost makeCost(double dRows, double dCpu, double dIo, double dNetwork, double dMemory) { + return new DrillCostBase(dRows, dCpu, dIo, dNetwork, dMemory); + } + public RelOptCost makeCost(double dRows, double dCpu, double dIo, double dNetwork) { - return new DrillCostBase(dRows, dCpu, dIo, dNetwork); + return new DrillCostBase(dRows, dCpu, dIo, dNetwork, 0); } public RelOptCost makeCost(double dRows, double dCpu, double dIo) { - return new DrillCostBase(dRows, dCpu, dIo, 0); + return new DrillCostBase(dRows, dCpu, dIo, 0, 0); } public RelOptCost makeHugeCost() { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java index 34c0b5a..c1b1fba 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java @@ -17,27 +17,13 @@ */ package org.apache.drill.exec.planner.physical; -import java.io.IOException; -import java.util.BitSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import net.hydromatic.linq4j.Ord; -import net.hydromatic.optiq.util.BitSets; - -import org.apache.drill.common.expression.ExpressionPosition; -import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.common.expression.FunctionCall; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.logical.data.NamedExpression; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.expr.holders.IntHolder; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.config.HashAggregate; import org.apache.drill.exec.planner.cost.DrillCostBase; import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory; -import org.apache.drill.exec.planner.logical.DrillParseContext; -import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.AggregateCall; import org.eigenbase.rel.AggregateRelBase; @@ -49,7 +35,9 @@ import org.eigenbase.relopt.RelOptCost; import org.eigenbase.relopt.RelOptPlanner; import org.eigenbase.relopt.RelTraitSet; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.BitSet; +import java.util.List; public class HashAggPrel extends AggPrelBase implements Prel{ @@ -84,8 +72,24 @@ public class HashAggPrel extends AggPrelBase implements Prel{ // add cpu cost for computing the aggregate functions cpuCost += DrillCostBase.FUNC_CPU_COST * numAggrFields * inputRows; double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints - DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory(); - return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0 /* network cost */); + + // TODO: use distinct row count + // + hash table template stuff + double factor = PrelUtil.getPlannerSettings(planner).getOptions() + .getOption(ExecConstants.HASH_AGG_TABLE_FACTOR_KEY).float_val; + long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions() + .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).num_val; + + // table + hashValues + links + double memCost = + ( + (fieldWidth * numGroupByFields) + + IntHolder.WIDTH + + IntHolder.WIDTH + ) * inputRows * factor; + + DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory(); + return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0 /* network cost */, memCost); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java index 471e165..e2ee57f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java @@ -17,8 +17,7 @@ */ package org.apache.drill.exec.planner.physical; -import java.util.logging.Logger; - +import com.google.common.collect.ImmutableList; import org.apache.drill.exec.planner.logical.DrillAggregateRel; import org.apache.drill.exec.planner.logical.RelOptHelper; import org.apache.drill.exec.planner.physical.AggPrelBase.OperatorPhase; @@ -30,7 +29,7 @@ import org.eigenbase.relopt.RelTrait; import org.eigenbase.relopt.RelTraitSet; import org.eigenbase.trace.EigenbaseTrace; -import com.google.common.collect.ImmutableList; +import java.util.logging.Logger; public class HashAggPrule extends AggPruleBase { public static final RelOptRule INSTANCE = new HashAggPrule(); @@ -42,11 +41,14 @@ public class HashAggPrule extends AggPruleBase { @Override public boolean matches(RelOptRuleCall call) { - return PrelUtil.getPlannerSettings(call.getPlanner()).isHashAggEnabled(); + PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner()); + return settings.isMemoryEstimationEnabled() || settings.isHashAggEnabled(); } @Override public void onMatch(RelOptRuleCall call) { + if (!PrelUtil.getPlannerSettings(call.getPlanner()).isHashAggEnabled()) return; + final DrillAggregateRel aggregate = (DrillAggregateRel) call.rel(0); final RelNode input = call.rel(1); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java index 136eb4c..aaf18b1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java @@ -17,11 +17,11 @@ */ package org.apache.drill.exec.planner.physical; -import java.io.IOException; -import java.util.List; - +import com.google.common.collect.Lists; import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.logical.data.JoinCondition; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.expr.holders.IntHolder; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.config.HashJoinPOP; import org.apache.drill.exec.planner.cost.DrillCostBase; @@ -40,7 +40,8 @@ import org.eigenbase.relopt.RelTraitSet; import org.eigenbase.rex.RexNode; import org.eigenbase.util.Pair; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.List; public class HashJoinPrel extends JoinPrel { @@ -78,8 +79,22 @@ public class HashJoinPrel extends JoinPrel { double joinConditionCost = DrillCostBase.COMPARE_CPU_COST * this.getLeftKeys().size(); double cpuCost = joinConditionCost * (buildRowCount + probeRowCount) + cpuCostBuild + cpuCostProbe; - DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory(); - return costFactory.makeCost(buildRowCount + probeRowCount, cpuCost, 0, 0); + + double factor = PrelUtil.getPlannerSettings(planner).getOptions() + .getOption(ExecConstants.HASH_JOIN_TABLE_FACTOR_KEY).float_val; + long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions() + .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).num_val; + + // table + hashValues + links + double memCost = + ( + (fieldWidth * this.getRightKeys().size()) + + IntHolder.WIDTH + + IntHolder.WIDTH + ) * buildRowCount * factor; + + DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory(); + return costFactory.makeCost(buildRowCount + probeRowCount, cpuCost, 0, 0, memCost); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java index 9ae4783..8588425 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java @@ -17,21 +17,15 @@ */ package org.apache.drill.exec.planner.physical; -import java.util.logging.Logger; - import org.apache.drill.exec.planner.logical.DrillJoinRel; import org.apache.drill.exec.planner.logical.RelOptHelper; import org.eigenbase.rel.InvalidRelException; -import org.eigenbase.rel.RelCollation; import org.eigenbase.rel.RelNode; -import org.eigenbase.rel.metadata.RelMetadataQuery; import org.eigenbase.relopt.RelOptRule; import org.eigenbase.relopt.RelOptRuleCall; -import org.eigenbase.relopt.RelTraitSet; -import org.eigenbase.relopt.volcano.RelSubset; import org.eigenbase.trace.EigenbaseTrace; -import com.google.common.collect.ImmutableList; +import java.util.logging.Logger; public class HashJoinPrule extends JoinPruleBase { public static final RelOptRule INSTANCE = new HashJoinPrule(); @@ -44,11 +38,14 @@ public class HashJoinPrule extends JoinPruleBase { @Override public boolean matches(RelOptRuleCall call) { - return PrelUtil.getPlannerSettings(call.getPlanner()).isHashJoinEnabled(); + PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner()); + return settings.isMemoryEstimationEnabled() || settings.isHashJoinEnabled(); } @Override public void onMatch(RelOptRuleCall call) { + if (!PrelUtil.getPlannerSettings(call.getPlanner()).isHashJoinEnabled()) return; + final DrillJoinRel join = (DrillJoinRel) call.rel(0); final RelNode left = join.getLeft(); final RelNode right = join.getRight(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java index 998ecc0..5ec52c9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java @@ -53,6 +53,10 @@ public class PlannerSettings implements Context{ this.options = options; } + public OptionManager getOptions() { + return options; + } + public boolean isSingleMode() { return options.getOption(EXCHANGE.getOptionName()).bool_val; } @@ -113,6 +117,10 @@ public class PlannerSettings implements Context{ return options.getOption(ExecConstants.SLICE_TARGET).num_val; } + public boolean isMemoryEstimationEnabled() { + return options.getOption(ExecConstants.ENABLE_MEMORY_ESTIMATION_KEY).bool_val; + } + @Override public <T> T unwrap(Class<T> clazz) { if(clazz == PlannerSettings.class){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java index 3ef33ae..0926f60 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java @@ -17,12 +17,9 @@ */ package org.apache.drill.exec.planner.physical; -import java.io.IOException; -import java.util.Iterator; - +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.config.ExternalSort; -import org.apache.drill.exec.physical.config.SingleMergeExchange; import org.apache.drill.exec.physical.config.Sort; import org.apache.drill.exec.planner.cost.DrillCostBase; import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory; @@ -38,6 +35,9 @@ import org.eigenbase.relopt.RelOptPlanner; import org.eigenbase.relopt.RelTraitSet; import org.eigenbase.rex.RexNode; +import java.io.IOException; +import java.util.Iterator; + public class SortPrel extends SortRel implements Prel { /** Creates a DrillSortRel. */ @@ -63,8 +63,17 @@ public class SortPrel extends SortRel implements Prel { int numSortFields = this.collation.getFieldCollations().size(); double cpuCost = DrillCostBase.COMPARE_CPU_COST * numSortFields * inputRows * (Math.log(inputRows)/Math.log(2)); double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints - DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory(); - return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0); + + // TODO: use rowWidth instead of avgFieldWidth * numFields + // avgFieldWidth * numFields * inputRows + double numFields = this.getRowType().getFieldCount(); + long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions() + .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).num_val; + + double memCost = fieldWidth * numFields * inputRows; + + DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory(); + return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0, memCost); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/MemoryEstimationVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/MemoryEstimationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/MemoryEstimationVisitor.java new file mode 100644 index 0000000..0fd1dd0 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/MemoryEstimationVisitor.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.physical.visitor; + +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.planner.cost.DrillCostBase; +import org.apache.drill.exec.planner.physical.Prel; +import org.apache.drill.exec.server.options.OptionManager; +import org.eigenbase.rel.metadata.RelMetadataQuery; + +public class MemoryEstimationVisitor extends BasePrelVisitor<Double, Void, RuntimeException> { + + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MemoryEstimationVisitor.class); + + public static boolean enoughMemory(Prel prel, OptionManager options, int numDrillbits) { + long allottedMemory = options.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val * numDrillbits; + long estimatedMemory = (long) Math.ceil(prel.accept(new MemoryEstimationVisitor(), null) / (1024.0 * 1024.0)); + estimatedMemory += options.getOption(ExecConstants.NON_BLOCKING_OPERATORS_MEMORY_KEY).num_val * numDrillbits; + + if (estimatedMemory > allottedMemory) { + logger.debug("Estimated memory (" + estimatedMemory + ") exceeds maximum allowed (" + allottedMemory + ")"); + } else { + logger.debug("Estimated memory (" + estimatedMemory + ") within maximum allowed (" + allottedMemory + ")"); + } + return estimatedMemory <= allottedMemory; + } + + public static Double estimateMemory(Prel prel) { + return prel.accept(new MemoryEstimationVisitor(), null); + } + + public MemoryEstimationVisitor() { + } + + @Override + public Double visitPrel(Prel prel, Void value) throws RuntimeException { + return ((DrillCostBase) RelMetadataQuery.getCumulativeCost(prel)).getMemory(); +// return findCost(prel); + } + + private double findCost(Prel prel) { + DrillCostBase cost = (DrillCostBase) RelMetadataQuery.getNonCumulativeCost(prel); + double memory = cost.getMemory(); + + for (Prel child : prel) { + memory += findCost(child); + } + return memory; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java index 177331d..c5d342f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java @@ -17,9 +17,9 @@ */ package org.apache.drill.exec.planner.sql.handlers; -import java.io.IOException; -import java.util.Collection; -import java.util.List; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import net.hydromatic.optiq.tools.Planner; import net.hydromatic.optiq.tools.RelConversionException; @@ -44,27 +44,28 @@ import org.apache.drill.exec.planner.physical.PhysicalPlanCreator; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.physical.Prel; import org.apache.drill.exec.planner.physical.explain.PrelSequencer; +import org.apache.drill.exec.planner.physical.visitor.ComplexToJsonPrelVisitor; import org.apache.drill.exec.planner.physical.visitor.ExcessiveExchangeIdentifier; import org.apache.drill.exec.planner.physical.visitor.FinalColumnReorderer; -import org.apache.drill.exec.planner.physical.visitor.ComplexToJsonPrelVisitor; import org.apache.drill.exec.planner.physical.visitor.JoinPrelRenameVisitor; +import org.apache.drill.exec.planner.physical.visitor.MemoryEstimationVisitor; import org.apache.drill.exec.planner.physical.visitor.ProducerConsumerPrelVisitor; import org.apache.drill.exec.planner.physical.visitor.RelUniqifier; import org.apache.drill.exec.planner.physical.visitor.SelectionVectorPrelVisitor; -import org.apache.drill.exec.planner.sql.DrillOperatorTable; import org.apache.drill.exec.planner.physical.visitor.StarColumnConverter; import org.apache.drill.exec.planner.sql.DrillSqlWorker; +import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.util.Pointer; import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.RelOptUtil; import org.eigenbase.relopt.RelTraitSet; -import org.eigenbase.rex.RexBuilder; import org.eigenbase.sql.SqlExplainLevel; import org.eigenbase.sql.SqlNode; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.Collection; +import java.util.List; public class DefaultSqlHandler extends AbstractSqlHandler { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DefaultSqlHandler.class); @@ -155,6 +156,18 @@ public class DefaultSqlHandler extends AbstractSqlHandler { Preconditions.checkArgument(drel.getConvention() == DrillRel.DRILL_LOGICAL); RelTraitSet traits = drel.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON); Prel phyRelNode = (Prel) planner.transform(DrillSqlWorker.PHYSICAL_MEM_RULES, traits, drel); + OptionManager queryOptions = context.getOptions(); + + if (context.getPlannerSettings().isMemoryEstimationEnabled() + && !MemoryEstimationVisitor.enoughMemory(phyRelNode, queryOptions, context.getActiveEndpoints().size())) { + log("Not enough memory for this plan", phyRelNode); + logger.debug("Re-planning without hash operations."); + + queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHJOIN.getOptionName(), false)); + queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHAGG.getOptionName(), false)); + + phyRelNode = (Prel) planner.transform(DrillSqlWorker.PHYSICAL_MEM_RULES, traits, drel); + } /* The order of the following transformation is important */ @@ -234,7 +247,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler { PlanPropertiesBuilder propsBuilder = PlanProperties.builder(); propsBuilder.type(PlanType.APACHE_DRILL_PHYSICAL); propsBuilder.version(1); - propsBuilder.options(new JSONOptions(context.getOptions().getSessionOptionList())); + propsBuilder.options(new JSONOptions(context.getOptions().getOptionList())); propsBuilder.resultMode(ResultMode.EXEC); propsBuilder.generator(this.getClass().getSimpleName(), ""); return new PhysicalPlan(propsBuilder.build(), getPops(op)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java index eafc8d9..028f569 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java @@ -17,18 +17,18 @@ */ package org.apache.drill.exec.planner.sql.handlers; -import java.io.IOException; - import net.hydromatic.optiq.tools.RelConversionException; import net.hydromatic.optiq.tools.ValidationException; - import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.planner.sql.DirectPlan; +import org.apache.drill.exec.server.options.OptionValue; import org.eigenbase.sql.SqlLiteral; import org.eigenbase.sql.SqlNode; import org.eigenbase.sql.SqlSetOption; +import java.io.IOException; + public class SetOptionHandler extends AbstractSqlHandler{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SetOptionHandler.class); @@ -47,18 +47,22 @@ public class SetOptionHandler extends AbstractSqlHandler{ String scope = option.getScope(); String name = option.getName(); SqlNode value = option.getValue(); - if(value instanceof SqlLiteral){ - switch(scope.toLowerCase()){ - case "session": - context.getOptions().setOption(option.getName(), (SqlLiteral) value); - break; - case "system": - context.getOptions().getSystemManager().setOption(name, (SqlLiteral) value); - break; - default: - throw new ValidationException("Invalid OPTION scope. Scope must be SESSION or SYSTEM."); + OptionValue.OptionType type; + if (value instanceof SqlLiteral) { + switch (scope.toLowerCase()) { + case "session": + type = OptionValue.OptionType.SESSION; + break; + case "system": + type = OptionValue.OptionType.SYSTEM; + break; +// case "query": +// type = OptionValue.OptionType.QUERY; +// break; + default: + throw new ValidationException("Invalid OPTION scope. Scope must be SESSION or SYSTEM."); } - + context.getOptions().setOption(name, (SqlLiteral) value, type); }else{ throw new ValidationException("Sql options can only be literals."); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java index 13414da..e7be380 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java @@ -17,17 +17,15 @@ */ package org.apache.drill.exec.rpc.user; -import java.util.Map; - +import com.google.common.collect.Maps; import net.hydromatic.optiq.SchemaPlus; - import org.apache.drill.exec.proto.UserBitShared.UserCredentials; import org.apache.drill.exec.proto.UserProtos.Property; import org.apache.drill.exec.proto.UserProtos.UserProperties; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.SessionOptionManager; -import com.google.common.collect.Maps; +import java.util.Map; public class UserSession { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UserSession.class); @@ -39,7 +37,7 @@ public class UserSession { private boolean supportComplexTypes = false; private UserCredentials credentials; private Map<String, String> properties; - private OptionManager options; + private OptionManager sessionOptions; public static class Builder { UserSession userSession; @@ -54,7 +52,7 @@ public class UserSession { } public Builder withOptionManager(OptionManager systemOptions) { - userSession.options = new SessionOptionManager(systemOptions); + userSession.sessionOptions = new SessionOptionManager(systemOptions); return this; } @@ -91,8 +89,8 @@ public class UserSession { return supportComplexTypes; } - public OptionManager getOptions(){ - return options; + public OptionManager getOptions() { + return sessionOptions; } public DrillUser getUser(){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionManager.java new file mode 100644 index 0000000..aaf5719 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionManager.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.options; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import org.eigenbase.sql.SqlLiteral; + +import java.util.Iterator; +import java.util.Map; + +public class FragmentOptionManager implements OptionManager { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentOptionManager.class); + + ImmutableMap<String, OptionValue> options; + OptionManager systemOptions; + + public FragmentOptionManager(OptionManager systemOptions, OptionList options) { + Map<String, OptionValue> tmp = Maps.newHashMap(); + for(OptionValue v : options){ + tmp.put(v.name, v); + } + this.options = ImmutableMap.copyOf(tmp); + this.systemOptions = systemOptions; + } + + @Override + public Iterator<OptionValue> iterator() { + return Iterables.concat(systemOptions, options.values()).iterator(); + } + + @Override + public OptionValue getOption(String name) { + OptionValue value = options.get(name); + if (value == null && systemOptions != null) { + value = systemOptions.getOption(name); + } + return value; + } + + @Override + public void setOption(OptionValue value) throws SetOptionException { + throw new UnsupportedOperationException(); + } + + @Override + public void setOption(String name, SqlLiteral literal, OptionValue.OptionType type) throws SetOptionException { + throw new UnsupportedOperationException(); + } + + @Override + public OptionAdmin getAdmin() { + throw new UnsupportedOperationException(); + } + + @Override + public OptionManager getSystemManager() { + throw new UnsupportedOperationException(); + } + + @Override + public OptionList getOptionList() { + throw new UnsupportedOperationException(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java deleted file mode 100644 index 49d08d9..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.server.options; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.eigenbase.sql.SqlLiteral; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; - -public class FragmentOptionsManager implements OptionManager{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentOptionsManager.class); - - ImmutableMap<String, OptionValue> options; - OptionManager systemOptions; - - public FragmentOptionsManager(OptionManager systemOptions, OptionList options){ - Map<String, OptionValue> tmp = Maps.newHashMap(); - for(OptionValue v : options){ - tmp.put(v.name, v); - } - this.options = ImmutableMap.copyOf(tmp); - this.systemOptions = systemOptions; - } - - @Override - public Iterator<OptionValue> iterator() { - return Iterables.concat(systemOptions, options.values()).iterator(); - } - - @Override - public OptionValue getOption(String name) { - OptionValue value = options.get(name); - if (value == null && systemOptions != null) { - value = systemOptions.getOption(name); - } - return value; - } - - @Override - public void setOption(OptionValue value) throws SetOptionException { - throw new UnsupportedOperationException(); - } - - @Override - public void setOption(String name, SqlLiteral literal) throws SetOptionException { - throw new UnsupportedOperationException(); - } - - @Override - public OptionAdmin getAdmin() { - throw new UnsupportedOperationException(); - } - - @Override - public OptionManager getSystemManager() { - throw new UnsupportedOperationException(); - } - - @Override - public OptionList getSessionOptionList() { - throw new UnsupportedOperationException(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionManager.java index 3833833..83af7db 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionManager.java @@ -22,10 +22,10 @@ import org.eigenbase.sql.SqlLiteral; public interface OptionManager extends Iterable<OptionValue>{ public OptionValue getOption(String name); public void setOption(OptionValue value) throws SetOptionException; - public void setOption(String name, SqlLiteral literal) throws SetOptionException; + public void setOption(String name, SqlLiteral literal, OptionValue.OptionType type) throws SetOptionException; public OptionAdmin getAdmin(); public OptionManager getSystemManager(); - public OptionList getSessionOptionList(); + public OptionList getOptionList(); public interface OptionAdmin{ public void registerOptionType(OptionValidator validator); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java index 1f45522..7401246 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.server.options; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.google.common.base.Preconditions; @@ -25,7 +26,7 @@ import com.google.common.base.Preconditions; public class OptionValue{ public static enum OptionType { - BOOT, SYSTEM, SESSION + BOOT, SYSTEM, SESSION, QUERY } public static enum Kind { @@ -58,6 +59,21 @@ public class OptionValue{ public OptionValue(){} + public static OptionValue createOption(Kind kind, OptionType type, String name, String val) { + switch (kind) { + case BOOLEAN: + return createBoolean(type, name, Boolean.valueOf(val)); + case LONG: + return createLong(type, name, Long.valueOf(val)); + case STRING: + return createString(type, name, val); + case DOUBLE: + return createDouble(type, name, Double.valueOf(val)); + } + return null; + } + + private OptionValue(Kind kind, OptionType type, String name, Long num_val, String string_val, Boolean bool_val, Double float_val) { super(); Preconditions.checkArgument(num_val != null || string_val != null || bool_val != null || float_val != null); @@ -71,6 +87,21 @@ public class OptionValue{ this.type = type; } + @JsonIgnore + public Object getValue() { + switch (kind) { + case BOOLEAN: + return bool_val; + case LONG: + return num_val; + case STRING: + return string_val; + case DOUBLE: + return float_val; + } + return null; + } + @Override public int hashCode() { final int prime = 31; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java new file mode 100644 index 0000000..8b5306f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.options; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import org.eigenbase.sql.SqlLiteral; + +import java.util.Iterator; +import java.util.Map; + +public class QueryOptionManager implements OptionManager { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SessionOptionManager.class); + + private Map<String, OptionValue> options = Maps.newConcurrentMap(); + private OptionManager sessionOptions; + + public QueryOptionManager(OptionManager sessionOptions) { + super(); + this.sessionOptions = sessionOptions; + } + + @Override + public Iterator<OptionValue> iterator() { + return Iterables.concat(sessionOptions, options.values()).iterator(); + } + + @Override + public OptionValue getOption(String name) { + OptionValue opt = options.get(name); + if (opt == null && sessionOptions != null) { + return sessionOptions.getOption(name); + } else { + return opt; + } + } + + @Override + public void setOption(OptionValue value) { + sessionOptions.getAdmin().validate(value); + setValidatedOption(value); + } + + @Override + public void setOption(String name, SqlLiteral literal, OptionValue.OptionType type) { + OptionValue val = sessionOptions.getAdmin().validate(name, literal); + val.type = type; + setValidatedOption(val); + } + + private void setValidatedOption(OptionValue value) { + if (value.type == OptionValue.OptionType.QUERY) { + options.put(value.name, value); + } else { + sessionOptions.setOption(value); + } + } + + @Override + public OptionManager.OptionAdmin getAdmin() { + return sessionOptions.getAdmin(); + } + + @Override + public OptionManager getSystemManager() { + return sessionOptions.getSystemManager(); + } + + @Override + public OptionList getOptionList() { + OptionList list = new OptionList(); + for (OptionValue o : options.values()) { + list.add(o); + } + return list; + } + + public OptionList getSessionOptionList() { + return sessionOptions.getOptionList(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java index b5c914e..53c26c1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java @@ -17,14 +17,13 @@ */ package org.apache.drill.exec.server.options; -import java.util.Iterator; -import java.util.Map; - +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; import org.apache.drill.exec.server.options.OptionValue.OptionType; import org.eigenbase.sql.SqlLiteral; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; +import java.util.Iterator; +import java.util.Map; public class SessionOptionManager implements OptionManager{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SessionOptionManager.class); @@ -59,30 +58,21 @@ public class SessionOptionManager implements OptionManager{ } @Override - public OptionList getSessionOptionList() { - OptionList list = new OptionList(); - for(OptionValue o : options.values()){ - list.add(o); - } - return list; + public void setOption(String name, SqlLiteral literal, OptionType type) { + OptionValue val = systemOptions.getAdmin().validate(name, literal); + val.type = type; + setValidatedOption(val); } - private void setValidatedOption(OptionValue value){ - if(value.type == OptionType.SYSTEM){ - systemOptions.setOption(value); - }else{ + private void setValidatedOption(OptionValue value) { + if (value.type == OptionType.SESSION) { options.put(value.name, value); + } else { + systemOptions.setOption(value); } } @Override - public void setOption(String name, SqlLiteral literal) { - OptionValue val = systemOptions.getAdmin().validate(name, literal); - val.type = OptionValue.OptionType.SESSION; - setValidatedOption(val); - } - - @Override public OptionAdmin getAdmin() { return systemOptions.getAdmin(); } @@ -92,4 +82,13 @@ public class SessionOptionManager implements OptionManager{ return systemOptions; } + @Override + public OptionList getOptionList() { + OptionList list = new OptionList(); + for (OptionValue o : options.values()) { + list.add(o); + } + return list; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index e49b107..eddb818 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -17,28 +17,28 @@ */ package org.apache.drill.exec.server.options; -import java.io.IOException; -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; - +import com.google.common.collect.Maps; +import org.apache.commons.collections.IteratorUtils; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.cache.DistributedCache.CacheConfig; import org.apache.drill.exec.compile.QueryClassLoader; -import org.apache.drill.exec.planner.fragment.SimpleParallelizer; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.server.options.OptionValue.OptionType; import org.apache.drill.exec.store.sys.PStore; import org.apache.drill.exec.store.sys.PStoreConfig; import org.apache.drill.exec.store.sys.PStoreProvider; -import org.apache.drill.exec.work.foreman.Foreman; import org.eigenbase.sql.SqlLiteral; -import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; public class SystemOptionManager implements OptionManager{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemOptionManager.class); + public static final CacheConfig<String, OptionValue> OPTION_CACHE = CacheConfig // .newBuilder(OptionValue.class) // .name("sys.options") // @@ -72,6 +72,12 @@ public class SystemOptionManager implements OptionManager{ ExecConstants.SMALL_QUEUE_SIZE, ExecConstants.MIN_HASH_TABLE_SIZE, ExecConstants.MAX_HASH_TABLE_SIZE, + ExecConstants.ENABLE_MEMORY_ESTIMATION, + ExecConstants.MAX_QUERY_MEMORY_PER_NODE, + ExecConstants.NON_BLOCKING_OPERATORS_MEMORY, + ExecConstants.HASH_JOIN_TABLE_FACTOR, + ExecConstants.HASH_AGG_TABLE_FACTOR, + ExecConstants.AVERAGE_FIELD_WIDTH, QueryClassLoader.JAVA_COMPILER_VALIDATOR, QueryClassLoader.JAVA_COMPILER_JANINO_MAXSIZE, QueryClassLoader.JAVA_COMPILER_DEBUG, @@ -133,21 +139,22 @@ public class SystemOptionManager implements OptionManager{ @Override public void setOption(OptionValue value) { - admin.validate(value); assert value.type == OptionType.SYSTEM; + admin.validate(value); options.put(value.name, value); } @Override - public void setOption(String name, SqlLiteral literal) { + public void setOption(String name, SqlLiteral literal, OptionType type) { + assert type == OptionValue.OptionType.SYSTEM; OptionValue v = admin.validate(name, literal); - v.type = OptionValue.OptionType.SYSTEM; + v.type = type; options.put(name, v); } @Override - public OptionList getSessionOptionList() { - throw new UnsupportedOperationException(); + public OptionList getOptionList() { + return (OptionList) IteratorUtils.toList(iterator()); } @Override @@ -160,9 +167,6 @@ public class SystemOptionManager implements OptionManager{ return admin; } - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemOptionManager.class); - - private class SystemOptionAdmin implements OptionAdmin{ public SystemOptionAdmin(){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java index a59c125..2f43374 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java @@ -17,14 +17,14 @@ */ package org.apache.drill.exec.server.options; -import java.math.BigDecimal; - import org.apache.drill.common.exceptions.ExpressionParsingException; import org.apache.drill.exec.server.options.OptionValue.Kind; import org.apache.drill.exec.server.options.OptionValue.OptionType; import org.eigenbase.sql.SqlLiteral; import org.eigenbase.util.NlsString; +import java.math.BigDecimal; + public class TypeValidators { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeValidators.class); @@ -38,13 +38,32 @@ public class TypeValidators { } @Override - public void extraValidate(OptionValue v) throws ExpressionParsingException { + public void validate(OptionValue v) throws ExpressionParsingException { + super.validate(v); if (v.num_val > max || v.num_val < 0) throw new ExpressionParsingException(String.format("Option %s must be between %d and %d.", getOptionName(), 0, max)); } } + public static class PowerOfTwoLongValidator extends PositiveLongValidator { + + public PowerOfTwoLongValidator(String name, long max, long def) { + super(name, max, def); + } + + @Override + public void validate(OptionValue v) throws ExpressionParsingException { + super.validate(v); + if (!isPowerOfTwo(v.num_val)) + throw new ExpressionParsingException(String.format("Option %s must be a power of two.", getOptionName())); + } + + private boolean isPowerOfTwo(long num) { + return (num & (num - 1)) == 0; + } + } + public static class RangeDoubleValidator extends DoubleValidator { private final double min; private final double max; @@ -56,7 +75,8 @@ public class TypeValidators { } @Override - public void extraValidate(OptionValue v) throws ExpressionParsingException { + public void validate(OptionValue v) throws ExpressionParsingException { + super.validate(v); if (v.float_val > max || v.float_val < min) throw new ExpressionParsingException(String.format("Option %s must be between %d and %d.", getOptionName(), min, max)); @@ -112,7 +132,7 @@ public class TypeValidators { } @Override - public final void validate(OptionValue v) throws ExpressionParsingException { + public void validate(OptionValue v) throws ExpressionParsingException { if (v.kind != kind) throw new ExpressionParsingException(String.format("Option %s must be of type %s but you tried to set to %s.", getOptionName(), kind.name(), v.kind.name())); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java index c2fabcb..db07ab4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java @@ -17,21 +17,75 @@ */ package org.apache.drill.exec.server.rest; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.collect.Lists; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.proto.CoordinationProtos; +import org.apache.drill.exec.work.WorkManager; +import org.glassfish.jersey.server.mvc.Viewable; + +import javax.inject.Inject; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; -import org.glassfish.jersey.server.mvc.Viewable; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; @Path("/") public class DrillRoot { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRoot.class); + @Inject + WorkManager work; + @GET @Produces(MediaType.TEXT_HTML) - public Viewable getHello() { - String status = "Running!"; - return new Viewable("/rest/index.ftl", status); + public Viewable getStats() { + return new Viewable("/rest/index.ftl", getStatsJSON()); } + @GET + @Path("/stats.json") + @Produces(MediaType.APPLICATION_JSON) + public List<Stat> getStatsJSON() { + List<Stat> stats = Lists.newLinkedList(); + stats.add(new Stat("Number of Drill Bits", work.getContext().getBits().size())); + int number = 0; + for (CoordinationProtos.DrillbitEndpoint bit : work.getContext().getBits()) { + String initialized = bit.isInitialized() ? " initialized" : " not initialized"; + stats.add(new Stat("Bit #" + number, bit.getAddress() + initialized)); + ++number; + } + stats.add(new Stat("Data Port Address", work.getContext().getEndpoint().getAddress() + + ":" + work.getContext().getEndpoint().getDataPort())); + stats.add(new Stat("User Port Address", work.getContext().getEndpoint().getAddress() + + ":" + work.getContext().getEndpoint().getUserPort())); + stats.add(new Stat("Control Port Address", work.getContext().getEndpoint().getAddress() + + ":" + work.getContext().getEndpoint().getControlPort())); + stats.add(new Stat("Maximum Direct Memory", DrillConfig.getMaxDirectMemory())); + + return stats; + } + + @XmlRootElement + public class Stat { + private String name; + private Object value; + + @JsonCreator + public Stat(String name, Object value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public Object getValue() { + return value; + } + + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java index 4ec4182..79150d9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java @@ -17,22 +17,116 @@ */ package org.apache.drill.exec.server.rest; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.OptionValue.Kind; +import org.apache.drill.exec.work.WorkManager; import org.glassfish.jersey.server.mvc.Viewable; +import javax.inject.Inject; +import javax.ws.rs.Consumes; +import javax.ws.rs.FormParam; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.LinkedList; +import java.util.List; -@Path("/status") +@Path("/") public class StatusResources { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StatusResources.class); + @Inject + WorkManager work; + @GET + @Path("/status") @Produces(MediaType.TEXT_HTML) public Viewable getStatus() { String status = "Running!"; return new Viewable("/rest/status.ftl", status); } + @GET + @Path("/options.json") + @Produces(MediaType.APPLICATION_JSON) + public List getSystemOptionsJSON() { + List<OptionWrapper> options = new LinkedList<>(); + for (OptionValue option : work.getContext().getOptionManager()) { + options.add(new OptionWrapper(option.name, option.getValue(), option.type, option.kind)); + } + return options; + } + + @GET + @Path("/options") + @Produces(MediaType.TEXT_HTML) + public Viewable getSystemOptions() { + return new Viewable("/rest/options.ftl", getSystemOptionsJSON()); + } + + @POST + @Path("/option/{optionName}") + @Consumes("application/x-www-form-urlencoded") + @Produces(MediaType.TEXT_HTML) + public Viewable updateSystemOption(@FormParam("name") String name, @FormParam("value") String value, + @FormParam("kind") String kind) { + try { + work.getContext() + .getOptionManager() + .setOption(OptionValue.createOption( + OptionValue.Kind.valueOf(kind), + OptionValue.OptionType.SYSTEM, + name, + value)); + } catch (Exception e) { + logger.debug("Could not update.", e); + } + return getSystemOptions(); + } + + @XmlRootElement + public class OptionWrapper { + + private String name; + private Object value; + private OptionValue.OptionType type; + private String kind; + + @JsonCreator + public OptionWrapper(String name, Object value, OptionValue.OptionType type, Kind kind) { + this.name = name; + this.value = value; + this.type = type; + this.kind = kind.name(); + } + + public String getName() { + return name; + } + + @JsonIgnore + public String getValueAsString() { + return value.toString(); + } + + public Object getValue() { + return value; + } + + public OptionValue.OptionType getType() { + return type; + } + + public String getKind() { + return kind; + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index b1ed8a5..9521e65 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -350,7 +350,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{ } } - QueryWorkUnit work = parallelizer.getFragments(context.getOptions().getSessionOptionList(), context.getCurrentEndpoint(), + QueryWorkUnit work = parallelizer.getFragments(context.getOptions().getOptionList(), context.getCurrentEndpoint(), queryId, context.getActiveEndpoints(), context.getPlanReader(), rootFragment, planningSet); this.context.getWorkBus().setFragmentStatusListener(work.getRootFragment().getHandle().getQueryId(), fragmentManager); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/resources/rest/generic.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/generic.ftl b/exec/java-exec/src/main/resources/rest/generic.ftl index 4b26926..0543755 100644 --- a/exec/java-exec/src/main/resources/rest/generic.ftl +++ b/exec/java-exec/src/main/resources/rest/generic.ftl @@ -50,7 +50,7 @@ <span class="icon-bar"></span> <span class="icon-bar"></span> </button> - <a class="navbar-brand">Apache Drill</a> + <a class="navbar-brand" href="/">Apache Drill</a> </div> <div class="navbar-collapse collapse"> <ul class="nav navbar-nav"> @@ -60,6 +60,10 @@ <li><a href="/metrics">Metrics</a></li> <li><a href="/threads">Threads</a></li> </ul> + <ul class="nav navbar-nav navbar-right"> + <li><a href="/options">Options</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/DRILL/Apache+Drill+Wiki";>Wiki</a> + </ul> </div> </div> </div> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/resources/rest/index.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/index.ftl b/exec/java-exec/src/main/resources/rest/index.ftl index c6e4a4e..99e9d8c 100644 --- a/exec/java-exec/src/main/resources/rest/index.ftl +++ b/exec/java-exec/src/main/resources/rest/index.ftl @@ -17,7 +17,18 @@ <a href="/queries">back</a><br/> <div class="page-header"> </div> - <p class="lead"> Read about Apache Drill <a href="http://incubator.apache.org/drill/drill_overview.html"; rel="nofollow">here</a>.</p> + <div class="table-responsive"> + <table class="table table-hover"> + <tbody> + <#list model as stat> + <tr> + <td style="border:none;"><b>${stat.getName()}</b></td> + <td style="border:none; font-family: Courier;">${stat.getValue()}</td> + </tr> + </#list> + </tbody> + </table> + </div> </#macro> <@page_html/> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/resources/rest/options.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/options.ftl b/exec/java-exec/src/main/resources/rest/options.ftl new file mode 100644 index 0000000..0092faa --- /dev/null +++ b/exec/java-exec/src/main/resources/rest/options.ftl @@ -0,0 +1,47 @@ +<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<#include "*/generic.ftl"> +<#macro page_head> +</#macro> + +<#macro page_body> + <a href="/queries">back</a><br/> + <div class="page-header"> + </div> + <h4>System options</h4> + <div align="right"> + <a href="https://cwiki.apache.org/confluence/display/DRILL/Planning+and+Execution+Options";>Documentation</a> + </div> + <div class="table-responsive"> + <table class="table table-hover"> + <tbody> + <#list model as option> + <tr> + <td style="border:none;">${option.getName()}</td> + <td style="border:none;"> + <form class="form-inline" role="form" action="/option/${option.getName()}" method="POST"> + <div class="form-group"> + <input type="text" class="form-control" name="value" value="${option.getValueAsString()}"> + <input type="hidden" class="form-control" name="kind" value="${option.getKind()}"> + <input type="hidden" class="form-control" name="name" value="${option.getName()}"> + </div> + <button type="submit" class="btn btn-default">Update</button> + </form> + </td> + </tr> + </#list> + </tbody> + </table> + </div> +</#macro> + +<@page_html/> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/main/resources/rest/status.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/status.ftl b/exec/java-exec/src/main/resources/rest/status.ftl index bda01ec..cafa523 100644 --- a/exec/java-exec/src/main/resources/rest/status.ftl +++ b/exec/java-exec/src/main/resources/rest/status.ftl @@ -20,6 +20,7 @@ <strong>${model}</strong> </div> </div> + <a href="/status/options"> System Options </a> </#macro> <@page_html/> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba027c7f/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java index e438ec2..f01a706 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java @@ -17,15 +17,15 @@ */ package org.apache.drill; -import java.io.IOException; -import java.net.URL; - +import com.codahale.metrics.MetricRegistry; +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Resources; import mockit.Mocked; import mockit.NonStrictExpectations; import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.jdbc.SimpleOptiqSchema; import net.hydromatic.optiq.tools.Frameworks; - import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.ExecTest; @@ -41,7 +41,7 @@ import org.apache.drill.exec.planner.sql.DrillSqlWorker; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.server.options.QueryOptionManager; import org.apache.drill.exec.server.options.SessionOptionManager; import org.apache.drill.exec.server.options.SystemOptionManager; import org.apache.drill.exec.store.StoragePluginRegistry; @@ -49,10 +49,8 @@ import org.apache.drill.exec.store.sys.local.LocalPStoreProvider; import org.junit.Rule; import org.junit.rules.TestRule; -import com.codahale.metrics.MetricRegistry; -import com.google.common.base.Charsets; -import com.google.common.collect.ImmutableList; -import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URL; public class PlanningBase extends ExecTest{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PlanningBase.class); @@ -77,9 +75,10 @@ public class PlanningBase extends ExecTest{ final LocalPStoreProvider provider = new LocalPStoreProvider(config); provider.start(); - final SystemOptionManager opt = new SystemOptionManager(config, provider); - opt.init(); - final OptionManager sess = new SessionOptionManager(opt); + final SystemOptionManager systemOptions = new SystemOptionManager(config, provider); + systemOptions.init(); + final SessionOptionManager sessionOptions = new SessionOptionManager(systemOptions); + final QueryOptionManager queryOptions = new QueryOptionManager(sessionOptions); new NonStrictExpectations() { { @@ -90,7 +89,7 @@ public class PlanningBase extends ExecTest{ dbContext.getConfig(); result = config; dbContext.getOptionManager(); - result = opt; + result = systemOptions; dbContext.getCache(); result = cache; dbContext.getPersistentStoreProvider(); @@ -121,9 +120,9 @@ public class PlanningBase extends ExecTest{ context.getActiveEndpoints(); result = ImmutableList.of(DrillbitEndpoint.getDefaultInstance()); context.getPlannerSettings(); - result = new PlannerSettings(sess); + result = new PlannerSettings(queryOptions); context.getOptions(); - result = sess; + result = queryOptions; context.getConfig(); result = config; context.getCache();
