Repository: asterixdb Updated Branches: refs/heads/master f27f8e2a0 -> 78dc83b12
[NO ISSUE][API] Add Execution Plans to Query Services - user model changes: no - storage format changes: no - interface changes: yes - Add execution plans parameters to query service. - Remove HTML code generation from APIFramework. Change-Id: I99215243aae2cb96174671d109084a82af877334 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2566 Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Michael Blow <mb...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/78dc83b1 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/78dc83b1 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/78dc83b1 Branch: refs/heads/master Commit: 78dc83b12bc26a1b27fbb614c95d604906a1f6db Parents: f27f8e2 Author: Murtadha Hubail <mhub...@apache.org> Authored: Wed Apr 4 09:45:58 2018 +0300 Committer: Murtadha Hubail <mhub...@apache.org> Committed: Wed Apr 4 09:47:21 2018 -0700 ---------------------------------------------------------------------- asterixdb/asterix-algebra/pom.xml | 4 + .../asterix/translator/ExecutionPlans.java | 70 ++++++++++ .../translator/ExecutionPlansHtmlPrintUtil.java | 69 ++++++++++ .../translator/ExecutionPlansJsonPrintUtil.java | 86 +++++++++++++ .../asterix/translator/IStatementExecutor.java | 7 + .../asterix/translator/SessionConfig.java | 16 +-- .../apache/asterix/api/common/APIFramework.java | 129 +++++++++---------- .../http/server/AbstractQueryApiServlet.java | 3 +- .../api/http/server/NCQueryServiceServlet.java | 1 + .../api/http/server/QueryServiceServlet.java | 58 ++++++++- .../message/ExecuteStatementRequestMessage.java | 1 + .../ExecuteStatementResponseMessage.java | 11 ++ .../asterix/app/translator/QueryTranslator.java | 16 +++ .../asterix/test/common/ResultExtractor.java | 4 +- .../java/org/apache/hyracks/util/JSONUtil.java | 2 +- 15 files changed, 397 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-algebra/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/pom.xml b/asterixdb/asterix-algebra/pom.xml index 5a61c96..79dc7bc 100644 --- a/asterixdb/asterix-algebra/pom.xml +++ b/asterixdb/asterix-algebra/pom.xml @@ -242,5 +242,9 @@ <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java new file mode 100644 index 0000000..d77164c --- /dev/null +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.translator; + +import java.io.Serializable; + +public class ExecutionPlans implements Serializable { + + private String expressionTree; + private String rewrittenExpressionTree; + private String logicalPlan; + private String optimizedLogicalPlan; + private String job; + + public String getExpressionTree() { + return expressionTree; + } + + public void setExpressionTree(String expressionTree) { + this.expressionTree = expressionTree; + } + + public String getRewrittenExpressionTree() { + return rewrittenExpressionTree; + } + + public void setRewrittenExpressionTree(String rewrittenExpressionTree) { + this.rewrittenExpressionTree = rewrittenExpressionTree; + } + + public String getLogicalPlan() { + return logicalPlan; + } + + public void setLogicalPlan(String logicalPlan) { + this.logicalPlan = logicalPlan; + } + + public String getOptimizedLogicalPlan() { + return optimizedLogicalPlan; + } + + public void setOptimizedLogicalPlan(String optimizedLogicalPlan) { + this.optimizedLogicalPlan = optimizedLogicalPlan; + } + + public String getJob() { + return job; + } + + public void setJob(String job) { + this.job = job; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java new file mode 100644 index 0000000..88e8255 --- /dev/null +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.translator; + +import java.io.PrintWriter; + +public class ExecutionPlansHtmlPrintUtil { + + private static final String LOGICAL_PLAN_LBL = "Logical plan"; + private static final String EXPRESSION_TREE_LBL = "Expression tree"; + private static final String REWRITTEN_EXPRESSION_TREE_LBL = "Rewritten expression tree"; + private static final String OPTIMIZED_LOGICAL_PLAN_LBL = "Optimized logical plan"; + private static final String JOB_LBL = "Job"; + + private ExecutionPlansHtmlPrintUtil() { + } + + public static void print(PrintWriter output, ExecutionPlans plans) { + printNonNull(output, EXPRESSION_TREE_LBL, plans.getExpressionTree()); + printNonNull(output, REWRITTEN_EXPRESSION_TREE_LBL, plans.getRewrittenExpressionTree()); + printNonNull(output, LOGICAL_PLAN_LBL, plans.getLogicalPlan()); + printNonNull(output, OPTIMIZED_LOGICAL_PLAN_LBL, plans.getOptimizedLogicalPlan()); + printNonNull(output, JOB_LBL, plans.getJob()); + } + + private static void printNonNull(PrintWriter output, String lbl, String value) { + if (value != null) { + printFieldPrefix(output, lbl); + output.print(value); + printFieldPostfix(output); + } + } + + private static void printFieldPrefix(PrintWriter output, String lbl) { + output.println(); + output.println("<h4>" + lbl + ":</h4>"); + switch (lbl) { + case LOGICAL_PLAN_LBL: + output.println("<pre class=query-plan>"); + break; + case OPTIMIZED_LOGICAL_PLAN_LBL: + output.println("<pre class=query-optimized-plan>"); + break; + default: + output.println("<pre>"); + break; + } + } + + private static void printFieldPostfix(PrintWriter output) { + output.println("</pre>"); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java new file mode 100644 index 0000000..5c47ca2 --- /dev/null +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.translator; + +import static org.apache.asterix.translator.SessionConfig.PlanFormat.STRING; + +import org.apache.hyracks.util.JSONUtil; + +public class ExecutionPlansJsonPrintUtil { + + private static final String LOGICAL_PLAN_LBL = "logicalPlan"; + private static final String EXPRESSION_TREE_LBL = "expressionTree"; + private static final String REWRITTEN_EXPRESSION_TREE_LBL = "rewrittenExpressionTree"; + private static final String OPTIMIZED_LOGICAL_PLAN_LBL = "optimizedLogicalPlan"; + private static final String JOB_LBL = "job"; + + private ExecutionPlansJsonPrintUtil() { + } + + public static String asJson(ExecutionPlans plans, SessionConfig.PlanFormat format) { + final StringBuilder output = new StringBuilder(); + appendOutputPrefix(output); + // TODO only string is currently supported for expression trees + appendNonNull(output, EXPRESSION_TREE_LBL, plans.getExpressionTree(), STRING); + appendNonNull(output, REWRITTEN_EXPRESSION_TREE_LBL, plans.getRewrittenExpressionTree(), STRING); + appendNonNull(output, LOGICAL_PLAN_LBL, plans.getLogicalPlan(), format); + appendNonNull(output, OPTIMIZED_LOGICAL_PLAN_LBL, plans.getOptimizedLogicalPlan(), format); + appendNonNull(output, JOB_LBL, plans.getJob(), format); + appendOutputPostfix(output); + return output.toString(); + } + + private static void appendNonNull(StringBuilder builder, String lbl, String value, + SessionConfig.PlanFormat format) { + if (value != null) { + printFieldPrefix(builder, lbl); + switch (format) { + case JSON: + builder.append(value); + break; + case STRING: + JSONUtil.quoteAndEscape(builder, value); + break; + default: + throw new IllegalStateException("Unrecognized plan format: " + format); + } + printFieldPostfix(builder); + } + } + + private static void appendOutputPrefix(StringBuilder builder) { + builder.append("{"); + } + + private static void printFieldPrefix(StringBuilder builder, String lbl) { + builder.append("\"" + lbl + "\": "); + } + + private static void printFieldPostfix(StringBuilder builder) { + builder.append(","); + } + + private static void appendOutputPostfix(StringBuilder builder) { + // remove extra comma if needed + if (builder.length() > 1) { + builder.deleteCharAt(builder.length() - 1); + } + builder.append("}"); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java index d76c421..0ff877b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java @@ -142,4 +142,11 @@ public interface IStatementExecutor { */ String getActiveDataverseName(String dataverse); + /** + * Gets the execution plans that are generated during query compilation + * + * @return the executions plans + */ + ExecutionPlans getExecutionPlans(); + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java index cb6d8e5..89619e5 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java @@ -63,7 +63,7 @@ public class SessionConfig implements Serializable { if (fmtString != null) { String format = ("JSON".equalsIgnoreCase(fmtString) || "CLEAN_JSON".equalsIgnoreCase(fmtString)) ? "JSON" : fmtString; - return PlanFormat.valueOf(format); + return PlanFormat.valueOf(format.toUpperCase()); } } catch (IllegalArgumentException e) { logger.log(Level.INFO, fmtString + ": unsupported " + label + ", using " + defaultFmt + "instead", e); @@ -129,7 +129,7 @@ public class SessionConfig implements Serializable { // Output format. private final OutputFormat fmt; - private final PlanFormat lpfmt; + private final PlanFormat planFormat; // Standard execution flags. private final boolean executeQuery; @@ -143,8 +143,8 @@ public class SessionConfig implements Serializable { this(fmt, PlanFormat.STRING); } - public SessionConfig(OutputFormat fmt, PlanFormat lpfmt) { - this(fmt, true, true, true, lpfmt); + public SessionConfig(OutputFormat fmt, PlanFormat planFormat) { + this(fmt, true, true, true, planFormat); } /** @@ -168,13 +168,13 @@ public class SessionConfig implements Serializable { } public SessionConfig(OutputFormat fmt, boolean optimize, boolean executeQuery, boolean generateJobSpec, - PlanFormat lpfmt) { + PlanFormat planFormat) { this.fmt = fmt; this.optimize = optimize; this.executeQuery = executeQuery; this.generateJobSpec = generateJobSpec; this.flags = new HashMap<>(); - this.lpfmt = lpfmt; + this.planFormat = planFormat; } /** @@ -187,8 +187,8 @@ public class SessionConfig implements Serializable { /** * Retrieve the PlanFormat for this execution. */ - public PlanFormat getLpfmt() { - return this.lpfmt; + public PlanFormat getPlanFormat() { + return this.planFormat; } /** http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index ad715a4..537625d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -20,6 +20,7 @@ package org.apache.asterix.api.common; import java.io.IOException; import java.io.PrintWriter; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -67,6 +68,7 @@ import org.apache.asterix.optimizer.base.FuzzyUtils; import org.apache.asterix.optimizer.rules.am.AbstractIntroduceAccessMethodRule; import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement; +import org.apache.asterix.translator.ExecutionPlans; import org.apache.asterix.translator.IStatementExecutor.Stats; import org.apache.asterix.translator.SessionConfig; import org.apache.asterix.translator.SessionOutput; @@ -120,8 +122,7 @@ public class APIFramework { private static final int MIN_FRAME_LIMIT_FOR_JOIN = 5; // one for query, two for intermediate results, one for final result, and one for reading an inverted list private static final int MIN_FRAME_LIMIT_FOR_TEXTSEARCH = 5; - private static final String LPLAN = "Logical plan"; - private static final String OPLAN = "Optimized logical plan"; + private static final ObjectWriter OBJECT_WRITER = new ObjectMapper().writerWithDefaultPrettyPrinter(); // A white list of supported configurable parameters. private static final Set<String> CONFIGURABLE_PARAMETER_NAMES = @@ -137,12 +138,14 @@ public class APIFramework { private final IAstPrintVisitorFactory astPrintVisitorFactory; private final ILangExpressionToPlanTranslatorFactory translatorFactory; private final IRuleSetFactory ruleSetFactory; + private final ExecutionPlans executionPlans; public APIFramework(ILangCompilationProvider compilationProvider) { this.rewriterFactory = compilationProvider.getRewriterFactory(); this.astPrintVisitorFactory = compilationProvider.getAstPrintVisitorFactory(); this.translatorFactory = compilationProvider.getExpressionToPlanTranslatorFactory(); this.ruleSetFactory = compilationProvider.getRuleSetFactory(); + executionPlans = new ExecutionPlans(); } private static class OptimizationContextFactory implements IOptimizationContextFactory { @@ -165,27 +168,6 @@ public class APIFramework { } } - private void printPlanPrefix(SessionOutput output, String planName) { - if (output.config().is(SessionConfig.FORMAT_HTML)) { - output.out().println("<h4>" + planName + ":</h4>"); - if (LPLAN.equalsIgnoreCase(planName)) { - output.out().println("<pre class = query-plan>"); - } else if (OPLAN.equalsIgnoreCase(planName)) { - output.out().println("<pre class = query-optimized-plan>"); - } else { - output.out().println("<pre>"); - } - } else { - output.out().println("----------" + planName + ":"); - } - } - - private void printPlanPostfix(SessionOutput output) { - if (output.config().is(SessionConfig.FORMAT_HTML)) { - output.out().println("</pre>"); - } - } - public Pair<IReturningStatement, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions, MetadataProvider metadataProvider, IReturningStatement q, SessionOutput output, boolean inlineUdfs) throws CompilationException { @@ -194,10 +176,7 @@ public class APIFramework { } SessionConfig conf = output.config(); if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_EXPR_TREE)) { - output.out().println(); - printPlanPrefix(output, "Expression tree"); - q.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0); - printPlanPostfix(output); + generateExpressionTree(q); } IQueryRewriter rw = rewriterFactory.createQueryRewriter(); rw.rewrite(declaredFunctions, q, metadataProvider, new LangRewritingContext(q.getVarCounter()), inlineUdfs); @@ -213,14 +192,9 @@ public class APIFramework { final boolean isLoad = statement != null && statement.getKind() == Statement.Kind.LOAD; SessionConfig conf = output.config(); - if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) { - output.out().println(); - - printPlanPrefix(output, "Rewritten expression tree"); - if (isQuery) { - query.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0); - } - printPlanPostfix(output); + if (isQuery && !conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) + && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) { + generateRewrittenExpressionTree(query); } final TxnId txnId = metadataProvider.getTxnIdFactory().create(); @@ -230,14 +204,9 @@ public class APIFramework { ILogicalPlan plan = isLoad ? t.translateLoad(statement) : t.translate(query, outputDatasetName, statement); - if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) { - output.out().println(); - - printPlanPrefix(output, "Logical plan"); - if (isQuery || isLoad) { - PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(output.config().getLpfmt(), output.out()), 0); - } - printPlanPostfix(output); + if ((isQuery || isLoad) && !conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) + && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) { + generateLogicalPlan(plan, output.config().getPlanFormat()); } CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties(); Map<String, String> querySpecificConfig = validateConfig(metadataProvider.getConfig()); @@ -273,12 +242,9 @@ public class APIFramework { AlgebricksAppendable buffer = new AlgebricksAppendable(output.out()); PlanPrettyPrinter.printPhysicalOps(plan, buffer, 0); } else { - printPlanPrefix(output, "Optimized logical plan"); if (isQuery || isLoad) { - PlanPrettyPrinter.printPlan(plan, - getPrettyPrintVisitor(output.config().getLpfmt(), output.out()), 0); + generateOptimizedLogicalPlan(plan, output.config().getPlanFormat()); } - printPlanPostfix(output); } } } @@ -327,8 +293,9 @@ public class APIFramework { ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf); spec.setRequiredClusterCapacity(jobRequiredCapacity); } - - printJobSpec(query, spec, conf, output); + if (isQuery && conf.is(SessionConfig.OOB_HYRACKS_JOB)) { + generateJob(spec); + } return spec; } @@ -373,23 +340,6 @@ public class APIFramework { } } - protected void printJobSpec(Query rwQ, JobSpecification spec, SessionConfig conf, SessionOutput output) - throws AlgebricksException { - if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) { - printPlanPrefix(output, "Hyracks job"); - if (rwQ != null) { - try { - final ObjectWriter objectWriter = new ObjectMapper().writerWithDefaultPrettyPrinter(); - output.out().println(objectWriter.writeValueAsString(spec.toJSON())); - } catch (IOException e) { - throw new AlgebricksException(e); - } - output.out().println(spec.getUserConstraints()); - } - printPlanPostfix(output); - } - } - private AbstractLogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor(SessionConfig.PlanFormat planFormat, PrintWriter out) { return planFormat.equals(SessionConfig.PlanFormat.JSON) ? new LogicalOperatorPrettyPrintVisitorJson(out) @@ -429,6 +379,10 @@ public class APIFramework { } } + public ExecutionPlans getExecutionPlans() { + return executionPlans; + } + // Chooses the location constraints, i.e., whether to use storage parallelism or use a user-sepcified number // of cores. private static AlgebricksAbsolutePartitionConstraint chooseLocations(IClusterInfoCollector clusterInfoCollector, @@ -524,6 +478,49 @@ public class APIFramework { return config; } + private void generateExpressionTree(IReturningStatement statement) throws CompilationException { + final StringWriter stringWriter = new StringWriter(); + try (PrintWriter writer = new PrintWriter(stringWriter)) { + statement.accept(astPrintVisitorFactory.createLangVisitor(writer), 0); + executionPlans.setExpressionTree(stringWriter.toString()); + } + } + + private void generateRewrittenExpressionTree(IReturningStatement statement) throws CompilationException { + final StringWriter stringWriter = new StringWriter(); + try (PrintWriter writer = new PrintWriter(stringWriter)) { + statement.accept(astPrintVisitorFactory.createLangVisitor(writer), 0); + executionPlans.setRewrittenExpressionTree(stringWriter.toString()); + } + } + + private void generateLogicalPlan(ILogicalPlan plan, SessionConfig.PlanFormat format) throws AlgebricksException { + final StringWriter stringWriter = new StringWriter(); + try (PrintWriter writer = new PrintWriter(stringWriter)) { + PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(format, writer), 0); + executionPlans.setLogicalPlan(stringWriter.toString()); + } + } + + private void generateOptimizedLogicalPlan(ILogicalPlan plan, SessionConfig.PlanFormat format) + throws AlgebricksException { + final StringWriter stringWriter = new StringWriter(); + try (PrintWriter writer = new PrintWriter(stringWriter)) { + PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(format, writer), 0); + executionPlans.setOptimizedLogicalPlan(stringWriter.toString()); + } + } + + private void generateJob(JobSpecification spec) { + final StringWriter stringWriter = new StringWriter(); + try (PrintWriter writer = new PrintWriter(stringWriter)) { + writer.println(OBJECT_WRITER.writeValueAsString(spec.toJSON())); + executionPlans.setJob(stringWriter.toString()); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + public static AlgebricksAbsolutePartitionConstraint getJobLocations(JobSpecification spec, INodeJobTracker jobTracker, AlgebricksAbsolutePartitionConstraint clusterLocations) { final Set<String> jobParticipatingNodes = jobTracker.getJobParticipatingNodes(spec); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java index b8c737d..bd096dd 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java @@ -51,7 +51,8 @@ public class AbstractQueryApiServlet extends AbstractServlet { RESULTS("results"), HANDLE("handle"), ERRORS("errors"), - METRICS("metrics"); + METRICS("metrics"), + PLANS("plans"); private final String str; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java index a420efc..1713ca5 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java @@ -132,6 +132,7 @@ public class NCQueryServiceServlet extends QueryServiceServlet { } else { sessionOutput.out().append(responseMsg.getResult()); } + printExecutionPlans(sessionOutput, responseMsg.getExecutionPlans()); } private void cancelQuery(INCMessageBroker messageBroker, String nodeId, String clientContextID, Exception exception, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index 56359e3..714bb53 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -45,6 +45,8 @@ import org.apache.asterix.lang.aql.parser.TokenMgrError; import org.apache.asterix.lang.common.base.IParser; import org.apache.asterix.lang.common.base.Statement; import org.apache.asterix.metadata.MetadataManager; +import org.apache.asterix.translator.ExecutionPlans; +import org.apache.asterix.translator.ExecutionPlansJsonPrintUtil; import org.apache.asterix.translator.IRequestParameters; import org.apache.asterix.translator.IStatementExecutor; import org.apache.asterix.translator.IStatementExecutor.ResultDelivery; @@ -73,6 +75,7 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; + import io.netty.handler.codec.http.HttpResponseStatus; public class QueryServiceServlet extends AbstractQueryApiServlet { @@ -141,7 +144,12 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { MODE("mode"), TIMEOUT("timeout"), PLAN_FORMAT("plan-format"), - MAX_RESULT_READS("max-result-reads"); + MAX_RESULT_READS("max-result-reads"), + EXPRESSION_TREE("expression-tree"), + REWRITTEN_EXPRESSION_TREE("rewritten-expression-tree"), + LOGICAL_PLAN("logical-plan"), + OPTIMIZED_LOGICAL_PLAN("optimized-logical-plan"), + JOB("job"); private final String str; @@ -198,6 +206,12 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { String clientContextID; String mode; String maxResultReads; + String planFormat; + boolean expressionTree; + boolean rewrittenExpressionTree; + boolean logicalPlan; + boolean optimizedLogicalPlan; + boolean job; @Override public String toString() { @@ -213,6 +227,12 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { on.put("format", format); on.put("timeout", timeout); on.put("maxResultReads", maxResultReads); + on.put("planFormat", planFormat); + on.put("expressionTree", expressionTree); + on.put("rewrittenExpressionTree", rewrittenExpressionTree); + on.put("logicalPlan", logicalPlan); + on.put("optimizedLogicalPlan", optimizedLogicalPlan); + on.put("job", job); return om.writer(new MinimalPrettyPrinter()).writeValueAsString(on); } catch (JsonProcessingException e) { // NOSONAR LOGGER.debug("unexpected exception marshalling {} instance to json", getClass(), e); @@ -307,9 +327,15 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { SessionOutput.ResultAppender appendStatus = ResultUtil.createResultStatusAppender(); SessionConfig.OutputFormat format = getFormat(param.format); - //TODO:get the parameters from UI.Currently set to clean_json. - SessionConfig sessionConfig = new SessionConfig(format); + final SessionConfig.PlanFormat planFormat = + SessionConfig.PlanFormat.get(param.planFormat, param.planFormat, SessionConfig.PlanFormat.JSON, LOGGER); + SessionConfig sessionConfig = new SessionConfig(format, planFormat); sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true); + sessionConfig.set(SessionConfig.OOB_EXPR_TREE, param.expressionTree); + sessionConfig.set(SessionConfig.OOB_REWRITTEN_EXPR_TREE, param.rewrittenExpressionTree); + sessionConfig.set(SessionConfig.OOB_LOGICAL_PLAN, param.logicalPlan); + sessionConfig.set(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN, param.optimizedLogicalPlan); + sessionConfig.set(SessionConfig.OOB_HYRACKS_JOB, param.job); sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, param.pretty); sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD, format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON); @@ -391,6 +417,13 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { param.clientContextID = getOptText(jsonRequest, Parameter.CLIENT_ID.str()); param.timeout = getOptText(jsonRequest, Parameter.TIMEOUT.str()); param.maxResultReads = getOptText(jsonRequest, Parameter.MAX_RESULT_READS.str()); + param.planFormat = getOptText(jsonRequest, Parameter.PLAN_FORMAT.str()); + param.expressionTree = getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE.str(), false); + param.rewrittenExpressionTree = + getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE.str(), false); + param.logicalPlan = getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN.str(), false); + param.optimizedLogicalPlan = getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN.str(), false); + param.job = getOptBoolean(jsonRequest, Parameter.JOB.str(), false); } catch (JsonParseException | JsonMappingException e) { // if the JSON parsing fails, the statement is empty and we get an empty statement error GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e); @@ -406,6 +439,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { param.clientContextID = request.getParameter(Parameter.CLIENT_ID.str()); param.timeout = request.getParameter(Parameter.TIMEOUT.str()); param.maxResultReads = request.getParameter(Parameter.MAX_RESULT_READS.str()); + param.planFormat = request.getParameter(Parameter.PLAN_FORMAT.str()); } return param; } @@ -533,6 +567,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { getHyracksDataset(), resultProperties, stats, null, param.clientContextID, optionalParameters); translator.compileAndExecute(getHyracksClientConnection(), queryCtx, requestParameters); execution.end(); + printExecutionPlans(sessionOutput, translator.getExecutionPlans()); } protected void handleExecuteStatementException(Throwable t, RequestExecutionState state, RequestParameters param) { @@ -566,4 +601,21 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); } } + + protected void printExecutionPlans(SessionOutput output, ExecutionPlans executionPlans) { + final PrintWriter pw = output.out(); + pw.print("\t\""); + pw.print(ResultFields.PLANS.str()); + pw.print("\":"); + final SessionConfig.PlanFormat planFormat = output.config().getPlanFormat(); + switch (planFormat) { + case JSON: + case STRING: + pw.print(ExecutionPlansJsonPrintUtil.asJson(executionPlans, planFormat)); + break; + default: + throw new IllegalStateException("Unrecognized plan format: " + planFormat); + } + pw.print(",\n"); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java index 53d4f3f..d295304 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java @@ -131,6 +131,7 @@ public final class ExecuteStatementRequestMessage implements ICcAddressedMessage responseMsg.setResult(outWriter.toString()); responseMsg.setMetadata(outMetadata); responseMsg.setStats(stats); + responseMsg.setExecutionPlans(translator.getExecutionPlans()); } catch (AlgebricksException | HyracksException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) { // we trust that "our" exceptions are serializable and have a comprehensible error message http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java index 7475be4..94dd541 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java @@ -23,6 +23,7 @@ import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.messaging.api.INcAddressedMessage; import org.apache.asterix.common.messaging.api.MessageFuture; import org.apache.asterix.messaging.NCMessageBroker; +import org.apache.asterix.translator.ExecutionPlans; import org.apache.asterix.translator.IStatementExecutor; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -39,6 +40,8 @@ public final class ExecuteStatementResponseMessage implements INcAddressedMessag private Throwable error; + private ExecutionPlans executionPlans; + public ExecuteStatementResponseMessage(long requestMessageId) { this.requestMessageId = requestMessageId; } @@ -84,6 +87,14 @@ public final class ExecuteStatementResponseMessage implements INcAddressedMessag this.stats = stats; } + public ExecutionPlans getExecutionPlans() { + return executionPlans; + } + + public void setExecutionPlans(ExecutionPlans executionPlans) { + this.executionPlans = executionPlans; + } + @Override public String toString() { return String.format("%s(id=%s): %d characters", getClass().getSimpleName(), requestMessageId, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 453bcb5..6a89bda 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -156,6 +156,8 @@ import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement; import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement; import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement; import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement; +import org.apache.asterix.translator.ExecutionPlans; +import org.apache.asterix.translator.ExecutionPlansHtmlPrintUtil; import org.apache.asterix.translator.IRequestParameters; import org.apache.asterix.translator.IStatementExecutor; import org.apache.asterix.translator.IStatementExecutorContext; @@ -1773,6 +1775,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen new CompiledLoadFromFileStatement(dataverseName, loadStmt.getDatasetName().getValue(), loadStmt.getAdapter(), loadStmt.getProperties(), loadStmt.dataIsAlreadySorted()); JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls); + afterCompile(); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; if (spec != null) { @@ -1864,6 +1867,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen stmtDelete.getDatasetName().getValue(), stmtDelete.getCondition(), stmtDelete.getVarCounter(), stmtDelete.getQuery()); JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs); + afterCompile(); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; @@ -2370,6 +2374,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen metadataProvider.setMetadataTxnContext(mdTxnCtx); try { final JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, null); + afterCompile(); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; return query.isExplain() || !sessionConfig.isExecuteQuery() ? null : jobSpec; @@ -2778,6 +2783,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen return (dataverse != null) ? dataverse : activeDataverse.getDataverseName(); } + @Override + public ExecutionPlans getExecutionPlans() { + return apiFramework.getExecutionPlans(); + } + public String getActiveDataverse(Identifier dataverse) { return getActiveDataverseName(dataverse != null ? dataverse.getValue() : null); } @@ -2812,4 +2822,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen IStatementRewriter rewriter = rewriterFactory.createStatementRewriter(); rewriter.rewrite(stmt); } + + protected void afterCompile() { + if (sessionOutput.config().is(SessionConfig.FORMAT_HTML)) { + ExecutionPlansHtmlPrintUtil.print(sessionOutput.out(), getExecutionPlans()); + } + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java index 890667a..d93555d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java @@ -53,7 +53,8 @@ public class ResultExtractor { SIGNATURE("signature"), STATUS("status"), TYPE("type"), - ERRORS("errors"); + ERRORS("errors"), + PLANS("plans"); private static final Map<String, ResultField> fields = new HashMap<>(); @@ -162,6 +163,7 @@ public class ResultExtractor { case SIGNATURE: case STATUS: case TYPE: + case PLANS: resultBuilder.append(OBJECT_MAPPER.writeValueAsString(fieldValue)); break; default: http://git-wip-us.apache.org/repos/asf/asterixdb/blob/78dc83b1/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java index 6085c1c..baa3174 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java @@ -149,7 +149,7 @@ public class JSONUtil { return quoteAndEscape(new StringBuilder(), str).toString(); } - private static StringBuilder quoteAndEscape(StringBuilder sb, String str) { + public static StringBuilder quoteAndEscape(StringBuilder sb, String str) { return escape(sb.append('"'), str).append('"'); }
