Author: heyongqiang
Date: Thu Nov 10 02:18:47 2011
New Revision: 1200098
URL: http://svn.apache.org/viewvc?rev=1200098&view=rev
Log:
HIVE-2546 add explain formatted (namit via He Yongqiang)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
hive/trunk/ql/src/test/queries/clientpositive/input4.q
hive/trunk/ql/src/test/queries/clientpositive/join0.q
hive/trunk/ql/src/test/results/clientpositive/input4.q.out
hive/trunk/ql/src/test/results/clientpositive/join0.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java?rev=1200098&r1=1200097&r2=1200098&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java Thu
Nov 10 02:18:47 2011
@@ -40,7 +40,8 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.StringUtils;
-
+import org.json.JSONException;
+import org.json.JSONObject;
/**
* ExplainTask implementation.
@@ -53,6 +54,47 @@ public class ExplainTask extends Task<Ex
super();
}
+ static public JSONObject getJSONPlan(PrintStream out, ExplainWork work)
+ throws Exception {
+ // If the user asked for a formatted output, dump the json output
+ // in the output stream
+ JSONObject outJSONObject = new JSONObject();
+ boolean jsonOutput = work.isFormatted();
+ if (jsonOutput) {
+ out = null;
+ }
+
+ // Print out the parse AST
+ String jsonAST = outputAST(work.getAstStringTree(), out, jsonOutput, 0);
+ if (out != null) {
+ out.println();
+ }
+
+ if (jsonOutput) {
+ outJSONObject.put("ABSTRACT SYNTAX TREE", jsonAST);
+ }
+
+ JSONObject jsonDependencies = outputDependencies(out, jsonOutput,
+ work.getRootTasks(), 0);
+
+ if (out != null) {
+ out.println();
+ }
+
+ if (jsonOutput) {
+ outJSONObject.put("STAGE DEPENDENCIES", jsonDependencies);
+ }
+
+ // Go over all the tasks and dump out the plans
+ JSONObject jsonPlan = outputStagePlans(out, work, work.getRootTasks(), 0);
+
+ if (jsonOutput) {
+ outJSONObject.put("STAGE PLANS", jsonPlan);
+ }
+
+ return jsonOutput ? outJSONObject : null;
+ }
+
@Override
public int execute(DriverContext driverContext) {
@@ -62,29 +104,27 @@ public class ExplainTask extends Task<Ex
OutputStream outS = resFile.getFileSystem(conf).create(resFile);
out = new PrintStream(outS);
- // Print out the parse AST
- outputAST(work.getAstStringTree(), out, 0);
- out.println();
+ JSONObject jsonPlan = getJSONPlan(out, work);
- outputDependencies(out, work.getRootTasks(), 0);
- out.println();
+ if (work.isFormatted()) {
+ out.print(jsonPlan);
+ }
- // Go over all the tasks and dump out the plans
- outputStagePlans(out, work.getRootTasks(), 0);
out.close();
out = null;
-
return (0);
- } catch (Exception e) {
- console.printError("Failed with exception " + e.getMessage(), "\n"
- + StringUtils.stringifyException(e));
+ }
+ catch (Exception e) {
+ console.printError("Failed with exception " + e.getMessage(),
+ "\n" + StringUtils.stringifyException(e));
return (1);
- } finally {
+ }
+ finally {
IOUtils.closeStream(out);
}
}
- private String indentString(int indent) {
+ private static String indentString(int indent) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < indent; ++i) {
sb.append(" ");
@@ -93,73 +133,112 @@ public class ExplainTask extends Task<Ex
return sb.toString();
}
- private void outputMap(Map<?, ?> mp, String header, PrintStream out,
- boolean extended, int indent) throws Exception {
+ private static JSONObject outputMap(Map<?, ?> mp, String header, PrintStream
out,
+ boolean extended, boolean jsonOutput, int indent) throws Exception {
boolean first_el = true;
TreeMap<Object, Object> tree = new TreeMap<Object, Object>();
tree.putAll(mp);
+ JSONObject json = jsonOutput ? new JSONObject() : null;
for (Entry<?, ?> ent : tree.entrySet()) {
- if (first_el) {
+ if (first_el && (out != null)) {
out.println(header);
}
first_el = false;
// Print the key
- out.print(indentString(indent));
- out.printf("%s ", ent.getKey().toString());
+ if (out != null) {
+ out.print(indentString(indent));
+ out.printf("%s ", ent.getKey().toString());
+ }
+
// Print the value
if (isPrintable(ent.getValue())) {
- out.print(ent.getValue());
- out.println();
- } else if (ent.getValue() instanceof List
- || ent.getValue() instanceof Map) {
- out.print(ent.getValue().toString());
- out.println();
- } else if (ent.getValue() instanceof Serializable) {
- out.println();
- outputPlan((Serializable) ent.getValue(), out, extended, indent + 2);
- } else {
- out.println();
+ if (out != null) {
+ out.print(ent.getValue());
+ out.println();
+ }
+ if (jsonOutput) {
+ json.put(ent.getKey().toString(), ent.getValue().toString());
+ }
+ }
+ else if (ent.getValue() instanceof List || ent.getValue() instanceof
Map) {
+ if (out != null) {
+ out.print(ent.getValue().toString());
+ out.println();
+ }
+ if (jsonOutput) {
+ json.put(ent.getKey().toString(), ent.getValue().toString());
+ }
+ }
+ else if (ent.getValue() instanceof Serializable) {
+ if (out != null) {
+ out.println();
+ }
+ JSONObject jsonOut = outputPlan((Serializable) ent.getValue(), out,
+ extended, jsonOutput, jsonOutput ? 0 : indent + 2);
+ if (jsonOutput) {
+ json.put(ent.getKey().toString(), jsonOut);
+ }
+ }
+ else {
+ if (out != null) {
+ out.println();
+ }
}
}
+
+ return jsonOutput ? json : null;
}
- private void outputList(List<?> l, String header, PrintStream out,
- boolean extended, int indent) throws Exception {
+ private static String outputList(List<?> l, String header, PrintStream out,
+ boolean extended, boolean jsonOutput, int indent) throws Exception {
boolean first_el = true;
boolean nl = false;
+ StringBuffer s = new StringBuffer();
for (Object o : l) {
- if (first_el) {
+ if (first_el && (out != null)) {
out.print(header);
}
if (isPrintable(o)) {
- if (!first_el) {
- out.print(", ");
- } else {
- out.print(" ");
+ String delim = first_el ? " " : ", ";
+ if (out != null) {
+ out.print(delim);
+ out.print(o);
}
- out.print(o);
+ if (jsonOutput) {
+ s.append(delim);
+ s.append(o);
+ }
nl = true;
- } else if (o instanceof Serializable) {
- if (first_el) {
+ }
+ else if (o instanceof Serializable) {
+ if (first_el && (out != null)) {
out.println();
}
- outputPlan((Serializable) o, out, extended, indent + 2);
+ JSONObject jsonOut = outputPlan((Serializable) o, out, extended,
+ jsonOutput, jsonOutput ? 0 : indent + 2);
+ if (jsonOutput) {
+ if (!first_el) {
+ s.append(", ");
+ }
+ s.append(jsonOut);
+ }
}
first_el = false;
}
- if (nl) {
+ if (nl && (out != null)) {
out.println();
}
+ return jsonOutput ? s.toString() : null;
}
- private boolean isPrintable(Object val) {
+ private static boolean isPrintable(Object val) {
if (val instanceof Boolean || val instanceof String
|| val instanceof Integer || val instanceof Byte
|| val instanceof Float || val instanceof Double) {
@@ -173,33 +252,56 @@ public class ExplainTask extends Task<Ex
return false;
}
- private void outputPlan(Serializable work, PrintStream out, boolean extended,
- int indent) throws Exception {
+ private static JSONObject outputPlan(Serializable work, PrintStream out,
+ boolean extended, boolean jsonOutput, int indent) throws Exception {
// Check if work has an explain annotation
Annotation note = work.getClass().getAnnotation(Explain.class);
+ String keyJSONObject = null;
+
if (note instanceof Explain) {
Explain xpl_note = (Explain) note;
if (extended || xpl_note.normalExplain()) {
- out.print(indentString(indent));
- out.println(xpl_note.displayName());
+ keyJSONObject = xpl_note.displayName();
+ if (out != null) {
+ out.print(indentString(indent));
+ out.println(xpl_note.displayName());
+ }
}
}
+ JSONObject json = jsonOutput ? new JSONObject() : null;
// If this is an operator then we need to call the plan generation on the
- // conf and then
- // the children
+ // conf and then the children
if (work instanceof Operator) {
Operator<? extends Serializable> operator = (Operator<? extends
Serializable>) work;
if (operator.getConf() != null) {
- outputPlan(operator.getConf(), out, extended, indent);
+ JSONObject jsonOut = outputPlan(operator.getConf(), out, extended,
+ jsonOutput, jsonOutput ? 0 : indent);
+ if (jsonOutput) {
+ json.put(operator.getOperatorId(), jsonOut);
+ }
}
+
if (operator.getChildOperators() != null) {
for (Operator<? extends Serializable> op :
operator.getChildOperators()) {
- outputPlan(op, out, extended, indent + 2);
+ JSONObject jsonOut = outputPlan(op, out, extended, jsonOutput,
jsonOutput ? 0 : indent + 2);
+ if (jsonOutput) {
+ json.put(operator.getOperatorId(), jsonOut);
+ }
}
}
- return;
+
+ if (jsonOutput) {
+ if (keyJSONObject != null) {
+ JSONObject ret = new JSONObject();
+ ret.put(keyJSONObject, json);
+ return ret;
+ }
+
+ return json;
+ }
+ return null;
}
// We look at all methods that generate values for explain
@@ -207,7 +309,7 @@ public class ExplainTask extends Task<Ex
Arrays.sort(methods, new MethodComparator());
for (Method m : methods) {
- int prop_indents = indent + 2;
+ int prop_indents = jsonOutput ? 0 : indent + 2;
note = m.getAnnotation(Explain.class);
if (note instanceof Explain) {
@@ -224,179 +326,286 @@ public class ExplainTask extends Task<Ex
String header = null;
if (!xpl_note.displayName().equals("")) {
header = indentString(prop_indents) + xpl_note.displayName() + ":";
- } else {
+ }
+ else {
prop_indents = indent;
header = indentString(prop_indents);
}
if (isPrintable(val)) {
-
- out.printf("%s ", header);
- out.println(val);
+ if (out != null) {
+ out.printf("%s ", header);
+ out.println(val);
+ }
+ if (jsonOutput) {
+ json.put(header, val.toString());
+ }
continue;
}
// Try this as a map
try {
// Go through the map and print out the stuff
Map<?, ?> mp = (Map<?, ?>) val;
- outputMap(mp, header, out, extended, prop_indents + 2);
+ JSONObject jsonOut = outputMap(mp, header, out, extended,
jsonOutput,
+ jsonOutput ? 0 : prop_indents + 2);
+ if (jsonOutput) {
+ json.put(header, jsonOut);
+ }
continue;
- } catch (ClassCastException ce) {
+ }
+ catch (ClassCastException ce) {
// Ignore - all this means is that this is not a map
}
// Try this as a list
try {
List<?> l = (List<?>) val;
- outputList(l, header, out, extended, prop_indents + 2);
-
+ String jsonOut = outputList(l, header, out, extended, jsonOutput,
+ jsonOutput ? 0 : prop_indents + 2);
+ if (jsonOutput) {
+ json.put(header, jsonOut);
+ }
continue;
- } catch (ClassCastException ce) {
+ }
+ catch (ClassCastException ce) {
// Ignore
}
// Finally check if it is serializable
try {
Serializable s = (Serializable) val;
- out.println(header);
- outputPlan(s, out, extended, prop_indents + 2);
-
+ if (out != null) {
+ out.println(header);
+ }
+ JSONObject jsonOut = outputPlan(s, out, extended, jsonOutput,
+ jsonOutput ? 0 : prop_indents + 2);
+ if (jsonOutput) {
+ json.put(header, jsonOut);
+ }
continue;
- } catch (ClassCastException ce) {
+ }
+ catch (ClassCastException ce) {
// Ignore
}
}
}
}
+
+ if (jsonOutput) {
+ if (keyJSONObject != null) {
+ JSONObject ret = new JSONObject();
+ ret.put(keyJSONObject, json);
+ return ret;
+ }
+
+ return json;
+ }
+
+ return null;
}
- private void outputPlan(Task<? extends Serializable> task, PrintStream out,
- boolean extended, HashSet<Task<? extends Serializable>> displayedSet,
+ private static JSONObject outputPlan(Task<? extends Serializable> task,
+ PrintStream out, JSONObject parentJSON, boolean extended,
+ boolean jsonOutput, HashSet<Task<? extends Serializable>> displayedSet,
int indent) throws Exception {
if (displayedSet.contains(task)) {
- return;
+ return null;
}
displayedSet.add(task);
- out.print(indentString(indent));
- out.printf("Stage: %s\n", task.getId());
+ if (out != null) {
+ out.print(indentString(indent));
+ out.printf("Stage: %s\n", task.getId());
+ }
+
// Start by getting the work part of the task and call the output plan for
// the work
- outputPlan(task.getWork(), out, extended, indent + 2);
- out.println();
+ JSONObject jsonOutputPlan = outputPlan(task.getWork(), out, extended,
+ jsonOutput, jsonOutput ? 0 : indent + 2);
+
+ if (out != null) {
+ out.println();
+ }
+
+ if (jsonOutput) {
+ parentJSON.put(task.getId(), jsonOutputPlan);
+ }
+
if (task instanceof ConditionalTask
&& ((ConditionalTask) task).getListTasks() != null) {
- for (Task<? extends Serializable> con : ((ConditionalTask) task)
- .getListTasks()) {
- outputPlan(con, out, extended, displayedSet, indent);
+ for (Task<? extends Serializable> con : ((ConditionalTask)
task).getListTasks()) {
+ outputPlan(con, out, parentJSON, extended, jsonOutput, displayedSet,
+ jsonOutput ? 0 : indent);
}
}
if (task.getChildTasks() != null) {
for (Task<? extends Serializable> child : task.getChildTasks()) {
- outputPlan(child, out, extended, displayedSet, indent);
+ outputPlan(child, out, parentJSON, extended, jsonOutput, displayedSet,
+ jsonOutput ? 0 : indent);
}
}
+ return null;
}
- private final Set<Task<? extends Serializable>> dependeciesTaskSet =
- new HashSet<Task<? extends Serializable>>();
-
- private void outputDependencies(Task<? extends Serializable> task,
- PrintStream out, int indent, boolean rootTskCandidate) throws Exception {
+ private static JSONObject outputDependencies(Task<? extends Serializable>
task,
+ Set<Task<? extends Serializable>> dependeciesTaskSet, PrintStream out,
+ JSONObject parentJson, boolean jsonOutput, int indent,
+ boolean rootTskCandidate) throws Exception {
if (dependeciesTaskSet.contains(task)) {
- return;
+ return null;
}
dependeciesTaskSet.add(task);
boolean first = true;
- out.print(indentString(indent));
- out.printf("%s", task.getId());
+ JSONObject json = jsonOutput ? new JSONObject() : null;
+ if (out != null) {
+ out.print(indentString(indent));
+ out.printf("%s", task.getId());
+ }
+
if ((task.getParentTasks() == null || task.getParentTasks().isEmpty())) {
if (rootTskCandidate) {
- out.print(" is a root stage");
+ if (out != null) {
+ out.print(" is a root stage");
+ }
+
+ if (jsonOutput) {
+ json.put("ROOT STAGE", "TRUE");
+ }
}
- } else {
- out.print(" depends on stages: ");
+ }
+ else {
+ StringBuffer s = new StringBuffer();
first = true;
for (Task<? extends Serializable> parent : task.getParentTasks()) {
if (!first) {
- out.print(", ");
+ s.append(", ");
}
first = false;
- out.print(parent.getId());
+ s.append(parent.getId());
}
- }
- Task<? extends Serializable> cuurBackupTask = task.getBackupTask();
- if(cuurBackupTask != null) {
- out.print(" has a backup stage: ");
- if (!first) {
- out.print(", ");
+ if (out != null) {
+ out.print(" depends on stages: ");
+ out.print(s.toString());
+ }
+ if (jsonOutput) {
+ json.put("DEPENDENT STAGES", s.toString());
}
- first = false;
- out.print(cuurBackupTask.getId());
}
+ Task<? extends Serializable> currBackupTask = task.getBackupTask();
+ if (currBackupTask != null) {
+ if (out != null) {
+ out.print(" has a backup stage: ");
+ out.print(currBackupTask.getId());
+ }
+ if (jsonOutput) {
+ json.put("BACKUP STAGE", currBackupTask.getId());
+ }
+ }
if (task instanceof ConditionalTask
&& ((ConditionalTask) task).getListTasks() != null) {
- out.print(" , consists of ");
+ StringBuffer s = new StringBuffer();
first = true;
- for (Task<? extends Serializable> con : ((ConditionalTask) task)
- .getListTasks()) {
+ for (Task<? extends Serializable> con : ((ConditionalTask)
task).getListTasks()) {
if (!first) {
- out.print(", ");
+ s.append(", ");
}
first = false;
- out.print(con.getId());
+ s.append(con.getId());
}
- }
+ if (out != null) {
+ out.print(" , consists of ");
+ out.print(s.toString());
+ }
+ if (jsonOutput) {
+ json.put("CONDITIONAL CHILD TASKS", s.toString());
+ }
+ }
- out.println();
+ if (out != null) {
+ out.println();
+ }
if (task instanceof ConditionalTask
&& ((ConditionalTask) task).getListTasks() != null) {
- for (Task<? extends Serializable> con : ((ConditionalTask) task)
- .getListTasks()) {
- outputDependencies(con, out, indent, false);
+ for (Task<? extends Serializable> con : ((ConditionalTask)
task).getListTasks()) {
+ JSONObject jsonOut = outputDependencies(con, dependeciesTaskSet, out,
+ parentJson, jsonOutput, jsonOutput ? 0 : indent, false);
+ if (jsonOutput && (jsonOut != null)) {
+ parentJson.put(con.getId(), jsonOut);
+ }
}
}
if (task.getChildTasks() != null) {
for (Task<? extends Serializable> child : task.getChildTasks()) {
- outputDependencies(child, out, indent, true);
+ JSONObject jsonOut = outputDependencies(child, dependeciesTaskSet, out,
+ parentJson, jsonOutput, jsonOutput ? 0 : indent, true);
+ if (jsonOutput && (jsonOut != null)) {
+ parentJson.put(child.getId(), jsonOut);
+ }
}
}
-
+ return jsonOutput ? json : null;
}
- public void outputAST(String treeString, PrintStream out, int indent) {
- out.print(indentString(indent));
- out.println("ABSTRACT SYNTAX TREE:");
- out.print(indentString(indent + 2));
- out.println(treeString);
+ public static String outputAST(String treeString, PrintStream out,
+ boolean jsonOutput, int indent) throws JSONException {
+ if (out != null) {
+ out.print(indentString(indent));
+ out.println("ABSTRACT SYNTAX TREE:");
+ out.print(indentString(indent + 2));
+ out.println(treeString);
+ }
+
+ return jsonOutput ? treeString : null;
}
- public void outputDependencies(PrintStream out,
- List<Task<? extends Serializable>> rootTasks, int indent) throws
Exception {
- out.print(indentString(indent));
- out.println("STAGE DEPENDENCIES:");
+ public static JSONObject outputDependencies(PrintStream out, boolean
jsonOutput,
+ List<Task<? extends Serializable>> rootTasks, int indent)
+ throws Exception {
+ if (out != null) {
+ out.print(indentString(indent));
+ out.println("STAGE DEPENDENCIES:");
+ }
+
+ JSONObject json = jsonOutput ? new JSONObject() : null;
+ Set<Task<? extends Serializable>> dependenciesTaskSet =
+ new HashSet<Task<? extends Serializable>>();
+
for (Task<? extends Serializable> rootTask : rootTasks) {
- outputDependencies(rootTask, out, indent + 2, true);
+ JSONObject jsonOut = outputDependencies(rootTask,
+ dependenciesTaskSet, out, json, jsonOutput,
+ jsonOutput ? 0 : indent + 2, true);
+ if (jsonOutput && (jsonOut != null)) {
+ json.put(rootTask.getId(), jsonOut);
+ }
}
+
+ return jsonOutput ? json : null;
}
- public void outputStagePlans(PrintStream out,
- List<Task<? extends Serializable>> rootTasks, int indent) throws
Exception {
- out.print(indentString(indent));
- out.println("STAGE PLANS:");
- HashSet<Task<? extends Serializable>> displayedSet =
- new HashSet<Task<? extends Serializable>>();
+ public static JSONObject outputStagePlans(PrintStream out, ExplainWork work,
+ List<Task<? extends Serializable>> rootTasks, int indent)
+ throws Exception {
+ boolean jsonOutput = work.isFormatted();
+ if (out != null) {
+ out.print(indentString(indent));
+ out.println("STAGE PLANS:");
+ }
+
+ JSONObject json = jsonOutput ? new JSONObject() : null;
+ HashSet<Task<? extends Serializable>> displayedSet = new HashSet<Task<?
extends Serializable>>();
for (Task<? extends Serializable> rootTask : rootTasks) {
- outputPlan(rootTask, out, work.getExtended(), displayedSet, indent + 2);
+ outputPlan(rootTask, out, json, work.getExtended(), jsonOutput,
+ displayedSet, jsonOutput ? 0 : indent + 2);
}
+ return jsonOutput ? json : null;
}
/**
@@ -425,7 +634,7 @@ public class ExplainTask extends Task<Ex
protected void localizeMRTmpFilesImpl(Context ctx) {
// explain task has nothing to localize
// we don't expect to enter this code path at all
- throw new RuntimeException ("Unexpected call");
+ throw new RuntimeException("Unexpected call");
}
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java?rev=1200098&r1=1200097&r2=1200098&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
Thu Nov 10 02:18:47 2011
@@ -47,10 +47,13 @@ public class ExplainSemanticAnalyzer ext
.getChild(0));
sem.analyze((ASTNode) ast.getChild(0), ctx);
sem.validate();
-
+
boolean extended = false;
- if (ast.getChildCount() > 1) {
- extended = true;
+ boolean formatted = false;
+ if (ast.getChildCount() == 2) {
+ int explainOptions = ast.getChild(1).getType();
+ formatted = (explainOptions == HiveParser.KW_FORMATTED);
+ extended = (explainOptions == HiveParser.KW_EXTENDED);
}
ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
@@ -65,7 +68,12 @@ public class ExplainSemanticAnalyzer ext
tasks.add(fetchTask);
}
- rootTasks.add(TaskFactory.get(new ExplainWork(ctx.getResFile().toString(),
- tasks, ((ASTNode) ast.getChild(0)).toStringTree(), extended), conf));
+ rootTasks.add(
+ TaskFactory.get(new ExplainWork(ctx.getResFile().toString(),
+ tasks,
+ ((ASTNode) ast.getChild(0)).toStringTree(),
+ extended,
+ formatted),
+ conf));
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g?rev=1200098&r1=1200097&r2=1200098&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g Thu Nov 10
02:18:47 2011
@@ -286,7 +286,8 @@ statement
explainStatement
@init { msgs.push("explain statement"); }
@after { msgs.pop(); }
- : KW_EXPLAIN (isExtended=KW_EXTENDED)? execStatement -> ^(TOK_EXPLAIN
execStatement $isExtended?)
+ : KW_EXPLAIN (explainOptions=KW_EXTENDED|explainOptions=KW_FORMATTED)?
execStatement
+ -> ^(TOK_EXPLAIN execStatement $explainOptions?)
;
execStatement
@@ -640,7 +641,6 @@ alterStatementSuffixRenameCol
: Identifier KW_CHANGE KW_COLUMN? oldName=Identifier newName=Identifier
colType (KW_COMMENT comment=StringLiteral)? alterStatementChangeColPosition?
->^(TOK_ALTERTABLE_RENAMECOL Identifier $oldName $newName colType
$comment? alterStatementChangeColPosition?)
;
-
alterStatementChangeColPosition
: first=KW_FIRST|KW_AFTER afterCol=Identifier
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java?rev=1200098&r1=1200097&r2=1200098&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java Thu
Nov 10 02:18:47 2011
@@ -35,17 +35,21 @@ public class ExplainWork implements Seri
private ArrayList<Task<? extends Serializable>> rootTasks;
private String astStringTree;
boolean extended;
+ boolean formatted;
public ExplainWork() {
}
public ExplainWork(String resFile,
- List<Task<? extends Serializable>> rootTasks, String astStringTree,
- boolean extended) {
+ List<Task<? extends Serializable>> rootTasks,
+ String astStringTree,
+ boolean extended,
+ boolean formatted) {
this.resFile = resFile;
this.rootTasks = new ArrayList<Task<? extends Serializable>>(rootTasks);
this.astStringTree = astStringTree;
this.extended = extended;
+ this.formatted = formatted;
}
public String getResFile() {
@@ -79,4 +83,12 @@ public class ExplainWork implements Seri
public void setExtended(boolean extended) {
this.extended = extended;
}
+
+ public boolean isFormatted() {
+ return formatted;
+ }
+
+ public void setFormatted(boolean formatted) {
+ this.formatted = formatted;
+ }
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/input4.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/input4.q?rev=1200098&r1=1200097&r2=1200098&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/input4.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/input4.q Thu Nov 10 02:18:47
2011
@@ -2,5 +2,7 @@ CREATE TABLE INPUT4(KEY STRING, VALUE ST
EXPLAIN
LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4;
LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4;
+EXPLAIN FORMATTED
+SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4;
SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4;
Modified: hive/trunk/ql/src/test/queries/clientpositive/join0.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/join0.q?rev=1200098&r1=1200097&r2=1200098&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/join0.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/join0.q Thu Nov 10 02:18:47
2011
@@ -6,6 +6,14 @@ SELECT src1.key as k1, src1.value as v1,
(SELECT * FROM src WHERE src.key < 10) src2
SORT BY k1, v1, k2, v2;
+EXPLAIN FORMATTED
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2;
+
SELECT src1.key as k1, src1.value as v1,
src2.key as k2, src2.value as v2 FROM
(SELECT * FROM src WHERE src.key < 10) src1
Modified: hive/trunk/ql/src/test/results/clientpositive/input4.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/input4.q.out?rev=1200098&r1=1200097&r2=1200098&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/input4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/input4.q.out Thu Nov 10
02:18:47 2011
@@ -19,8 +19,8 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
Copy
- source:
file:/Users/krishnak/Projects/hdp/sources/hive-git-apache/data/files/kv1.txt
- destination:
pfile:/Users/krishnak/Projects/hdp/sources/hive-git-apache/build/ql/scratchdir/hive_2011-03-20_23-49-16_008_2588436210508896099/-ext-10000
+ source: file:/data/users/njain/hive_commit1/data/files/kv1.txt
+ destination:
pfile:/data/users/njain/hive_commit1/build/ql/scratchdir/hive_2011-11-03_16-47-14_258_1757057361379982466/-ext-10000
Stage: Stage-1
Move Operator
@@ -39,14 +39,21 @@ PREHOOK: Output: default@input4
POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE
INPUT4
POSTHOOK: type: LOAD
POSTHOOK: Output: default@input4
+PREHOOK: query: EXPLAIN FORMATTED
+SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN FORMATTED
+SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4
+POSTHOOK: type: QUERY
+{"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Alias -> Map Operator
Tree:":{"input4":{"TS_0":{"SEL_1":{"FS_2":{"File Output
Operator":{"GlobalTableId:":"0","compressed:":"false","table:":{"input
format:":"org.apache.hadoop.mapred.TextInputFormat","output
format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}}}}}},"Percentage
Sample:":{}}},"Stage-0":{"Fetch Operator":{"limit:":"-1"}}},"STAGE
DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"ROOT
STAGE":"TRUE"}},"ABSTRACT SYNTAX TREE":"(TOK_QUERY (TOK_FROM (TOK_TABREF
(TOK_TABNAME INPUT4))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL INPUT4) VALUE)) (TOK_SELEXPR (.
(TOK_TABLE_OR_COL INPUT4) KEY)))))"}
PREHOOK: query: SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4
PREHOOK: type: QUERY
PREHOOK: Input: default@input4
-PREHOOK: Output:
file:/var/folders/67/67R3POPtF90VG63KSmCbcU++F0U/-Tmp-/krishnak/hive_2011-03-20_23-49-17_477_5960644229997823855/-mr-10000
+PREHOOK: Output:
file:/tmp/njain/hive_2011-11-03_16-47-15_010_2383066747887433640/-mr-10000
POSTHOOK: query: SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4
POSTHOOK: type: QUERY
POSTHOOK: Input: default@input4
-POSTHOOK: Output:
file:/var/folders/67/67R3POPtF90VG63KSmCbcU++F0U/-Tmp-/krishnak/hive_2011-03-20_23-49-17_477_5960644229997823855/-mr-10000
+POSTHOOK: Output:
file:/tmp/njain/hive_2011-11-03_16-47-15_010_2383066747887433640/-mr-10000
val_238 238
val_86 86
val_311 311
Modified: hive/trunk/ql/src/test/results/clientpositive/join0.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/join0.q.out?rev=1200098&r1=1200097&r2=1200098&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/join0.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/join0.q.out Thu Nov 10
02:18:47 2011
@@ -100,7 +100,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
-
file:/tmp/sdong/hive_2011-02-10_16-01-47_793_8777845351054616084/-mr-10002
+
file:/var/folders/bZ/bZe+iKfoFTuPoShRd6dy6-tOU9Y/-Tmp-/njain/hive_2011-11-03_16-13-27_794_5039563178190087904/-mr-10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -136,6 +136,23 @@ STAGE PLANS:
limit: -1
+PREHOOK: query: EXPLAIN FORMATTED
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN FORMATTED
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2
+POSTHOOK: type: QUERY
+{"STAGE PLANS":{"Stage-2":{"Map Reduce":{"Reduce Operator
Tree:":{"OP_11":{"FS_12":{"File Output
Operator":{"GlobalTableId:":"0","compressed:":"false","table:":{"input
format:":"org.apache.hadoop.mapred.TextInputFormat","output
format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}}}},"Alias
-> Map Operator
Tree:":{"file:/var/folders/bZ/bZe+iKfoFTuPoShRd6dy6-tOU9Y/-Tmp-/njain/hive_2011-11-03_16-13-28_171_2888498791907064539/-mr-10002":{"TS_16":{"RS_10":{"Reduce
Output Operator":{"Map-reduce partition columns:":"","sort
order:":"++++","tag:":"-1","value
expressions:":"{\"type:\":\"string\",\"expr:\":\"_col0\"},
{\"type:\":\"string\",\"expr:\":\"_col1\"},
{\"type:\":\"string\",\"expr:\":\"_col2\"},
{\"type:\":\"string\",\"expr:\":\"_col3\"}","key
expressions:":"{\"type:\":\"string\",\"expr:\":\"_col0\"},
{\"type:\":\"string\",\"expr:\":\"_col1\"},
{\"type:\":\"string\",\"expr:\":\"_col2\"},
{\"type:\":\"string\",\"expr:\":\"_col3\"}"}}}}},"Percentage Sample:":
{}}},"Stage-1":{"Map Reduce":{"Reduce Operator
Tree:":{"JOIN_8":{"SEL_9":{"FS_15":{"File Output
Operator":{"GlobalTableId:":"0","compressed:":"false","table:":{"input
format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output
format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat"}}}}}},"Alias
-> Map Operator Tree:":{"src2:src":{"TS_0":{"FIL_13":{"SEL_2":{"RS_7":{"Reduce
Output Operator":{"Map-reduce partition columns:":"","sort
order:":"","tag:":"1","value
expressions:":"{\"type:\":\"string\",\"expr:\":\"_col0\"},
{\"type:\":\"string\",\"expr:\":\"_col1\"}","key
expressions:":""}}}}}},"src1:src":{"TS_3":{"FIL_14":{"SEL_5":{"RS_6":{"Reduce
Output Operator":{"Map-reduce partition columns:":"","sort
order:":"","tag:":"0","value
expressions:":"{\"type:\":\"string\",\"expr:\":\"_col0\"},
{\"type:\":\"string\",\"expr:\":\"_col1\"}","key
expressions:":""}}}}}}},"Percentage Sample:":{}}},"Stage-0":{"Fetch
Operator":{"limit:":"-1"}}},"STAGE DEPENDENCIES":{"Stag
e-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-1":{"ROOT
STAGE":"TRUE"},"Stage-0":{"ROOT STAGE":"TRUE"}},"ABSTRACT SYNTAX
TREE":"(TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM
(TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR
TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (.
(TOK_TABLE_OR_COL src) key) 10)))) src1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM
(TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR
TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (.
(TOK_TABLE_OR_COL src) key) 10)))) src2))) (TOK_INSERT (TOK_DESTINATION
(TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1)
key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) v1) (TOK_SELEXPR (.
(TOK_TABLE_OR_COL src2) key) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value)
v2)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k1))
(TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v1)) (TOK_TABSORTCOLNAMEASC
(TOK_TABLE_OR_COL k2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v2)))))"}
PREHOOK: query: SELECT src1.key as k1, src1.value as v1,
src2.key as k2, src2.value as v2 FROM
(SELECT * FROM src WHERE src.key < 10) src1
@@ -144,7 +161,7 @@ PREHOOK: query: SELECT src1.key as k1, s
SORT BY k1, v1, k2, v2
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_16-01-47_898_1006010715784348203/-mr-10000
+PREHOOK: Output:
file:/var/folders/bZ/bZe+iKfoFTuPoShRd6dy6-tOU9Y/-Tmp-/njain/hive_2011-11-03_16-13-28_323_1828259293437597011/-mr-10000
POSTHOOK: query: SELECT src1.key as k1, src1.value as v1,
src2.key as k2, src2.value as v2 FROM
(SELECT * FROM src WHERE src.key < 10) src1
@@ -153,7 +170,7 @@ POSTHOOK: query: SELECT src1.key as k1,
SORT BY k1, v1, k2, v2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_16-01-47_898_1006010715784348203/-mr-10000
+POSTHOOK: Output:
file:/var/folders/bZ/bZe+iKfoFTuPoShRd6dy6-tOU9Y/-Tmp-/njain/hive_2011-11-03_16-13-28_323_1828259293437597011/-mr-10000
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
