[
https://issues.apache.org/jira/browse/DRILL-5260?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15881269#comment-15881269
]
ASF GitHub Bot commented on DRILL-5260:
---------------------------------------
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/753#discussion_r102618884
--- Diff:
exec/java-exec/src/test/java/org/apache/drill/test/ProfileParser.java ---
@@ -138,9 +414,208 @@ public long getMetric(int id) {
}
}
- public Map<Integer,OpInfo> getOpInfo( ) {
+ /**
+ * Information about an operator definition: the plan-time information
+ * that appears in the plan portion of the profile. Also holds the
+ * "actuals" from the minor fragment portion of the profile.
+ * Allows integrating the "planned" vs. "actual" performance of the
+ * query.
+ */
+
+ public static class OpDefInfo {
+ public String opName;
+ public boolean isInferred;
+ public int majorId;
+ public int stepId;
+ public String args;
+ public List<FieldDef> columns;
+ public int globalLevel;
+ public int localLevel;
+ public int id;
+ public int branchId;
+ public boolean isBranchRoot;
+ public double estMemoryCost;
+ public double estNetCost;
+ public double estIOCost;
+ public double estCpuCost;
+ public double estRowCost;
+ public double estRows;
+ public String name;
+ public long actualMemory;
+ public int actualBatches;
+ public long actualRows;
+ public OpDefInfo inferredParent;
+ public List<OperatorProfile> opExecs = new ArrayList<>( );
+ public List<OpDefInfo> children = new ArrayList<>( );
+
+ // 00-00 Screen : rowType = RecordType(VARCHAR(10) Year,
VARCHAR(65536) Month, VARCHAR(100) Devices, VARCHAR(100) Tier, VARCHAR(100)
LOB, CHAR(10) Gateway, BIGINT Day, BIGINT Hour, INTEGER Week, VARCHAR(100)
Week_end_date, BIGINT Usage_Cnt): \
+ // rowcount = 100.0, cumulative cost = {7.42124276972414E9 rows,
7.663067406383167E10 cpu, 0.0 io, 2.24645048816E10 network, 2.692766612982188E8
memory}, id = 129302
+ //
+ // 00-01 Project(Year=[$0], Month=[$1], Devices=[$2], Tier=[$3],
LOB=[$4], Gateway=[$5], Day=[$6], Hour=[$7], Week=[$8], Week_end_date=[$9],
Usage_Cnt=[$10]) :
+ // rowType = RecordType(VARCHAR(10) Year, VARCHAR(65536) Month,
VARCHAR(100) Devices, VARCHAR(100) Tier, VARCHAR(100) LOB, CHAR(10) Gateway,
BIGINT Day, BIGINT Hour, INTEGER Week, VARCHAR(100) Week_end_date, BIGINT
Usage_Cnt): rowcount = 100.0, cumulative cost = {7.42124275972414E9 rows,
7.663067405383167E10 cpu, 0.0 io, 2.24645048816E10 network, 2.692766612982188E8
memory}, id = 129301
+
+ public OpDefInfo(String plan) {
+ Pattern p = Pattern.compile(
"^(\\d+)-(\\d+)(\\s+)(\\w+)(?:\\((.*)\\))?\\s*:\\s*(.*)$" );
+ Matcher m = p.matcher(plan);
+ if (!m.matches()) {
+ throw new IllegalStateException( "Could not parse plan: " + plan );
+ }
+ majorId = Integer.parseInt(m.group(1));
+ stepId = Integer.parseInt(m.group(2));
+ name = m.group(4);
+ args = m.group(5);
+ String tail = m.group(6);
+ String indent = m.group(3);
+ globalLevel = (indent.length() - 4) / 2;
+
+ p = Pattern.compile("rowType = RecordType\\((.*)\\): (rowcount .*)");
+ m = p.matcher(tail);
+ if ( m.matches() ) {
+ columns = parseCols(m.group(1));
+ tail = m.group(2);
+ }
+
+ p = Pattern.compile( "rowcount = ([\\d.E]+), cumulative cost =
\\{([\\d.E]+) rows, ([\\d.E]+) cpu, ([\\d.E]+) io, ([\\d.E]+) network,
([\\d.E]+) memory\\}, id = (\\d+)");
+ m = p.matcher(tail);
+ if (! m.matches()) {
+ throw new IllegalStateException("Could not parse costs: " + tail );
+ }
+ estRows = Double.parseDouble(m.group(1));
+ estRowCost = Double.parseDouble(m.group(2));
+ estCpuCost = Double.parseDouble(m.group(3));
+ estIOCost = Double.parseDouble(m.group(4));
+ estNetCost = Double.parseDouble(m.group(5));
+ estMemoryCost = Double.parseDouble(m.group(6));
+ id = Integer.parseInt(m.group(7));
+ }
+
+ public void printTree(String indent) {
+ new TreePrinter().visit(this);
+ }
+
+ public OpDefInfo(int major, int id) {
+ majorId = major;
+ stepId = id;
+ }
+
+ @Override
+ public String toString() {
+ String head = "[OpDefInfo " + majorId + "-" + stepId + ": " + name;
+ if ( isInferred ) {
+ head += " (" + opName + ")";
+ }
+ return head + "]";
+ }
+ }
+
+ /**
+ * Visit a tree of operator definitions to support printing,
+ * analysis and other tasks.
+ */
+
+ public static class TreeVisitor
+ {
+ public void visit(OpDefInfo root) {
+ visit(root, 0);
+ }
+ public void visit(OpDefInfo node, int indent) {
+ visitOp( node, indent );
+ if (node.children.isEmpty()) {
+ return;
+ }
+ if ( node.children.size() == 1) {
+ visit(node.children.get(0), indent);
+ return;
+ }
+ indent++;
+ int i = 0;
+ for (OpDefInfo child : node.children) {
+ visitSubtree(node, i++, indent);
+ visit(child, indent+1);
+ }
+ }
+
+ protected void visitOp(OpDefInfo node, int indent) {
+ }
+
+ protected void visitSubtree(OpDefInfo node, int i, int indent) {
+ }
+
+ public String indentString(int indent, String pad) {
+ StringBuilder buf = new StringBuilder();
+ for (int i = 0; i < indent; i++) {
+ buf.append( pad );
+ }
+ return buf.toString();
+ }
+
+ public String indentString(int indent) {
+ return indentString(indent, " ");
+ }
+
+ public String subtreeLabel(OpDefInfo node, int branch) {
+ if (node.name.equals("HashJoin")) {
+ return (branch == 0) ? "Probe" : "Build";
+ } else {
+ return "Input " + (branch + 1);
+ }
+ }
+ }
+
+ /**
+ * Print the operator tree for analysis.
+ */
+
+ public static class TreePrinter extends TreeVisitor
+ {
+ @Override
+ protected void visitOp(OpDefInfo node, int indent) {
+ System.out.print( indentString(indent) );
+ System.out.println( node.toString() );
+ }
+
+ @Override
+ protected void visitSubtree(OpDefInfo node, int i, int indent) {
+ System.out.print( indentString(indent) );
+ System.out.println(subtreeLabel(node, i));
+ }
+ }
+
+ /**
+ * Print out the tree showing a comparison of estimated vs.
+ * actual costs.
+ */
+
+ public static class CostPrinter extends TreeVisitor
+ {
+ @Override
+ protected void visitOp(OpDefInfo node, int indentLevel) {
+ System.out.print(String.format("%02d-%02d ", node.majorId,
node.stepId));
+ String indent = indentString(indentLevel, ". ");
+ System.out.print( indent + node.name );
+ if (node.opName != null) {
+ System.out.print( " (" + node.opName + ")" );
+ }
+ System.out.println( );
+ indent = indentString(15);
+ System.out.print( indent );
+ System.out.println(String.format(" Estimate: %,15.0f rows, %,7.0f
MB",
+ node.estRows, node.estMemoryCost / 1024 / 1024) );
+ System.out.print( indent );
+ System.out.println(String.format(" Actual: %,15d rows, %,7d MB",
+ node.actualRows, node.actualMemory / 1024 /
1024));
+ }
+
+ @Override
+ protected void visitSubtree(OpDefInfo node, int i, int indent) {
+ System.out.print( indentString(indent) + " " );
+ System.out.println(subtreeLabel(node, i));
+ }
+ }
+
+ public Map<Integer,OperatorProfile> getOpInfo( ) {
Map<Integer,String> ops = getOperators( );
--- End diff --
This looks like it's getting the operatorProfile of root fragment only ? If
yes it would be helpful to rename the method and add some documentation here.
> Refinements to new "Cluster Fixture" test framework
> ---------------------------------------------------
>
> Key: DRILL-5260
> URL: https://issues.apache.org/jira/browse/DRILL-5260
> Project: Apache Drill
> Issue Type: Improvement
> Affects Versions: 1.10
> Reporter: Paul Rogers
> Assignee: Paul Rogers
> Priority: Minor
> Fix For: 1.10
>
>
> Roll-up of a number of enhancements to the cluster fixture framework.
> * Config option to suppress printing of CSV and other output. (Allows
> printing for single tests, not printing when running from Maven.)
> * Parsing of query profiles to extract plan and run time information.
> * Fix bug in log fixture when enabling logging for a package.
> * Improved ZK support.
> * Set up the new CTTAS default temporary workspace for tests.
> * Revise TestDrillbitResiliance to use the new framework.
> * Revise TestWindowFrame to to use the new framework.
> * Revise TestMergeJoinWithSchemaChanges to use the new framework.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
