[
https://issues.apache.org/jira/browse/DRILL-5260?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15881580#comment-15881580
]
ASF GitHub Bot commented on DRILL-5260:
---------------------------------------
Github user paul-rogers commented on a diff in the pull request:
https://github.com/apache/drill/pull/753#discussion_r102845391
--- Diff:
exec/java-exec/src/test/java/org/apache/drill/test/ProfileParser.java ---
@@ -42,44 +48,207 @@
public class ProfileParser {
JsonObject profile;
+ String query;
List<String> plans;
+ List<OpDefInfo> operations;
+ Map<Integer,FragInfo> fragments = new HashMap<>();
+ private List<OpDefInfo> topoOrder;
public ProfileParser( File file ) throws IOException {
try (FileReader fileReader = new FileReader(file);
JsonReader reader = Json.createReader(fileReader)) {
profile = (JsonObject) reader.read();
}
+
+ parse();
+ }
+
+ private void parse() {
+ parseQuery();
+ parsePlans();
+ buildFrags();
+ parseFragProfiles();
+ mapOpProfiles();
+ aggregateOpers();
+ buildTree();
+ }
+
+ private void parseQuery() {
+ query = profile.getString("query");
+ query = query.replace("//n", "\n");
+ }
+
+ /**
+ * Parse a text version of the plan as it appears in the JSON
+ * query profile.
+ */
+
+ private static class PlanParser {
+
+ List<String> plans = new ArrayList<>();
+ List<OpDefInfo> operations = new ArrayList<>();
+ List<OpDefInfo> sorted = new ArrayList<>();
+
+ public void parsePlans(String plan) {
+ plans = new ArrayList<>( );
+ String parts[] = plan.split("\n");
+ for (String part : parts) {
+ plans.add(part);
+ OpDefInfo opDef = new OpDefInfo( part );
+ operations.add(opDef);
+ }
+ sortList();
+ }
+
+ public void sortList() {
+ List<OpDefInfo> raw = new ArrayList<>( );
+ raw.addAll( operations );
+ Collections.sort( raw, new Comparator<OpDefInfo>() {
+ @Override
+ public int compare(OpDefInfo o1, OpDefInfo o2) {
+ int result = Integer.compare(o1.majorId, o2.majorId);
+ if ( result == 0 ) {
+ result = Integer.compare(o1.stepId, o2.stepId);
+ }
+ return result;
+ }
+ });
+ int currentFrag = 0;
+ int currentStep = 0;
+ for ( OpDefInfo opDef : raw ) {
+ if ( currentFrag < opDef.majorId ) {
+ currentFrag++;
+ OpDefInfo sender = new OpDefInfo( currentFrag, 0 );
+ sender.isInferred = true;
+ sender.name = "Sender";
+ sorted.add(sender);
+ currentStep = 1;
+ opDef.inferredParent = sender;
+ sender.children.add( opDef );
+ }
+ if ( opDef.stepId > currentStep ) {
+ OpDefInfo unknown = new OpDefInfo( currentFrag, currentStep );
+ unknown.isInferred = true;
+ unknown.name = "Unknown";
+ sorted.add(unknown);
+ opDef.inferredParent = unknown;
+ unknown.children.add( opDef );
+ }
+ sorted.add( opDef );
+ currentStep = opDef.stepId + 1;
+ }
+ }
+ }
+
+ /**
+ * Parse the plan portion of the query profile.
+ */
+
+ private void parsePlans() {
+ PlanParser parser = new PlanParser();
+ String plan = getPlan( );
+ parser.parsePlans(plan);
+ plans = parser.plans;
+ topoOrder = parser.operations;
+ operations = parser.sorted;
+ }
+
+ private void buildFrags() {
+ for (OpDefInfo opDef : operations) {
+ FragInfo major = fragments.get(opDef.majorId);
+ if (major == null) {
+ major = new FragInfo(opDef.majorId);
+ fragments.put(opDef.majorId, major);
+ }
+ major.ops.add(opDef);
+ }
+ }
+
+ private static List<FieldDef> parseCols(String cols) {
+ String parts[] = cols.split( ", " );
+ List<FieldDef> fields = new ArrayList<>( );
+ for ( String part : parts ) {
+ String halves[] = part.split( " " );
+ fields.add( new FieldDef( halves[1], halves[0] ) );
+ }
+ return fields;
+ }
+
+ private void parseFragProfiles() {
+ JsonArray frags = getFragmentProfile( );
+ for (JsonObject fragProfile : frags.getValuesAs(JsonObject.class)) {
+ int mId = fragProfile.getInt("majorFragmentId");
+ FragInfo major = fragments.get(mId);
+ major.parse(fragProfile);
+ }
+ }
+
+ private void mapOpProfiles() {
+ for (FragInfo major : fragments.values()) {
+ for (MinorFragInfo minor : major.minors) {
+ minor.mapOpProfiles(major);
+ }
+ }
+ }
+
+ /**
+ * A typical plan has many operator details across multiple
+ * minor fragments. Aggregate these totals to the "master"
+ * definition of each operator.
+ */
+
+ private void aggregateOpers() {
+ for (FragInfo major : fragments.values()) {
+ for (OpDefInfo opDef : major.ops) {
+ for ( OperatorProfile op : opDef.opExecs) {
+ Preconditions.checkState( major.id == op.majorFragId );
+ Preconditions.checkState( opDef.stepId == op.opId );
+ opDef.actualRows += op.records;
+ opDef.actualBatches += op.batches;
+ opDef.actualMemory += op.peakMem * 1024 * 1024;
--- End diff --
OK. Multiplications by constant powers of 2 is pretty fast so didn't worry
about it originally.
> Refinements to new "Cluster Fixture" test framework
> ---------------------------------------------------
>
> Key: DRILL-5260
> URL: https://issues.apache.org/jira/browse/DRILL-5260
> Project: Apache Drill
> Issue Type: Improvement
> Affects Versions: 1.10
> Reporter: Paul Rogers
> Assignee: Paul Rogers
> Priority: Minor
> Fix For: 1.10
>
>
> Roll-up of a number of enhancements to the cluster fixture framework.
> * Config option to suppress printing of CSV and other output. (Allows
> printing for single tests, not printing when running from Maven.)
> * Parsing of query profiles to extract plan and run time information.
> * Fix bug in log fixture when enabling logging for a package.
> * Improved ZK support.
> * Set up the new CTTAS default temporary workspace for tests.
> * Revise TestDrillbitResiliance to use the new framework.
> * Revise TestWindowFrame to to use the new framework.
> * Revise TestMergeJoinWithSchemaChanges to use the new framework.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
