thomasrebele commented on a change in pull request #2486: URL: https://github.com/apache/calcite/pull/2486#discussion_r689428607
########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); + * + * volcanoPlanner.findBestExpr(); + * + * // after the optimization, adds the final best plan + * visualizerListener.getVisualizer().addFinalPlan(); + * // writes the output to files + * visualizerListener.getVisualizer().writeToFile(outputDirectory, ""); + * </pre> + */ +public class VolcanoRuleMatchVisualizer { + + VolcanoPlanner volcanoPlanner; + + // a sequence of ruleMatch ID to represent the order of rule match + List<String> ruleMatchSequence = new ArrayList<>(); + // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch + Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>(); + // map of nodeID to the ruleID it's first added + Map<String, String> nodeAddedInRule = new HashMap<>(); + + // a map of relNode ID to the actual RelNode object + // contains all the relNodes appear during the optimization + // all RelNode are immutable in Calcite, therefore only new nodes will be added Review comment: The children of the RelNodes can be modified by RelNode#replaceInput, so I would prefer to rephrase it. Maybe we just remove this line? ########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); + * + * volcanoPlanner.findBestExpr(); + * + * // after the optimization, adds the final best plan + * visualizerListener.getVisualizer().addFinalPlan(); + * // writes the output to files + * visualizerListener.getVisualizer().writeToFile(outputDirectory, ""); + * </pre> + */ +public class VolcanoRuleMatchVisualizer { + + VolcanoPlanner volcanoPlanner; + + // a sequence of ruleMatch ID to represent the order of rule match + List<String> ruleMatchSequence = new ArrayList<>(); + // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch + Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>(); + // map of nodeID to the ruleID it's first added + Map<String, String> nodeAddedInRule = new HashMap<>(); + + // a map of relNode ID to the actual RelNode object + // contains all the relNodes appear during the optimization + // all RelNode are immutable in Calcite, therefore only new nodes will be added + Map<String, RelNode> allNodes = new HashMap<>(); + + public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) { + this.volcanoPlanner = volcanoPlanner; + } + + public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) { + + // store the current state snapshot + // nodes contained in the sets + // and inputs of relNodes (and relSubsets) + Map<String, String> setLabels = new HashMap<>(); + Map<String, String> setOriginalRel = new HashMap<>(); + Map<String, Set<String>> nodesInSet = new HashMap<>(); + Map<String, Set<String>> nodeInputs = new HashMap<>(); + + // newNodes appeared after this ruleCall + Set<String> newNodes = new HashSet<>(); + + // populate current snapshot, and fill in the allNodes map + volcanoPlanner.allSets.forEach(set -> { + String setID = "set-" + set.id; + String setLabel = getSetLabel(set); + setLabels.put(setID, setLabel); + setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId())); + + nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>())); + + Consumer<RelNode> addNode = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodesInSet.get(setID).add(nodeID); + + if (!allNodes.containsKey(nodeID)) { + newNodes.add(nodeID); + allNodes.put(nodeID, rel); + } + }; + + Consumer<RelNode> addLink = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodeInputs.put(nodeID, new HashSet<>()); Review comment: Assigning the new HashSet to a variable and using it below instead of nodeinputs.get(nodeId) would save a lot of lookups. ########## File path: core/src/main/resources/volcano-viz/viz-template.html ########## @@ -0,0 +1,323 @@ +<!doctype html> +<html lang="en"> +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> +<meta charset="utf-8"> +<title>Calcite Rule Match Visualization</title> + +<script src="https://d3js.org/d3.v5.min.js"; charset="utf-8"></script> +<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js";></script> +<script src="https://d3js.org/d3-zoom.v1.min.js";></script> +<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js";></script> +<script src="volcano-viz-data.js"></script> + +<style id="css"> + body { + height: 100%; + margin: 0 0; + color: #333; + font-weight: 300; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serf; + } + + li a { + display: block; + /* and you can use padding for additional space if needs, as a clickable area / or other styling */ + padding: 5px 20px; + } + + section { + margin-bottom: 3em; + } + + section p { + text-align: justify; + } + + svg { + border: 1px solid #ccc; + overflow: hidden; + margin: 0 auto; + } + + pre { + border: 1px solid #ccc; + } + + .clusters rect { + fill: #FFFFE0; + stroke: #999; + stroke-width: 1.5px; + } + + text { + font-weight: 300; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serf; + font-size: 2em; + } + + .node rect { + stroke: #999; + fill: #fff; + stroke-width: 1.5px; + } + + .edgePath path { + stroke: #333; + stroke-width: 2px; + } + + .container { + display: flex; + align-items: center; + } + + .column1 { + flex: 0 0 300px; + } + + .column2 { + flex: 0 0 1000px; + } + + .tippy-content { + word-break: break-all; + word-wrap: break-word; + } +</style> + +<div class="container"> + <div class="column1"> + <div style="width: 100%; text-align: center"> + <div id="current-rule" style="display: block"></div> + <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button> + <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button> + </div> + <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto"> + </ul> + </div> + <div class="column2"> + <svg id="svg-canvas" width="1200px" height="800px"></svg> + </div> +</div> + +<script id="js"> + + var allNodes = data.allNodes; + var ruleMatchSequence = data.ruleMatchSequence; + var ruleMatchInfoMap = data.ruleMatchInfoMap; + var nodeAddedInRule = data.nodeAddedInRule; + + /* + * Graph data and D3 JS render related variables + */ + + // Create the input graph + var g = new dagreD3.graphlib.Graph({ + compound: true + }) + .setGraph({ + rankdir: 'LR' + }) + .setDefaultEdgeLabel(function () { + return {}; + }); + + // Create the renderer + var render = new dagreD3.render(); + + // Set up an SVG group so that we can translate the final graph. + var svg = d3.select("svg"); + var svgGroup = svg.append("g"); + + // Set up zoom support + var svg = d3.select("svg") + .attr("width", "1200px") + .attr("height", "800px") + .call(d3.zoom().on("zoom", function () { + svgGroup.attr("transform", d3.event.transform) + })); + + /* + * Global State + */ + + var currentRuleID = undefined; + + /* + * Event Handler functions + */ + + var setCurrentRule = (ruleMatchID) => { + // un-highlight previous entry + var prevRuleID = currentRuleID; + if (prevRuleID !== undefined) { + var prevRuleElement = document.getElementById(prevRuleID); + prevRuleElement.style.backgroundColor = "#FFFFFF"; + } + + currentRuleID = ruleMatchID; + document.getElementById('current-rule').innerText = currentRuleID; + + var currentRuleElement = document.getElementById(currentRuleID); + currentRuleElement.style.backgroundColor = "#D3D3D3"; + + var ruleIndex = ruleMatchSequence.indexOf(currentRuleID); + + document.getElementById("prev-button").disabled = false; + document.getElementById("next-button").disabled = false; + + if (ruleIndex === 0) { + document.getElementById("prev-button").disabled = true; + } + if (ruleIndex === ruleMatchSequence.length - 1) { + document.getElementById("next-button").disabled = true; + } + + createGraph(ruleMatchID); + } + + var createGraph = (ruleMatchID) => { + var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID] + console.log(ruleMatchInfo); + + // remove previous rendered view and clear graph model + d3.select("svg g").selectAll("*").remove(); + g.nodes().slice().forEach(nodeID => g.removeNode(nodeID)); Review comment: I think it would be better to update the graph instead of resetting it. We can leave it like this for the moment and change it when we add a functionality that would benefit from keeping the existing nodes. ########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); + * + * volcanoPlanner.findBestExpr(); + * + * // after the optimization, adds the final best plan + * visualizerListener.getVisualizer().addFinalPlan(); + * // writes the output to files + * visualizerListener.getVisualizer().writeToFile(outputDirectory, ""); + * </pre> + */ +public class VolcanoRuleMatchVisualizer { + + VolcanoPlanner volcanoPlanner; + + // a sequence of ruleMatch ID to represent the order of rule match + List<String> ruleMatchSequence = new ArrayList<>(); + // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch + Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>(); + // map of nodeID to the ruleID it's first added + Map<String, String> nodeAddedInRule = new HashMap<>(); + + // a map of relNode ID to the actual RelNode object + // contains all the relNodes appear during the optimization + // all RelNode are immutable in Calcite, therefore only new nodes will be added + Map<String, RelNode> allNodes = new HashMap<>(); + + public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) { + this.volcanoPlanner = volcanoPlanner; + } + + public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) { + + // store the current state snapshot + // nodes contained in the sets + // and inputs of relNodes (and relSubsets) + Map<String, String> setLabels = new HashMap<>(); + Map<String, String> setOriginalRel = new HashMap<>(); + Map<String, Set<String>> nodesInSet = new HashMap<>(); + Map<String, Set<String>> nodeInputs = new HashMap<>(); + + // newNodes appeared after this ruleCall + Set<String> newNodes = new HashSet<>(); + + // populate current snapshot, and fill in the allNodes map + volcanoPlanner.allSets.forEach(set -> { + String setID = "set-" + set.id; + String setLabel = getSetLabel(set); + setLabels.put(setID, setLabel); + setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId())); + + nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>())); + + Consumer<RelNode> addNode = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodesInSet.get(setID).add(nodeID); + + if (!allNodes.containsKey(nodeID)) { + newNodes.add(nodeID); + allNodes.put(nodeID, rel); + } + }; + + Consumer<RelNode> addLink = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodeInputs.put(nodeID, new HashSet<>()); + if (rel instanceof RelSubset) { + RelSubset relSubset = (RelSubset) rel; + relSubset.getRelList().stream() + .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet())) + .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId()))); + relSubset.set.subsets.stream() + .filter(other -> !other.equals(relSubset)) + .filter(other -> other.getTraitSet().satisfies(relSubset.getTraitSet())) + .forEach(other -> nodeInputs.get(nodeID).add(String.valueOf(other.getId()))); + } else { + rel.getInputs().forEach(input -> nodeInputs.get(nodeID) + .add(String.valueOf(input.getId()))); + } + }; + + set.rels.forEach(addNode); + set.subsets.forEach(addNode); + set.rels.forEach(addLink); + set.subsets.forEach(addLink); + }); + + // get the matched nodes of this rule + Set<String> matchedNodeIDs = matchedRels.stream() + .map(rel -> String.valueOf(rel.getId())) + .collect(Collectors.toSet()); + + // get importance 0 rels as of right now + Set<String> importanceZeroNodes = new HashSet<>(); Review comment: Maybe it's better to keep the term `prunedNodes` used by the VolcanoPlanner? ########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); Review comment: how about adding the listener when constructing the visualizer? ########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); + * + * volcanoPlanner.findBestExpr(); + * + * // after the optimization, adds the final best plan + * visualizerListener.getVisualizer().addFinalPlan(); + * // writes the output to files + * visualizerListener.getVisualizer().writeToFile(outputDirectory, ""); + * </pre> + */ +public class VolcanoRuleMatchVisualizer { + + VolcanoPlanner volcanoPlanner; + + // a sequence of ruleMatch ID to represent the order of rule match + List<String> ruleMatchSequence = new ArrayList<>(); + // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch + Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>(); + // map of nodeID to the ruleID it's first added + Map<String, String> nodeAddedInRule = new HashMap<>(); + + // a map of relNode ID to the actual RelNode object + // contains all the relNodes appear during the optimization + // all RelNode are immutable in Calcite, therefore only new nodes will be added + Map<String, RelNode> allNodes = new HashMap<>(); + + public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) { + this.volcanoPlanner = volcanoPlanner; + } + + public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) { + + // store the current state snapshot + // nodes contained in the sets + // and inputs of relNodes (and relSubsets) + Map<String, String> setLabels = new HashMap<>(); + Map<String, String> setOriginalRel = new HashMap<>(); + Map<String, Set<String>> nodesInSet = new HashMap<>(); + Map<String, Set<String>> nodeInputs = new HashMap<>(); + + // newNodes appeared after this ruleCall + Set<String> newNodes = new HashSet<>(); + + // populate current snapshot, and fill in the allNodes map + volcanoPlanner.allSets.forEach(set -> { + String setID = "set-" + set.id; + String setLabel = getSetLabel(set); + setLabels.put(setID, setLabel); + setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId())); + + nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>())); + + Consumer<RelNode> addNode = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodesInSet.get(setID).add(nodeID); + + if (!allNodes.containsKey(nodeID)) { + newNodes.add(nodeID); + allNodes.put(nodeID, rel); + } + }; + + Consumer<RelNode> addLink = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodeInputs.put(nodeID, new HashSet<>()); + if (rel instanceof RelSubset) { + RelSubset relSubset = (RelSubset) rel; + relSubset.getRelList().stream() + .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet())) + .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId()))); + relSubset.set.subsets.stream() + .filter(other -> !other.equals(relSubset)) + .filter(other -> other.getTraitSet().satisfies(relSubset.getTraitSet())) + .forEach(other -> nodeInputs.get(nodeID).add(String.valueOf(other.getId()))); + } else { + rel.getInputs().forEach(input -> nodeInputs.get(nodeID) + .add(String.valueOf(input.getId()))); + } + }; + + set.rels.forEach(addNode); + set.subsets.forEach(addNode); + set.rels.forEach(addLink); + set.subsets.forEach(addLink); + }); + + // get the matched nodes of this rule + Set<String> matchedNodeIDs = matchedRels.stream() + .map(rel -> String.valueOf(rel.getId())) + .collect(Collectors.toSet()); + + // get importance 0 rels as of right now + Set<String> importanceZeroNodes = new HashSet<>(); + volcanoPlanner.prunedNodes + .forEach(rel -> importanceZeroNodes.add(Integer.toString(rel.getId()))); + + VisualizerRuleMatchInfo ruleMatchInfo = + new VisualizerRuleMatchInfo(setLabels, setOriginalRel, nodesInSet, + nodeInputs, matchedNodeIDs, newNodes, importanceZeroNodes); + + ruleMatchSequence.add(ruleCallID); + ruleInfoMap.put(ruleCallID, ruleMatchInfo); + + newNodes.forEach(newNode -> nodeAddedInRule.put(newNode, ruleCallID)); + } + + /** + * Add a final plan to the variable. + */ + public void addFinalPlan() { + assert !ruleMatchSequence.contains("FINAL"); + + Set<RelNode> finalPlanNodes = new HashSet<>(); + Deque<RelSubset> subsetsToVisit = new LinkedList<>(); + subsetsToVisit.add((RelSubset) volcanoPlanner.getRoot()); + + RelSubset subset; + while ((subset = subsetsToVisit.poll()) != null) { + // add subset itself to the highlight list + finalPlanNodes.add(subset); + // highlight its best node if it exists + RelNode best = subset.getBest(); + if (best == null) { + continue; + } + finalPlanNodes.add(best); + // recursively visit the input relSubsets of the best node + best.getInputs().stream().map(rel -> (RelSubset) rel).forEach(subsetsToVisit::add); + } + + this.addRuleMatch("FINAL", new ArrayList<>(finalPlanNodes)); + } + + private String getSetLabel(RelSet set) { + return "set-" + set.id + " "; + } + + private String getJsonStringResult() { + try { + Map<String, VisualizerNodeInfo> nodeInfoMap = new HashMap<>(); + for (String nodeID : allNodes.keySet()) { + RelNode relNode = allNodes.get(nodeID); + RelNode root = volcanoPlanner.getRoot(); + if (root == null) { + throw new RuntimeException("volcano planner root is null"); + } + RelOptCluster cluster = root.getCluster(); + RelOptCost cost = volcanoPlanner.getCost(relNode, cluster.getMetadataQuery()); + Double rowCount = + relNode.getCluster().getMetadataQuery().getRowCount(relNode); + + VisualizerNodeInfo nodeInfo; + if (relNode instanceof RelSubset) { + RelSubset relSubset = (RelSubset) relNode; + String nodeLabel = "subset#" + relSubset.getId() + "-set#" + relSubset.set.id + "-\n" + + relSubset.getTraitSet().toString(); + String relIDs = relSubset.getRelList().stream() + .map(i -> "#" + i.getId()).collect(joining(", ")); + String explanation = "rels: [" + relIDs + "]"; + nodeInfo = + new VisualizerNodeInfo(nodeLabel, true, explanation, cost, rowCount); + } else { + InputExcludedRelWriter relWriter = new InputExcludedRelWriter(); + relNode.explain(relWriter); + String inputIDs = relNode.getInputs().stream() + .map(i -> "#" + i.getId()).collect(joining(", ")); + String explanation = relWriter.toString() + ", inputs: [" + inputIDs + "]"; + + String nodeLabel = "#" + relNode.getId() + "-" + relNode.getRelTypeName(); + nodeInfo = new VisualizerNodeInfo(nodeLabel, false, explanation, cost, + rowCount); + } + + nodeInfoMap.put(nodeID, nodeInfo); + } + + HashMap<String, Object> data = new HashMap<>(); + data.put("allNodes", nodeInfoMap); + data.put("ruleMatchSequence", ruleMatchSequence); + data.put("ruleMatchInfoMap", ruleInfoMap); + data.put("nodeAddedInRule", nodeAddedInRule); + + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.writeValueAsString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Writes the HTML and JS files of the rule match visualization. + * <p> + * The old files with the same name will be replaced. + * + * @param outputDirectory directory of the output files + * @param suffix file name suffix, can be null + */ + public void writeToFile(String outputDirectory, String suffix) { + // default HTML template is under "resources" + writeToFile("volcano-viz", outputDirectory, suffix); + } + + public void writeToFile(String templateDirectory, String outputDirectory, String suffix) { + try { + String templatePath = Paths.get(templateDirectory).resolve("viz-template.html").toString(); + String htmlTemplate = IOUtils.toString(getClass().getResourceAsStream(templatePath), Review comment: Calling getClass().getResourceAsStream(...) tries to access a wrong path, while getClass().getClassLoader().getResourceAsStream() seems to work. ########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); + * + * volcanoPlanner.findBestExpr(); + * + * // after the optimization, adds the final best plan + * visualizerListener.getVisualizer().addFinalPlan(); Review comment: we could execute this step automatically when calling writeToFile(...) ########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); + * + * volcanoPlanner.findBestExpr(); + * + * // after the optimization, adds the final best plan + * visualizerListener.getVisualizer().addFinalPlan(); + * // writes the output to files + * visualizerListener.getVisualizer().writeToFile(outputDirectory, ""); + * </pre> + */ +public class VolcanoRuleMatchVisualizer { + + VolcanoPlanner volcanoPlanner; + + // a sequence of ruleMatch ID to represent the order of rule match + List<String> ruleMatchSequence = new ArrayList<>(); + // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch + Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>(); + // map of nodeID to the ruleID it's first added + Map<String, String> nodeAddedInRule = new HashMap<>(); + + // a map of relNode ID to the actual RelNode object + // contains all the relNodes appear during the optimization + // all RelNode are immutable in Calcite, therefore only new nodes will be added + Map<String, RelNode> allNodes = new HashMap<>(); + + public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) { + this.volcanoPlanner = volcanoPlanner; + } + + public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) { + + // store the current state snapshot + // nodes contained in the sets + // and inputs of relNodes (and relSubsets) + Map<String, String> setLabels = new HashMap<>(); + Map<String, String> setOriginalRel = new HashMap<>(); + Map<String, Set<String>> nodesInSet = new HashMap<>(); + Map<String, Set<String>> nodeInputs = new HashMap<>(); + + // newNodes appeared after this ruleCall + Set<String> newNodes = new HashSet<>(); + + // populate current snapshot, and fill in the allNodes map + volcanoPlanner.allSets.forEach(set -> { + String setID = "set-" + set.id; + String setLabel = getSetLabel(set); + setLabels.put(setID, setLabel); + setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId())); + + nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>())); + + Consumer<RelNode> addNode = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodesInSet.get(setID).add(nodeID); + + if (!allNodes.containsKey(nodeID)) { + newNodes.add(nodeID); + allNodes.put(nodeID, rel); + } + }; + + Consumer<RelNode> addLink = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodeInputs.put(nodeID, new HashSet<>()); + if (rel instanceof RelSubset) { + RelSubset relSubset = (RelSubset) rel; + relSubset.getRelList().stream() + .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet())) + .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId()))); Review comment: What's the advantage over using `relSubset.getRels().forEach(input -> relInputs.add(String.valueOf(input.getId())));`? ########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); + * + * volcanoPlanner.findBestExpr(); + * + * // after the optimization, adds the final best plan + * visualizerListener.getVisualizer().addFinalPlan(); + * // writes the output to files + * visualizerListener.getVisualizer().writeToFile(outputDirectory, ""); + * </pre> + */ +public class VolcanoRuleMatchVisualizer { + + VolcanoPlanner volcanoPlanner; + + // a sequence of ruleMatch ID to represent the order of rule match + List<String> ruleMatchSequence = new ArrayList<>(); + // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch + Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>(); + // map of nodeID to the ruleID it's first added + Map<String, String> nodeAddedInRule = new HashMap<>(); + + // a map of relNode ID to the actual RelNode object + // contains all the relNodes appear during the optimization + // all RelNode are immutable in Calcite, therefore only new nodes will be added + Map<String, RelNode> allNodes = new HashMap<>(); + + public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) { + this.volcanoPlanner = volcanoPlanner; + } + + public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) { + + // store the current state snapshot + // nodes contained in the sets + // and inputs of relNodes (and relSubsets) + Map<String, String> setLabels = new HashMap<>(); + Map<String, String> setOriginalRel = new HashMap<>(); + Map<String, Set<String>> nodesInSet = new HashMap<>(); + Map<String, Set<String>> nodeInputs = new HashMap<>(); + + // newNodes appeared after this ruleCall + Set<String> newNodes = new HashSet<>(); + + // populate current snapshot, and fill in the allNodes map + volcanoPlanner.allSets.forEach(set -> { + String setID = "set-" + set.id; + String setLabel = getSetLabel(set); + setLabels.put(setID, setLabel); + setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId())); + + nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>())); + + Consumer<RelNode> addNode = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodesInSet.get(setID).add(nodeID); + + if (!allNodes.containsKey(nodeID)) { + newNodes.add(nodeID); + allNodes.put(nodeID, rel); + } + }; + + Consumer<RelNode> addLink = rel -> { + String nodeID = String.valueOf(rel.getId()); + nodeInputs.put(nodeID, new HashSet<>()); + if (rel instanceof RelSubset) { + RelSubset relSubset = (RelSubset) rel; + relSubset.getRelList().stream() + .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet())) + .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId()))); + relSubset.set.subsets.stream() + .filter(other -> !other.equals(relSubset)) + .filter(other -> other.getTraitSet().satisfies(relSubset.getTraitSet())) + .forEach(other -> nodeInputs.get(nodeID).add(String.valueOf(other.getId()))); Review comment: What's the advantage over using the following? ``` relSubset.getSubsetsSatisfyingThis() .filter(other -> !other.equals(relSubset)) .forEach(input -> relInputs.add(String.valueOf(input.getId()))); ``` ########## File path: core/src/main/resources/volcano-viz/viz-template.html ########## @@ -0,0 +1,323 @@ +<!doctype html> +<html lang="en"> +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> +<meta charset="utf-8"> +<title>Calcite Rule Match Visualization</title> + +<script src="https://d3js.org/d3.v5.min.js"; charset="utf-8"></script> +<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js";></script> +<script src="https://d3js.org/d3-zoom.v1.min.js";></script> +<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js";></script> +<script src="volcano-viz-data.js"></script> + +<style id="css"> + body { + height: 100%; + margin: 0 0; + color: #333; + font-weight: 300; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serf; + } + + li a { + display: block; + /* and you can use padding for additional space if needs, as a clickable area / or other styling */ + padding: 5px 20px; + } + + section { + margin-bottom: 3em; + } + + section p { + text-align: justify; + } + + svg { + border: 1px solid #ccc; + overflow: hidden; + margin: 0 auto; + } + + pre { + border: 1px solid #ccc; + } + + .clusters rect { + fill: #FFFFE0; + stroke: #999; + stroke-width: 1.5px; + } + + text { + font-weight: 300; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serf; + font-size: 2em; + } + + .node rect { + stroke: #999; + fill: #fff; + stroke-width: 1.5px; + } + + .edgePath path { + stroke: #333; + stroke-width: 2px; + } + + .container { + display: flex; + align-items: center; + } + + .column1 { + flex: 0 0 300px; + } + + .column2 { + flex: 0 0 1000px; + } + + .tippy-content { + word-break: break-all; + word-wrap: break-word; + } +</style> + +<div class="container"> + <div class="column1"> + <div style="width: 100%; text-align: center"> + <div id="current-rule" style="display: block"></div> + <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button> + <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button> + </div> + <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto"> + </ul> + </div> + <div class="column2"> + <svg id="svg-canvas" width="1200px" height="800px"></svg> + </div> +</div> + +<script id="js"> + + var allNodes = data.allNodes; + var ruleMatchSequence = data.ruleMatchSequence; + var ruleMatchInfoMap = data.ruleMatchInfoMap; + var nodeAddedInRule = data.nodeAddedInRule; + + /* + * Graph data and D3 JS render related variables + */ + + // Create the input graph + var g = new dagreD3.graphlib.Graph({ + compound: true + }) + .setGraph({ + rankdir: 'LR' + }) + .setDefaultEdgeLabel(function () { + return {}; + }); + + // Create the renderer + var render = new dagreD3.render(); + + // Set up an SVG group so that we can translate the final graph. + var svg = d3.select("svg"); + var svgGroup = svg.append("g"); + + // Set up zoom support + var svg = d3.select("svg") + .attr("width", "1200px") + .attr("height", "800px") + .call(d3.zoom().on("zoom", function () { + svgGroup.attr("transform", d3.event.transform) + })); + + /* + * Global State + */ + + var currentRuleID = undefined; + + /* + * Event Handler functions + */ + + var setCurrentRule = (ruleMatchID) => { + // un-highlight previous entry + var prevRuleID = currentRuleID; + if (prevRuleID !== undefined) { + var prevRuleElement = document.getElementById(prevRuleID); + prevRuleElement.style.backgroundColor = "#FFFFFF"; + } + + currentRuleID = ruleMatchID; + document.getElementById('current-rule').innerText = currentRuleID; + + var currentRuleElement = document.getElementById(currentRuleID); + currentRuleElement.style.backgroundColor = "#D3D3D3"; + + var ruleIndex = ruleMatchSequence.indexOf(currentRuleID); + + document.getElementById("prev-button").disabled = false; + document.getElementById("next-button").disabled = false; Review comment: We can set the correct value directly (without using an if). ########## File path: core/src/main/resources/volcano-viz/viz-template.html ########## @@ -0,0 +1,323 @@ +<!doctype html> +<html lang="en"> +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> +<meta charset="utf-8"> +<title>Calcite Rule Match Visualization</title> + +<script src="https://d3js.org/d3.v5.min.js"; charset="utf-8"></script> Review comment: Maybe we should switch to a more recent version. The latest version of d3 is 7.0.0. I tried v6 and v7 and this breaks only the panning and zooming. Probably not too difficult to fix. ########## File path: core/src/main/resources/volcano-viz/volcano-viz-data.js ########## @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// sample generated visualizer data used by the visualizer JavaScript code +var data = { + + allNodes: { + "node1": { + label: "TableScan-1", // node label being displayed + isSubset: false, // if the node is a RelSubset + explanation: "table=t1", // additional explanation of properties + finalCost: "100 cpu, 100io" // final cost (in string) of the node at the end of optimization + }, + "node2": { + label: "Filter-2", + isSubset: false, + explanation: "condition=c", + finalCost: "200cpu, 100io" + }, + "node3": { + label: "TableSink-3", + isSubset: false, + explanation: "table=t2", + finalCost: "20cpu, 20io" + }, + "node4": { + label: "IndexTableScan-4", + isSubset: false, + explanation: "table=t1, condition=c", + finalCost: "10cpu, 10io" + }, + }, + + ruleMatchSequence: [ + "INITIAL", + "IndexTableScanRule#1", + "FINAL" + ], + + ruleMatchInfoMap: { + "INITIAL": { + setLabels: { + "set1": "set1", + "set2": "set2", + "set3": "set3", + }, + setOriginalRel: { + "set1": "node1", + "set2": "node2", + "set3": "node3", + }, Review comment: The setLabels and setOriginalRel could be moved to the root level (i.e., to line 47). This requires changing the js code, though. ########## File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.volcano; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.tools.visualizer.InputExcludedRelWriter; +import org.apache.calcite.tools.visualizer.VisualizerNodeInfo; +import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.joining; + +/** + * This is tool to visualize the rule match process of the VolcanoPlanner. + * + * + * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase. + * Then writes the output to a file after the optimization ends. + * + * <pre> + * // construct the visualizer and attach a listener to VolcanoPlanner + * VolcanoRuleMatchVisualizerListener visualizerListener = + * new VolcanoRuleMatchVisualizerListener(volcanoPlanner); + * volcanoPlanner.addListener(visualizerListener); + * + * volcanoPlanner.findBestExpr(); + * + * // after the optimization, adds the final best plan + * visualizerListener.getVisualizer().addFinalPlan(); + * // writes the output to files + * visualizerListener.getVisualizer().writeToFile(outputDirectory, ""); + * </pre> + */ +public class VolcanoRuleMatchVisualizer { + + VolcanoPlanner volcanoPlanner; + + // a sequence of ruleMatch ID to represent the order of rule match + List<String> ruleMatchSequence = new ArrayList<>(); + // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch + Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>(); + // map of nodeID to the ruleID it's first added + Map<String, String> nodeAddedInRule = new HashMap<>(); + + // a map of relNode ID to the actual RelNode object + // contains all the relNodes appear during the optimization + // all RelNode are immutable in Calcite, therefore only new nodes will be added + Map<String, RelNode> allNodes = new HashMap<>(); + + public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) { + this.volcanoPlanner = volcanoPlanner; + } + + public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) { + + // store the current state snapshot + // nodes contained in the sets + // and inputs of relNodes (and relSubsets) + Map<String, String> setLabels = new HashMap<>(); + Map<String, String> setOriginalRel = new HashMap<>(); + Map<String, Set<String>> nodesInSet = new HashMap<>(); + Map<String, Set<String>> nodeInputs = new HashMap<>(); Review comment: Using TreeMaps would make the output more stable (i.e., no reordering when adding new nodes). ########## File path: core/src/main/resources/volcano-viz/viz-template.html ########## @@ -0,0 +1,323 @@ +<!doctype html> +<html lang="en"> +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> +<meta charset="utf-8"> +<title>Calcite Rule Match Visualization</title> + +<script src="https://d3js.org/d3.v5.min.js"; charset="utf-8"></script> +<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js";></script> +<script src="https://d3js.org/d3-zoom.v1.min.js";></script> +<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js";></script> +<script src="volcano-viz-data.js"></script> + +<style id="css"> + body { + height: 100%; + margin: 0 0; + color: #333; + font-weight: 300; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serf; + } + + li a { + display: block; + /* and you can use padding for additional space if needs, as a clickable area / or other styling */ + padding: 5px 20px; + } + + section { + margin-bottom: 3em; + } + + section p { + text-align: justify; + } + + svg { + border: 1px solid #ccc; + overflow: hidden; + margin: 0 auto; + } + + pre { + border: 1px solid #ccc; + } + + .clusters rect { + fill: #FFFFE0; + stroke: #999; + stroke-width: 1.5px; + } + + text { + font-weight: 300; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serf; + font-size: 2em; + } + + .node rect { + stroke: #999; + fill: #fff; + stroke-width: 1.5px; + } + + .edgePath path { + stroke: #333; + stroke-width: 2px; + } + + .container { + display: flex; + align-items: center; + } + + .column1 { + flex: 0 0 300px; + } + + .column2 { + flex: 0 0 1000px; + } + + .tippy-content { + word-break: break-all; + word-wrap: break-word; + } +</style> + +<div class="container"> + <div class="column1"> + <div style="width: 100%; text-align: center"> + <div id="current-rule" style="display: block"></div> + <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button> + <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button> + </div> + <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto"> + </ul> + </div> + <div class="column2"> + <svg id="svg-canvas" width="1200px" height="800px"></svg> + </div> +</div> + +<script id="js"> + + var allNodes = data.allNodes; + var ruleMatchSequence = data.ruleMatchSequence; + var ruleMatchInfoMap = data.ruleMatchInfoMap; + var nodeAddedInRule = data.nodeAddedInRule; + + /* + * Graph data and D3 JS render related variables + */ + + // Create the input graph + var g = new dagreD3.graphlib.Graph({ + compound: true + }) + .setGraph({ + rankdir: 'LR' + }) + .setDefaultEdgeLabel(function () { + return {}; + }); + + // Create the renderer + var render = new dagreD3.render(); + + // Set up an SVG group so that we can translate the final graph. + var svg = d3.select("svg"); + var svgGroup = svg.append("g"); + + // Set up zoom support + var svg = d3.select("svg") + .attr("width", "1200px") + .attr("height", "800px") + .call(d3.zoom().on("zoom", function () { + svgGroup.attr("transform", d3.event.transform) + })); + + /* + * Global State + */ + + var currentRuleID = undefined; + + /* + * Event Handler functions + */ + + var setCurrentRule = (ruleMatchID) => { + // un-highlight previous entry + var prevRuleID = currentRuleID; + if (prevRuleID !== undefined) { + var prevRuleElement = document.getElementById(prevRuleID); + prevRuleElement.style.backgroundColor = "#FFFFFF"; + } + + currentRuleID = ruleMatchID; + document.getElementById('current-rule').innerText = currentRuleID; + + var currentRuleElement = document.getElementById(currentRuleID); + currentRuleElement.style.backgroundColor = "#D3D3D3"; + + var ruleIndex = ruleMatchSequence.indexOf(currentRuleID); + + document.getElementById("prev-button").disabled = false; + document.getElementById("next-button").disabled = false; + + if (ruleIndex === 0) { + document.getElementById("prev-button").disabled = true; + } + if (ruleIndex === ruleMatchSequence.length - 1) { + document.getElementById("next-button").disabled = true; + } + + createGraph(ruleMatchID); + } + + var createGraph = (ruleMatchID) => { + var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID] + console.log(ruleMatchInfo); + + // remove previous rendered view and clear graph model + d3.select("svg g").selectAll("*").remove(); + g.nodes().slice().forEach(nodeID => g.removeNode(nodeID)); + + // create nodes and sets + for (var setID in ruleMatchInfo.nodesInSet) { + // add set + var setLabel = ruleMatchInfo.setLabels[setID]; + if (setLabel === null || setLabel === undefined) { + setLabel = setID; + } + g.setNode(setID, { + label: setLabel, + clusterLabelPos: 'top' + }); + // add nodes and node-set parent relationship + var nodes = ruleMatchInfo.nodesInSet[setID]; + + nodes.forEach(nodeID => { + nodeInfo = allNodes[nodeID]; + var nodeLabel; + if (ruleMatchID === "FINAL") { + nodeLabel = nodeInfo.label + "--" + nodeInfo.finalCost; + } else { + nodeLabel = nodeInfo.label; + } + var nodeStyle; + if (ruleMatchInfo.importanceZeroNodes + && ruleMatchInfo.importanceZeroNodes.includes(nodeID)) { + nodeStyle = "fill: #D3D3D3" Review comment: Using a class instead of explicit CSS attributes would allow to change the colors later. The same applies to the other styles set by the js code. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
