Repository: asterixdb Updated Branches: refs/heads/master fd9c6020d -> 8351d2531
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8351d253/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java index 75e714b..445fbd8 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java @@ -181,6 +181,23 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations return index.getKeyFieldNames().size(); } + /** + * ====== + * | SK | Bloom filter + * ====== + * ====== ====== + * | SK | PK | comparators, type traits + * ====== ====== + * ====== ........ + * | SK | Filter | field access evaluators + * ====== ........ + * ====== ====== ........ + * | SK | PK | Filter | record fields + * ====== ====== ........ + * ====== ========= ........ ........ + * | PK | Payload | Meta | Filter | enforced record + * ====== ========= ........ ........ + */ @Override @SuppressWarnings("rawtypes") protected void setSecondaryRecDescAndComparators() throws AlgebricksException { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8351d253/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java new file mode 100644 index 0000000..1bbda27 --- /dev/null +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.algebricks.core.utils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class DotFormatBuilder { + private final StringBuilder stringBuilder; + private final Set<Node> nodes; + private final List<Edge> edges; + + public DotFormatBuilder(StringValue graphName) { + this.edges = new ArrayList<>(); + this.nodes = new HashSet<>(); + this.stringBuilder = new StringBuilder(); + this.stringBuilder.append("digraph ").append(graphName).append(" {\n").append("rankdir=BT;\n"); + this.stringBuilder.append("node [style=\"rounded,filled\",shape=box];\n"); + } + + public String getDotDocument() { + // print edges first + for (Edge edge : edges) { + stringBuilder.append(edge); + } + // print nodes + for (Node node : nodes) { + stringBuilder.append(node); + } + stringBuilder.append("\n}"); + return stringBuilder.toString(); + } + + // point of entry method + public Node createNode(StringValue nodeId, StringValue nodeLabel) { + Node node = new Node(nodeId, nodeLabel); + for (Node existingNode : nodes) { + if (node.equals(existingNode)) { + existingNode.setNodeLabel(nodeLabel); + return existingNode; + } + } + nodes.add(node); + return node; + } + + // point of entry method + public Edge createEdge(final Node source, final Node destination) { + // sanity checks if any variable is null? + if (source == null || destination == null || !nodes.contains(source) || !nodes.contains(destination)) { + return null; + } + + // append to edges list + Edge newEdge = new Edge(source, destination); + edges.add(newEdge); + + return newEdge; + } + + public class Node { + private final StringValue nodeId; + private HashMap<String,AttributeValue> attributes = new HashMap<>(); + + // no instantiation + private Node(StringValue nodeId, StringValue nodeLabel) { + this.nodeId = nodeId; + setNodeLabel(nodeLabel); + } + + public StringValue getNodeId() { + return nodeId; + } + + public AttributeValue getNodeLabel() { + return attributes.get(Attribute.LABEL); + } + + public Node setNodeLabel(StringValue nodeLabel) { + if (nodeLabel != null) { + attributes.put(Attribute.LABEL, nodeLabel); + } + return this; + } + + public Node setFillColor(Color color) { + if (color != null) { + attributes.put(Attribute.COLOR, color); + } + return this; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Node)) { + return false; + } + Node otherNode = (Node) other; + + return nodeId.getValue().equals(otherNode.nodeId.getValue()); + } + + @Override + public int hashCode() { + return nodeId.getValue().hashCode(); + } + + @Override + public String toString() { + StringBuilder nodeString = new StringBuilder(); + nodeString.append(nodeId).append(" ["); + for (Map.Entry attribute : attributes.entrySet()) { + nodeString.append(attribute.getKey()).append("=").append(attribute.getValue()).append(","); + } + // remove last "," + if (nodeString.charAt(nodeString.length() - 1) == ',') { + nodeString.deleteCharAt(nodeString.length() - 1); + } + nodeString.append("];\n"); + + return nodeString.toString(); + } + } + + public class Edge { + private final Node source; + private final Node destination; + private final HashMap<String,AttributeValue> attributes = new HashMap<>(); + + // no instantiation + private Edge(Node source, Node destination) { + this.source = source; + this.destination = destination; + } + + public Edge setLabel(StringValue edgeLabel) { + if (edgeLabel != null) { + attributes.put(Attribute.LABEL, edgeLabel); + } + return this; + } + + public Edge setColor(Color color) { + if (color != null) { + attributes.put(Attribute.COLOR, color); + } + return this; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Edge)) { + return false; + } + Edge otherEdge = (Edge) other; + + return source.equals(otherEdge.source) && destination.equals(otherEdge.destination); + } + + @Override + public int hashCode() { + return source.hashCode() ^ destination.hashCode(); + } + + @Override + public String toString() { + StringBuilder edgeString = new StringBuilder(); + edgeString.append(source.getNodeId()).append("->").append(destination.getNodeId()).append(" ["); + for (Map.Entry attribute : attributes.entrySet()) { + edgeString.append(attribute.getKey()).append("=").append(attribute.getValue()).append(","); + } + // remove last "," + if (edgeString.charAt(edgeString.length() - 1) == ',') { + edgeString.deleteCharAt(edgeString.length() - 1); + } + edgeString.append("];\n"); + + return edgeString.toString(); + } + } + + public abstract static class AttributeValue { + private final String value; + + // no instantiation + private AttributeValue(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return value; + } + } + + public static final class StringValue extends AttributeValue { + // no instantiation + private StringValue (String value) { + super(value); + } + + public static StringValue of(String value) { + String newValue = value; + if (value == null) { + newValue = ""; + } + return new StringValue("\"" + newValue.replace("\"","\'").trim() + "\""); + } + } + + public static final class Color extends AttributeValue { + public static final Color RED = new Color("red"); + public static final Color SKYBLUE = new Color("skyblue"); + + // no instantiation + private Color (String color) { + super(color); + } + } + + private class Attribute { + private static final String COLOR = "color"; + private static final String LABEL = "label"; + + // no instantiation + private Attribute() { + } + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8351d253/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java new file mode 100644 index 0000000..392bf44 --- /dev/null +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.algebricks.core.utils; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang3.mutable.Mutable; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan; +import org.apache.hyracks.algebricks.core.algebra.base.IPhysicalOperator; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractReplicateOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator; +import org.apache.hyracks.api.dataflow.ActivityId; +import org.apache.hyracks.api.dataflow.ConnectorDescriptorId; +import org.apache.hyracks.api.dataflow.IActivity; +import org.apache.hyracks.api.dataflow.IConnectorDescriptor; +import org.apache.hyracks.api.dataflow.IOperatorDescriptor; +import org.apache.hyracks.api.job.JobActivityGraph; +import org.apache.hyracks.api.job.JobSpecification; + +public class DotFormatGenerator { + + private DotFormatGenerator() { + } + + /** + * Generates DOT format for {@link JobActivityGraph} that can be visualized + * using any DOT format visualizer. + * + * @param jobActivityGraph The job activity graph + * @return DOT format + */ + public static String generate(final JobActivityGraph jobActivityGraph) { + final DotFormatBuilder graphBuilder = + new DotFormatBuilder(DotFormatBuilder.StringValue.of("JobActivityGraph")); + List<IConnectorDescriptor> connectors; + IActivity activity; + ActivityId fromActivityId; + ActivityId toActivityId; + String fromFullClassName; + String toFullClassName; + String fromClassName; + String toClassName; + DotFormatBuilder.Node fromNode; + DotFormatBuilder.Node toNode; + final Set<Pair<ActivityId, ActivityId>> activitiesPairedSet = new HashSet<>(); + final Map<ActivityId, IActivity> activityMap = jobActivityGraph.getActivityMap(); + final Map<ActivityId, List<IConnectorDescriptor>> activityInputMap = jobActivityGraph.getActivityInputMap(); + final Map<ActivityId, List<IConnectorDescriptor>> activityOutputMap = jobActivityGraph.getActivityOutputMap(); + + // go through each activity. First, map its input -> activity, then activity -> its output + for (Map.Entry<ActivityId, IActivity> entry : activityMap.entrySet()) { + toFullClassName = entry.getValue().getClass().getName(); + toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1); + toActivityId = entry.getValue().getActivityId(); + toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()), + DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName)); + // process input -> to activity + connectors = activityInputMap.get(entry.getKey()); + if (connectors != null) { + for (IConnectorDescriptor connector : connectors) { + fromActivityId = jobActivityGraph.getProducerActivity(connector.getConnectorId()); + activity = activityMap.get(fromActivityId); + fromFullClassName = activity.getClass().getName(); + fromClassName = fromFullClassName.substring(fromFullClassName.lastIndexOf('.') + 1); + fromNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(fromActivityId.toString()), + DotFormatBuilder.StringValue.of(fromActivityId.toString() + "-" + fromClassName)); + Pair<ActivityId, ActivityId> newPair = new ImmutablePair<>(fromActivityId, toActivityId); + if (!activitiesPairedSet.contains(newPair)) { + activitiesPairedSet.add(newPair); + graphBuilder.createEdge(fromNode, toNode); + } + } + } + + // process from activity -> output + fromActivityId = toActivityId; + fromNode = toNode; + connectors = activityOutputMap.get(entry.getKey()); + if (connectors != null) { + for (IConnectorDescriptor connector : connectors) { + toActivityId = jobActivityGraph.getConsumerActivity(connector.getConnectorId()); + activity = activityMap.get(toActivityId); + toFullClassName = activity.getClass().getName(); + toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1); + toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()), + DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName)); + Pair<ActivityId, ActivityId> newPair = new ImmutablePair<>(fromActivityId, toActivityId); + if (!activitiesPairedSet.contains(newPair)) { + activitiesPairedSet.add(newPair); + graphBuilder.createEdge(fromNode, toNode); + } + } + } + } + + final Map<ActivityId, Set<ActivityId>> blocked2BlockerMap = jobActivityGraph.getBlocked2BlockerMap(); + IActivity blockedActivity; + for (Map.Entry<ActivityId, Set<ActivityId>> entry : blocked2BlockerMap.entrySet()) { + blockedActivity = activityMap.get(entry.getKey()); + toFullClassName = blockedActivity.getClass().getName(); + toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1); + toActivityId = entry.getKey(); + toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()), + DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName)); + for (ActivityId blockingActivityId : entry.getValue()) { + fromActivityId = blockingActivityId; + activity = activityMap.get(fromActivityId); + fromFullClassName = activity.getClass().getName(); + fromClassName = fromFullClassName.substring(fromFullClassName.lastIndexOf('.') + 1); + fromNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(fromActivityId.toString()), + DotFormatBuilder.StringValue.of(fromActivityId.toString() + "-" + fromClassName)); + Pair<ActivityId, ActivityId> newPair = new ImmutablePair<>(fromActivityId, toActivityId); + if (!activitiesPairedSet.contains(newPair)) { + activitiesPairedSet.add(newPair); + graphBuilder.createEdge(fromNode, toNode).setColor(DotFormatBuilder.Color.RED); + } + } + } + + return graphBuilder.getDotDocument(); + } + + /** + * Generates DOT format for {@link JobSpecification} that can be visualized + * using any DOT format visualizer. + * + * @param jobSpecification The job specification + * @return DOT format + */ + public static String generate(final JobSpecification jobSpecification) { + final DotFormatBuilder graphBuilder = + new DotFormatBuilder(DotFormatBuilder.StringValue.of("JobSpecification")); + final Map<ConnectorDescriptorId, IConnectorDescriptor> connectorMap = jobSpecification.getConnectorMap(); + final Map<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>> + cOp = jobSpecification.getConnectorOperatorMap(); + ConnectorDescriptorId connectorId; + IConnectorDescriptor connector; + IOperatorDescriptor leftOperator; + IOperatorDescriptor rightOperator; + DotFormatBuilder.Node sourceNode; + DotFormatBuilder.Node destinationNode; + String source; + String destination; + String edgeLabel; + for (Map.Entry<ConnectorDescriptorId, + Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>> entry : cOp.entrySet()) { + connectorId = entry.getKey(); + connector = connectorMap.get(connectorId); + edgeLabel = connector.getClass().getName().substring(connector.getClass().getName().lastIndexOf(".") + 1); + edgeLabel += "-" + connectorId; + leftOperator = entry.getValue().getLeft().getLeft(); + rightOperator = entry.getValue().getRight().getLeft(); + source = leftOperator.getClass().getName().substring( + leftOperator.getClass().getName().lastIndexOf(".") + 1); + sourceNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(leftOperator.toString()), + DotFormatBuilder.StringValue.of(leftOperator.toString() + "-" + source)); + destination = rightOperator.getClass().getName().substring( + rightOperator.getClass().getName().lastIndexOf(".") + 1); + destinationNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(rightOperator.toString()), + DotFormatBuilder.StringValue.of(rightOperator.toString() + "-" + destination)); + graphBuilder.createEdge(sourceNode, destinationNode).setLabel(DotFormatBuilder.StringValue.of(edgeLabel)); + } + + return graphBuilder.getDotDocument(); + } + + /** + * Generates DOT format for {@link ILogicalPlan} that can be visualized + * using any DOT format visualizer. + * + * @param plan The logical plan + * @param dotVisitor The DOT visitor + * @return DOT format + * @throws AlgebricksException + */ + public static String generate(ILogicalPlan plan, LogicalOperatorDotVisitor dotVisitor) throws AlgebricksException { + final DotFormatBuilder graphBuilder = new DotFormatBuilder(DotFormatBuilder.StringValue.of("Plan")); + ILogicalOperator root = plan.getRoots().get(0).getValue(); + generateNode(graphBuilder, root, dotVisitor, new HashSet<>()); + return graphBuilder.getDotDocument(); + } + + public static void generateNode(DotFormatBuilder dotBuilder, ILogicalOperator op, + LogicalOperatorDotVisitor dotVisitor, Set<ILogicalOperator> operatorsVisited) + throws AlgebricksException { + DotFormatBuilder.StringValue destinationNodeLabel = formatStringOf(op, dotVisitor); + DotFormatBuilder.Node destinationNode = dotBuilder.createNode(DotFormatBuilder.StringValue.of( + Integer.toString(op.hashCode())), destinationNodeLabel); + DotFormatBuilder.StringValue sourceNodeLabel; + DotFormatBuilder.Node sourceNode; + for (Mutable<ILogicalOperator> child : op.getInputs()) { + sourceNodeLabel = formatStringOf(child.getValue(), dotVisitor); + sourceNode = dotBuilder.createNode(DotFormatBuilder.StringValue.of( + Integer.toString(child.getValue().hashCode())), sourceNodeLabel); + dotBuilder.createEdge(sourceNode, destinationNode); + if (!operatorsVisited.contains(child.getValue())) { + generateNode(dotBuilder, child.getValue(), dotVisitor, operatorsVisited); + } + } + if (((AbstractLogicalOperator) op).hasNestedPlans()) { + ILogicalOperator nestedOperator; + for (ILogicalPlan nestedPlan : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) { + nestedOperator = nestedPlan.getRoots().get(0).getValue(); + sourceNodeLabel = formatStringOf(nestedOperator, dotVisitor); + sourceNode = dotBuilder.createNode(DotFormatBuilder.StringValue.of( + Integer.toString(nestedOperator.hashCode())), sourceNodeLabel); + dotBuilder.createEdge(sourceNode, destinationNode). + setLabel(DotFormatBuilder.StringValue.of("subplan")); + if (!operatorsVisited.contains(nestedOperator)) { + generateNode(dotBuilder, nestedOperator, dotVisitor, operatorsVisited); + } + } + } + if (!(op instanceof ExchangeOperator)) { + destinationNode.setFillColor(DotFormatBuilder.Color.SKYBLUE); + } + + // replicate/split operator + if (op.getOperatorTag() == LogicalOperatorTag.REPLICATE || op.getOperatorTag() == LogicalOperatorTag.SPLIT) { + AbstractReplicateOperator replicateOperator = (AbstractReplicateOperator) op; + ILogicalOperator replicateOutput; + sourceNode = destinationNode; + for (int i = 0; i < replicateOperator.getOutputs().size(); i++) { + replicateOutput = replicateOperator.getOutputs().get(i).getValue(); + destinationNodeLabel = formatStringOf(replicateOutput, dotVisitor); + destinationNode = dotBuilder.createNode(DotFormatBuilder.StringValue.of( + Integer.toString(replicateOutput.hashCode())), destinationNodeLabel); + if (replicateOperator.getOutputMaterializationFlags()[i]) { + dotBuilder.createEdge(sourceNode, destinationNode).setColor(DotFormatBuilder.Color.RED); + } else { + dotBuilder.createEdge(sourceNode, destinationNode); + } + } + } + + operatorsVisited.add(op); + } + + private static DotFormatBuilder.StringValue formatStringOf(ILogicalOperator operator, + LogicalOperatorDotVisitor dotVisitor) throws AlgebricksException { + String formattedString = operator.accept(dotVisitor, null).trim(); + IPhysicalOperator physicalOperator = ((AbstractLogicalOperator)operator).getPhysicalOperator(); + if (physicalOperator != null) { + formattedString += "\\n" + physicalOperator.toString().trim() + " |" + operator.getExecutionMode() + "|"; + } else { + formattedString += "\\n|" + operator.getExecutionMode() + "|"; + } + + return DotFormatBuilder.StringValue.of(formattedString); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8351d253/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java new file mode 100644 index 0000000..a54ff63 --- /dev/null +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/LogicalOperatorDotVisitor.java @@ -0,0 +1,492 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.algebricks.core.utils; + +import java.util.List; + +import org.apache.commons.lang3.mutable.Mutable; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.common.utils.Triple; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator.Kind; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.SplitOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator; +import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; + +public class LogicalOperatorDotVisitor implements ILogicalOperatorVisitor<String, Void> { + + private final StringBuilder stringBuilder; + + public LogicalOperatorDotVisitor() { + stringBuilder = new StringBuilder(); + } + + @Override + public String toString() { + return ""; + } + + private CharSequence str(Object o) { + return String.valueOf(o); + } + + @Override + public String visitAggregateOperator(AggregateOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("aggregate ").append(str(op.getVariables())).append(" <- "); + pprintExprList(op.getExpressions()); + return stringBuilder.toString(); + } + + @Override + public String visitRunningAggregateOperator(RunningAggregateOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("running-aggregate ").append(str(op.getVariables())).append(" <- "); + pprintExprList(op.getExpressions()); + return stringBuilder.toString(); + } + + @Override + public String visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("empty-tuple-source"); + return stringBuilder.toString(); + } + + @Override + public String visitGroupByOperator(GroupByOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("group by").append(op.isGroupAll() ? " (all)" : "").append(" ("); + pprintVeList(op.getGroupByList()); + stringBuilder.append(") decor ("); + pprintVeList(op.getDecorList()); + stringBuilder.append(")"); + return stringBuilder.toString(); + } + + @Override + public String visitDistinctOperator(DistinctOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("distinct ("); + pprintExprList(op.getExpressions()); + stringBuilder.append(")"); + return stringBuilder.toString(); + } + + @Override + public String visitInnerJoinOperator(InnerJoinOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("join (").append(op.getCondition().getValue().toString()).append(")"); + return stringBuilder.toString(); + } + + @Override + public String visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("left outer join (").append(op.getCondition().getValue().toString()).append(")"); + return stringBuilder.toString(); + } + + @Override + public String visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("nested tuple source"); + return stringBuilder.toString(); + } + + @Override + public String visitOrderOperator(OrderOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("order "); + for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) { + if (op.getTopK() != -1) { + stringBuilder.append("(topK: ").append(op.getTopK()).append(") "); + } + String fst = getOrderString(p.first); + stringBuilder.append("(").append(fst).append(", ").append(p.second.getValue().toString()).append(") "); + } + return stringBuilder.toString(); + } + + private String getOrderString(OrderOperator.IOrder first) { + switch (first.getKind()) { + case ASC: + return "ASC"; + case DESC: + return "DESC"; + default: + return first.getExpressionRef().toString(); + } + } + + @Override + public String visitAssignOperator(AssignOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("assign ").append(str(op.getVariables())).append(" <- "); + pprintExprList(op.getExpressions()); + return stringBuilder.toString(); + } + + @Override + public String visitWriteOperator(WriteOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("write "); + pprintExprList(op.getExpressions()); + return stringBuilder.toString(); + } + + @Override + public String visitDistributeResultOperator(DistributeResultOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("distribute result "); + pprintExprList(op.getExpressions()); + return stringBuilder.toString(); + } + + @Override + public String visitWriteResultOperator(WriteResultOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("load ").append(str(op.getDataSource())).append(" from ") + .append(op.getPayloadExpression().getValue().toString()).append(" partitioned by "); + pprintExprList(op.getKeyExpressions()); + return stringBuilder.toString(); + } + + @Override + public String visitSelectOperator(SelectOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("select (").append(op.getCondition().getValue().toString()).append(")"); + return stringBuilder.toString(); + } + + @Override + public String visitProjectOperator(ProjectOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("project ").append("(").append(op.getVariables()).append(")"); + return stringBuilder.toString(); + } + + @Override + public String visitSubplanOperator(SubplanOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("subplan {}"); + return stringBuilder.toString(); + } + + @Override + public String visitUnionOperator(UnionAllOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("union"); + for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> v : op.getVariableMappings()) { + stringBuilder.append(" (").append(v.first).append(", ").append(v.second).append(", ").append(v.third) + .append(")"); + } + return stringBuilder.toString(); + } + + @Override + public String visitIntersectOperator(IntersectOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("intersect ("); + stringBuilder.append('['); + for (int i = 0; i < op.getOutputVars().size(); i++) { + if (i > 0) { + stringBuilder.append(", "); + } + stringBuilder.append(str(op.getOutputVars().get(i))); + } + stringBuilder.append("] <- ["); + for (int i = 0; i < op.getNumInput(); i++) { + if (i > 0) { + stringBuilder.append(", "); + } + stringBuilder.append('['); + for (int j = 0; j < op.getInputVariables(i).size(); j++) { + if (j > 0) { + stringBuilder.append(", "); + } + stringBuilder.append(str(op.getInputVariables(i).get(j))); + } + stringBuilder.append(']'); + } + stringBuilder.append("])"); + return stringBuilder.toString(); + } + + @Override + public String visitUnnestOperator(UnnestOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("unnest ").append(op.getVariable()); + if (op.getPositionalVariable() != null) { + stringBuilder.append(" at ").append(op.getPositionalVariable()); + } + stringBuilder.append(" <- ").append(op.getExpressionRef().getValue().toString()); + return stringBuilder.toString(); + } + + @Override + public String visitLeftOuterUnnestOperator(LeftOuterUnnestOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("outer-unnest ").append(op.getVariable()); + if (op.getPositionalVariable() != null) { + stringBuilder.append(" at ").append(op.getPositionalVariable()); + } + stringBuilder.append(" <- ").append(op.getExpressionRef().getValue().toString()); + return stringBuilder.toString(); + } + + @Override + public String visitUnnestMapOperator(UnnestMapOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + return printAbstractUnnestMapOperator(op, "unnest-map"); + } + + @Override + public String visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void noArgs) + throws AlgebricksException { + stringBuilder.setLength(0); + return printAbstractUnnestMapOperator(op, "left-outer-unnest-map"); + } + + private String printAbstractUnnestMapOperator(AbstractUnnestMapOperator op, String opSignature) { + stringBuilder.append(opSignature).append(" ").append(op.getVariables()).append(" <- ") + .append(op.getExpressionRef().getValue().toString()); + appendFilterInformation(stringBuilder, op.getMinFilterVars(), op.getMaxFilterVars()); + return stringBuilder.toString(); + } + + @Override + public String visitDataScanOperator(DataSourceScanOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("data-scan ").append(op.getProjectVariables()).append("<-").append(op.getVariables()) + .append(" <- ").append(op.getDataSource()); + appendFilterInformation(stringBuilder, op.getMinFilterVars(), op.getMaxFilterVars()); + return stringBuilder.toString(); + } + + private String appendFilterInformation(StringBuilder plan, List<LogicalVariable> minFilterVars, + List<LogicalVariable> maxFilterVars) { + if (minFilterVars != null || maxFilterVars != null) { + plan.append(" with filter on"); + } + if (minFilterVars != null) { + plan.append(" min:").append(minFilterVars); + } + if (maxFilterVars != null) { + plan.append(" max:").append(maxFilterVars); + } + return stringBuilder.toString(); + } + + @Override + public String visitLimitOperator(LimitOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("limit ").append(op.getMaxObjects().getValue().toString()); + ILogicalExpression offset = op.getOffset().getValue(); + if (offset != null) { + stringBuilder.append(", ").append(offset.toString()); + } + return stringBuilder.toString(); + } + + @Override + public String visitExchangeOperator(ExchangeOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("exchange"); + return stringBuilder.toString(); + } + + @Override + public String visitScriptOperator(ScriptOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("script (in: ").append(op.getInputVariables()).append(") (out: " ) + .append(op.getOutputVariables()).append(")"); + return stringBuilder.toString(); + } + + @Override + public String visitReplicateOperator(ReplicateOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("replicate"); + return stringBuilder.toString(); + } + + @Override + public String visitSplitOperator(SplitOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + Mutable<ILogicalExpression> branchingExpression = op.getBranchingExpression(); + stringBuilder.append("split ").append(branchingExpression.getValue().toString()); + return stringBuilder.toString(); + } + + @Override + public String visitMaterializeOperator(MaterializeOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("materialize"); + return stringBuilder.toString(); + } + + @Override + public String visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void noArgs) + throws AlgebricksException { + stringBuilder.setLength(0); + String header = getIndexOpString(op.getOperation()); + stringBuilder.append(header).append(str(op.getDataSource())).append(" from record: ") + .append(op.getPayloadExpression().getValue().toString()); + if (op.getAdditionalNonFilteringExpressions() != null) { + stringBuilder.append(", meta: "); + pprintExprList(op.getAdditionalNonFilteringExpressions()); + } + stringBuilder.append(" partitioned by "); + pprintExprList(op.getPrimaryKeyExpressions()); + if (op.getOperation() == Kind.UPSERT) { + stringBuilder.append(" out: ([record-before-upsert:").append(op.getBeforeOpRecordVar()); + if (op.getBeforeOpAdditionalNonFilteringVars() != null) { + stringBuilder.append(", additional-before-upsert: ").append(op.getBeforeOpAdditionalNonFilteringVars()); + } + stringBuilder.append("]) "); + } + if (op.isBulkload()) { + stringBuilder.append(" [bulkload]"); + } + return stringBuilder.toString(); + } + + @Override + public String visitIndexInsertDeleteUpsertOperator(IndexInsertDeleteUpsertOperator op, Void noArgs) + throws AlgebricksException { + stringBuilder.setLength(0); + String header = getIndexOpString(op.getOperation()); + stringBuilder.append(header).append(op.getIndexName()).append(" on ") + .append(str(op.getDataSourceIndex().getDataSource())).append(" from "); + if (op.getOperation() == Kind.UPSERT) { + stringBuilder.append(" replace:"); + pprintExprList(op.getPrevSecondaryKeyExprs()); + stringBuilder.append(" with:"); + pprintExprList(op.getSecondaryKeyExpressions()); + } else { + pprintExprList(op.getSecondaryKeyExpressions()); + } + if (op.isBulkload()) { + stringBuilder.append(" [bulkload]"); + } + return stringBuilder.toString(); + } + + private String getIndexOpString(Kind opKind) { + switch (opKind) { + case DELETE: + return "delete from "; + case INSERT: + return "insert into "; + case UPSERT: + return "upsert into "; + } + return null; + } + + @Override + public String visitTokenizeOperator(TokenizeOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("tokenize ").append(str(op.getTokenizeVars())).append(" <- "); + pprintExprList(op.getSecondaryKeyExpressions()); + return stringBuilder.toString(); + } + + @Override + public String visitSinkOperator(SinkOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append("sink"); + return stringBuilder.toString(); + } + + @Override + public String visitDelegateOperator(DelegateOperator op, Void noArgs) throws AlgebricksException { + stringBuilder.setLength(0); + stringBuilder.append(op.toString()); + return stringBuilder.toString(); + } + + private void pprintExprList(List<Mutable<ILogicalExpression>> expressions) { + stringBuilder.append("["); + boolean first = true; + for (Mutable<ILogicalExpression> exprRef : expressions) { + if (first) { + first = false; + } else { + stringBuilder.append(", "); + } + stringBuilder.append(exprRef.getValue().toString()); + } + stringBuilder.append("]"); + } + + private void pprintVeList(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList) { + stringBuilder.append("["); + boolean fst = true; + for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) { + if (fst) { + fst = false; + } else { + stringBuilder.append("; "); + } + if (ve.first != null) { + stringBuilder.append(ve.first).append(" := ").append(ve.second); + } else { + stringBuilder.append(ve.second.getValue().toString()); + } + } + stringBuilder.append("]"); + } +}
