This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 29cf33b Enhance StarTreeIndexViewer to support multiple trees (#6569)
29cf33b is described below
commit 29cf33b280116c4cf7d3305204c3531f2261b1f9
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Sun Feb 14 15:13:41 2021 -0800
Enhance StarTreeIndexViewer to support multiple trees (#6569)
Enhance the StarTreeIndexViewers to support:
- Visualize multiple star-trees
- Include dimension name in the node
- Fix a bug of miscounting the top nodes
---
.../pinot/core/startree/OffHeapStarTreeNode.java | 4 +-
.../tests/StarTreeClusterIntegrationTest.java | 2 +-
.../apache/pinot/tools/StarTreeIndexViewer.java | 217 +++++++++++----------
pinot-tools/src/main/resources/star-tree.html | 2 +-
4 files changed, 115 insertions(+), 110 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/startree/OffHeapStarTreeNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/startree/OffHeapStarTreeNode.java
index e918680..0104ac6 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/startree/OffHeapStarTreeNode.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/startree/OffHeapStarTreeNode.java
@@ -39,7 +39,7 @@ public class OffHeapStarTreeNode implements StarTreeNode {
public OffHeapStarTreeNode(PinotDataBuffer dataBuffer, int nodeId) {
_dataBuffer = dataBuffer;
- long offset = nodeId * SERIALIZABLE_SIZE_IN_BYTES;
+ long offset = (long) nodeId * SERIALIZABLE_SIZE_IN_BYTES;
_dimensionId = dataBuffer.getInt(offset);
offset += Integer.BYTES;
@@ -77,7 +77,7 @@ public class OffHeapStarTreeNode implements StarTreeNode {
if (_firstChildId == INVALID_ID) {
return INVALID_ID;
} else {
- return _dataBuffer.getInt(_firstChildId * SERIALIZABLE_SIZE_IN_BYTES);
+ return _dataBuffer.getInt((long) _firstChildId *
SERIALIZABLE_SIZE_IN_BYTES);
}
}
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java
index 2d2585b..0d29bee 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java
@@ -172,7 +172,7 @@ public class StarTreeClusterIntegrationTest extends
BaseClusterIntegrationTest {
functionColumnPairs.add(new
AggregationFunctionColumnPair(functionType, metric).toColumnName());
}
}
- return new StarTreeIndexConfig(dimensions, null, functionColumnPairs, 0);
+ return new StarTreeIndexConfig(dimensions, null, functionColumnPairs, 100);
}
@Test
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/StarTreeIndexViewer.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/StarTreeIndexViewer.java
index 2afd8db..f237a2d 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/StarTreeIndexViewer.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/StarTreeIndexViewer.java
@@ -26,11 +26,9 @@ import java.io.InputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
@@ -39,15 +37,13 @@ import javax.ws.rs.core.MediaType;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.segment.ReadMode;
-import org.apache.pinot.core.common.DataSource;
import org.apache.pinot.core.indexsegment.IndexSegment;
import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
-import org.apache.pinot.core.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.core.segment.index.readers.Dictionary;
import org.apache.pinot.core.startree.StarTree;
import org.apache.pinot.core.startree.StarTreeNode;
+import org.apache.pinot.core.startree.v2.StarTreeV2;
import org.apache.pinot.spi.utils.JsonUtils;
-import org.glassfish.grizzly.http.server.HttpServer;
import org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpServerFactory;
import org.glassfish.jersey.server.ResourceConfig;
import org.slf4j.Logger;
@@ -55,6 +51,57 @@ import org.slf4j.LoggerFactory;
public class StarTreeIndexViewer {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(StarTreeIndexViewer.class);
+
+ /*
+ * MAX num children to show in the UI
+ */
+ private static final int MAX_CHILDREN = 100;
+
+ public StarTreeIndexViewer(File segmentDir)
+ throws Exception {
+ IndexSegment indexSegment = ImmutableSegmentLoader.load(segmentDir,
ReadMode.heap);
+ List<StarTreeV2> starTrees = indexSegment.getStarTrees();
+ if (starTrees == null) {
+ throw new IllegalStateException("Cannot find star-tree in segment
directory: " + segmentDir);
+ }
+
+ StarTreeJsonNode rootJsonNode = new StarTreeJsonNode("ROOT");
+ int numStarTrees = starTrees.size();
+ if (numStarTrees == 1) {
+ new SingleTreeViewer(starTrees.get(0)).build(rootJsonNode);
+ } else {
+ for (int i = 0; i < numStarTrees; i++) {
+ StarTreeJsonNode singleTreeRootJsonNode = new
StarTreeJsonNode("Star-Tree-" + i);
+ new SingleTreeViewer(starTrees.get(i)).build(singleTreeRootJsonNode);
+ rootJsonNode.addChild(singleTreeRootJsonNode);
+ }
+ }
+
+ indexSegment.destroy();
+ String rootJsonString = JsonUtils.objectToPrettyString(rootJsonNode);
+ LOGGER.info(rootJsonString);
+ startServer(rootJsonString);
+ }
+
+ private void startServer(String jsonString) {
+
GrizzlyHttpServerFactory.createHttpServer(URI.create("http://0.0.0.0:8090/"),
new StarTreeResource(jsonString));
+ LOGGER.info("Go to http://localhost:8090/ to view the star-trees");
+ }
+
+ public static void main(String[] args)
+ throws Exception {
+ if (args.length != 1) {
+ LOGGER.error("USAGE: StarIndexViewer <segmentDirectory>");
+ System.exit(1);
+ }
+ String segmentDir = args[0];
+ if (!new File(segmentDir).isDirectory()) {
+ throw new FileNotFoundException("Missing directory: " + segmentDir);
+ }
+ new StarTreeIndexViewer(new File(segmentDir));
+ }
+
private static class StarTreeJsonNode {
String _name;
List<StarTreeJsonNode> _children;
@@ -80,107 +127,74 @@ public class StarTreeIndexViewer {
}
}
- private static final Logger LOGGER =
LoggerFactory.getLogger(StarTreeIndexViewer.class);
-
- /*
- * MAX num children to show in the UI
- */
- static int MAX_CHILDREN = 100;
- private List<String> _dimensionNames;
- private Map<String, Dictionary> dictionaries;
-
- public StarTreeIndexViewer(File segmentDir)
- throws Exception {
- IndexSegment indexSegment = ImmutableSegmentLoader.load(segmentDir,
ReadMode.heap);
-
- dictionaries = new HashMap<>();
- SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
-
- for (String columnName : metadata.getAllColumns()) {
- DataSource dataSource = indexSegment.getDataSource(columnName);
- dictionaries.put(columnName, dataSource.getDictionary());
+ private static class SingleTreeViewer {
+ private final StarTreeNode _rootNode;
+ private final List<String> _dimensionNames;
+ private final List<Dictionary> _dimensionDictionaries;
+
+ public SingleTreeViewer(StarTreeV2 starTreeV2) {
+ StarTree starTree = starTreeV2.getStarTree();
+ _rootNode = starTree.getRoot();
+ _dimensionNames = starTree.getDimensionNames();
+ int numDimensions = _dimensionNames.size();
+ _dimensionDictionaries = new ArrayList<>(numDimensions);
+ for (String dimension : _dimensionNames) {
+
_dimensionDictionaries.add(starTreeV2.getDataSource(dimension).getDictionary());
+ }
}
- StarTree tree = indexSegment.getStarTrees().get(0).getStarTree();
- _dimensionNames = tree.getDimensionNames();
- StarTreeJsonNode jsonRoot = new StarTreeJsonNode("ROOT");
- build(tree.getRoot(), jsonRoot);
- indexSegment.destroy();
- String writeValueAsString = JsonUtils.objectToPrettyString(jsonRoot);
- LOGGER.info(writeValueAsString);
- startServer(segmentDir, writeValueAsString);
- }
-
- private int build(StarTreeNode indexNode, StarTreeJsonNode json) {
- Iterator<? extends StarTreeNode> childrenIterator =
indexNode.getChildrenIterator();
- if (!childrenIterator.hasNext()) {
- return 0;
+ public void build(StarTreeJsonNode rootJsonNode) {
+ build(_rootNode, rootJsonNode);
}
- int childDimensionId = indexNode.getChildDimensionId();
- String childDimensionName = _dimensionNames.get(childDimensionId);
- Dictionary dictionary = dictionaries.get(childDimensionName);
- int totalChildNodes = indexNode.getNumChildren();
- Comparator<Pair<String, Integer>> comparator = new Comparator<Pair<String,
Integer>>() {
-
- @Override
- public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
- return -1 * Integer.compare(o1.getRight(), o2.getRight());
- }
- };
- MinMaxPriorityQueue<Pair<String, Integer>> queue =
-
MinMaxPriorityQueue.orderedBy(comparator).maximumSize(MAX_CHILDREN).create();
- StarTreeJsonNode allNode = null;
-
- while (childrenIterator.hasNext()) {
- StarTreeNode childIndexNode = childrenIterator.next();
- int childDimensionValueId = childIndexNode.getDimensionValue();
- String childDimensionValue = "ALL";
- if (childDimensionValueId != StarTreeNode.ALL) {
- childDimensionValue = dictionary.get(childDimensionValueId).toString();
+ @SuppressWarnings("UnstableApiUsage")
+ private long build(StarTreeNode treeNode, StarTreeJsonNode jsonNode) {
+ if (treeNode.isLeaf()) {
+ return 0;
}
- StarTreeJsonNode childJson = new StarTreeJsonNode(childDimensionValue);
- totalChildNodes += build(childIndexNode, childJson);
- if (childDimensionValueId != StarTreeNode.ALL) {
- json.addChild(childJson);
- queue.add(ImmutablePair.of(childDimensionValue, totalChildNodes));
- } else {
- allNode = childJson;
- }
- }
- //put ALL node at the end
- if (allNode != null) {
- json.addChild(allNode);
- }
- if (totalChildNodes > MAX_CHILDREN) {
- Iterator<Pair<String, Integer>> qIterator = queue.iterator();
- Set<String> topKDimensions = new HashSet<>();
- topKDimensions.add("ALL");
- while (qIterator.hasNext()) {
- topKDimensions.add(qIterator.next().getKey());
+
+ int childDimensionId = treeNode.getChildDimensionId();
+ String childDimension = _dimensionNames.get(childDimensionId);
+ Dictionary dictionary = _dimensionDictionaries.get(childDimensionId);
+ int numChildNodes = treeNode.getNumChildren();
+ long totalChildNodes = numChildNodes;
+
+ Comparator<Pair<String, Long>> comparator = (o1, o2) ->
Long.compare(o2.getRight(), o1.getRight());
+ MinMaxPriorityQueue<Pair<String, Long>> queue =
+
MinMaxPriorityQueue.orderedBy(comparator).maximumSize(MAX_CHILDREN).create();
+ StarTreeJsonNode starJsonNode = null;
+
+ Iterator<? extends StarTreeNode> childrenIterator =
treeNode.getChildrenIterator();
+ while (childrenIterator.hasNext()) {
+ StarTreeNode childTreeNode = childrenIterator.next();
+ int childDimensionValueId = childTreeNode.getDimensionValue();
+ String childDimensionValue =
+ childDimensionValueId != StarTreeNode.ALL ?
dictionary.getStringValue(childDimensionValueId) : "ALL";
+ String childJsonNodeName = childDimension + ": " + childDimensionValue;
+ StarTreeJsonNode childJsonNode = new
StarTreeJsonNode(childJsonNodeName);
+ long result = build(childTreeNode, childJsonNode);
+ totalChildNodes += result;
+ if (childDimensionValueId != StarTreeNode.ALL) {
+ jsonNode.addChild(childJsonNode);
+ queue.add(ImmutablePair.of(childJsonNodeName, result));
+ } else {
+ starJsonNode = childJsonNode;
+ }
}
- Iterator<StarTreeJsonNode> iterator = json.getChildren().iterator();
- while (iterator.hasNext()) {
- StarTreeJsonNode next = iterator.next();
- if (!topKDimensions.contains(next.getName())) {
- iterator.remove();
+ if (numChildNodes > MAX_CHILDREN) {
+ // Keep the nodes with most child nodes
+ Set<String> nodeNamesToKeep = new HashSet<>();
+ for (Pair<String, Long> pair : queue) {
+ nodeNamesToKeep.add(pair.getKey());
}
+ jsonNode.getChildren().removeIf(next ->
!nodeNamesToKeep.contains(next.getName()));
}
+ // Add star-node at last
+ if (starJsonNode != null) {
+ jsonNode.addChild(starJsonNode);
+ }
+ return totalChildNodes;
}
- return totalChildNodes;
- }
-
- public static void main(String[] args)
- throws Exception {
- if (args.length != 1) {
- LOGGER.error("USAGE: StarIndexViewer <segmentDirectory>");
- System.exit(1);
- }
- String segmentDir = args[0];
- if (!new File(segmentDir).exists()) {
- throw new FileNotFoundException("Missing directory:" + segmentDir);
- }
- new StarTreeIndexViewer(new File(segmentDir));
}
private static class StarTreeResource extends ResourceConfig {
@@ -191,15 +205,6 @@ public class StarTreeIndexViewer {
}
}
- private void startServer(final File segmentDirectory, final String json)
- throws Exception {
- int httpPort = 8090;
- URI baseUri = URI.create("http://0.0.0.0:" + Integer.toString(httpPort) +
"/");
- HttpServer httpServer = GrizzlyHttpServerFactory.createHttpServer(baseUri,
new StarTreeResource(json));
-
- LOGGER.info("Go to http://{}:{}/ to view the star tree", "localhost",
httpPort);
- }
-
@Path("/")
public static final class StarTreeViewRestResource {
public static String json;
diff --git a/pinot-tools/src/main/resources/star-tree.html
b/pinot-tools/src/main/resources/star-tree.html
index c39d7b7..7385606 100644
--- a/pinot-tools/src/main/resources/star-tree.html
+++ b/pinot-tools/src/main/resources/star-tree.html
@@ -49,7 +49,7 @@
<script>
var margin = {top: 20, right: 120, bottom: 20, left: 120},
- width = 960 - margin.right - margin.left,
+ width = 5000 - margin.right - margin.left,
height = 15000 - margin.top - margin.bottom;
var i = 0,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]