This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 61511fbd15 Add logs indicating the star-tree config diff to understand
the rebuild purpose (#13627)
61511fbd15 is described below
commit 61511fbd15d767f5ad6b2c3f60ea098926de27f1
Author: Shounak kulkarni <[email protected]>
AuthorDate: Tue Aug 13 12:22:13 2024 +0500
Add logs indicating the star-tree config diff to understand the rebuild
purpose (#13627)
---
.../local/startree/v2/builder/MultipleTreesBuilder.java | 10 ++++++++++
.../startree/v2/builder/StarTreeIndexSeparator.java | 17 ++++++++++++++++-
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
index 35343dc3f1..361e6fa67d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
@@ -90,6 +90,15 @@ public class MultipleTreesBuilder implements Closeable {
_metadataProperties =
CommonsConfigurationUtils.fromFile(new File(_segmentDirectory,
V1Constants.MetadataKeys.METADATA_FILE_NAME));
_separator = getSeparator();
+ // log the updated star-tree configs
+ if (LOGGER.isDebugEnabled()) {
+ StringBuilder logUpdatedStarTrees = new StringBuilder();
+ logUpdatedStarTrees.append("Updated star-tree configs :");
+ for (StarTreeV2BuilderConfig startree : _builderConfigs) {
+ logUpdatedStarTrees.append("\n").append(startree);
+ }
+ LOGGER.debug(logUpdatedStarTrees.toString());
+ }
_segment = ImmutableSegmentLoader.load(indexDir, ReadMode.mmap);
}
@@ -127,6 +136,7 @@ public class MultipleTreesBuilder implements Closeable {
throws Exception {
List<StarTreeV2Metadata> starTreeMetadataList = new
SegmentMetadataImpl(_indexDir).getStarTreeV2MetadataList();
if (starTreeMetadataList == null) {
+ LOGGER.info("No existing star-tree. Building all new start-trees.");
return null;
}
try {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparator.java
index 29e677f5d5..51f01a0b48 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparator.java
@@ -36,12 +36,16 @@ import org.apache.pinot.segment.spi.V1Constants;
import
org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Metadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The {@code StarTreeIndexSeparator} pulls out the individual star-trees from
the common star-tree index file
*/
public class StarTreeIndexSeparator implements Closeable {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(StarTreeIndexSeparator.class);
+
private final List<Map<StarTreeIndexMapUtils.IndexKey,
StarTreeIndexMapUtils.IndexValue>> _indexMapList;
private final List<StarTreeV2BuilderConfig> _builderConfigList;
private final List<Integer> _numDocsList;
@@ -58,9 +62,18 @@ public class StarTreeIndexSeparator implements Closeable {
_builderConfigList = new ArrayList<>(numStarTrees);
_numDocsList = new ArrayList<>(numStarTrees);
for (StarTreeV2Metadata starTreeMetadata : starTreeMetadataList) {
-
_builderConfigList.add(StarTreeV2BuilderConfig.fromMetadata(starTreeMetadata));
+ StarTreeV2BuilderConfig config =
StarTreeV2BuilderConfig.fromMetadata(starTreeMetadata);
+ _builderConfigList.add(config);
_numDocsList.add(starTreeMetadata.getNumDocs());
}
+ if (LOGGER.isDebugEnabled()) {
+ StringBuilder logExistingStarTrees = new StringBuilder();
+ logExistingStarTrees.append("Existing star-tree configs :");
+ for (StarTreeV2BuilderConfig config : _builderConfigList) {
+ logExistingStarTrees.append("\n").append(config);
+ }
+ LOGGER.debug(logExistingStarTrees.toString());
+ }
_indexFileChannel = new RandomAccessFile(indexFile, "r").getChannel();
}
@@ -77,8 +90,10 @@ public class StarTreeIndexSeparator implements Closeable {
throws IOException {
int treeIndex = _builderConfigList.indexOf(builderConfig);
if (treeIndex == -1) {
+ LOGGER.info("No existing star-tree found for config: {}", builderConfig);
return -1;
}
+ LOGGER.info("Separating star-tree for config: {}", builderConfig);
separate(starTreeOutputDir, treeIndex);
return _numDocsList.get(treeIndex);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
