This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch execution-stats in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit de9d74cdf7860d0eef5f005aac6cc20a185bd08e Author: kishoreg <[email protected]> AuthorDate: Mon Nov 19 18:28:53 2018 -0800 Adding execution stats for numSegmentsQueried/Processed/Matched --- .../response/broker/BrokerResponseNative.java | 25 +++++++++++++++++++++- .../com/linkedin/pinot/common/utils/DataTable.java | 3 +++ .../core/operator/CombineGroupByOperator.java | 2 ++ .../pinot/core/operator/CombineOperator.java | 2 ++ .../pinot/core/operator/ExecutionStatistics.java | 20 +++++++++++++++-- .../operator/blocks/IntermediateResultsBlock.java | 21 ++++++++++++++++++ .../core/query/reduce/BrokerReduceService.java | 13 +++++++++++ 7 files changed, 83 insertions(+), 3 deletions(-) diff --git a/pinot-common/src/main/java/com/linkedin/pinot/common/response/broker/BrokerResponseNative.java b/pinot-common/src/main/java/com/linkedin/pinot/common/response/broker/BrokerResponseNative.java index e4a251c..43b0ade 100644 --- a/pinot-common/src/main/java/com/linkedin/pinot/common/response/broker/BrokerResponseNative.java +++ b/pinot-common/src/main/java/com/linkedin/pinot/common/response/broker/BrokerResponseNative.java @@ -40,7 +40,7 @@ import org.json.JSONObject; * Supports serialization via JSON. */ @JsonPropertyOrder({"selectionResults", "aggregationResults", "exceptions", "numServersQueried", "numServersResponded", - "numDocsScanned", "numEntriesScannedInFilter", "numEntriesScannedPostFilter", "totalDocs", "numGroupsLimitReached", + "numDocsScanned", "numEntriesScannedInFilter", "numEntriesScannedPostFilter", "numSegmentsProcessed", "numSegmentsWithNoMatch", "totalDocs", "numGroupsLimitReached", "timeUsedMs", "segmentStatistics", "traceInfo"}) public class BrokerResponseNative implements BrokerResponse { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -54,6 +54,9 @@ public class BrokerResponseNative implements BrokerResponse { private long _numDocsScanned = 0L; private long _numEntriesScannedInFilter = 0L; private long _numEntriesScannedPostFilter = 0L; + private long _numSegmentsProcessed = 0L; + private long _numSegmentsWithNoMatch = 0L; + private long _totalDocs = 0L; private boolean _numGroupsLimitReached = false; private long _timeUsedMs = 0L; @@ -173,6 +176,26 @@ public class BrokerResponseNative implements BrokerResponse { _numEntriesScannedPostFilter = numEntriesScannedPostFilter; } + @JsonProperty("numSegmentsProcessed") + public long getNumSegmentsProcessed() { + return _numSegmentsProcessed; + } + + @JsonProperty("numSegmentsProcessed") + public void setNumSegmentsProcessed(long numSegmentsProcessed) { + _numSegmentsProcessed = numSegmentsProcessed; + } + + @JsonProperty("numSegmentsWithNoMatch") + public long getNumSegmentsWithNoMatch() { + return _numSegmentsWithNoMatch; + } + + @JsonProperty("numSegmentsWithNoMatch") + public void setNumSegmentsWithNoMatch(long numSegmentsWithNoMatch) { + _numSegmentsWithNoMatch = numSegmentsWithNoMatch; + } + @JsonProperty("totalDocs") @Override public long getTotalDocs() { diff --git a/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java b/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java index caaa397..09066bf 100644 --- a/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java +++ b/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java @@ -29,6 +29,9 @@ public interface DataTable { String NUM_DOCS_SCANNED_METADATA_KEY = "numDocsScanned"; String NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY = "numEntriesScannedInFilter"; String NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY = "numEntriesScannedPostFilter"; + String NUM_SEGMENTS_QUERIED = "numSegmentsQueried"; + String NUM_SEGMENTS_PROCESSED = "numSegmentsProcessed"; + String NUM_SEGMENTS_WITH_NO_MATCH = "numSegmentsWithNoMatch"; String TOTAL_DOCS_METADATA_KEY = "totalDocs"; String NUM_GROUPS_LIMIT_REACHED_KEY = "numGroupsLimitReached"; String TIME_USED_MS_METADATA_KEY = "timeUsedMs"; diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java index 023b7eb..e4593f8 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java @@ -192,6 +192,8 @@ public class CombineGroupByOperator extends BaseOperator<IntermediateResultsBloc mergedBlock.setNumDocsScanned(executionStatistics.getNumDocsScanned()); mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter()); mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter()); + mergedBlock.setNumSegmentsProcessed(executionStatistics.getNumSegmentsProcessed()); + mergedBlock.setNumSegmentsWithNoMatch(executionStatistics.getNumSegmentsWithNoMatch()); mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs()); // NOTE: numGroups might go slightly over numGroupsLimit because the comparison is not atomic if (numGroups.get() >= _numGroupsLimit) { diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java index 107a687..c7148ec 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java @@ -187,6 +187,8 @@ public class CombineOperator extends BaseOperator<IntermediateResultsBlock> { mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter()); mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter()); mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs()); + mergedBlock.setNumSegmentsProcessed(executionStatistics.getNumSegmentsProcessed()); + mergedBlock.setNumSegmentsWithNoMatch(executionStatistics.getNumSegmentsWithNoMatch()); return mergedBlock; } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java index 2a1d9eb..0a2919d 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java @@ -23,7 +23,9 @@ public class ExecutionStatistics { private long _numEntriesScannedInFilter; private long _numEntriesScannedPostFilter; private long _numTotalRawDocs; - + private long _numSegmentsProcessed; + private long _numSegmentsWithNoMatch; + public ExecutionStatistics() { } @@ -33,6 +35,8 @@ public class ExecutionStatistics { _numEntriesScannedInFilter = numEntriesScannedInFilter; _numEntriesScannedPostFilter = numEntriesScannedPostFilter; _numTotalRawDocs = numTotalRawDocs; + _numSegmentsProcessed = 1; + _numSegmentsWithNoMatch = (numDocsScanned == 0) ? 1 : 0; } public long getNumDocsScanned() { @@ -51,6 +55,14 @@ public class ExecutionStatistics { return _numTotalRawDocs; } + public long getNumSegmentsProcessed() { + return _numSegmentsProcessed; + } + + public long getNumSegmentsWithNoMatch() { + return _numSegmentsWithNoMatch; + } + /** * Merge another execution statistics into the current one. * @@ -61,6 +73,8 @@ public class ExecutionStatistics { _numEntriesScannedInFilter += executionStatisticsToMerge._numEntriesScannedInFilter; _numEntriesScannedPostFilter += executionStatisticsToMerge._numEntriesScannedPostFilter; _numTotalRawDocs += executionStatisticsToMerge._numTotalRawDocs; + _numSegmentsProcessed += executionStatisticsToMerge._numSegmentsProcessed; + _numSegmentsWithNoMatch += executionStatisticsToMerge._numSegmentsWithNoMatch; } @Override @@ -69,6 +83,8 @@ public class ExecutionStatistics { + "\n numDocsScanned: " + _numDocsScanned + "\n numEntriesScannedInFilter: " + _numEntriesScannedInFilter + "\n numEntriesScannedPostFilter: " + _numEntriesScannedPostFilter - + "\n numTotalRawDocs: " + _numTotalRawDocs; + + "\n numTotalRawDocs: " + _numTotalRawDocs + + "\n numSegmentsProcessed: " + _numSegmentsProcessed + + "\n numSegmentsWithNoMatch: " + _numSegmentsWithNoMatch; } } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java index 80da626..aebbf5e 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java @@ -53,6 +53,8 @@ public class IntermediateResultsBlock implements Block { private long _numEntriesScannedInFilter; private long _numEntriesScannedPostFilter; private long _numTotalRawDocs; + private long _numSegmentsProcessed; + private long _numSegmentsWithNoMatch; private boolean _numGroupsLimitReached; /** @@ -172,6 +174,22 @@ public class IntermediateResultsBlock implements Block { public void setNumEntriesScannedPostFilter(long numEntriesScannedPostFilter) { _numEntriesScannedPostFilter = numEntriesScannedPostFilter; } + + public long getNumSegmentsProcessed() { + return _numSegmentsProcessed; + } + + public void setNumSegmentsProcessed(long numSegmentsProcessed) { + _numSegmentsProcessed = numSegmentsProcessed; + } + + public long getNumSegmentsWithNoMatch() { + return _numSegmentsWithNoMatch; + } + + public void setNumSegmentsWithNoMatch(long numSegmentsWithNoMatch) { + _numSegmentsWithNoMatch = numSegmentsWithNoMatch; + } public void setNumTotalRawDocs(long numTotalRawDocs) { _numTotalRawDocs = numTotalRawDocs; @@ -279,6 +297,9 @@ public class IntermediateResultsBlock implements Block { .put(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY, String.valueOf(_numEntriesScannedInFilter)); dataTable.getMetadata() .put(DataTable.NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY, String.valueOf(_numEntriesScannedPostFilter)); + dataTable.getMetadata().put(DataTable.NUM_SEGMENTS_PROCESSED, String.valueOf(_numSegmentsProcessed)); + dataTable.getMetadata().put(DataTable.NUM_SEGMENTS_WITH_NO_MATCH, String.valueOf(_numSegmentsWithNoMatch)); + dataTable.getMetadata().put(DataTable.TOTAL_DOCS_METADATA_KEY, String.valueOf(_numTotalRawDocs)); if (_numGroupsLimitReached) { dataTable.getMetadata().put(DataTable.NUM_GROUPS_LIMIT_REACHED_KEY, "true"); diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/query/reduce/BrokerReduceService.java b/pinot-core/src/main/java/com/linkedin/pinot/core/query/reduce/BrokerReduceService.java index 1c43b40..483ae9d 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/query/reduce/BrokerReduceService.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/query/reduce/BrokerReduceService.java @@ -76,6 +76,8 @@ public class BrokerReduceService implements ReduceService<BrokerResponseNative> long numDocsScanned = 0L; long numEntriesScannedInFilter = 0L; long numEntriesScannedPostFilter = 0L; + long numSegmentsProcessed = 0L; + long numSegmentsWithNoMatch = 0L; long numTotalRawDocs = 0L; boolean numGroupsLimitReached = false; @@ -116,6 +118,15 @@ public class BrokerReduceService implements ReduceService<BrokerResponseNative> if (numEntriesScannedPostFilterString != null) { numEntriesScannedPostFilter += Long.parseLong(numEntriesScannedPostFilterString); } + String numSegmentsProcessedString = metadata.get(DataTable.NUM_SEGMENTS_PROCESSED); + if (numSegmentsProcessedString != null) { + numSegmentsProcessed += Long.parseLong(numSegmentsProcessedString); + } + String numSegmentsWithNoMatchString = metadata.get(DataTable.NUM_SEGMENTS_WITH_NO_MATCH); + if (numSegmentsProcessedString != null) { + numSegmentsWithNoMatch += Long.parseLong(numSegmentsWithNoMatchString); + } + String numTotalRawDocsString = metadata.get(DataTable.TOTAL_DOCS_METADATA_KEY); if (numTotalRawDocsString != null) { numTotalRawDocs += Long.parseLong(numTotalRawDocsString); @@ -143,6 +154,8 @@ public class BrokerReduceService implements ReduceService<BrokerResponseNative> brokerResponseNative.setNumDocsScanned(numDocsScanned); brokerResponseNative.setNumEntriesScannedInFilter(numEntriesScannedInFilter); brokerResponseNative.setNumEntriesScannedPostFilter(numEntriesScannedPostFilter); + brokerResponseNative.setNumSegmentsProcessed(numSegmentsProcessed); + brokerResponseNative.setNumSegmentsWithNoMatch(numSegmentsWithNoMatch); brokerResponseNative.setTotalDocs(numTotalRawDocs); brokerResponseNative.setNumGroupsLimitReached(numGroupsLimitReached); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
