This is an automated email from the ASF dual-hosted git repository. sunithabeeram pushed a commit to branch trackIndexWork in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 8f373f1026ac583bf5f9bd85060c3e213617abf9 Author: Sunitha Beeram <[email protected]> AuthorDate: Fri Nov 16 16:10:39 2018 -0800 [PINOT-7370] Track inverted index loaded per query --- .../main/java/com/linkedin/pinot/common/utils/DataTable.java | 1 + .../linkedin/pinot/core/operator/CombineGroupByOperator.java | 1 + .../com/linkedin/pinot/core/operator/CombineOperator.java | 1 + .../com/linkedin/pinot/core/operator/DocIdSetOperator.java | 3 ++- .../linkedin/pinot/core/operator/ExecutionStatistics.java | 12 ++++++++++-- .../pinot/core/operator/blocks/IntermediateResultsBlock.java | 6 ++++++ .../pinot/core/operator/docidsets/AndBlockDocIdSet.java | 9 +++++++++ .../pinot/core/operator/docidsets/BitmapDocIdSet.java | 12 +++++++++--- .../core/operator/docidsets/EmptyFilterBlockDocIdSet.java | 5 +++++ .../pinot/core/operator/docidsets/FilterBlockDocIdSet.java | 5 +++++ .../pinot/core/operator/docidsets/OrBlockDocIdSet.java | 9 +++++++++ .../core/operator/docidsets/ScanBasedMultiValueDocIdSet.java | 5 +++++ .../operator/docidsets/ScanBasedSingleValueDocIdSet.java | 5 +++++ .../pinot/core/operator/docidsets/SizeBasedDocIdSet.java | 5 +++++ .../pinot/core/operator/docidsets/SortedDocIdSet.java | 5 +++++ .../pinot/core/operator/docidsets/StarTreeDocIdSet.java | 6 ++++++ .../core/operator/query/AggregationGroupByOperator.java | 4 ++-- .../pinot/core/operator/query/AggregationOperator.java | 4 ++-- .../operator/query/DictionaryBasedAggregationOperator.java | 3 ++- .../pinot/core/operator/query/EmptySelectionOperator.java | 2 +- .../operator/query/MetadataBasedAggregationOperator.java | 3 ++- .../pinot/core/operator/query/SelectionOnlyOperator.java | 4 ++-- .../pinot/core/operator/query/SelectionOrderByOperator.java | 4 ++-- .../linkedin/pinot/core/query/scheduler/QueryScheduler.java | 6 ++++-- .../pinot/operator/filter/FilterOperatorTestUtils.java | 5 +++++ 25 files changed, 106 insertions(+), 19 deletions(-) diff --git a/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java b/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java index caaa397..9c6a7e9 100644 --- a/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java +++ b/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java @@ -27,6 +27,7 @@ import javax.annotation.Nonnull; public interface DataTable { String EXCEPTION_METADATA_KEY = "Exception"; String NUM_DOCS_SCANNED_METADATA_KEY = "numDocsScanned"; + String NUM_INDICES_LOADED_KEY = "numIndicesLoaded"; String NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY = "numEntriesScannedInFilter"; String NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY = "numEntriesScannedPostFilter"; String TOTAL_DOCS_METADATA_KEY = "totalDocs"; diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java index 023b7eb..f552269 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java @@ -190,6 +190,7 @@ public class CombineGroupByOperator extends BaseOperator<IntermediateResultsBloc } } mergedBlock.setNumDocsScanned(executionStatistics.getNumDocsScanned()); + mergedBlock.setNumIndicesLoaded(executionStatistics.getNumIndicesLoaded()); mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter()); mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter()); mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs()); diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java index 107a687..c2b81b7 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java @@ -184,6 +184,7 @@ public class CombineOperator extends BaseOperator<IntermediateResultsBlock> { } } mergedBlock.setNumDocsScanned(executionStatistics.getNumDocsScanned()); + mergedBlock.setNumIndicesLoaded(executionStatistics.getNumIndicesLoaded()); mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter()); mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter()); mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs()); diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/DocIdSetOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/DocIdSetOperator.java index 03858a3..cbb88a1 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/DocIdSetOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/DocIdSetOperator.java @@ -88,6 +88,7 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> { @Override public ExecutionStatistics getExecutionStatistics() { - return new ExecutionStatistics(0L, _filterBlockDocIdSet.getNumEntriesScannedInFilter(), 0L, 0L); + return new ExecutionStatistics(0L, _filterBlockDocIdSet.getNumIndicesLoaded(), + _filterBlockDocIdSet.getNumEntriesScannedInFilter(), 0L, 0L); } } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java index 2a1d9eb..eb0674f 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java @@ -20,6 +20,7 @@ package com.linkedin.pinot.core.operator; */ public class ExecutionStatistics { private long _numDocsScanned; + private long _numIndicesLoaded; private long _numEntriesScannedInFilter; private long _numEntriesScannedPostFilter; private long _numTotalRawDocs; @@ -27,9 +28,10 @@ public class ExecutionStatistics { public ExecutionStatistics() { } - public ExecutionStatistics(long numDocsScanned, long numEntriesScannedInFilter, long numEntriesScannedPostFilter, - long numTotalRawDocs) { + public ExecutionStatistics(long numDocsScanned, long numIndicesLoaded, long numEntriesScannedInFilter, + long numEntriesScannedPostFilter, long numTotalRawDocs) { _numDocsScanned = numDocsScanned; + _numIndicesLoaded = numIndicesLoaded; _numEntriesScannedInFilter = numEntriesScannedInFilter; _numEntriesScannedPostFilter = numEntriesScannedPostFilter; _numTotalRawDocs = numTotalRawDocs; @@ -39,6 +41,10 @@ public class ExecutionStatistics { return _numDocsScanned; } + public long getNumIndicesLoaded() { + return _numIndicesLoaded; + } + public long getNumEntriesScannedInFilter() { return _numEntriesScannedInFilter; } @@ -58,6 +64,7 @@ public class ExecutionStatistics { */ public void merge(ExecutionStatistics executionStatisticsToMerge) { _numDocsScanned += executionStatisticsToMerge._numDocsScanned; + _numIndicesLoaded += executionStatisticsToMerge._numIndicesLoaded; _numEntriesScannedInFilter += executionStatisticsToMerge._numEntriesScannedInFilter; _numEntriesScannedPostFilter += executionStatisticsToMerge._numEntriesScannedPostFilter; _numTotalRawDocs += executionStatisticsToMerge._numTotalRawDocs; @@ -67,6 +74,7 @@ public class ExecutionStatistics { public String toString() { return "Execution Statistics:" + "\n numDocsScanned: " + _numDocsScanned + + "\n numIndicesLoaded: " + _numIndicesLoaded + "\n numEntriesScannedInFilter: " + _numEntriesScannedInFilter + "\n numEntriesScannedPostFilter: " + _numEntriesScannedPostFilter + "\n numTotalRawDocs: " + _numTotalRawDocs; diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java index 80da626..06e50bc 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java @@ -50,6 +50,7 @@ public class IntermediateResultsBlock implements Block { private List<Map<String, Object>> _combinedAggregationGroupByResult; private List<ProcessingException> _processingExceptions; private long _numDocsScanned; + private long _numIndicesLoaded; private long _numEntriesScannedInFilter; private long _numEntriesScannedPostFilter; private long _numTotalRawDocs; @@ -165,6 +166,10 @@ public class IntermediateResultsBlock implements Block { _numDocsScanned = numDocsScanned; } + public void setNumIndicesLoaded(long numIndicesLoaded) { + _numIndicesLoaded = numIndicesLoaded; + } + public void setNumEntriesScannedInFilter(long numEntriesScannedInFilter) { _numEntriesScannedInFilter = numEntriesScannedInFilter; } @@ -275,6 +280,7 @@ public class IntermediateResultsBlock implements Block { private DataTable attachMetadataToDataTable(DataTable dataTable) { dataTable.getMetadata().put(DataTable.NUM_DOCS_SCANNED_METADATA_KEY, String.valueOf(_numDocsScanned)); + dataTable.getMetadata().put(DataTable.NUM_INDICES_LOADED_KEY, String.valueOf(_numIndicesLoaded)); dataTable.getMetadata() .put(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY, String.valueOf(_numEntriesScannedInFilter)); dataTable.getMetadata() diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/AndBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/AndBlockDocIdSet.java index f6cefd3..85adc6e 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/AndBlockDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/AndBlockDocIdSet.java @@ -259,4 +259,13 @@ public final class AndBlockDocIdSet implements FilterBlockDocIdSet { } return numEntriesScannedInFilter; } + + @Override + public long getNumIndicesLoaded() { + long loaded = 0; + for (FilterBlockDocIdSet docIdSet : blockDocIdSets) { + loaded += docIdSet.getNumIndicesLoaded(); + } + return loaded; + } } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapDocIdSet.java index ba56f47..556eab3 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapDocIdSet.java @@ -23,19 +23,20 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap; public class BitmapDocIdSet implements FilterBlockDocIdSet { private final ImmutableRoaringBitmap _bitmap; + private final long _bitMapCount; private int _startDocId; // Inclusive private int _endDocId; public BitmapDocIdSet(ImmutableRoaringBitmap[] bitmaps, int startDocId, int endDocId, boolean exclusive) { - int numBitmaps = bitmaps.length; - if (numBitmaps > 1) { + _bitMapCount = bitmaps.length; + if (_bitMapCount > 1) { MutableRoaringBitmap orBitmap = MutableRoaringBitmap.or(bitmaps); if (exclusive) { orBitmap.flip(startDocId, endDocId + 1); } _bitmap = orBitmap; - } else if (numBitmaps == 1) { + } else if (_bitMapCount == 1) { if (exclusive) { // NOTE: cannot use ImmutableRoaringBitmap.flip() because the library has a bug in that method // TODO: the bug has been fixed in the latest version of ImmutableRoaringBitmap, update the version @@ -83,6 +84,11 @@ public class BitmapDocIdSet implements FilterBlockDocIdSet { } @Override + public long getNumIndicesLoaded() { + return _bitMapCount; + } + + @Override public BlockDocIdIterator iterator() { BitmapDocIdIterator bitmapDocIdIterator = new BitmapDocIdIterator(_bitmap.getIntIterator()); bitmapDocIdIterator.setStartDocId(_startDocId); diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/EmptyFilterBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/EmptyFilterBlockDocIdSet.java index af02c83..87c64c2 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/EmptyFilterBlockDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/EmptyFilterBlockDocIdSet.java @@ -56,6 +56,11 @@ public final class EmptyFilterBlockDocIdSet implements FilterBlockDocIdSet { } @Override + public long getNumIndicesLoaded() { + return 0L; + } + + @Override public BlockDocIdIterator iterator() { return EmptyBlockDocIdIterator.getInstance(); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/FilterBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/FilterBlockDocIdSet.java index 7dcf864..22721c3 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/FilterBlockDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/FilterBlockDocIdSet.java @@ -68,4 +68,9 @@ public interface FilterBlockDocIdSet extends BlockDocIdSet { * Returns the number of entries scanned in filtering phase. */ long getNumEntriesScannedInFilter(); + + /** + * Returns the number of indices loaded in filtering phase. + */ + long getNumIndicesLoaded(); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/OrBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/OrBlockDocIdSet.java index bd18cb4..17de4bf 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/OrBlockDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/OrBlockDocIdSet.java @@ -72,6 +72,15 @@ public final class OrBlockDocIdSet implements FilterBlockDocIdSet { } @Override + public long getNumIndicesLoaded() { + long loaded = 0; + for (FilterBlockDocIdSet docIdSet : _docIdSets) { + loaded += docIdSet.getNumIndicesLoaded(); + } + return loaded; + } + + @Override public BlockDocIdIterator iterator() { boolean useBitmapOr = false; for (BlockDocIdSet docIdSet : _docIdSets) { diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedMultiValueDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedMultiValueDocIdSet.java index 2fdd038..c5162b8 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedMultiValueDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedMultiValueDocIdSet.java @@ -72,6 +72,11 @@ public class ScanBasedMultiValueDocIdSet implements FilterBlockDocIdSet { } @Override + public long getNumIndicesLoaded() { + return 0L; + } + + @Override public ScanBasedDocIdIterator iterator() { return blockValSetBlockDocIdIterator; } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedSingleValueDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedSingleValueDocIdSet.java index 6594586..ccb5d55 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedSingleValueDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedSingleValueDocIdSet.java @@ -73,6 +73,11 @@ public class ScanBasedSingleValueDocIdSet implements FilterBlockDocIdSet { } @Override + public long getNumIndicesLoaded() { + return 0L; + } + + @Override public ScanBasedDocIdIterator iterator() { return blockValSetBlockDocIdIterator; } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SizeBasedDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SizeBasedDocIdSet.java index bbd6626..d5d0098 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SizeBasedDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SizeBasedDocIdSet.java @@ -50,6 +50,11 @@ public final class SizeBasedDocIdSet implements FilterBlockDocIdSet { } @Override + public long getNumIndicesLoaded() { + return 0L; + } + + @Override public BlockDocIdIterator iterator() { return new SizeBasedDocIdIterator(_maxDocId); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SortedDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SortedDocIdSet.java index 13c8b20..8514029 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SortedDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SortedDocIdSet.java @@ -78,6 +78,11 @@ public class SortedDocIdSet implements FilterBlockDocIdSet { } @Override + public long getNumIndicesLoaded() { + return 0L; + } + + @Override public BlockDocIdIterator iterator() { if (pairs == null || pairs.isEmpty()) { return EmptyBlockDocIdIterator.getInstance(); diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/StarTreeDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/StarTreeDocIdSet.java index 1b14c5c..0c55688 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/StarTreeDocIdSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/StarTreeDocIdSet.java @@ -89,6 +89,12 @@ public class StarTreeDocIdSet implements FilterBlockDocIdSet { } @Override + public long getNumIndicesLoaded() { + // TODO: this must be fixed + return 0L; + } + + @Override public BlockDocIdIterator iterator() { return new RangeBasedDocIdIterator(minDocId, maxDocId); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationGroupByOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationGroupByOperator.java index a5a3b47..2dff034 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationGroupByOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationGroupByOperator.java @@ -83,8 +83,8 @@ public class AggregationGroupByOperator extends BaseOperator<IntermediateResults long numEntriesScannedInFilter = _transformOperator.getExecutionStatistics().getNumEntriesScannedInFilter(); long numEntriesScannedPostFilter = numDocsScanned * _transformOperator.getNumColumnsProjected(); _executionStatistics = - new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, - _numTotalRawDocs); + new ExecutionStatistics(numDocsScanned,_transformOperator.getExecutionStatistics().getNumIndicesLoaded(), + numEntriesScannedInFilter, numEntriesScannedPostFilter, _numTotalRawDocs); // Build intermediate result block based on aggregation group-by result from the executor return new IntermediateResultsBlock(_functionContexts, groupByResult); diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationOperator.java index 776f122..e5d3910 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationOperator.java @@ -71,8 +71,8 @@ public class AggregationOperator extends BaseOperator<IntermediateResultsBlock> long numEntriesScannedInFilter = _transformOperator.getExecutionStatistics().getNumEntriesScannedInFilter(); long numEntriesScannedPostFilter = numDocsScanned * _transformOperator.getNumColumnsProjected(); _executionStatistics = - new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, - _numTotalRawDocs); + new ExecutionStatistics(numDocsScanned, _transformOperator.getExecutionStatistics().getNumIndicesLoaded(), + numEntriesScannedInFilter, numEntriesScannedPostFilter, _numTotalRawDocs); // Build intermediate result block based on aggregation result from the executor return new IntermediateResultsBlock(_functionContexts, aggregationResult, false); diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/DictionaryBasedAggregationOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/DictionaryBasedAggregationOperator.java index 749ff7e..840a6a7 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/DictionaryBasedAggregationOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/DictionaryBasedAggregationOperator.java @@ -95,7 +95,8 @@ public class DictionaryBasedAggregationOperator extends BaseOperator<Intermediat // Create execution statistics. Set numDocsScanned to totalRawDocs for backward compatibility. _executionStatistics = - new ExecutionStatistics(_totalRawDocs, 0/* numEntriesScannedInFilter */, 0/* numEntriesScannedPostFilter */, + new ExecutionStatistics(_totalRawDocs, 0/*numIndicesLoaded*/, + 0/* numEntriesScannedInFilter */, 0/* numEntriesScannedPostFilter */, _totalRawDocs); // Build intermediate result block based on aggregation result from the executor. diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/EmptySelectionOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/EmptySelectionOperator.java index 025bc79..c421547 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/EmptySelectionOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/EmptySelectionOperator.java @@ -42,7 +42,7 @@ public class EmptySelectionOperator extends BaseOperator<IntermediateResultsBloc List<String> selectionColumns = SelectionOperatorUtils.getSelectionColumns(selection.getSelectionColumns(), indexSegment); _dataSchema = SelectionOperatorUtils.extractDataSchema(null, selectionColumns, indexSegment); - _executionStatistics = new ExecutionStatistics(0L, 0L, 0L, indexSegment.getSegmentMetadata().getTotalRawDocs()); + _executionStatistics = new ExecutionStatistics(0L, 0L, 0L, 0L, indexSegment.getSegmentMetadata().getTotalRawDocs()); } @Override diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/MetadataBasedAggregationOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/MetadataBasedAggregationOperator.java index ce9087b..b1650f1 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/MetadataBasedAggregationOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/MetadataBasedAggregationOperator.java @@ -70,7 +70,8 @@ public class MetadataBasedAggregationOperator extends BaseOperator<IntermediateR // Create execution statistics. Set numDocsScanned to totalRawDocs for backward compatibility. _executionStatistics = - new ExecutionStatistics(totalRawDocs, 0/*numEntriesScannedInFilter*/, 0/*numEntriesScannedPostFilter*/, + new ExecutionStatistics(totalRawDocs, 0/*numIndicesLoaded*/, + 0/*numEntriesScannedInFilter*/, 0/*numEntriesScannedPostFilter*/, totalRawDocs); // Build intermediate result block based on aggregation result from the executor. diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOnlyOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOnlyOperator.java index 5f2051f..8ebb313 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOnlyOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOnlyOperator.java @@ -88,8 +88,8 @@ public class SelectionOnlyOperator extends BaseOperator<IntermediateResultsBlock long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumColumnsProjected(); long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs(); _executionStatistics = - new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, - numTotalRawDocs); + new ExecutionStatistics(numDocsScanned, _projectionOperator.getExecutionStatistics().getNumIndicesLoaded(), + numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs); return new IntermediateResultsBlock(_dataSchema, _rowEvents); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOrderByOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOrderByOperator.java index 2e5f35b..84dfe35 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOrderByOperator.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOrderByOperator.java @@ -98,8 +98,8 @@ public class SelectionOrderByOperator extends BaseOperator<IntermediateResultsBl long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumColumnsProjected(); long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs(); _executionStatistics = - new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, - numTotalRawDocs); + new ExecutionStatistics(numDocsScanned, _projectionOperator.getExecutionStatistics().getNumIndicesLoaded(), + numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs); return new IntermediateResultsBlock(_selectionOperatorService.getDataSchema(), _selectionOperatorService.getRows()); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/query/scheduler/QueryScheduler.java b/pinot-core/src/main/java/com/linkedin/pinot/core/query/scheduler/QueryScheduler.java index 1edb97c..411d218 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/query/scheduler/QueryScheduler.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/query/scheduler/QueryScheduler.java @@ -143,6 +143,8 @@ public abstract class QueryScheduler { String tableNameWithType = queryRequest.getTableNameWithType(); long numDocsScanned = Long.parseLong(dataTableMetadata.getOrDefault(DataTable.NUM_DOCS_SCANNED_METADATA_KEY, INVALID_NUM_SCANNED)); + long numIndicesLoaded = + Long.parseLong(dataTableMetadata.getOrDefault(DataTable.NUM_INDICES_LOADED_KEY, "0")); long numEntriesScannedInFilter = Long.parseLong( dataTableMetadata.getOrDefault(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY, INVALID_NUM_SCANNED)); long numEntriesScannedPostFilter = Long.parseLong( @@ -161,11 +163,11 @@ public abstract class QueryScheduler { TimerContext timerContext = queryRequest.getTimerContext(); LOGGER.info( - "Processed requestId={},table={},reqSegments={},prunedToSegmentCount={},totalExecMs={},totalTimeMs={},broker={},numDocsScanned={},scanInFilter={},scanPostFilter={},sched={}", + "Processed requestId={},table={},reqSegments={},prunedToSegmentCount={},totalExecMs={},totalTimeMs={},broker={},numDocsScanned={},numIndicesLoaded={},scanInFilter={},scanPostFilter={},sched={}", requestId, tableNameWithType, queryRequest.getSegmentsToQuery().size(), queryRequest.getSegmentCountAfterPruning(), timerContext.getPhaseDurationMs(ServerQueryPhase.QUERY_PROCESSING), timerContext.getPhaseDurationMs(ServerQueryPhase.TOTAL_QUERY_TIME), queryRequest.getBrokerId(), numDocsScanned, - numEntriesScannedInFilter, numEntriesScannedPostFilter, name()); + numIndicesLoaded, numEntriesScannedInFilter, numEntriesScannedPostFilter, name()); serverMetrics.addMeteredTableValue(tableNameWithType, ServerMeter.NUM_SEGMENTS_SEARCHED, queryRequest.getSegmentCountAfterPruning()); diff --git a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java b/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java index 58fb7db..76cd00c 100644 --- a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java +++ b/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java @@ -65,6 +65,11 @@ public class FilterOperatorTestUtils { } @Override + public long getNumIndicesLoaded() { + return 0; + } + + @Override public BlockDocIdIterator iterator() { return new ArrayBasedDocIdIterator(docIds, docIds.length); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
