This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch release-0.3.0 in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 6f00be37ff467ddb163a6836c6ab6fb478072a60 Author: Jialiang Li <j...@linkedin.com> AuthorDate: Thu Mar 5 13:24:19 2020 -0800 Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator (#5114) * Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator * Update bitmap range start/end id * Address comments --- .../org/apache/pinot/core/operator/DocIdSetOperator.java | 8 +++++++- .../pinot/core/operator/filter/ExpressionFilterOperator.java | 7 ++++--- .../integration/tests/BaseClusterIntegrationTestSet.java | 12 ++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java index 8528ef9..5db7db4 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java @@ -49,11 +49,17 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> { private FilterBlockDocIdSet _filterBlockDocIdSet; private BlockDocIdIterator _blockDocIdIterator; private int _currentDocId = 0; + private boolean _threadLocal = true; public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet) { + this(filterOperator, maxSizeOfDocIdSet, true); + } + + public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet, boolean threadLocal) { Preconditions.checkArgument(maxSizeOfDocIdSet > 0 && maxSizeOfDocIdSet <= DocIdSetPlanNode.MAX_DOC_PER_CALL); _filterOperator = filterOperator; _maxSizeOfDocIdSet = maxSizeOfDocIdSet; + _threadLocal = threadLocal; } @Override @@ -69,7 +75,7 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> { } int pos = 0; - int[] docIds = THREAD_LOCAL_DOC_IDS.get(); + int[] docIds = _threadLocal? THREAD_LOCAL_DOC_IDS.get(): new int[_maxSizeOfDocIdSet]; for (int i = 0; i < _maxSizeOfDocIdSet; i++) { _currentDocId = _blockDocIdIterator.next(); if (_currentDocId == Constants.EOF) { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java index 1f7a6d3..9078403 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java @@ -151,7 +151,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator { throw new UnsupportedOperationException("Filter on expressions that return multi-values is not yet supported"); } _blockDocIdIterator.setStartDocId(0); - _blockDocIdIterator.setEndDocId(_expressionFilterOperator._numDocs - 1); + _blockDocIdIterator.setEndDocId(_expressionFilterOperator._numDocs); } @Override @@ -223,8 +223,9 @@ public class ExpressionFilterOperator extends BaseFilterOperator { _currentBlockEndDocId = _currentBlockStartDocId + DocIdSetPlanNode.MAX_DOC_PER_CALL; _currentBlockEndDocId = Math.min(_currentBlockEndDocId, _endDocId); MutableRoaringBitmap bitmapRange = new MutableRoaringBitmap(); - bitmapRange.add(_currentBlockStartDocId, _currentBlockEndDocId + 1); + bitmapRange.add(_currentBlockStartDocId, _currentBlockEndDocId); MutableRoaringBitmap matchedBitmap = evaluate(bitmapRange); + _intIterator = matchedBitmap.getIntIterator(); _numDocsScanned += (_currentBlockEndDocId - _currentBlockStartDocId); } @@ -290,7 +291,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator { private MutableRoaringBitmap evaluate(MutableRoaringBitmap answer) { BaseFilterOperator filterOperator = new BitmapWrappedFilterOperator(answer); - DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL); + DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL, false); ProjectionOperator projectionOperator = new ProjectionOperator(_expressionFilterOperator._dataSourceMap, docIdSetOperator); TransformOperator operator = diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java index 52b6fcb..52df943 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java @@ -170,6 +170,18 @@ public abstract class BaseClusterIntegrationTestSet extends BaseClusterIntegrati String query; List<String> h2queries; query = + "SELECT COUNT(*) FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay LIMIT 1"; + testSqlQuery(query, Collections.singletonList(query)); + query = + "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000"; + testSqlQuery(query, Collections.singletonList(query)); + query = + "SELECT COUNT(*) FROM mytable WHERE ArrDelay > CarrierDelay LIMIT 1"; + testSqlQuery(query, Collections.singletonList(query)); + query = + "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000"; + testSqlQuery(query, Collections.singletonList(query)); + query = "SELECT count(*) FROM mytable WHERE AirlineID > 20355 AND OriginState BETWEEN 'PA' AND 'DE' AND DepTime <> 2202 LIMIT 21"; testSqlQuery(query, Collections.singletonList(query)); query = --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org