This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch fixing_transform_docIds_thread_local_issue in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 82833f21a0c7b88c0083df1d5f90683e430cc47b Author: Xiang Fu <fx19880...@gmail.com> AuthorDate: Wed Mar 4 16:42:57 2020 -0800 Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator --- .../org/apache/pinot/core/operator/DocIdSetOperator.java | 8 +++++++- .../pinot/core/operator/filter/ExpressionFilterOperator.java | 2 +- .../integration/tests/BaseClusterIntegrationTestSet.java | 12 ++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java index 8528ef9..133db262 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java @@ -49,11 +49,17 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> { private FilterBlockDocIdSet _filterBlockDocIdSet; private BlockDocIdIterator _blockDocIdIterator; private int _currentDocId = 0; + private boolean _threadLocal = true; public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet) { + this(filterOperator, maxSizeOfDocIdSet, true); + } + + public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet, boolean threadLocal) { Preconditions.checkArgument(maxSizeOfDocIdSet > 0 && maxSizeOfDocIdSet <= DocIdSetPlanNode.MAX_DOC_PER_CALL); _filterOperator = filterOperator; _maxSizeOfDocIdSet = maxSizeOfDocIdSet; + _threadLocal = threadLocal; } @Override @@ -69,7 +75,7 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> { } int pos = 0; - int[] docIds = THREAD_LOCAL_DOC_IDS.get(); + int[] docIds = _threadLocal? THREAD_LOCAL_DOC_IDS.get(): new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; for (int i = 0; i < _maxSizeOfDocIdSet; i++) { _currentDocId = _blockDocIdIterator.next(); if (_currentDocId == Constants.EOF) { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java index 1f7a6d3..718d879 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java @@ -290,7 +290,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator { private MutableRoaringBitmap evaluate(MutableRoaringBitmap answer) { BaseFilterOperator filterOperator = new BitmapWrappedFilterOperator(answer); - DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL); + DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL, false); ProjectionOperator projectionOperator = new ProjectionOperator(_expressionFilterOperator._dataSourceMap, docIdSetOperator); TransformOperator operator = diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java index 52b6fcb..52df943 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java @@ -170,6 +170,18 @@ public abstract class BaseClusterIntegrationTestSet extends BaseClusterIntegrati String query; List<String> h2queries; query = + "SELECT COUNT(*) FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay LIMIT 1"; + testSqlQuery(query, Collections.singletonList(query)); + query = + "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000"; + testSqlQuery(query, Collections.singletonList(query)); + query = + "SELECT COUNT(*) FROM mytable WHERE ArrDelay > CarrierDelay LIMIT 1"; + testSqlQuery(query, Collections.singletonList(query)); + query = + "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000"; + testSqlQuery(query, Collections.singletonList(query)); + query = "SELECT count(*) FROM mytable WHERE AirlineID > 20355 AND OriginState BETWEEN 'PA' AND 'DE' AND DepTime <> 2202 LIMIT 21"; testSqlQuery(query, Collections.singletonList(query)); query = --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org