This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3168194 Implement NOT Operator (#8148)
3168194 is described below
commit 31681947ca1952e6ac226c99b91589b4c6bc4942
Author: Atri Sharma <[email protected]>
AuthorDate: Mon Feb 21 14:16:06 2022 +0530
Implement NOT Operator (#8148)
This PR implements the NOT operator.
NOT operator is implemented in a generic manner i.e. it can support most
underlying operators.
The operator operates by "skipping" documents that are valid for the
underlying predicate and iterating over all other valid docIDs.
Support for LIKE operator is not present yet due to Calcite's slightly
different parsing of the LIKE operator.
---
.../common/request/context/FilterContext.java | 2 +-
.../request/context/RequestContextUtils.java | 3 +
.../pinot/pql/parsers/pql2/ast/FilterKind.java | 1 +
.../operator/dociditerators/NotDocIdIterator.java | 66 ++++++
.../pinot/core/operator/docidsets/NotDocIdSet.java | 49 ++--
.../core/operator/filter/FilterOperatorUtils.java | 17 ++
.../core/operator/filter/NotFilterOperator.java | 64 ++++++
.../org/apache/pinot/core/plan/FilterPlanNode.java | 5 +
.../optimizer/filter/NumericalFilterOptimizer.java | 1 +
.../dociditerators/NotDocIdIteratorTest.java | 103 +++++++++
.../operator/filter/NotFilterOperatorTest.java | 46 ++++
.../filter/NumericalFilterOptimizerTest.java | 6 +
.../pinot/queries/NotOperatorQueriesTest.java | 247 +++++++++++++++++++++
13 files changed, 581 insertions(+), 29 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/FilterContext.java
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/FilterContext.java
index 90e12d2..9ade2a8 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/FilterContext.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/FilterContext.java
@@ -30,7 +30,7 @@ import
org.apache.pinot.common.request.context.predicate.Predicate;
*/
public class FilterContext {
public enum Type {
- AND, OR, PREDICATE
+ AND, OR, NOT, PREDICATE
}
private final Type _type;
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
index 3620ef0..ccc4589 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
@@ -183,6 +183,9 @@ public class RequestContextUtils {
children.add(getFilter(operand));
}
return new FilterContext(FilterContext.Type.OR, children, null);
+ case NOT:
+ assert numOperands == 1;
+ return new FilterContext(FilterContext.Type.NOT,
Collections.singletonList(getFilter(operands.get(0))), null);
case EQUALS:
return new FilterContext(FilterContext.Type.PREDICATE, null,
new EqPredicate(getExpression(operands.get(0)),
getStringValue(operands.get(1))));
diff --git
a/pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java
b/pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java
index ff49eb3..601b639 100644
---
a/pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java
+++
b/pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java
@@ -21,6 +21,7 @@ package org.apache.pinot.pql.parsers.pql2.ast;
public enum FilterKind {
AND,
OR,
+ NOT,
EQUALS,
NOT_EQUALS,
GREATER_THAN,
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/NotDocIdIterator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/NotDocIdIterator.java
new file mode 100644
index 0000000..5ce0de3
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/NotDocIdIterator.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.dociditerators;
+
+import org.apache.pinot.core.common.BlockDocIdIterator;
+import org.apache.pinot.segment.spi.Constants;
+
+
+/**
+ * The iterator performs a linear pass through the underlying child iterator
and returns
+ * the complement of the result set.
+ */
+public class NotDocIdIterator implements BlockDocIdIterator {
+ private final BlockDocIdIterator _childDocIdIterator;
+ private final int _numDocs;
+ private int _nextDocId;
+ private int _nextNonMatchingDocId;
+
+ public NotDocIdIterator(BlockDocIdIterator childDocIdIterator, int numDocs) {
+ _childDocIdIterator = childDocIdIterator;
+ _nextDocId = 0;
+
+ int currentDocIdFromChildIterator = childDocIdIterator.next();
+ _nextNonMatchingDocId = currentDocIdFromChildIterator == Constants.EOF ?
numDocs : currentDocIdFromChildIterator;
+ _numDocs = numDocs;
+ }
+
+ @Override
+ public int next() {
+ while (_nextDocId == _nextNonMatchingDocId) {
+ _nextDocId++;
+ int nextNonMatchingDocId = _childDocIdIterator.next();
+ _nextNonMatchingDocId = nextNonMatchingDocId == Constants.EOF ? _numDocs
: nextNonMatchingDocId;
+ }
+ if (_nextDocId >= _numDocs) {
+ return Constants.EOF;
+ }
+ return _nextDocId++;
+ }
+
+ @Override
+ public int advance(int targetDocId) {
+ _nextDocId = targetDocId;
+ if (targetDocId > _nextNonMatchingDocId) {
+ int nextNonMatchingDocId = _childDocIdIterator.advance(targetDocId);
+ _nextNonMatchingDocId = nextNonMatchingDocId == Constants.EOF ? _numDocs
: nextNonMatchingDocId;
+ }
+ return next();
+ }
+}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/NotDocIdSet.java
similarity index 53%
copy from
pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java
copy to
pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/NotDocIdSet.java
index ff49eb3..1fe6462 100644
---
a/pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/NotDocIdSet.java
@@ -16,35 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.pql.parsers.pql2.ast;
+package org.apache.pinot.core.operator.docidsets;
-public enum FilterKind {
- AND,
- OR,
- EQUALS,
- NOT_EQUALS,
- GREATER_THAN,
- GREATER_THAN_OR_EQUAL,
- LESS_THAN,
- LESS_THAN_OR_EQUAL,
- LIKE,
- BETWEEN,
- RANGE,
- IN,
- NOT_IN,
- REGEXP_LIKE,
- IS_NULL,
- IS_NOT_NULL,
- TEXT_MATCH,
- JSON_MATCH;
+import org.apache.pinot.core.common.BlockDocIdIterator;
+import org.apache.pinot.core.operator.dociditerators.NotDocIdIterator;
- /**
- * Helper method that returns true if the enum maps to a Range.
- *
- * @return True if the enum is of Range type, false otherwise.
- */
- public boolean isRange() {
- return this == GREATER_THAN || this == GREATER_THAN_OR_EQUAL || this ==
LESS_THAN || this == LESS_THAN_OR_EQUAL
- || this == BETWEEN || this == RANGE;
+
+public class NotDocIdSet implements FilterBlockDocIdSet {
+ private final FilterBlockDocIdSet _childDocIdSet;
+ private final int _numDocs;
+
+ public NotDocIdSet(FilterBlockDocIdSet childDocIdSet, int numDocs) {
+ _childDocIdSet = childDocIdSet;
+ _numDocs = numDocs;
+ }
+
+ @Override
+ public BlockDocIdIterator iterator() {
+ return new NotDocIdIterator(_childDocIdSet.iterator(), _numDocs);
+ }
+
+ @Override
+ public long getNumEntriesScannedInFilter() {
+ return _childDocIdSet.getNumEntriesScannedInFilter();
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
index c1c3031..4b6feb0 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
@@ -135,6 +135,20 @@ public class FilterOperatorUtils {
}
/**
+ * Returns the NOT filter operator or equivalent filter operator.
+ */
+ public static BaseFilterOperator getNotFilterOperator(BaseFilterOperator
filterOperator, int numDocs,
+ @Nullable Map<String, String> debugOptions) {
+ if (filterOperator.isResultMatchingAll()) {
+ return EmptyFilterOperator.getInstance();
+ } else if (filterOperator.isResultEmpty()) {
+ return new MatchAllFilterOperator(numDocs);
+ }
+
+ return new NotFilterOperator(filterOperator, numDocs);
+ }
+
+ /**
* For AND filter operator, reorders its child filter operators based on the
their cost and puts the ones with
* inverted index first in order to reduce the number of documents to be
processed.
* <p>Special filter operators such as {@link MatchAllFilterOperator} and
{@link EmptyFilterOperator} should be
@@ -165,6 +179,9 @@ public class FilterOperatorUtils {
if (filterOperator instanceof OrFilterOperator) {
return 4;
}
+ if (filterOperator instanceof NotFilterOperator) {
+ return getPriority((NotFilterOperator) ((NotFilterOperator)
filterOperator).getChildFilterOperator());
+ }
if (filterOperator instanceof ScanBasedFilterOperator) {
return getScanBasedFilterPriority((ScanBasedFilterOperator)
filterOperator, 5, debugOptions);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/NotFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/NotFilterOperator.java
new file mode 100644
index 0000000..50a03b8
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/NotFilterOperator.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.filter;
+
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.pinot.core.common.Operator;
+import org.apache.pinot.core.operator.blocks.FilterBlock;
+import org.apache.pinot.core.operator.docidsets.NotDocIdSet;
+
+
+public class NotFilterOperator extends BaseFilterOperator {
+ private static final String OPERATOR_NAME = "NotFilterOperator";
+ private static final String EXPLAIN_NAME = "FILTER_NOT";
+ private final BaseFilterOperator _filterOperator;
+ private final int _numDocs;
+
+ public NotFilterOperator(BaseFilterOperator filterOperator, int numDocs) {
+ _filterOperator = filterOperator;
+ _numDocs = numDocs;
+ }
+
+ @Override
+ public String getOperatorName() {
+ return OPERATOR_NAME;
+ }
+
+ @Override
+ public List<Operator> getChildOperators() {
+ return Collections.singletonList(_filterOperator);
+ }
+
+ @Nullable
+ @Override
+ public String toExplainString() {
+ return EXPLAIN_NAME;
+ }
+
+ @Override
+ protected FilterBlock getNextBlock() {
+ return new FilterBlock(new
NotDocIdSet(_filterOperator.nextBlock().getBlockDocIdSet(), _numDocs));
+ }
+
+ public Operator getChildFilterOperator() {
+ return _filterOperator;
+ }
+}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
index 014a35d..d0b5300 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
@@ -174,6 +174,11 @@ public class FilterPlanNode implements PlanNode {
}
}
return FilterOperatorUtils.getOrFilterOperator(childFilterOperators,
_numDocs, _queryContext.getDebugOptions());
+ case NOT:
+ childFilters = filter.getChildren();
+ assert childFilters.size() == 1;
+ BaseFilterOperator childFilterOperator =
constructPhysicalOperator(childFilters.get(0));
+ return FilterOperatorUtils.getNotFilterOperator(childFilterOperator,
_numDocs, null);
case PREDICATE:
Predicate predicate = filter.getPredicate();
ExpressionContext lhs = predicate.getLhs();
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/NumericalFilterOptimizer.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/NumericalFilterOptimizer.java
index d8385cd..16b4e57 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/NumericalFilterOptimizer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/NumericalFilterOptimizer.java
@@ -87,6 +87,7 @@ public class NumericalFilterOptimizer implements
FilterOptimizer {
switch (kind) {
case AND:
case OR:
+ case NOT:
// Recursively traverse the expression tree to find an operator node
that can be rewritten.
operands.forEach(operand -> optimize(operand, schema));
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/dociditerators/NotDocIdIteratorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/dociditerators/NotDocIdIteratorTest.java
new file mode 100644
index 0000000..6f16302
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/dociditerators/NotDocIdIteratorTest.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.dociditerators;
+
+import org.apache.pinot.core.common.BlockDocIdIterator;
+import org.apache.pinot.segment.spi.Constants;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+
+public class NotDocIdIteratorTest {
+ @Test
+ public void testNotDocIdIterator() {
+ // OR result: [0, 1, 2, 4, 5, 6, 8, 10, 13, 15, 16, 17, 18, 19, 20]
+ int[] docIds1 = new int[]{1, 4, 6, 10, 15, 17, 18, 20};
+ int[] docIds2 = new int[]{0, 1, 5, 8, 15, 18};
+ int[] docIds3 = new int[]{1, 2, 6, 13, 16, 19};
+ int[] docIds4 = new int[]{0, 1, 2, 3, 4, 5};
+
+ MutableRoaringBitmap bitmap1 = new MutableRoaringBitmap();
+ bitmap1.add(docIds1);
+ MutableRoaringBitmap bitmap2 = new MutableRoaringBitmap();
+ bitmap2.add(docIds2);
+ MutableRoaringBitmap bitmap3 = new MutableRoaringBitmap();
+ bitmap3.add(docIds3);
+ MutableRoaringBitmap bitmap4 = new MutableRoaringBitmap();
+ bitmap4.add(docIds4);
+ OrDocIdIterator orDocIdIterator = new OrDocIdIterator(new
BlockDocIdIterator[]{
+ new RangelessBitmapDocIdIterator(bitmap1), new
RangelessBitmapDocIdIterator(
+ bitmap2), new RangelessBitmapDocIdIterator(bitmap3)
+ });
+ NotDocIdIterator notDocIdIterator = new NotDocIdIterator(new
RangelessBitmapDocIdIterator(bitmap1), 25);
+
+ assertEquals(notDocIdIterator.advance(1), 2);
+ assertEquals(notDocIdIterator.next(), 3);
+ assertEquals(notDocIdIterator.next(), 5);
+ assertEquals(notDocIdIterator.advance(7), 7);
+ assertEquals(notDocIdIterator.advance(13), 13);
+ assertEquals(notDocIdIterator.next(), 14);
+ assertEquals(notDocIdIterator.advance(18), 19);
+ assertEquals(notDocIdIterator.advance(21), 21);
+ assertEquals(notDocIdIterator.advance(26), Constants.EOF);
+
+ notDocIdIterator = new NotDocIdIterator(new
RangelessBitmapDocIdIterator(bitmap1), 25);
+ assertEquals(notDocIdIterator.next(), 0);
+ assertEquals(notDocIdIterator.next(), 2);
+ assertEquals(notDocIdIterator.next(), 3);
+ assertEquals(notDocIdIterator.next(), 5);
+ assertEquals(notDocIdIterator.next(), 7);
+ assertEquals(notDocIdIterator.next(), 8);
+ assertEquals(notDocIdIterator.next(), 9);
+ assertEquals(notDocIdIterator.next(), 11);
+ assertEquals(notDocIdIterator.next(), 12);
+ assertEquals(notDocIdIterator.next(), 13);
+ assertEquals(notDocIdIterator.next(), 14);
+ assertEquals(notDocIdIterator.next(), 16);
+ assertEquals(notDocIdIterator.next(), 19);
+ assertEquals(notDocIdIterator.next(), 21);
+ assertEquals(notDocIdIterator.next(), 22);
+ assertEquals(notDocIdIterator.next(), 23);
+ assertEquals(notDocIdIterator.next(), 24);
+ assertEquals(notDocIdIterator.next(), Constants.EOF);
+
+ notDocIdIterator = new NotDocIdIterator(orDocIdIterator, 25);
+ assertEquals(notDocIdIterator.next(), 3);
+ assertEquals(notDocIdIterator.next(), 7);
+ assertEquals(notDocIdIterator.next(), 9);
+ assertEquals(notDocIdIterator.next(), 11);
+ assertEquals(notDocIdIterator.next(), 12);
+ assertEquals(notDocIdIterator.next(), 14);
+ assertEquals(notDocIdIterator.next(), 21);
+ assertEquals(notDocIdIterator.next(), 22);
+ assertEquals(notDocIdIterator.next(), 23);
+ assertEquals(notDocIdIterator.next(), 24);
+ assertEquals(notDocIdIterator.next(), Constants.EOF);
+
+ notDocIdIterator = new NotDocIdIterator(new
RangelessBitmapDocIdIterator(bitmap4), 6);
+ assertEquals(notDocIdIterator.next(), Constants.EOF);
+
+ notDocIdIterator = new NotDocIdIterator(new
RangelessBitmapDocIdIterator(bitmap4), 9);
+ assertEquals(notDocIdIterator.next(), 6);
+ assertEquals(notDocIdIterator.next(), 7);
+ assertEquals(notDocIdIterator.next(), 8);
+ }
+}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/NotFilterOperatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/NotFilterOperatorTest.java
new file mode 100644
index 0000000..893389b
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/NotFilterOperatorTest.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.filter;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import org.apache.pinot.core.common.BlockDocIdIterator;
+import org.apache.pinot.segment.spi.Constants;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class NotFilterOperatorTest {
+
+ @Test
+ public void testNotOperator() {
+ int[] docIds1 = new int[]{2, 3, 10, 15, 16, 17, 18, 21, 22, 23, 24, 26,
28};
+ Set<Integer> expectedResult = new HashSet();
+ expectedResult.addAll(Arrays.asList(0, 1, 4, 5, 6, 7, 8, 9, 11, 12, 13,
14, 19, 20, 25, 27, 29));
+ Iterator<Integer> expectedIterator = expectedResult.iterator();
+ NotFilterOperator notFilterOperator = new NotFilterOperator(new
TestFilterOperator(docIds1), 30);
+ BlockDocIdIterator iterator =
notFilterOperator.nextBlock().getBlockDocIdSet().iterator();
+ int docId;
+ while ((docId = iterator.next()) != Constants.EOF) {
+ Assert.assertEquals(docId, expectedIterator.next().intValue());
+ }
+ }
+}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/NumericalFilterOptimizerTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/NumericalFilterOptimizerTest.java
index b54adc3..f7c467f 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/NumericalFilterOptimizerTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/NumericalFilterOptimizerTest.java
@@ -48,6 +48,12 @@ public class NumericalFilterOptimizerTest {
"Expression(type:FUNCTION, functionCall:Function(operator:EQUALS,
operands:[Expression(type:IDENTIFIER, "
+ "identifier:Identifier(name:intColumn)),
Expression(type:LITERAL, literal:<Literal longValue:5>)]))");
+ // Test a query containing NOT operator
+ Assert.assertEquals(rewrite("SELECT * FROM testTable WHERE NOT intColumn =
5.0"),
+ "Expression(type:FUNCTION, functionCall:Function(operator:NOT,
operands:[Expression(type:FUNCTION, "
+ + "functionCall:Function(operator:EQUALS,
operands:[Expression(type:IDENTIFIER, identifier:Identifier"
+ + "(name:intColumn)), Expression(type:LITERAL, literal:<Literal
longValue:5>)]))]))");
+
// Test int column equals invalid decimal value.
Assert.assertEquals(rewrite("SELECT * FROM testTable WHERE intColumn =
5.5"),
"Expression(type:LITERAL, literal:<Literal boolValue:false>)");
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/NotOperatorQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/NotOperatorQueriesTest.java
new file mode 100644
index 0000000..ca00dd5
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/NotOperatorQueriesTest.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.response.broker.BrokerResponseNative;
+import org.apache.pinot.common.response.broker.ResultTable;
+import org.apache.pinot.core.common.Operator;
+import org.apache.pinot.core.operator.blocks.IntermediateResultsBlock;
+import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.spi.config.table.FSTType;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.data.readers.RecordReader;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class NotOperatorQueriesTest extends BaseQueriesTest {
+ private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(),
"NotOperatorQueriesTest");
+ private static final String TABLE_NAME = "MyTable";
+ private static final String SEGMENT_NAME = "testSegment";
+ private static final String FIRST_INT_COL_NAME = "FIRST_INT_COL";
+ private static final String SECOND_INT_COL_NAME = "SECOND_INT_COL";
+ private static final String DOMAIN_NAMES_COL = "DOMAIN_NAMES";
+ private static final Integer INT_BASE_VALUE = 1000;
+ private static final Integer NUM_ROWS = 1024;
+
+ private IndexSegment _indexSegment;
+ private List<IndexSegment> _indexSegments;
+
+ @Override
+ protected String getFilter() {
+ return "";
+ }
+
+ @Override
+ protected IndexSegment getIndexSegment() {
+ return _indexSegment;
+ }
+
+ @Override
+ protected List<IndexSegment> getIndexSegments() {
+ return _indexSegments;
+ }
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ FileUtils.deleteQuietly(INDEX_DIR);
+
+ List<IndexSegment> segments = new ArrayList<>();
+ buildSegment();
+ IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
+ Set<String> invertedIndexCols = new HashSet<>();
+ invertedIndexCols.add(FIRST_INT_COL_NAME);
+ indexLoadingConfig.setInvertedIndexColumns(invertedIndexCols);
+ Set<String> fstIndexCols = new HashSet<>();
+ fstIndexCols.add(DOMAIN_NAMES_COL);
+ indexLoadingConfig.setFSTIndexColumns(fstIndexCols);
+ indexLoadingConfig.setFSTIndexType(FSTType.LUCENE);
+ ImmutableSegment segment = ImmutableSegmentLoader.load(new File(INDEX_DIR,
SEGMENT_NAME), indexLoadingConfig);
+ segments.add(segment);
+
+ _indexSegment = segment;
+ _indexSegments = segments;
+ }
+
+ private List<String> getDomainNames() {
+ return Arrays.asList("www.domain1.com", "www.domain1.co.ab",
"www.domain1.co.bc", "www.domain1.co.cd",
+ "www.sd.domain1.com", "www.sd.domain1.co.ab", "www.sd.domain1.co.bc",
"www.sd.domain1.co.cd", "www.domain2.com",
+ "www.domain2.co.ab", "www.domain2.co.bc", "www.domain2.co.cd",
"www.sd.domain2.com", "www.sd.domain2.co.ab",
+ "www.sd.domain2.co.bc", "www.sd.domain2.co.cd");
+ }
+
+ private List<GenericRow> createTestData(int numRows) {
+ List<GenericRow> rows = new ArrayList<>();
+ List<String> domainNames = getDomainNames();
+ for (int i = 0; i < numRows; i++) {
+ String domain = domainNames.get(i % domainNames.size());
+ GenericRow row = new GenericRow();
+ row.putField(FIRST_INT_COL_NAME, i);
+ row.putField(SECOND_INT_COL_NAME, INT_BASE_VALUE + i);
+ row.putField(DOMAIN_NAMES_COL, domain);
+ rows.add(row);
+ }
+ return rows;
+ }
+
+ private void buildSegment()
+ throws Exception {
+ List<GenericRow> rows = createTestData(NUM_ROWS);
+ List<FieldConfig> fieldConfigs = new ArrayList<>();
+ fieldConfigs.add(
+ new FieldConfig(DOMAIN_NAMES_COL, FieldConfig.EncodingType.DICTIONARY,
FieldConfig.IndexType.FST, null, null));
+ fieldConfigs.add(
+ new FieldConfig(FIRST_INT_COL_NAME,
FieldConfig.EncodingType.DICTIONARY, FieldConfig.IndexType.INVERTED, null,
+ null));
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+
.setInvertedIndexColumns(Arrays.asList(FIRST_INT_COL_NAME)).setFieldConfigList(fieldConfigs).build();
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension(FIRST_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(DOMAIN_NAMES_COL, FieldSpec.DataType.STRING)
+ .addMetric(SECOND_INT_COL_NAME, FieldSpec.DataType.INT).build();
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ config.setOutDir(INDEX_DIR.getPath());
+ config.setTableName(TABLE_NAME);
+ config.setSegmentName(SEGMENT_NAME);
+
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ try (RecordReader recordReader = new GenericRowRecordReader(rows)) {
+ driver.init(config, recordReader);
+ driver.build();
+ }
+ }
+
+ @AfterClass
+ public void tearDown() {
+ _indexSegment.destroy();
+ FileUtils.deleteQuietly(INDEX_DIR);
+ }
+
+ private void testSelectionResults(String query, int expectedResultSize,
List<Serializable[]> expectedResults) {
+ Operator<IntermediateResultsBlock> operator =
getOperatorForSqlQuery(query);
+ IntermediateResultsBlock operatorResult = operator.nextBlock();
+ List<Object[]> resultset = (List<Object[]>)
operatorResult.getSelectionResult();
+ Assert.assertNotNull(resultset);
+ Assert.assertEquals(resultset.size(), expectedResultSize);
+ if (expectedResults != null) {
+ for (int i = 0; i < expectedResultSize; i++) {
+ Object[] actualRow = resultset.get(i);
+ Object[] expectedRow = expectedResults.get(i);
+ Assert.assertEquals(actualRow.length, expectedRow.length);
+ for (int j = 0; j < actualRow.length; j++) {
+ Object actualColValue = actualRow[j];
+ Object expectedColValue = expectedRow[j];
+ Assert.assertEquals(actualColValue, expectedColValue);
+ }
+ }
+ }
+ }
+
+ private void testOptimizedQueryHelper(String query, int expectedResultSize,
List<Serializable[]> expectedResults) {
+ BrokerResponseNative brokerResponseNative =
getBrokerResponseForOptimizedSqlQuery(query, null);
+ ResultTable resultTable = brokerResponseNative.getResultTable();
+ List<Object[]> results = resultTable.getRows();
+ Assert.assertEquals(results.size(), expectedResultSize);
+
+ if (expectedResults != null) {
+ for (int i = 0; i < expectedResults.size(); i++) {
+ Object[] actualRow = results.get(i);
+ Serializable[] expectedRow = expectedResults.get(i);
+ Assert.assertEquals(actualRow, expectedRow);
+ }
+ }
+ }
+
+ @Test
+ public void testLikeBasedNotOperator() {
+ String query =
+ "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
REGEXP_LIKE(DOMAIN_NAMES, 'www.domain1.*') LIMIT "
+ + "50000";
+ testSelectionResults(query, 768, null);
+ testOptimizedQueryHelper(query, 1536, null);
+
+ query = "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
REGEXP_LIKE(DOMAIN_NAMES, 'www.sd.domain1.*') "
+ + "LIMIT 50000";
+ testSelectionResults(query, 768, null);
+ testOptimizedQueryHelper(query, 1536, null);
+
+ query =
+ "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
REGEXP_LIKE(DOMAIN_NAMES, '.*domain1.*') LIMIT "
+ + "50000";
+ testSelectionResults(query, 512, null);
+ testOptimizedQueryHelper(query, 1024, null);
+
+ query = "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
REGEXP_LIKE(DOMAIN_NAMES, '.*domain.*') LIMIT "
+ + "50000";
+ testSelectionResults(query, 0, null);
+ testOptimizedQueryHelper(query, 0, null);
+
+ query =
+ "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
REGEXP_LIKE(DOMAIN_NAMES, '.*com') LIMIT 50000";
+ testSelectionResults(query, 768, null);
+ testOptimizedQueryHelper(query, 1536, null);
+ }
+
+ @Test
+ public void testWeirdPredicates() {
+ String query = "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE
NOT FIRST_INT_COL = 5 LIMIT 50000";
+ testSelectionResults(query, 1023, null);
+
+ query = "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
FIRST_INT_COL < 5 LIMIT " + "50000";
+ testSelectionResults(query, 1019, null);
+
+ query = "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
FIRST_INT_COL > 5 LIMIT " + "50000";
+ testSelectionResults(query, 6, null);
+ }
+
+ @Test
+ public void testCompositePredicates() {
+ String query =
+ "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
(FIRST_INT_COL > 5 AND SECOND_INT_COL < 1009) "
+ + "LIMIT " + "50000";
+ testSelectionResults(query, 1021, null);
+
+ query = "SELECT FIRST_INT_COL, SECOND_INT_COL FROM MyTable WHERE NOT
(FIRST_INT_COL < 5 OR SECOND_INT_COL > 2000) "
+ + "LIMIT " + "50000";
+ testSelectionResults(query, 996, null);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
