This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new bde7223cf29 Implement canProduceBitmaps and getBitmaps for AND and OR
filter operators (#19038)
bde7223cf29 is described below
commit bde7223cf2973b93bb9bce3a8f03d356411baa78
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Tue Jul 21 23:58:08 2026 -0700
Implement canProduceBitmaps and getBitmaps for AND and OR filter operators
(#19038)
---
.../core/operator/filter/AndFilterOperator.java | 25 +++-
.../core/operator/filter/OrFilterOperator.java | 25 +++-
.../operator/filter/AndFilterOperatorTest.java | 155 +++++++++++++++------
.../core/operator/filter/OrFilterOperatorTest.java | 136 +++++++++++++-----
4 files changed, 255 insertions(+), 86 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/AndFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/AndFilterOperator.java
index e47cff90589..53ed6e4b53f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/AndFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/AndFilterOperator.java
@@ -103,11 +103,7 @@ public class AndFilterOperator extends BaseFilterOperator {
@Override
public boolean canOptimizeCount() {
- boolean allChildrenCanProduceBitmaps = true;
- for (BaseFilterOperator child : _filterOperators) {
- allChildrenCanProduceBitmaps &= child.canProduceBitmaps();
- }
- return allChildrenCanProduceBitmaps;
+ return canProduceBitmaps();
}
@Override
@@ -123,6 +119,25 @@ public class AndFilterOperator extends BaseFilterOperator {
return BufferFastAggregation.andCardinality(bitmaps);
}
+ @Override
+ public boolean canProduceBitmaps() {
+ for (BaseFilterOperator child : _filterOperators) {
+ if (!child.canProduceBitmaps()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public BitmapCollection getBitmaps() {
+ ImmutableRoaringBitmap[] bitmaps = new
ImmutableRoaringBitmap[_filterOperators.size()];
+ for (int i = 0; i < _filterOperators.size(); i++) {
+ bitmaps[i] = _filterOperators.get(i).getBitmaps().reduce();
+ }
+ return new BitmapCollection(_numDocs, false,
BufferFastAggregation.and(bitmaps));
+ }
+
@Override
public List<Operator> getChildOperators() {
return new ArrayList<>(_filterOperators);
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/OrFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/OrFilterOperator.java
index 1aa635d3fc6..e2c92bfd0c0 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/OrFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/OrFilterOperator.java
@@ -112,11 +112,7 @@ public class OrFilterOperator extends BaseFilterOperator {
@Override
public boolean canOptimizeCount() {
- boolean allChildrenProduceBitmaps = true;
- for (BaseFilterOperator child : _filterOperators) {
- allChildrenProduceBitmaps &= child.canProduceBitmaps();
- }
- return allChildrenProduceBitmaps;
+ return canProduceBitmaps();
}
@Override
@@ -130,4 +126,23 @@ public class OrFilterOperator extends BaseFilterOperator {
}
return BufferFastAggregation.orCardinality(bitmaps);
}
+
+ @Override
+ public boolean canProduceBitmaps() {
+ for (BaseFilterOperator child : _filterOperators) {
+ if (!child.canProduceBitmaps()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public BitmapCollection getBitmaps() {
+ ImmutableRoaringBitmap[] bitmaps = new
ImmutableRoaringBitmap[_filterOperators.size()];
+ for (int i = 0; i < _filterOperators.size(); i++) {
+ bitmaps[i] = _filterOperators.get(i).getBitmaps().reduce();
+ }
+ return new BitmapCollection(_numDocs, false,
BufferFastAggregation.or(bitmaps));
+ }
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/AndFilterOperatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/AndFilterOperatorTest.java
index ad4b10cba50..a939447a41f 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/AndFilterOperatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/AndFilterOperatorTest.java
@@ -19,16 +19,18 @@
package org.apache.pinot.core.operator.filter;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import org.apache.pinot.core.common.BlockDocIdIterator;
import org.apache.pinot.core.operator.docidsets.EmptyDocIdSet;
import org.apache.pinot.core.operator.docidsets.MatchAllDocIdSet;
import org.apache.pinot.segment.spi.Constants;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
-import org.testng.Assert;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
public class AndFilterOperatorTest {
@@ -44,9 +46,9 @@ public class AndFilterOperatorTest {
AndFilterOperator andOperator = new AndFilterOperator(operators, null,
numDocs, false);
BlockDocIdIterator iterator =
andOperator.nextBlock().getBlockDocIdSet().iterator();
- Assert.assertEquals(iterator.next(), 3);
- Assert.assertEquals(iterator.next(), 28);
- Assert.assertEquals(iterator.next(), Constants.EOF);
+ assertEquals(iterator.next(), 3);
+ assertEquals(iterator.next(), 28);
+ assertEquals(iterator.next(), Constants.EOF);
}
@Test
@@ -63,9 +65,9 @@ public class AndFilterOperatorTest {
AndFilterOperator andOperator = new AndFilterOperator(operators, null,
numDocs, false);
BlockDocIdIterator iterator =
andOperator.nextBlock().getBlockDocIdSet().iterator();
- Assert.assertEquals(iterator.next(), 3);
- Assert.assertEquals(iterator.next(), 6);
- Assert.assertEquals(iterator.next(), Constants.EOF);
+ assertEquals(iterator.next(), 3);
+ assertEquals(iterator.next(), 6);
+ assertEquals(iterator.next(), Constants.EOF);
}
@Test
@@ -86,9 +88,9 @@ public class AndFilterOperatorTest {
AndFilterOperator andOperator = new AndFilterOperator(operators, null,
numDocs, false);
BlockDocIdIterator iterator =
andOperator.nextBlock().getBlockDocIdSet().iterator();
- Assert.assertEquals(iterator.next(), 3);
- Assert.assertEquals(iterator.next(), 6);
- Assert.assertEquals(iterator.next(), Constants.EOF);
+ assertEquals(iterator.next(), 3);
+ assertEquals(iterator.next(), 6);
+ assertEquals(iterator.next(), Constants.EOF);
}
@Test
@@ -122,18 +124,18 @@ public class AndFilterOperatorTest {
AndFilterOperator andFilterOperator2 = new
AndFilterOperator(childOperators2, null, numDocs, false);
BlockDocIdIterator iterator1 =
andFilterOperator1.getNextBlock().getBlockDocIdSet().iterator();
BlockDocIdIterator iterator2 =
andFilterOperator2.getNextBlock().getBlockDocIdSet().iterator();
- Assert.assertEquals(iterator1.next(), 0);
- Assert.assertEquals(iterator1.next(), 60);
- Assert.assertEquals(iterator1.next(), 120);
- Assert.assertEquals(iterator1.next(), 180);
+ assertEquals(iterator1.next(), 0);
+ assertEquals(iterator1.next(), 60);
+ assertEquals(iterator1.next(), 120);
+ assertEquals(iterator1.next(), 180);
- Assert.assertEquals(iterator2.next(), 0);
- Assert.assertEquals(iterator2.next(), 60);
- Assert.assertEquals(iterator2.next(), 120);
- Assert.assertEquals(iterator2.next(), 180);
+ assertEquals(iterator2.next(), 0);
+ assertEquals(iterator2.next(), 60);
+ assertEquals(iterator2.next(), 120);
+ assertEquals(iterator2.next(), 180);
for (int i = 0; i < numDocs / 10; i++) {
- Assert.assertEquals(iterator1.next(), iterator2.next());
+ assertEquals(iterator1.next(), iterator2.next());
}
}
@@ -155,11 +157,11 @@ public class AndFilterOperatorTest {
AndFilterOperator andOperator = new AndFilterOperator(operators, null,
numDocs, false);
BlockDocIdIterator iterator =
andOperator.nextBlock().getBlockDocIdSet().iterator();
- Assert.assertEquals(iterator.next(), 2);
- Assert.assertEquals(iterator.next(), 3);
- Assert.assertEquals(iterator.next(), 6);
- Assert.assertEquals(iterator.next(), 28);
- Assert.assertEquals(iterator.next(), Constants.EOF);
+ assertEquals(iterator.next(), 2);
+ assertEquals(iterator.next(), 3);
+ assertEquals(iterator.next(), 6);
+ assertEquals(iterator.next(), 28);
+ assertEquals(iterator.next(), Constants.EOF);
}
@Test
@@ -171,11 +173,11 @@ public class AndFilterOperatorTest {
int[] nullDocIds2 = new int[]{3, 4, 5, 6, 7};
AndFilterOperator andFilterOperator = new AndFilterOperator(
- Arrays.asList(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
+ List.of(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
new TestFilterOperator(docIds2, nullDocIds2, numDocs)), null,
numDocs, true);
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()),
List.of(1, 2));
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of(0, 7, 8, 9));
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()), List.of(1,
2));
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of(0, 7, 8, 9));
}
@Test
@@ -187,11 +189,11 @@ public class AndFilterOperatorTest {
int[] nullDocIds2 = new int[]{};
AndFilterOperator andFilterOperator = new AndFilterOperator(
- Arrays.asList(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
+ List.of(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
new TestFilterOperator(docIds2, nullDocIds2, numDocs)), null,
numDocs, false);
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()),
List.of(0));
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of(1, 2, 3));
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()),
List.of(0));
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of(1, 2, 3));
}
@Test
@@ -201,11 +203,11 @@ public class AndFilterOperatorTest {
int[] nullDocIds1 = new int[]{4, 5, 6};
AndFilterOperator andFilterOperator = new AndFilterOperator(
- Arrays.asList(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
EmptyFilterOperator.getInstance()), null,
+ List.of(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
EmptyFilterOperator.getInstance()), null,
numDocs, true);
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()),
List.of());
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()), List.of());
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
}
@@ -216,23 +218,23 @@ public class AndFilterOperatorTest {
int[] nullDocIds1 = new int[]{4, 5, 6};
AndFilterOperator andFilterOperator = new AndFilterOperator(
- Arrays.asList(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
new MatchAllFilterOperator(numDocs)), null,
+ List.of(new TestFilterOperator(docIds1, nullDocIds1, numDocs), new
MatchAllFilterOperator(numDocs)), null,
numDocs, true);
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()),
List.of(1, 2, 3));
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of(0, 7, 8, 9));
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()), List.of(1,
2, 3));
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of(0, 7, 8, 9));
}
@Test
public void testAndWithAllMatchesAll() {
int numDocs = 10;
AndFilterOperator andFilterOperator =
- new AndFilterOperator(Arrays.asList(new
MatchAllFilterOperator(numDocs), new MatchAllFilterOperator(numDocs)),
+ new AndFilterOperator(List.of(new MatchAllFilterOperator(numDocs), new
MatchAllFilterOperator(numDocs)),
null, numDocs, true);
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()),
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getTrues()),
List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
- Assert.assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of());
+ assertEquals(TestUtils.getDocIds(andFilterOperator.getFalses()),
List.of());
}
@Test
@@ -242,12 +244,12 @@ public class AndFilterOperatorTest {
int[] emptyDocIds = new int[0];
AndFilterOperator andFilterOperator = new AndFilterOperator(
- Arrays.asList(
+ List.of(
new TestFilterOperator(regularDocIds, numDocs),
new TestFilterOperator(emptyDocIds, numDocs)
), null, numDocs, false);
- Assert.assertEquals((andFilterOperator.getTrues()).getOptimizedDocIdSet(),
EmptyDocIdSet.getInstance());
+ assertEquals((andFilterOperator.getTrues()).getOptimizedDocIdSet(),
EmptyDocIdSet.getInstance());
}
@Test
@@ -256,11 +258,76 @@ public class AndFilterOperatorTest {
int numDocs2 = 50;
AndFilterOperator andFilterOperator = new AndFilterOperator(
- Arrays.asList(
+ List.of(
new MatchAllFilterOperator(numDocs),
new MatchAllFilterOperator(numDocs2)
), null, numDocs, false);
- Assert.assertTrue(andFilterOperator.getTrues() instanceof
MatchAllDocIdSet);
+ assertTrue(andFilterOperator.getTrues() instanceof MatchAllDocIdSet);
+ }
+
+ @Test
+ public void testCanProduceBitmapsWhenAllChildrenCan() {
+ int numDocs = 40;
+ AndFilterOperator andOperator = new AndFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 10), bitmapOp(numDocs, false,
3, 10, 20)), null, numDocs, false);
+ assertTrue(andOperator.canProduceBitmaps());
+ assertTrue(andOperator.canOptimizeCount());
+ }
+
+ @Test
+ public void testCannotProduceBitmapsWhenAnyChildCannot() {
+ int numDocs = 40;
+ AndFilterOperator andOperator = new AndFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 10), new TestFilterOperator(new
int[]{3, 10, 20}, numDocs)), null,
+ numDocs, false);
+ assertFalse(andOperator.canProduceBitmaps());
+ assertFalse(andOperator.canOptimizeCount());
+ }
+
+ @Test
+ public void testGetBitmapsIntersectionForTwoChildren() {
+ int numDocs = 40;
+ AndFilterOperator andOperator = new AndFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 10, 15, 16, 28),
bitmapOp(numDocs, false, 3, 6, 8, 20, 28)), null,
+ numDocs, false);
+ assertEquals(andOperator.getBitmaps().reduce().toArray(), new int[]{3,
28});
+ }
+
+ @Test
+ public void testGetBitmapsIntersectionForThreeChildren() {
+ int numDocs = 40;
+ AndFilterOperator andOperator = new AndFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 6, 10, 15, 16, 28),
bitmapOp(numDocs, false, 3, 6, 8, 20, 28),
+ bitmapOp(numDocs, false, 1, 2, 3, 6, 30)), null, numDocs, false);
+ assertEquals(andOperator.getBitmaps().reduce().toArray(), new int[]{3, 6});
+ }
+
+ @Test
+ public void testGetBitmapsWithExclusiveChild() {
+ int numDocs = 10;
+ // Second child is exclusive: it matches every doc except {2, 4}, so
reduce() must materialize its complement
+ // before the intersection. The AND is {1, 2, 3, 4, 5} minus {2, 4}.
+ AndFilterOperator andOperator = new AndFilterOperator(
+ List.of(bitmapOp(numDocs, false, 1, 2, 3, 4, 5), bitmapOp(numDocs,
true, 2, 4)), null, numDocs, false);
+ assertEquals(andOperator.getBitmaps().reduce().toArray(), new int[]{1, 3,
5});
+ }
+
+ @Test
+ public void testGetBitmapsWithNestedAnd() {
+ int numDocs = 40;
+ AndFilterOperator childAnd = new AndFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 6, 28), bitmapOp(numDocs,
false, 3, 6, 8, 28)), null, numDocs,
+ false);
+ AndFilterOperator andOperator =
+ new AndFilterOperator(List.of(childAnd, bitmapOp(numDocs, false, 3, 6,
30)), null, numDocs, false);
+ assertTrue(andOperator.canProduceBitmaps());
+ assertEquals(andOperator.getBitmaps().reduce().toArray(), new int[]{3, 6});
+ }
+
+ private static BitmapBasedFilterOperator bitmapOp(int numDocs, boolean
exclusive, int... docIds) {
+ MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+ bitmap.add(docIds);
+ return new BitmapBasedFilterOperator(bitmap.toImmutableRoaringBitmap(),
exclusive, numDocs);
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/OrFilterOperatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/OrFilterOperatorTest.java
index 6287cf51f66..7c95bd9e683 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/OrFilterOperatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/OrFilterOperatorTest.java
@@ -19,7 +19,6 @@
package org.apache.pinot.core.operator.filter;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.TreeSet;
@@ -28,9 +27,13 @@ import org.apache.pinot.core.common.BlockDocIdIterator;
import org.apache.pinot.core.operator.docidsets.EmptyDocIdSet;
import org.apache.pinot.core.operator.docidsets.MatchAllDocIdSet;
import org.apache.pinot.segment.spi.Constants;
-import org.testng.Assert;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
public class OrFilterOperatorTest {
@@ -40,8 +43,8 @@ public class OrFilterOperatorTest {
int[] docIds1 = new int[]{2, 3, 10, 15, 16, 28};
int[] docIds2 = new int[]{3, 6, 8, 20, 28};
TreeSet<Integer> treeSet = new TreeSet<>();
- treeSet.addAll(Arrays.asList(ArrayUtils.toObject(docIds1)));
- treeSet.addAll(Arrays.asList(ArrayUtils.toObject(docIds2)));
+ treeSet.addAll(List.of(ArrayUtils.toObject(docIds1)));
+ treeSet.addAll(List.of(ArrayUtils.toObject(docIds2)));
Iterator<Integer> expectedIterator = treeSet.iterator();
List<BaseFilterOperator> operators = new ArrayList<>();
@@ -52,8 +55,9 @@ public class OrFilterOperatorTest {
BlockDocIdIterator iterator =
orOperator.nextBlock().getBlockDocIdSet().iterator();
int docId;
while ((docId = iterator.next()) != Constants.EOF) {
- Assert.assertEquals(docId, expectedIterator.next().intValue());
+ assertEquals(docId, expectedIterator.next().intValue());
}
+ assertFalse(expectedIterator.hasNext());
}
@Test
@@ -63,9 +67,9 @@ public class OrFilterOperatorTest {
int[] docIds2 = new int[]{3, 6, 8, 20, 28};
int[] docIds3 = new int[]{1, 2, 3, 6, 30};
TreeSet<Integer> treeSet = new TreeSet<>();
- treeSet.addAll(Arrays.asList(ArrayUtils.toObject(docIds1)));
- treeSet.addAll(Arrays.asList(ArrayUtils.toObject(docIds2)));
- treeSet.addAll(Arrays.asList(ArrayUtils.toObject(docIds3)));
+ treeSet.addAll(List.of(ArrayUtils.toObject(docIds1)));
+ treeSet.addAll(List.of(ArrayUtils.toObject(docIds2)));
+ treeSet.addAll(List.of(ArrayUtils.toObject(docIds3)));
Iterator<Integer> expectedIterator = treeSet.iterator();
List<BaseFilterOperator> operators = new ArrayList<>();
@@ -77,8 +81,9 @@ public class OrFilterOperatorTest {
BlockDocIdIterator iterator =
orOperator.nextBlock().getBlockDocIdSet().iterator();
int docId;
while ((docId = iterator.next()) != Constants.EOF) {
- Assert.assertEquals(docId, expectedIterator.next().intValue());
+ assertEquals(docId, expectedIterator.next().intValue());
}
+ assertFalse(expectedIterator.hasNext());
}
@Test
@@ -88,9 +93,9 @@ public class OrFilterOperatorTest {
int[] docIds2 = new int[]{3, 6, 8, 20, 28};
int[] docIds3 = new int[]{1, 2, 3, 6, 30};
TreeSet<Integer> treeSet = new TreeSet<>();
- treeSet.addAll(Arrays.asList(ArrayUtils.toObject(docIds1)));
- treeSet.addAll(Arrays.asList(ArrayUtils.toObject(docIds2)));
- treeSet.addAll(Arrays.asList(ArrayUtils.toObject(docIds3)));
+ treeSet.addAll(List.of(ArrayUtils.toObject(docIds1)));
+ treeSet.addAll(List.of(ArrayUtils.toObject(docIds2)));
+ treeSet.addAll(List.of(ArrayUtils.toObject(docIds3)));
Iterator<Integer> expectedIterator = treeSet.iterator();
List<BaseFilterOperator> childOperators = new ArrayList<>();
@@ -106,8 +111,9 @@ public class OrFilterOperatorTest {
BlockDocIdIterator iterator =
orOperator.nextBlock().getBlockDocIdSet().iterator();
int docId;
while ((docId = iterator.next()) != Constants.EOF) {
- Assert.assertEquals(docId, expectedIterator.next().intValue());
+ assertEquals(docId, expectedIterator.next().intValue());
}
+ assertFalse(expectedIterator.hasNext());
}
@Test
@@ -119,11 +125,11 @@ public class OrFilterOperatorTest {
int[] nullDocIds2 = new int[]{3, 4, 5, 6, 7};
OrFilterOperator orFilterOperator = new OrFilterOperator(
- Arrays.asList(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
+ List.of(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
new TestFilterOperator(docIds2, nullDocIds2, numDocs)), null,
numDocs, true);
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()),
List.of(0, 1, 2, 3));
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()),
List.of(8, 9));
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()), List.of(0,
1, 2, 3));
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()), List.of(8,
9));
}
@Test
@@ -135,11 +141,11 @@ public class OrFilterOperatorTest {
int[] nullDocIds2 = new int[]{};
OrFilterOperator orFilterOperator = new OrFilterOperator(
- Arrays.asList(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
+ List.of(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
new TestFilterOperator(docIds2, nullDocIds2, numDocs)), null,
numDocs, false);
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()),
List.of(0, 1, 2, 3));
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()),
List.of(4, 5, 6, 7, 8, 9));
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()), List.of(0,
1, 2, 3));
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()), List.of(4,
5, 6, 7, 8, 9));
}
@Test
@@ -149,11 +155,11 @@ public class OrFilterOperatorTest {
int[] nullDocIds1 = new int[]{4, 5, 6};
OrFilterOperator orFilterOperator = new OrFilterOperator(
- Arrays.asList(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
EmptyFilterOperator.getInstance()), null,
+ List.of(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
EmptyFilterOperator.getInstance()), null,
numDocs, true);
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()),
Arrays.asList(1, 2, 3));
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()),
Arrays.asList(0, 7, 8, 9));
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()), List.of(1,
2, 3));
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()), List.of(0,
7, 8, 9));
}
@Test
@@ -163,11 +169,11 @@ public class OrFilterOperatorTest {
int[] nullDocIds1 = new int[]{4, 5, 6};
OrFilterOperator orFilterOperator = new OrFilterOperator(
- Arrays.asList(new TestFilterOperator(docIds1, nullDocIds1, numDocs),
new MatchAllFilterOperator(numDocs)), null,
+ List.of(new TestFilterOperator(docIds1, nullDocIds1, numDocs), new
MatchAllFilterOperator(numDocs)), null,
numDocs, true);
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()),
Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()),
List.of());
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()), List.of(0,
1, 2, 3, 4, 5, 6, 7, 8, 9));
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()), List.of());
}
@Test
@@ -175,11 +181,11 @@ public class OrFilterOperatorTest {
int numDocs = 10;
OrFilterOperator orFilterOperator =
- new OrFilterOperator(Arrays.asList(EmptyFilterOperator.getInstance(),
EmptyFilterOperator.getInstance()), null,
+ new OrFilterOperator(List.of(EmptyFilterOperator.getInstance(),
EmptyFilterOperator.getInstance()), null,
numDocs, true);
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()),
List.of());
- Assert.assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()),
Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getTrues()), List.of());
+ assertEquals(TestUtils.getDocIds(orFilterOperator.getFalses()), List.of(0,
1, 2, 3, 4, 5, 6, 7, 8, 9));
}
@Test
@@ -188,12 +194,12 @@ public class OrFilterOperatorTest {
int[] regularDocIds = new int[]{1, 2, 3};
OrFilterOperator orFilterOperator = new OrFilterOperator(
- Arrays.asList(
+ List.of(
new TestFilterOperator(regularDocIds, numDocs),
new MatchAllFilterOperator(numDocs)
), null, numDocs, false);
- Assert.assertTrue((orFilterOperator.getTrues()).getOptimizedDocIdSet()
instanceof MatchAllDocIdSet);
+ assertTrue((orFilterOperator.getTrues()).getOptimizedDocIdSet() instanceof
MatchAllDocIdSet);
}
@Test
@@ -202,11 +208,77 @@ public class OrFilterOperatorTest {
int[] emptyDocIds = new int[0];
OrFilterOperator orFilterOperator = new OrFilterOperator(
- Arrays.asList(
+ List.of(
new TestFilterOperator(emptyDocIds, numDocs),
new TestFilterOperator(emptyDocIds, numDocs)
), null, numDocs, false);
- Assert.assertTrue(orFilterOperator.getTrues().getOptimizedDocIdSet()
instanceof EmptyDocIdSet);
+ assertTrue(orFilterOperator.getTrues().getOptimizedDocIdSet() instanceof
EmptyDocIdSet);
+ }
+
+ @Test
+ public void testCanProduceBitmapsWhenAllChildrenCan() {
+ int numDocs = 40;
+ OrFilterOperator orOperator = new OrFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 10), bitmapOp(numDocs, false,
3, 10, 20)), null, numDocs, false);
+ assertTrue(orOperator.canProduceBitmaps());
+ assertTrue(orOperator.canOptimizeCount());
+ }
+
+ @Test
+ public void testCannotProduceBitmapsWhenAnyChildCannot() {
+ int numDocs = 40;
+ OrFilterOperator orOperator = new OrFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 10), new TestFilterOperator(new
int[]{3, 10, 20}, numDocs)), null,
+ numDocs, false);
+ assertFalse(orOperator.canProduceBitmaps());
+ assertFalse(orOperator.canOptimizeCount());
+ }
+
+ @Test
+ public void testGetBitmapsUnionForTwoChildren() {
+ int numDocs = 40;
+ OrFilterOperator orOperator = new OrFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 10, 15, 16, 28),
bitmapOp(numDocs, false, 3, 6, 8, 20, 28)), null,
+ numDocs, false);
+ assertEquals(orOperator.getBitmaps().reduce().toArray(), new int[]{2, 3,
6, 8, 10, 15, 16, 20, 28});
+ }
+
+ @Test
+ public void testGetBitmapsUnionForThreeChildren() {
+ int numDocs = 40;
+ OrFilterOperator orOperator = new OrFilterOperator(
+ List.of(bitmapOp(numDocs, false, 2, 3, 6), bitmapOp(numDocs, false, 3,
6, 8), bitmapOp(numDocs, false, 1)),
+ null, numDocs, false);
+ assertEquals(orOperator.getBitmaps().reduce().toArray(), new int[]{1, 2,
3, 6, 8});
+ }
+
+ @Test
+ public void testGetBitmapsWithExclusiveChild() {
+ int numDocs = 10;
+ // Second child is exclusive: it matches every doc except {0..7}, i.e. {8,
9}, so reduce() must materialize its
+ // complement before the union. The OR is {1, 3} union {8, 9}.
+ OrFilterOperator orOperator = new OrFilterOperator(
+ List.of(bitmapOp(numDocs, false, 1, 3), bitmapOp(numDocs, true, 0, 1,
2, 3, 4, 5, 6, 7)), null, numDocs,
+ false);
+ assertEquals(orOperator.getBitmaps().reduce().toArray(), new int[]{1, 3,
8, 9});
+ }
+
+ @Test
+ public void testGetBitmapsWithNestedOr() {
+ int numDocs = 40;
+ OrFilterOperator childOr =
+ new OrFilterOperator(List.of(bitmapOp(numDocs, false, 2, 3),
bitmapOp(numDocs, false, 6, 8)), null,
+ numDocs, false);
+ OrFilterOperator orOperator =
+ new OrFilterOperator(List.of(childOr, bitmapOp(numDocs, false, 1,
30)), null, numDocs, false);
+ assertTrue(orOperator.canProduceBitmaps());
+ assertEquals(orOperator.getBitmaps().reduce().toArray(), new int[]{1, 2,
3, 6, 8, 30});
+ }
+
+ private static BitmapBasedFilterOperator bitmapOp(int numDocs, boolean
exclusive, int... docIds) {
+ MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+ bitmap.add(docIds);
+ return new BitmapBasedFilterOperator(bitmap.toImmutableRoaringBitmap(),
exclusive, numDocs);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
