This is an automated email from the ASF dual-hosted git repository.
saurabhd336 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 57f50d3dec Json extract index filter support (#12683)
57f50d3dec is described below
commit 57f50d3dec2a2023effa67ca044f61dfd6561dad
Author: Saurabh Dubey <[email protected]>
AuthorDate: Tue Mar 26 09:22:29 2024 +0530
Json extract index filter support (#12683)
---------
Co-authored-by: Saurabh Dubey <[email protected]>
Co-authored-by: Saurabh Dubey
<[email protected]>
---
.../common/function/TransformFunctionType.java | 3 +-
.../JsonExtractIndexTransformFunction.java | 28 ++++++----
.../JsonExtractIndexTransformFunctionTest.java | 17 +++++++
.../realtime/impl/json/MutableJsonIndexImpl.java | 28 +++++++++-
.../readers/json/ImmutableJsonIndexReader.java | 46 +++++++++++++++--
.../segment/local/segment/index/JsonIndexTest.java | 59 ++++++++++++++--------
.../segment/spi/index/reader/JsonIndexReader.java | 3 +-
7 files changed, 147 insertions(+), 37 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
index 20bc26854c..88c269c6aa 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
@@ -117,7 +117,8 @@ public enum TransformFunctionType {
ReturnTypes.cascade(opBinding ->
positionalReturnTypeInferenceFromStringLiteral(opBinding, 2,
SqlTypeName.VARCHAR), SqlTypeTransforms.FORCE_NULLABLE),
OperandTypes.family(ImmutableList.of(SqlTypeFamily.ANY,
SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER,
- SqlTypeFamily.CHARACTER), ordinal -> ordinal > 2),
"json_extract_index"),
+ SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER), ordinal ->
ordinal > 2), "json_extract_index"),
+
JSON_EXTRACT_KEY("jsonExtractKey", ReturnTypes.TO_ARRAY,
OperandTypes.family(ImmutableList.of(SqlTypeFamily.ANY,
SqlTypeFamily.CHARACTER)), "json_extract_key"),
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractIndexTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractIndexTransformFunction.java
index 160ed36b0f..12e38ea5d6 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractIndexTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractIndexTransformFunction.java
@@ -53,11 +53,11 @@ public class JsonExtractIndexTransformFunction extends
BaseTransformFunction {
@Override
public void init(List<TransformFunction> arguments, Map<String,
ColumnContext> columnContextMap) {
- // Check that there are exactly 3 or 4 arguments
- if (arguments.size() < 3 || arguments.size() > 4) {
+ // Check that there are exactly 3 or 4 or 5 arguments
+ if (arguments.size() < 3 || arguments.size() > 5) {
throw new IllegalArgumentException(
- "Expected 3/4 arguments for transform function:
jsonExtractIndex(jsonFieldName, 'jsonPath', 'resultsType',"
- + " ['defaultValue'])");
+ "Expected 3/4/5 arguments for transform function:
jsonExtractIndex(jsonFieldName, 'jsonPath', 'resultsType',"
+ + " ['defaultValue'], ['jsonFilterExpression'])");
}
TransformFunction firstArgument = arguments.get(0);
@@ -76,13 +76,12 @@ public class JsonExtractIndexTransformFunction extends
BaseTransformFunction {
if (!(secondArgument instanceof LiteralTransformFunction)) {
throw new IllegalArgumentException("JSON path argument must be a
literal");
}
- String inputJsonPath = ((LiteralTransformFunction)
secondArgument).getStringLiteral();
+ _jsonPathString = ((LiteralTransformFunction)
secondArgument).getStringLiteral();
try {
- JsonPathCache.INSTANCE.getOrCompute(inputJsonPath);
+ JsonPathCache.INSTANCE.getOrCompute(_jsonPathString);
} catch (Exception e) {
throw new IllegalArgumentException("JSON path argument is not a valid
JSON path");
}
- _jsonPathString = inputJsonPath.substring(1); // remove $ prefix
TransformFunction thirdArgument = arguments.get(2);
if (!(thirdArgument instanceof LiteralTransformFunction)) {
@@ -90,14 +89,14 @@ public class JsonExtractIndexTransformFunction extends
BaseTransformFunction {
}
String resultsType = ((LiteralTransformFunction)
thirdArgument).getStringLiteral().toUpperCase();
boolean isSingleValue = !resultsType.endsWith("_ARRAY");
- if (isSingleValue && inputJsonPath.contains("[*]")) {
+ if (isSingleValue && _jsonPathString.contains("[*]")) {
throw new IllegalArgumentException(
"[*] syntax in json path is unsupported for singleValue field
json_extract_index");
}
DataType dataType = isSingleValue ? DataType.valueOf(resultsType)
: DataType.valueOf(resultsType.substring(0, resultsType.length() - 6));
- if (arguments.size() == 4) {
+ if (arguments.size() >= 4) {
TransformFunction fourthArgument = arguments.get(3);
if (!(fourthArgument instanceof LiteralTransformFunction)) {
throw new IllegalArgumentException("Default value must be a literal");
@@ -105,8 +104,17 @@ public class JsonExtractIndexTransformFunction extends
BaseTransformFunction {
_defaultValue = dataType.convert(((LiteralTransformFunction)
fourthArgument).getStringLiteral());
}
+ String filterJsonPath = null;
+ if (arguments.size() == 5) {
+ TransformFunction fifthArgument = arguments.get(4);
+ if (!(fifthArgument instanceof LiteralTransformFunction)) {
+ throw new IllegalArgumentException("JSON path filter argument must be
a literal");
+ }
+ filterJsonPath = ((LiteralTransformFunction)
fifthArgument).getStringLiteral();
+ }
+
_resultMetadata = new TransformResultMetadata(dataType, isSingleValue,
false);
- _valueToMatchingDocsMap =
_jsonIndexReader.getMatchingFlattenedDocsMap(_jsonPathString);
+ _valueToMatchingDocsMap =
_jsonIndexReader.getMatchingFlattenedDocsMap(_jsonPathString, filterJsonPath);
if (isSingleValue) {
// For single value result type, it's more efficient to use original
docIDs map
_jsonIndexReader.convertFlattenedDocIdsToDocIds(_valueToMatchingDocsMap);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/JsonExtractIndexTransformFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/JsonExtractIndexTransformFunctionTest.java
index c61084c430..d2cd792107 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/JsonExtractIndexTransformFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/JsonExtractIndexTransformFunctionTest.java
@@ -247,6 +247,23 @@ public class JsonExtractIndexTransformFunctionTest extends
BaseTransformFunction
String.format("jsonExtractIndex(%s,'%s','STRING_ARRAY')",
JSON_STRING_SV_COLUMN,
"$.arrayField[*].arrStringField"),
"$.arrayField[*].arrStringField", DataType.STRING, false
});
+
+ // MV with filters
+ testArguments.add(new Object[]{
+ String.format(
+ "jsonExtractIndex(%s,'%s','INT_ARRAY', '0',
'REGEXP_LIKE(\"$.arrayField[*].arrStringField\", ''.*y.*'')')",
+ JSON_STRING_SV_COLUMN,
+ "$.arrayField[*].arrIntField"), "$.arrayField[?(@.arrStringField
=~ /.*y.*/)].arrIntField", DataType.INT,
+ false
+ });
+
+ testArguments.add(new Object[]{
+ String.format(
+ "jsonExtractIndex(%s,'%s','STRING_ARRAY', '0',
'\"$.arrayField[*].arrIntField\" > 2')",
+ JSON_STRING_SV_COLUMN,
+ "$.arrayField[*].arrStringField"), "$.arrayField[?(@.arrIntField >
2)].arrStringField", DataType.STRING,
+ false
+ });
}
@Test(dataProvider = "testJsonExtractIndexDefaultValue")
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
index 8a5ca799cd..2f8cff69da 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
@@ -33,6 +33,7 @@ import java.util.PriorityQueue;
import java.util.TreeMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern;
+import javax.annotation.Nullable;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.FilterContext;
@@ -367,10 +368,32 @@ public class MutableJsonIndexImpl implements
MutableJsonIndex {
}
@Override
- public Map<String, RoaringBitmap> getMatchingFlattenedDocsMap(String
jsonPathKey) {
+ public Map<String, RoaringBitmap> getMatchingFlattenedDocsMap(String
jsonPathKey, @Nullable String filterString) {
Map<String, RoaringBitmap> valueToMatchingFlattenedDocIdsMap = new
HashMap<>();
_readLock.lock();
try {
+ RoaringBitmap filteredFlattenedDocIds = null;
+ FilterContext filter;
+ if (filterString != null) {
+ filter =
RequestContextUtils.getFilter(CalciteSqlParser.compileToExpression(filterString));
+ Preconditions.checkArgument(!filter.isConstant(), "Invalid json match
filter: " + filterString);
+ if (filter.getType() == FilterContext.Type.PREDICATE &&
isExclusive(filter.getPredicate().getType())) {
+ // Handle exclusive predicate separately because the flip can only
be applied to the
+ // unflattened doc ids in order to get the correct result, and it
cannot be nested
+ filteredFlattenedDocIds =
getMatchingFlattenedDocIds(filter.getPredicate());
+ filteredFlattenedDocIds.flip(0, (long) _nextFlattenedDocId);
+ } else {
+ filteredFlattenedDocIds = getMatchingFlattenedDocIds(filter);
+ }
+ }
+ // Support 2 formats:
+ // - JSONPath format (e.g. "$.a[1].b"='abc', "$[0]"=1, "$"='abc')
+ // - Legacy format (e.g. "a[1].b"='abc')
+ if (jsonPathKey.startsWith("$")) {
+ jsonPathKey = jsonPathKey.substring(1);
+ } else {
+ jsonPathKey = JsonUtils.KEY_SEPARATOR + jsonPathKey;
+ }
Pair<String, RoaringBitmap> result =
getKeyAndFlattenedDocIds(jsonPathKey);
jsonPathKey = result.getLeft();
RoaringBitmap arrayIndexFlattenDocIds = result.getRight();
@@ -380,6 +403,9 @@ public class MutableJsonIndexImpl implements
MutableJsonIndex {
Map<String, RoaringBitmap> subMap = getMatchingKeysMap(jsonPathKey);
for (Map.Entry<String, RoaringBitmap> entry : subMap.entrySet()) {
RoaringBitmap flattenedDocIds = entry.getValue().clone();
+ if (filteredFlattenedDocIds != null) {
+ flattenedDocIds.and(filteredFlattenedDocIds);
+ }
if (arrayIndexFlattenDocIds != null) {
flattenedDocIds.and(arrayIndexFlattenDocIds);
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java
index 9af37b50fb..2d67309a38 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.regex.Pattern;
+import javax.annotation.Nullable;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.FilterContext;
@@ -134,7 +135,8 @@ public class ImmutableJsonIndexReader implements
JsonIndexReader {
case AND: {
List<FilterContext> children = filter.getChildren();
int numChildren = children.size();
- MutableRoaringBitmap matchingDocIds =
getMatchingFlattenedDocIds(children.get(0));
+ MutableRoaringBitmap matchingDocIds =
+ getMatchingFlattenedDocIds(children.get(0));
for (int i = 1; i < numChildren; i++) {
matchingDocIds.and(getMatchingFlattenedDocIds(children.get(i)));
}
@@ -143,7 +145,8 @@ public class ImmutableJsonIndexReader implements
JsonIndexReader {
case OR: {
List<FilterContext> children = filter.getChildren();
int numChildren = children.size();
- MutableRoaringBitmap matchingDocIds =
getMatchingFlattenedDocIds(children.get(0));
+ MutableRoaringBitmap matchingDocIds =
+ getMatchingFlattenedDocIds(children.get(0));
for (int i = 1; i < numChildren; i++) {
matchingDocIds.or(getMatchingFlattenedDocIds(children.get(i)));
}
@@ -331,7 +334,40 @@ public class ImmutableJsonIndexReader implements
JsonIndexReader {
}
@Override
- public Map<String, RoaringBitmap> getMatchingFlattenedDocsMap(String
jsonPathKey) {
+ public Map<String, RoaringBitmap> getMatchingFlattenedDocsMap(String
jsonPathKey, @Nullable String filterString) {
+ RoaringBitmap filteredFlattenedDocIds = null;
+ if (filterString != null) {
+ FilterContext filter;
+ try {
+ filter =
RequestContextUtils.getFilter(CalciteSqlParser.compileToExpression(filterString));
+ Preconditions.checkArgument(!filter.isConstant());
+ } catch (Exception e) {
+ throw new BadQueryRequestException("Invalid json match filter: " +
filterString);
+ }
+ if (filter.getType() == FilterContext.Type.PREDICATE &&
isExclusive(filter.getPredicate().getType())) {
+ // Handle exclusive predicate separately because the flip can only be
applied to the
+ // unflattened doc ids in order to get the correct result, and it
cannot be nested
+ filteredFlattenedDocIds =
getMatchingFlattenedDocIds(filter.getPredicate()).toRoaringBitmap();
+ filteredFlattenedDocIds.flip(0, _numFlattenedDocs);
+ } else {
+ filteredFlattenedDocIds =
getMatchingFlattenedDocIds(filter).toRoaringBitmap();
+ }
+ }
+ // Support 2 formats:
+ // - JSONPath format (e.g. "$.a[1].b"='abc', "$[0]"=1, "$"='abc')
+ // - Legacy format (e.g. "a[1].b"='abc')
+ if (_version == BaseJsonIndexCreator.VERSION_2) {
+ if (jsonPathKey.startsWith("$")) {
+ jsonPathKey = jsonPathKey.substring(1);
+ } else {
+ jsonPathKey = JsonUtils.KEY_SEPARATOR + jsonPathKey;
+ }
+ } else {
+ // For V1 backward-compatibility
+ if (jsonPathKey.startsWith("$.")) {
+ jsonPathKey = jsonPathKey.substring(2);
+ }
+ }
Map<String, RoaringBitmap> result = new HashMap<>();
Pair<String, MutableRoaringBitmap> pathKey =
getKeyAndFlattenedDocIds(jsonPathKey);
if (pathKey.getRight() != null && pathKey.getRight().isEmpty()) {
@@ -347,6 +383,10 @@ public class ImmutableJsonIndexReader implements
JsonIndexReader {
for (int dictId = dictIds[0]; dictId < dictIds[1]; dictId++) {
String key = _dictionary.getStringValue(dictId);
RoaringBitmap docIds =
_invertedIndex.getDocIds(dictId).toRoaringBitmap();
+ if (filteredFlattenedDocIds != null) {
+ docIds.and(filteredFlattenedDocIds);
+ }
+
if (arrayIndexFlattenDocIds != null) {
docIds.and(arrayIndexFlattenDocIds);
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
index 1b2a910da3..14f8a79bc1 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
@@ -347,9 +347,21 @@ public class JsonIndexTest {
// CHECKSTYLE:ON
// @formatter: on
- String[] testKeys = new String[]{
- ".arrField[*].intKey01",
- ".arrField[*].stringKey01",
+ String[][] testKeys = new String[][]{
+ // Without filters
+ {"$.arrField[*].intKey01", null},
+ {"$.arrField[*].stringKey01", null},
+
+ // With regexp filter
+ {"$.arrField[*].intKey01", "REGEXP_LIKE(\"arrField..stringKey01\",
'.*f.*')"},
+ // With range filter
+ {"$.arrField[*].stringKey01", "\"arrField..intKey01\" > 2"},
+ // With AND filters
+ {"$.arrField[*].intKey01", "\"arrField..intKey01\" > 2 AND
REGEXP_LIKE(\"arrField..stringKey01\", "
+ + "'[a-b][a-b].*')"},
+ // Exclusive filters
+ {"$.arrField[*].intKey01", "\"arrField[*].stringKey01\" != 'bar'"},
+ {"$.arrField[*].stringKey01", "\"arrField[*].intKey01\" != '3'"},
};
String colName = "col";
@@ -371,18 +383,23 @@ public class JsonIndexTest {
int docIdValidLength = 2;
String[][][] expectedValues = new String[][][]{
{{"1", "1", "3", "5"}, {"1", "1", "6", "3"}},
- {{"abc", "foo", "bar", "fuzz"}, {"pqr", "foo", "test", "testf2"}}
+ {{"abc", "foo", "bar", "fuzz"}, {"pqr", "foo", "test", "testf2"}},
+ {{"1", "5"}, {"1", "3"}},
+ {{"bar", "fuzz"}, {"test", "testf2"}},
+ {{"3"}, {}},
+ {{"1", "1", "5"}, {"1", "1", "6", "3"}},
+ {{"abc", "foo", "fuzz"}, {"pqr", "foo", "test"}}
};
for (int i = 0; i < testKeys.length; i++) {
Map<String, RoaringBitmap> context =
- offHeapIndexReader.getMatchingFlattenedDocsMap(testKeys[i]);
+ offHeapIndexReader.getMatchingFlattenedDocsMap(testKeys[i][0],
testKeys[i][1]);
String[][] values = offHeapIndexReader.getValuesMV(docMask,
docIdValidLength, context);
for (int j = 0; j < docIdValidLength; j++) {
Assert.assertEquals(values[j], expectedValues[i][j]);
}
- context = mutableJsonIndex.getMatchingFlattenedDocsMap(testKeys[i]);
+ context =
mutableJsonIndex.getMatchingFlattenedDocsMap(testKeys[i][0], testKeys[i][1]);
values = mutableJsonIndex.getValuesMV(docMask, docIdValidLength,
context);
Assert.assertEquals(values, expectedValues[i]);
}
@@ -402,7 +419,7 @@ public class JsonIndexTest {
};
// CHECKSTYLE:ON
// @formatter: on
- String[] testKeys = new String[]{".field1", ".field2", ".field3",
".field4"};
+ String[] testKeys = new String[]{"$.field1", "$.field2", "$.field3",
"$.field4"};
String colName = "col";
try (
@@ -427,7 +444,7 @@ public class JsonIndexTest {
new String[][]{{"value1", "value2", "value1"}, {"value2", null,
"value4"}, {"value3", null, null},
{null, null, null}};
for (int i = 0; i < testKeys.length; i++) {
- Map<String, RoaringBitmap> context =
offHeapIndexReader.getMatchingFlattenedDocsMap(testKeys[i]);
+ Map<String, RoaringBitmap> context =
offHeapIndexReader.getMatchingFlattenedDocsMap(testKeys[i], null);
String[] values = offHeapIndexReader.getValuesSV(docMask,
docMask.length, context, true);
Assert.assertEquals(values, expectedValues[i]);
@@ -435,7 +452,7 @@ public class JsonIndexTest {
values = offHeapIndexReader.getValuesSV(docMask, docMask.length,
context, false);
Assert.assertEquals(values, expectedValues[i]);
- context = mutableJsonIndex.getMatchingFlattenedDocsMap(testKeys[i]);
+ context = mutableJsonIndex.getMatchingFlattenedDocsMap(testKeys[i],
null);
values = mutableJsonIndex.getValuesSV(docMask, docMask.length,
context, true);
Assert.assertEquals(values, expectedValues[i]);
@@ -448,7 +465,7 @@ public class JsonIndexTest {
docMask = new int[]{1, 2};
expectedValues = new String[][]{{"value2", "value1"}, {null,
"value4"}, {null, null}, {null, null}};
for (int i = 0; i < testKeys.length; i++) {
- Map<String, RoaringBitmap> context =
offHeapIndexReader.getMatchingFlattenedDocsMap(testKeys[i]);
+ Map<String, RoaringBitmap> context =
offHeapIndexReader.getMatchingFlattenedDocsMap(testKeys[i], null);
String[] values = offHeapIndexReader.getValuesSV(docMask,
docMask.length, context, true);
Assert.assertEquals(values, expectedValues[i]);
@@ -456,7 +473,7 @@ public class JsonIndexTest {
values = offHeapIndexReader.getValuesSV(docMask, docMask.length,
context, false);
Assert.assertEquals(values, expectedValues[i]);
- context = mutableJsonIndex.getMatchingFlattenedDocsMap(testKeys[i]);
+ context = mutableJsonIndex.getMatchingFlattenedDocsMap(testKeys[i],
null);
values = mutableJsonIndex.getValuesSV(docMask, docMask.length,
context, true);
Assert.assertEquals(values, expectedValues[i]);
@@ -466,7 +483,7 @@ public class JsonIndexTest {
}
// Immutable index, context is reused for the second method call
- Map<String, RoaringBitmap> context =
offHeapIndexReader.getMatchingFlattenedDocsMap(".field1");
+ Map<String, RoaringBitmap> context =
offHeapIndexReader.getMatchingFlattenedDocsMap("$.field1", null);
docMask = new int[]{0};
String[] values = offHeapIndexReader.getValuesSV(docMask,
docMask.length, context, true);
Assert.assertEquals(values, new String[]{"value1"});
@@ -483,7 +500,7 @@ public class JsonIndexTest {
Assert.assertEquals(values, new String[]{"value2", "value1"});
// Mutable index, context is reused for the second method call
- context = mutableJsonIndex.getMatchingFlattenedDocsMap(".field1");;
+ context = mutableJsonIndex.getMatchingFlattenedDocsMap("$.field1",
null);;
docMask = new int[]{0};
values = mutableJsonIndex.getValuesSV(docMask, docMask.length,
context, true);
Assert.assertEquals(values, new String[]{"value1"});
@@ -525,9 +542,9 @@ public class JsonIndexTest {
for (String record : records) {
mutableJsonIndex.add(record);
}
- Map<String, RoaringBitmap> onHeapRes =
onHeapIndexReader.getMatchingFlattenedDocsMap("");
- Map<String, RoaringBitmap> offHeapRes =
offHeapIndexReader.getMatchingFlattenedDocsMap("");
- Map<String, RoaringBitmap> mutableRes =
mutableJsonIndex.getMatchingFlattenedDocsMap("");
+ Map<String, RoaringBitmap> onHeapRes =
onHeapIndexReader.getMatchingFlattenedDocsMap("$", null);
+ Map<String, RoaringBitmap> offHeapRes =
offHeapIndexReader.getMatchingFlattenedDocsMap("$", null);
+ Map<String, RoaringBitmap> mutableRes =
mutableJsonIndex.getMatchingFlattenedDocsMap("$", null);
Map<String, RoaringBitmap> expectedRes =
Collections.singletonMap(JsonUtils.SKIPPED_VALUE_REPLACEMENT,
RoaringBitmap.bitmapOf(0));
Assert.assertEquals(expectedRes, onHeapRes);
@@ -563,8 +580,8 @@ public class JsonIndexTest {
File offHeapIndexFile = new File(INDEX_DIR, OFF_HEAP_COLUMN_NAME +
V1Constants.Indexes.JSON_INDEX_FILE_EXTENSION);
Assert.assertTrue(offHeapIndexFile.exists());
- String[] keys = {".foo[0].bar[1]", ".foo[1].bar[0]", ".foo2[0]",
".foo[100].bar[100]", ".foo[0].bar[*]",
- ".foo[*].bar[0]", ".foo[*].bar[*]"};
+ String[] keys = {"$.foo[0].bar[1]", "$.foo[1].bar[0]", "$.foo2[0]",
"$.foo[100].bar[100]", "$.foo[0].bar[*]",
+ "$.foo[*].bar[0]", "$.foo[*].bar[*]"};
List<Map<String, RoaringBitmap>> expected = List.of(
Map.of("y", RoaringBitmap.bitmapOf(0), "z",
RoaringBitmap.bitmapOf(1)),
Map.of("a", RoaringBitmap.bitmapOf(0)),
@@ -593,11 +610,11 @@ public class JsonIndexTest {
}
for (int i = 0; i < keys.length; i++) {
- Map<String, RoaringBitmap> onHeapRes =
onHeapIndexReader.getMatchingFlattenedDocsMap(keys[i]);
+ Map<String, RoaringBitmap> onHeapRes =
onHeapIndexReader.getMatchingFlattenedDocsMap(keys[i], null);
onHeapIndexReader.convertFlattenedDocIdsToDocIds(onHeapRes);
- Map<String, RoaringBitmap> offHeapRes =
offHeapIndexReader.getMatchingFlattenedDocsMap(keys[i]);
+ Map<String, RoaringBitmap> offHeapRes =
offHeapIndexReader.getMatchingFlattenedDocsMap(keys[i], null);
offHeapIndexReader.convertFlattenedDocIdsToDocIds(offHeapRes);
- Map<String, RoaringBitmap> mutableRes =
mutableJsonIndex.getMatchingFlattenedDocsMap(keys[i]);
+ Map<String, RoaringBitmap> mutableRes =
mutableJsonIndex.getMatchingFlattenedDocsMap(keys[i], null);
mutableJsonIndex.convertFlattenedDocIdsToDocIds(mutableRes);
Assert.assertEquals(expected.get(i), onHeapRes);
Assert.assertEquals(expected.get(i), offHeapRes);
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/JsonIndexReader.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/JsonIndexReader.java
index 73ef8450ee..44f7dc82c6 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/JsonIndexReader.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/JsonIndexReader.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.spi.index.reader;
import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.index.IndexReader;
import org.roaringbitmap.RoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
@@ -60,7 +61,7 @@ public interface JsonIndexReader extends IndexReader {
* For a JSON key, returns a Map from each value to the flattened docId
posting list. This map should be used to
* avoid reading and converting the posting list of flattened docIds to real
docIds
*/
- Map<String, RoaringBitmap> getMatchingFlattenedDocsMap(String key);
+ Map<String, RoaringBitmap> getMatchingFlattenedDocsMap(String key, @Nullable
String filterJsonString);
/**
* Converts the flattened docIds to real docIds using the map returned by
getMatchingFlattenedDocsMap
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
