This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e8bb63fa6ad Fix MutableJsonIndexImpl corrupting doc ID mapping on
unpaired UTF-16 surrogates (#18738)
e8bb63fa6ad is described below
commit e8bb63fa6ad7e6c14a1c84f36a1c32e8a307034b
Author: Samuel Papin <[email protected]>
AuthorDate: Wed Jun 24 15:57:50 2026 -0400
Fix MutableJsonIndexImpl corrupting doc ID mapping on unpaired UTF-16
surrogates (#18738)
Utf8.encodedLength throws on unpaired surrogates, aborting the posting-list
build loop after _docIdMapping has already been extended. This leaves the
two
counters permanently out of sync, silently shifting all subsequent
documents'
json_match results by one.
_bytesSize is only a memory heuristic, so a size calculation must never
abort indexing. Wraps the calls in estimateLength(), which falls back to
String.getBytes(UTF_8) — which never throws — for malformed strings.
Fixes https://github.com/apache/pinot/issues/18737
Co-authored-by: spapin <[email protected]>
---
.../local/realtime/impl/json/MutableJsonIndexImpl.java | 15 +++++++++++++--
.../pinot/segment/local/segment/index/JsonIndexTest.java | 16 ++++++++++++++++
2 files changed, 29 insertions(+), 2 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
index 9e0583ab0bf..b9f1dbe515a 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
@@ -24,6 +24,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -134,12 +135,12 @@ public class MutableJsonIndexImpl implements
MutableJsonIndex {
// Put both key and key-value into the posting list. Key is useful for
checking if a key exists in the json.
String key = entry.getKey();
_postingListMap.computeIfAbsent(key, k -> {
- _bytesSize += Utf8.encodedLength(key);
+ _bytesSize += estimateLength(key);
return new RoaringBitmap();
}).add(_nextFlattenedDocId);
String keyValue = key + JsonIndexCreator.KEY_VALUE_SEPARATOR +
entry.getValue();
_postingListMap.computeIfAbsent(keyValue, k -> {
- _bytesSize += Utf8.encodedLength(keyValue);
+ _bytesSize += estimateLength(keyValue);
return new RoaringBitmap();
}).add(_nextFlattenedDocId);
}
@@ -147,6 +148,16 @@ public class MutableJsonIndexImpl implements
MutableJsonIndex {
}
}
+ private static int estimateLength(String value) {
+ // Utf8.encodedLength counts bytes without allocating but throws on
malformed strings (e.g. unpaired surrogates).
+ // _bytesSize is only a heuristic, so fall back to getBytes (which never
throws) instead of failing.
+ try {
+ return Utf8.encodedLength(value);
+ } catch (IllegalArgumentException e) {
+ return value.getBytes(StandardCharsets.UTF_8).length;
+ }
+ }
+
@Override
public MutableRoaringBitmap getMatchingDocIds(String filterString) {
FilterContext filter;
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
index b8631bdc97b..b033e5dc6cc 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
@@ -98,6 +98,22 @@ public class JsonIndexTest implements
PinotBuffersAfterMethodCheckRule {
FileUtils.deleteDirectory(INDEX_DIR);
}
+ @Test
+ public void testAddRecordWithUnpairedSurrogateDoesNotShiftDocIds()
+ throws IOException {
+ JsonIndexConfig jsonIndexConfig = getIndexConfig();
+ try (MutableJsonIndexImpl mutableJsonIndex =
+ new MutableJsonIndexImpl(jsonIndexConfig, "table__0__1", "col")) {
+ mutableJsonIndex.add("{\"name\":\"first\"}");
+ mutableJsonIndex.add("{\"name\":\"" + '\uD800' + "\"}");
+ mutableJsonIndex.add("{\"name\":\"third\"}");
+
+ assertDocIds(mutableJsonIndex, "name='first'", ids(0));
+ assertDocIds(mutableJsonIndex, "name='" + '\uD800' + "'", ids(1));
+ assertDocIds(mutableJsonIndex, "name='third'", ids(2));
+ }
+ }
+
@Test
public void testSmallIndex()
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
