This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 14b564428ee Make Utf8Utils.encodedLength agree with encode() on
unpaired surrogates (#20153)
14b564428ee is described below
commit 14b564428ee6d939728e3111ffb2f13c957afa18
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Wed Nov 4 22:42:05 2026 -0800
Make Utf8Utils.encodedLength agree with encode() on unpaired surrogates
(#20153)
---
.../indexsegment/mutable/MutableSegmentImpl.java | 4 +-
.../converter/stats/CompactedColumnStatistics.java | 4 +-
.../stats/CompactedNoDictColumnStatistics.java | 6 +-
.../dictionary/StringOnHeapMutableDictionary.java | 5 +-
.../impl/forward/CLPMutableForwardIndex.java | 4 +-
.../impl/forward/SameValueMutableForwardIndex.java | 4 +-
.../realtime/impl/json/MutableJsonIndexImpl.java | 17 +----
.../SchemaConformingTransformer.java | 4 +-
.../creator/impl/SegmentDictionaryCreator.java | 2 +-
.../impl/openstruct/OpenStructColumnSplitter.java | 4 +-
.../stats/NoDictColumnStatisticsCollector.java | 4 +-
.../stats/StringColumnPreIndexStatsCollector.java | 6 +-
.../segment/index/loader/ForwardIndexHandler.java | 4 +-
.../defaultcolumn/DefaultColumnStatistics.java | 4 +-
.../PartitionIdVirtualColumnProvider.java | 4 +-
.../local/startree/StarTreeBuilderUtils.java | 4 +-
.../stats/MutableNoDictColumnStatisticsTest.java | 6 +-
.../defaultcolumn/DefaultColumnStatisticsTest.java | 6 +-
.../index/readers/ImmutableDictionaryTest.java | 4 +-
.../pinot/server/api/resources/TablesResource.java | 4 +-
.../java/org/apache/pinot/spi/utils/Utf8Utils.java | 78 +++++++++++++---------
.../org/apache/pinot/spi/utils/Utf8UtilsTest.java | 58 ++++++++++++++--
22 files changed, 144 insertions(+), 92 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
index 832b1c1cbd7..3920c0e2faf 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
@@ -19,7 +19,6 @@
package org.apache.pinot.segment.local.indexsegment.mutable;
import com.google.common.base.Preconditions;
-import com.google.common.base.Utf8;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
@@ -126,6 +125,7 @@ import org.apache.pinot.spi.utils.BooleanUtils;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.FixedIntArray;
import org.apache.pinot.spi.utils.MapUtils;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.apache.pinot.spi.utils.UuidUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.roaringbitmap.BatchIterator;
@@ -1720,7 +1720,7 @@ public class MutableSegmentImpl implements MutableSegment
{
switch (dataType) {
case STRING: {
for (Object value : values) {
- rowLength += Utf8.encodedLength((String) value);
+ rowLength += Utf8Utils.encodedLength((String) value);
}
_varByteMVMaxRowLengthInBytes =
Math.max(_varByteMVMaxRowLengthInBytes, rowLength);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedColumnStatistics.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedColumnStatistics.java
index a1464e584fc..9d2f33dc639 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedColumnStatistics.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedColumnStatistics.java
@@ -19,7 +19,6 @@
package org.apache.pinot.segment.local.realtime.converter.stats;
import com.google.common.base.Preconditions;
-import com.google.common.base.Utf8;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import java.math.BigDecimal;
@@ -31,6 +30,7 @@ import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.roaringbitmap.RoaringBitmap;
@@ -237,7 +237,7 @@ public class CompactedColumnStatistics extends
MutableColumnStatistics {
maxValue = values[_cardinality - 1];
isAscii = true;
for (String value : values) {
- int length = Utf8.encodedLength(value);
+ int length = Utf8Utils.encodedLength(value);
minElementLength = Math.min(minElementLength, length);
maxElementLength = Math.max(maxElementLength, length);
if (isAscii) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
index ff96b76f58b..25fd626072d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
@@ -19,13 +19,13 @@
package org.apache.pinot.segment.local.realtime.converter.stats;
import com.google.common.base.Preconditions;
-import com.google.common.base.Utf8;
import java.math.BigDecimal;
import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.roaringbitmap.IntIterator;
import org.roaringbitmap.RoaringBitmap;
@@ -259,7 +259,7 @@ public class CompactedNoDictColumnStatistics extends
MutableNoDictColumnStatisti
if (maxString == null || value.compareTo(maxString) > 0) {
maxString = value;
}
- int length = Utf8.encodedLength(value);
+ int length = Utf8Utils.encodedLength(value);
minElementLength = Math.min(minElementLength, length);
maxElementLength = Math.max(maxElementLength, length);
if (isAscii) {
@@ -406,7 +406,7 @@ public class CompactedNoDictColumnStatistics extends
MutableNoDictColumnStatisti
case BIG_DECIMAL:
return BigDecimalUtils.byteSize((BigDecimal) value);
case STRING:
- return Utf8.encodedLength((String) value);
+ return Utf8Utils.encodedLength((String) value);
case BYTES:
return ((ByteArray) value).length();
default:
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/StringOnHeapMutableDictionary.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/StringOnHeapMutableDictionary.java
index ad8ef63cd5a..721667c6027 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/StringOnHeapMutableDictionary.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/StringOnHeapMutableDictionary.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.realtime.impl.dictionary;
-import com.google.common.base.Utf8;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.ints.IntSets;
@@ -181,11 +180,11 @@ public class StringOnHeapMutableDictionary extends
BaseOnHeapMutableDictionary {
@Override
public int getValueSize(int dictId) {
- return Utf8.encodedLength(getStringValue(dictId));
+ return Utf8Utils.encodedLength(getStringValue(dictId));
}
private void updateStats(String value) {
- int utf8Length = Utf8.encodedLength(value);
+ int utf8Length = Utf8Utils.encodedLength(value);
if (_min == null) {
_min = value;
_max = value;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/CLPMutableForwardIndex.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/CLPMutableForwardIndex.java
index 2341a6e6471..c618bc4c43a 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/CLPMutableForwardIndex.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/CLPMutableForwardIndex.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.realtime.impl.forward;
-import com.google.common.base.Utf8;
import com.yscope.clp.compressorfrontend.BuiltInVariableHandlingRuleVersions;
import com.yscope.clp.compressorfrontend.EncodedMessage;
import com.yscope.clp.compressorfrontend.MessageDecoder;
@@ -32,6 +31,7 @@ import
org.apache.pinot.segment.spi.index.mutable.MutableDictionary;
import org.apache.pinot.segment.spi.index.mutable.MutableForwardIndex;
import org.apache.pinot.segment.spi.memory.PinotDataBufferMemoryManager;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.Utf8Utils;
public class CLPMutableForwardIndex implements MutableForwardIndex {
@@ -116,7 +116,7 @@ public class CLPMutableForwardIndex implements
MutableForwardIndex {
String[] dictVars;
Long[] encodedVars;
- int length = Utf8.encodedLength(value);
+ int length = Utf8Utils.encodedLength(value);
_lengthOfShortestElement = Math.min(_lengthOfShortestElement, length);
_lengthOfLongestElement = Math.max(_lengthOfLongestElement, length);
if (_isAscii) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/SameValueMutableForwardIndex.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/SameValueMutableForwardIndex.java
index 2e01a391fff..9d0659f9ed4 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/SameValueMutableForwardIndex.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/SameValueMutableForwardIndex.java
@@ -18,11 +18,11 @@
*/
package org.apache.pinot.segment.local.realtime.impl.forward;
-import com.google.common.base.Utf8;
import java.io.IOException;
import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.index.mutable.MutableForwardIndex;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.Utf8Utils;
/// SameValueMutableForwardIndex is used to wrap any MutableForwardIndex, but
store the same value. This is done to
@@ -36,7 +36,7 @@ public class SameValueMutableForwardIndex implements
MutableForwardIndex {
public SameValueMutableForwardIndex(Object actualValue, MutableForwardIndex
delegate) {
_actualValue = actualValue.toString();
_actualValues = new Object[]{_actualValue};
- _valueLength = Utf8.encodedLength(_actualValue);
+ _valueLength = Utf8Utils.encodedLength(_actualValue);
_delegate = delegate;
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
index f9fde3c6ea3..0cc6ff6d358 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/json/MutableJsonIndexImpl.java
@@ -19,12 +19,10 @@
package org.apache.pinot.segment.local.realtime.impl.json;
import com.google.common.base.Preconditions;
-import com.google.common.base.Utf8;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -59,6 +57,7 @@ import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.exception.BadQueryRequestException;
import org.apache.pinot.spi.query.QueryThreadContext;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.apache.pinot.sql.parsers.CalciteSqlParser;
import org.roaringbitmap.IntConsumer;
import org.roaringbitmap.RoaringBitmap;
@@ -150,12 +149,12 @@ public class MutableJsonIndexImpl implements
MutableJsonIndex {
// Put both key and key-value into the posting list. Key is useful for
checking if a key exists in the json.
String key = entry.getKey();
_postingListMap.computeIfAbsent(key, k -> {
- _bytesSize += estimateLength(key);
+ _bytesSize += Utf8Utils.encodedLength(key);
return new RoaringBitmap();
}).add(_nextFlattenedDocId);
String keyValue = key + JsonIndexCreator.KEY_VALUE_SEPARATOR +
entry.getValue();
_postingListMap.computeIfAbsent(keyValue, k -> {
- _bytesSize += estimateLength(keyValue);
+ _bytesSize += Utf8Utils.encodedLength(keyValue);
return new RoaringBitmap();
}).add(_nextFlattenedDocId);
}
@@ -163,16 +162,6 @@ public class MutableJsonIndexImpl implements
MutableJsonIndex {
}
}
- private static int estimateLength(String value) {
- // Utf8.encodedLength counts bytes without allocating but throws on
malformed strings (e.g. unpaired surrogates).
- // _bytesSize is only a heuristic, so fall back to getBytes (which never
throws) instead of failing.
- try {
- return Utf8.encodedLength(value);
- } catch (IllegalArgumentException e) {
- return value.getBytes(StandardCharsets.UTF_8).length;
- }
- }
-
@Override
public MutableRoaringBitmap getMatchingDocIds(String filterString) {
FilterContext filter;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformer.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformer.java
index 51183eca92f..518d28d2c56 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformer.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformer.java
@@ -20,7 +20,6 @@ package org.apache.pinot.segment.local.recordtransformer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Preconditions;
-import com.google.common.base.Utf8;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -48,6 +47,7 @@ import org.apache.pinot.spi.metrics.PinotMeter;
import org.apache.pinot.spi.recordtransformer.RecordTransformer;
import org.apache.pinot.spi.stream.StreamDataDecoderImpl;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -140,7 +140,7 @@ public class SchemaConformingTransformer implements
RecordTransformer {
unindexableExtrasFieldName != null ?
getAndValidateExtrasFieldType(schema, unindexableExtrasFieldName) : null;
_mergedTextIndexFieldSpec =
schema.getDimensionSpec(_transformerConfig.getMergedTextIndexField());
_schemaTree = validateSchemaAndCreateTree(schema, _transformerConfig);
- _jsonKeyValueSeparatorByteCount =
Utf8.encodedLength(_transformerConfig.getJsonKeyValueSeparator());
+ _jsonKeyValueSeparatorByteCount =
Utf8Utils.encodedLength(_transformerConfig.getJsonKeyValueSeparator());
_continueOnError = ingestionConfig.isContinueOnError();
_serverMetrics = ServerMetrics.get();
_throttledLogger = new ThrottledLogger(_logger, ingestionConfig);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
index e0849ce369d..9717e34d0fb 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
@@ -490,7 +490,7 @@ public class SegmentDictionaryCreator implements
IndexCreator {
_totalUncompressedValueSizeInBytes +=
BigDecimalUtils.byteSize((BigDecimal) value);
break;
case STRING:
- _totalUncompressedValueSizeInBytes +=
Utf8Utils.encodedLengthWithReplacement((String) value);
+ _totalUncompressedValueSizeInBytes += Utf8Utils.encodedLength((String)
value);
break;
case BYTES:
_totalUncompressedValueSizeInBytes += ((ByteArray)
value).getBytes().length;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
index 1ca8783bb4a..babcd5be606 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
@@ -19,7 +19,6 @@
package org.apache.pinot.segment.local.segment.creator.impl.openstruct;
import com.google.common.base.Preconditions;
-import com.google.common.base.Utf8;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -74,6 +73,7 @@ import org.apache.pinot.spi.data.OpenStructNaming;
import org.apache.pinot.spi.data.OpenStructTypeInference;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.PinotDataType;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.roaringbitmap.RoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -748,7 +748,7 @@ public class OpenStructColumnSplitter implements
ColumnarOpenStructIndexCreator
try {
String json = JsonUtils.objectToString(sparseEntries);
jsonPerDoc[docId] = json;
- maxLen = Math.max(maxLen, Utf8.encodedLength(json));
+ maxLen = Math.max(maxLen, Utf8Utils.encodedLength(json));
} catch (IOException e) {
throw new RuntimeException("Failed to serialize sparse entries for
docId " + docId, e);
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
index 6810d55e010..adac748fe6c 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
@@ -19,7 +19,6 @@
package org.apache.pinot.segment.local.segment.creator.impl.stats;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
-import com.google.common.base.Utf8;
import java.math.BigDecimal;
import java.util.HashSet;
import java.util.Set;
@@ -32,6 +31,7 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -205,7 +205,7 @@ public class NoDictColumnStatisticsCollector extends
AbstractColumnStatisticsCol
case BIG_DECIMAL:
return BigDecimalUtils.byteSize((BigDecimal) value);
case STRING:
- return Utf8.encodedLength((String) value);
+ return Utf8Utils.encodedLength((String) value);
case BYTES:
return ((byte[]) value).length;
default:
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
index 26a168d4986..b4f7fb50b11 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
@@ -19,7 +19,6 @@
package org.apache.pinot.segment.local.segment.creator.impl.stats;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Utf8;
import com.yscope.clp.compressorfrontend.BuiltInVariableHandlingRuleVersions;
import com.yscope.clp.compressorfrontend.EncodedMessage;
import com.yscope.clp.compressorfrontend.MessageEncoder;
@@ -32,6 +31,7 @@ import
org.apache.pinot.segment.spi.creator.StatsCollectorConfig;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.utils.Utf8Utils;
@@ -73,7 +73,7 @@ public class StringColumnPreIndexStatsCollector extends
AbstractColumnStatistics
if (_clpStatsCollector != null) {
_clpStatsCollector.collect(value);
}
- int length = Utf8.encodedLength(value);
+ int length = Utf8Utils.encodedLength(value);
if (_values.add(value)) {
_minLength = Math.min(_minLength, length);
_maxLength = Math.max(_maxLength, length);
@@ -96,7 +96,7 @@ public class StringColumnPreIndexStatsCollector extends
AbstractColumnStatistics
if (isPartitionEnabled()) {
updatePartition(value);
}
- int length = Utf8.encodedLength(value);
+ int length = Utf8Utils.encodedLength(value);
if (_isAscii) {
_isAscii = length == value.length();
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
index 88355ba28de..41cc548a58d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
@@ -1424,12 +1424,12 @@ public class ForwardIndexHandler extends
BaseIndexHandler {
case STRING:
if (singleValue) {
String value = forwardIndex.getString(docId, context);
- rowLength = Utf8Utils.encodedLengthWithReplacement(value);
+ rowLength = Utf8Utils.encodedLength(value);
stats.addElement(rowLength, isAscii(value));
} else {
int numValues = forwardIndex.getStringMV(docId, stringBuffer,
context);
for (int i = 0; i < numValues; i++) {
- int valueLength =
Utf8Utils.encodedLengthWithReplacement(stringBuffer[i]);
+ int valueLength = Utf8Utils.encodedLength(stringBuffer[i]);
rowLength += valueLength;
stats.addElement(valueLength, isAscii(stringBuffer[i]));
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatistics.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatistics.java
index 199195b4239..6abdb8b7c1f 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatistics.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatistics.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.segment.index.loader.defaultcolumn;
-import com.google.common.base.Utf8;
import java.math.BigDecimal;
import java.util.Set;
import javax.annotation.Nullable;
@@ -28,6 +27,7 @@ import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.Utf8Utils;
/// Column statistics for a default-value column.
@@ -68,7 +68,7 @@ public class DefaultColumnStatistics implements
ColumnStatistics {
case STRING:
String stringValue = (String) value;
_valueSet = new String[]{stringValue};
- length = Utf8.encodedLength(stringValue);
+ length = Utf8Utils.encodedLength(stringValue);
isAscii = length == stringValue.length();
break;
case BYTES:
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/PartitionIdVirtualColumnProvider.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/PartitionIdVirtualColumnProvider.java
index 9431617d1fe..f88eaa0ddcb 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/PartitionIdVirtualColumnProvider.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/PartitionIdVirtualColumnProvider.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.segment.virtualcolumn;
-import com.google.common.base.Utf8;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
@@ -37,6 +36,7 @@ import
org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.Utf8Utils;
/// Virtual column provider that returns the partition information of the
segment.
@@ -212,7 +212,7 @@ public class PartitionIdVirtualColumnProvider implements
VirtualColumnProvider {
@Override
public int getValueSize(int dictId) {
- return Utf8.encodedLength(_values.get(dictId));
+ return Utf8Utils.encodedLength(_values.get(dictId));
}
@Override
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
index ad2a5ad5d84..ca41f1d5742 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
@@ -19,7 +19,6 @@
package org.apache.pinot.segment.local.startree;
import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Utf8;
import java.io.File;
import java.io.IOException;
import java.nio.ByteOrder;
@@ -51,6 +50,7 @@ import
org.apache.pinot.segment.spi.utils.SegmentMetadataUtils;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.Utf8Utils;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -147,7 +147,7 @@ public class StarTreeBuilderUtils {
for (String dimension : dimensions) {
headerSizeInBytes += Integer.BYTES; // For dimension index
headerSizeInBytes += Integer.BYTES; // For length of dimension name
- headerSizeInBytes += Utf8.encodedLength(dimension); // For dimension name
+ headerSizeInBytes += Utf8Utils.encodedLength(dimension); // For
dimension name
}
headerSizeInBytes += Integer.BYTES; // For number of nodes.
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatisticsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatisticsTest.java
index 22ae84fb6b3..45baf0032bc 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatisticsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatisticsTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.realtime.converter.stats;
-import com.google.common.base.Utf8;
import java.math.BigDecimal;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
@@ -28,6 +27,7 @@ import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -304,12 +304,12 @@ public class MutableNoDictColumnStatisticsTest {
when(metadata.getMinValue()).thenReturn("hello");
when(metadata.getMaxValue()).thenReturn("héllo");
when(metadata.getMaxNumValuesPerMVEntry()).thenReturn(0);
-
when(metadata.getMaxRowLengthInBytes()).thenReturn(Utf8.encodedLength("héllo"));
+
when(metadata.getMaxRowLengthInBytes()).thenReturn(Utf8Utils.encodedLength("héllo"));
MutableForwardIndex forwardIndex = mock(MutableForwardIndex.class);
when(forwardIndex.isSingleValue()).thenReturn(true);
when(forwardIndex.getLengthOfShortestElement()).thenReturn(5);
-
when(forwardIndex.getLengthOfLongestElement()).thenReturn(Utf8.encodedLength("héllo"));
+
when(forwardIndex.getLengthOfLongestElement()).thenReturn(Utf8Utils.encodedLength("héllo"));
when(forwardIndex.isAscii()).thenReturn(false);
when(forwardIndex.getString(0)).thenReturn("hello");
when(forwardIndex.getString(1)).thenReturn("héllo");
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatisticsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatisticsTest.java
index 9ff9db8055f..175e7a084cc 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatisticsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatisticsTest.java
@@ -18,13 +18,13 @@
*/
package org.apache.pinot.segment.local.segment.index.loader.defaultcolumn;
-import com.google.common.base.Utf8;
import java.math.BigDecimal;
import java.util.Arrays;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -92,7 +92,7 @@ public class DefaultColumnStatisticsTest {
assertFalse(stats.isAscii());
break;
case STRING:
- expectedLength = Utf8.encodedLength((String)
fieldSpec.getDefaultNullValue());
+ expectedLength = Utf8Utils.encodedLength((String)
fieldSpec.getDefaultNullValue());
assertTrue(stats.isAscii()); // default "null" is ASCII
break;
case BYTES:
@@ -148,7 +148,7 @@ public class DefaultColumnStatisticsTest {
int expectedLength;
switch (storedType) {
case STRING:
- expectedLength = Utf8.encodedLength((String)
fieldSpec.getDefaultNullValue());
+ expectedLength = Utf8Utils.encodedLength((String)
fieldSpec.getDefaultNullValue());
assertTrue(stats.isAscii()); // default "null" is ASCII
break;
case BYTES:
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
index b4da5465a45..e4b0d86175c 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import com.google.common.base.Utf8;
import it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet;
import it.unimi.dsi.fastutil.floats.FloatOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
@@ -49,6 +48,7 @@ import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.FALFInterner;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -440,7 +440,7 @@ public class ImmutableDictionaryTest implements
PinotBuffersAfterMethodCheckRule
for (int i = 0; i < NUM_VALUES; i++) {
assertEquals(stringDictionary.get(i), _stringValues[i]);
assertEquals(stringDictionary.getStringValue(i), _stringValues[i]);
- assertEquals(stringDictionary.getValueSize(i),
Utf8.encodedLength(_stringValues[i]));
+ assertEquals(stringDictionary.getValueSize(i),
Utf8Utils.encodedLength(_stringValues[i]));
assertEquals(stringDictionary.indexOf(_stringValues[i]), i);
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index 1a0288d406c..82bfa64f570 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -21,7 +21,6 @@ package org.apache.pinot.server.api.resources;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Preconditions;
-import com.google.common.base.Utf8;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiKeyAuthDefinition;
import io.swagger.annotations.ApiOperation;
@@ -132,6 +131,7 @@ import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.CommonConstants;
import
org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.SegmentStateModel;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.slf4j.Logger;
@@ -272,7 +272,7 @@ public class TablesResource {
// For raw STRING/BYTES/BIG_DECIMAL column, set the
columnLength as the length of the max value.
Comparable<?> maxValue = columnMetadata.getMaxValue();
if (maxValue instanceof String) {
- columnLength = Utf8.encodedLength((String) maxValue);
+ columnLength = Utf8Utils.encodedLength((String) maxValue);
} else if (maxValue instanceof ByteArray) {
columnLength = ((ByteArray) maxValue).length();
} else if (maxValue instanceof BigDecimal) {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Utf8Utils.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Utf8Utils.java
index ea1e8604735..bd7bc1c525c 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Utf8Utils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Utf8Utils.java
@@ -18,45 +18,57 @@
*/
package org.apache.pinot.spi.utils;
-import com.google.common.base.Utf8;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
+import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
+/// UTF-8 helpers whose sizing and encoding agree: [#encodedLength] returns
exactly the length of the array that
+/// [#encode] produces for the same string.
public class Utf8Utils {
private Utf8Utils() {
}
+ // Byte order is irrelevant: isAscii() only ORs the bytes together
+ private static final VarHandle LONG_VIEW =
+ MethodHandles.byteArrayViewVarHandle(long[].class,
ByteOrder.nativeOrder());
+
public static byte[] encode(String s) {
return s.getBytes(StandardCharsets.UTF_8);
}
+ /// Returns the number of bytes in the UTF-8 encoding of `s`, which is
exactly `encode(s).length`. An unpaired
+ /// surrogate counts as the single replacement byte that [String#getBytes]
writes for it, so this never throws.
public static int encodedLength(String s) {
- return Utf8.encodedLength(s);
- }
-
- /// Returns the UTF-8 byte length using the same malformed UTF-16
replacement behavior as [String#getBytes()].
- /// Unlike [#encodedLength(String)], this method does not reject unpaired
surrogate code units.
- public static int encodedLengthWithReplacement(String s) {
- int length = 0;
- int index = 0;
- while (index < s.length()) {
- char character = s.charAt(index);
- if (character <= 0x7f) {
- length++;
- } else if (character <= 0x7ff) {
- length += 2;
- } else if (Character.isHighSurrogate(character) && index + 1 < s.length()
- && Character.isLowSurrogate(s.charAt(index + 1))) {
- length += 4;
- index++;
- } else if (Character.isSurrogate(character)) {
- length++;
- } else {
- length += 3;
+ int utf16Length = s.length();
+ // Fast path for the common all-ASCII prefix, one byte per char
+ int i = 0;
+ while (i < utf16Length && s.charAt(i) < 0x80) {
+ i++;
+ }
+ // Every char counts one byte up front. Add 1 for a 2-byte char
(branch-free: (0x7f - c) >>> 31 is 1 iff c >= 0x80),
+ // 2 for a 3-byte char or a surrogate pair, and nothing for an unpaired
surrogate, which encodes as the single
+ // replacement byte '?'. The surrogate ranges are compared inline instead
of through Character.isSurrogate() and
+ // similar helpers: C2 folds adjacent range checks that guard uncommon
traps into one unsigned comparison only when
+ // both sit in the same method, so a check inside an inlined helper stays
a separate compare-and-branch per char,
+ // which measured 1.6x slower on CJK text.
+ int utf8Length = utf16Length;
+ for (; i < utf16Length; i++) {
+ char c = s.charAt(i);
+ if (c < 0x800) {
+ utf8Length += (0x7f - c) >>> 31;
+ } else if (c < Character.MIN_SURROGATE || c > Character.MAX_SURROGATE) {
+ utf8Length += 2;
+ } else if (c <= Character.MAX_HIGH_SURROGATE && i + 1 < utf16Length) {
+ char next = s.charAt(i + 1);
+ if (next >= Character.MIN_LOW_SURROGATE && next <=
Character.MAX_LOW_SURROGATE) {
+ utf8Length += 2;
+ i++;
+ }
}
- index++;
}
- return length;
+ return utf8Length;
}
public static String decode(byte[] bytes) {
@@ -64,11 +76,17 @@ public class Utf8Utils {
}
public static boolean isAscii(byte[] bytes) {
- int or = 0;
- for (byte b : bytes) {
- // Do not check within loop because most values are ASCII
- or |= b;
+ int length = bytes.length;
+ int loopBound = length & ~0x7;
+ long or = 0;
+ int i = 0;
+ for (; i < loopBound; i += 8) {
+ or |= (long) LONG_VIEW.get(bytes, i);
+ }
+ for (; i < length; i++) {
+ or |= bytes[i];
}
- return (or & 0x80) == 0;
+ // Do not check within the loops because most values are ASCII
+ return (or & 0x8080808080808080L) == 0;
}
}
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/Utf8UtilsTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/Utf8UtilsTest.java
index 4e2e3943445..0fafd2c201e 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/Utf8UtilsTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/Utf8UtilsTest.java
@@ -18,19 +18,65 @@
*/
package org.apache.pinot.spi.utils;
-import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
-/// Tests UTF-8 sizing contracts used by segment metadata and compression
statistics.
+/// Tests [Utf8Utils].
public class Utf8UtilsTest {
+ @DataProvider
+ public Object[][] encodedLengths() {
+ return new Object[][]{
+ {"", 0},
+ {"ascii", 5},
+ // Two-byte characters
+ {"caf\u00e9", 5},
+ {"\u00e9\u00e9", 4},
+ // Three-byte characters
+ {"\u4e2d\u6587", 6},
+ {"a\u20acb", 5},
+ // Surrogate pairs encode to four bytes
+ {"\uD83D\uDE00", 4},
+ {"a\uD83D\uDE00b", 6},
+ // Unpaired surrogates encode to a single replacement byte
+ {"\uD800", 1},
+ {"abc\uD800", 4},
+ {"a\uD800b", 3},
+ {"\uDC00", 1},
+ {"\uDC00\uD800", 2},
+ {"\uD800\uD83D\uDE00", 5}
+ };
+ }
+
+ @Test(dataProvider = "encodedLengths")
+ public void testEncodedLength(String s, int expectedLength) {
+ assertEquals(Utf8Utils.encodedLength(s), expectedLength);
+ assertEquals(Utf8Utils.encode(s).length, expectedLength);
+ }
+
@Test
- public void testEncodedLengthWithReplacementMatchesJdkEncoding() {
- for (String value : new String[]{"ascii", "caf\u00e9", "\uD83D\uDE00",
"\uD800", "\uDC00", "a\uD800b"}) {
- assertEquals(Utf8Utils.encodedLengthWithReplacement(value),
value.getBytes(StandardCharsets.UTF_8).length,
- "Unexpected serialized UTF-8 length for: " + value);
+ public void testIsAscii() {
+ // Cover the empty array and every tail length around the 8-byte stride
+ for (int length = 0; length <= 24; length++) {
+ byte[] bytes = new byte[length];
+ Arrays.fill(bytes, (byte) 'a');
+ assertTrue(Utf8Utils.isAscii(bytes), "length: " + length);
+ for (int i = 0; i < length; i++) {
+ bytes[i] = 0x7f;
+ assertTrue(Utf8Utils.isAscii(bytes), "length: " + length + ", index: "
+ i);
+ bytes[i] = (byte) 0x80;
+ assertFalse(Utf8Utils.isAscii(bytes), "length: " + length + ", index:
" + i);
+ bytes[i] = (byte) 0xff;
+ assertFalse(Utf8Utils.isAscii(bytes), "length: " + length + ", index:
" + i);
+ bytes[i] = 'a';
+ }
}
+ assertTrue(Utf8Utils.isAscii(Utf8Utils.encode("ascii")));
+ assertFalse(Utf8Utils.isAscii(Utf8Utils.encode("caf\u00e9")));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
