This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 50dcbffb4df Refactor splitPart(limit) to avoid full array allocation
(#18892)
50dcbffb4df is described below
commit 50dcbffb4dfc73122a6213cbf32d3401d90715d5
Author: Akanksha kedia <[email protected]>
AuthorDate: Wed Jul 8 03:18:47 2026 +0530
Refactor splitPart(limit) to avoid full array allocation (#18892)
The 4-argument splitPart overload previously allocated a full String[]
via StringUtils.splitByWholeSeparator on every call, even when only a
single element was needed. This is wasteful in hot query paths where
splitPart is invoked per-row.
Replace the array-based implementation with index-based forward scanning
that extracts only the requested field without materializing all split
parts. The new implementation:
- Skips leading separators
- Collapses consecutive separators (matching splitByWholeSeparator
semantics)
- Handles trailing separators (producing one empty trailing token)
- For positive indices: single forward scan, O(index) work
- For negative indices: two passes (count then extract) but still no
String[] allocation
Falls back to the array-based path only for null/empty delimiters
(whitespace splitting) where the rules are complex.
All existing unit tests (172 cases including randomized fuzz) pass
unchanged, confirming behavioral equivalence.
---
.../common/function/scalar/StringFunctions.java | 154 +++++++++++++++++++--
1 file changed, 145 insertions(+), 9 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
index 132dc56cea1..d8630a842b8 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
@@ -709,22 +709,158 @@ public class StringFunctions {
}
/**
+ * Splits input by delimiter with a limit on the number of parts and returns
the part at the given index.
+ * Avoids allocating the full String array by scanning the input directly.
+ *
+ * <p>Replicates the semantics of {@link
StringUtils#splitByWholeSeparator(String, String, int)}:
+ * leading separators are stripped, consecutive separators in the middle are
collapsed,
+ * and trailing separators produce one empty trailing token.
+ *
* @param input the input String to be split into parts.
* @param delimiter the specified delimiter to split the input string.
- * @param limit the max count of parts that the input string can be splitted
into.
- * @param index the specified index for the splitted parts to be returned.
- * @return splits string on the delimiter with the limit count and returns
String at specified index from the split.
+ * @param limit the max count of parts that the input string can be split
into (0 or negative means unlimited).
+ * @param index the specified index for the split parts to be returned;
negative indices count from the end.
+ * @return the part at the given index, or "null" if the index is out of
bounds.
*/
@ScalarFunction
public static String splitPart(String input, String delimiter, int limit,
int index) {
- String[] splitString = StringUtils.splitByWholeSeparator(input, delimiter,
limit);
- if (index >= 0 && index < splitString.length) {
- return splitString[index];
- } else if (index < 0 && index >= -splitString.length) {
- return splitString[splitString.length + index];
- } else {
+ // Null/empty delimiter splits on whitespace — complex rules best handled
by Apache Commons.
+ if (delimiter == null || delimiter.isEmpty()) {
+ return splitPartArrayBased(StringUtils.splitByWholeSeparator(input,
delimiter, limit), index);
+ }
+
+ int inputLen = input.length();
+ int delimLen = delimiter.length();
+ if (inputLen == 0) {
+ return "null";
+ }
+
+ // Guard against Integer.MIN_VALUE (negation overflows)
+ if (index == Integer.MIN_VALUE) {
return "null";
}
+
+ // effectiveLimit: 0 or negative means unlimited
+ int effectiveLimit = limit <= 0 ? Integer.MAX_VALUE : limit;
+
+ if (index >= 0) {
+ return splitPartLimitedForward(input, delimiter, effectiveLimit, index,
inputLen, delimLen);
+ } else {
+ // For negative index, count total fields first then extract the target
+ int totalFields = countFieldsLimited(input, delimiter, effectiveLimit,
inputLen, delimLen);
+ if (totalFields == 0) {
+ return "null";
+ }
+ int adjustedIndex = totalFields + index;
+ if (adjustedIndex < 0) {
+ return "null";
+ }
+ return splitPartLimitedForward(input, delimiter, effectiveLimit,
adjustedIndex, inputLen, delimLen);
+ }
+ }
+
+ /**
+ * Counts the number of fields produced by splitting input with the given
delimiter and limit,
+ * following splitByWholeSeparator semantics (leading seps stripped,
consecutive collapsed,
+ * trailing seps produce one empty field). Does not allocate any String
objects.
+ */
+ private static int countFieldsLimited(String input, String delimiter, int
effectiveLimit,
+ int inputLen, int delimLen) {
+ int pos = 0;
+ // Skip leading separators
+ while (pos < inputLen && input.startsWith(delimiter, pos)) {
+ pos += delimLen;
+ }
+ if (pos >= inputLen) {
+ // Entire string is separators — produces a single empty trailing token
+ return 1;
+ }
+
+ int totalFields = 0;
+ while (pos < inputLen) {
+ totalFields++;
+ if (totalFields >= effectiveLimit) {
+ // Limit reached — this is the last field (remainder)
+ break;
+ }
+ // Scan past non-delimiter content to find the next delimiter
+ int end = input.indexOf(delimiter, pos);
+ if (end == -1) {
+ // No more delimiters — check for trailing delimiter (there isn't
one), so this is the last field
+ break;
+ }
+ // Advance past delimiter(s)
+ pos = end + delimLen;
+ while (pos < inputLen && input.startsWith(delimiter, pos)) {
+ pos += delimLen;
+ }
+ // If we've consumed to end after delimiters, there's a trailing empty
field
+ if (pos >= inputLen) {
+ totalFields++;
+ break;
+ }
+ }
+ return totalFields;
+ }
+
+ /**
+ * Extracts the field at the given positive index by scanning forward
through the input.
+ * Follows splitByWholeSeparator semantics: leading separators stripped,
consecutive collapsed,
+ * trailing separators produce one empty token. With limit, the last field
gets the remainder.
+ */
+ private static String splitPartLimitedForward(String input, String
delimiter, int effectiveLimit, int index,
+ int inputLen, int delimLen) {
+ int pos = 0;
+ // Skip leading separators
+ while (pos < inputLen && input.startsWith(delimiter, pos)) {
+ pos += delimLen;
+ }
+ if (pos >= inputLen) {
+ // Entire string is separators — single empty trailing token at index 0
+ return index == 0 ? "" : "null";
+ }
+
+ int fieldCount = 0;
+ while (pos <= inputLen) {
+ // Check if this is the last field due to limit
+ if (fieldCount + 1 >= effectiveLimit) {
+ // Limit reached — remainder from pos to end is the last field
+ return fieldCount == index ? input.substring(pos) : "null";
+ }
+
+ // Find the next delimiter
+ int end = input.indexOf(delimiter, pos);
+ if (end == -1) {
+ // No more delimiters — current field extends to end of input
+ if (fieldCount == index) {
+ return input.substring(pos);
+ }
+ // Check if input ends with delimiter characters that were already
consumed
+ // (this case is handled by the trailing-delimiter logic below)
+ return "null";
+ }
+
+ // Found a delimiter at 'end' — current field is [pos, end)
+ if (fieldCount == index) {
+ return input.substring(pos, end);
+ }
+
+ // Advance past delimiter and consecutive delimiters
+ pos = end + delimLen;
+ while (pos < inputLen && input.startsWith(delimiter, pos)) {
+ pos += delimLen;
+ }
+ fieldCount++;
+
+ // If we've consumed to the end after delimiters, there's a trailing
empty field
+ if (pos >= inputLen) {
+ if (fieldCount == index) {
+ return "";
+ }
+ return "null";
+ }
+ }
+ return "null";
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
