deniskuzZ commented on code in PR #6747:
URL: https://github.com/apache/hive/pull/6747#discussion_r3940782927
##########
serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java:
##########
@@ -402,105 +453,248 @@ public String getDetailedReadPositionString() {
sb.append(" at field start position ");
sb.append(startPositions[currentTopLevelFieldIndex]);
int currentFieldLength = startPositions[currentTopLevelFieldIndex + 1] -
- startPositions[currentTopLevelFieldIndex] - 1;
+ startPositions[currentTopLevelFieldIndex] - topLevelSeparatorLen;
sb.append(" for field length ");
sb.append(currentFieldLength);
}
return sb.toString();
}
+ /**
+ * Bytes at {@code buf[off..off+dlen)} equal to {@code delim[0..dlen)}?
+ *
+ * Caller has already checked {@code buf[off] == delim[0]}, so we start at
+ * index 1 — this is only ever invoked when the first byte matched, which
+ * keeps the multi-byte hot loop from paying for a tail compare on every
+ * mismatching input byte.
+ */
+ private static boolean matchesAt(byte[] buf, int off, byte[] delim, int
dlen) {
+ for (int i = 1; i < dlen; i++) {
+ if (buf[off + i] != delim[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Parse the byte[] and fill each field.
*
* This is an adapted version of the parse method in the LazyStruct class.
* They should parse things the same way.
+ *
+ * <p>Structure: pick one of three parsing algorithms based on the
(isEscaped,
+ * fieldDelimMulti) pair, run it to find each field's start position, then
+ * fill the trailing sentinel and record EOF. Each algorithm is a single-
+ * callsite private method — splitting them out is a readability move; the
+ * combined loops used to hide three flow-controls inside one big if/else.
+ *
+ * <p>The helpers write into {@link #startPositions} and (for the escape
+ * path) {@link #escapeCounts} directly, and stash the parse-stop cursor —
+ * how many fields they consumed, and where in the byte[] they stopped — on
+ * two instance fields {@link #parsedFieldCount} / {@link
#parsedFieldByteEnd}.
+ * Bundling those into a return value would cost a heap allocation on every
+ * row, which we can't afford here.
*/
private void topLevelParse() {
+ if (!isEscaped) {
+ if (fieldDelimMulti == null) {
+ parseSingleByteNoEscape();
+ } else {
+ parseMultiByteNoEscape();
+ }
+ } else {
+ parseSingleByteWithEscape();
+ }
- int fieldId = 0;
- int fieldByteBegin = start;
- int fieldByteEnd = start;
+ final int fieldId = parsedFieldCount;
+ final int fieldByteEnd = parsedFieldByteEnd;
+ /*
+ * All fields have been parsed, or the row has been consumed. Fill
+ * startPositions[fieldId..end] with a sentinel so that the shared
+ * "length = next.start - this.start - sepLen" arithmetic used by
+ * readField yields a negative length for missing fields —
+ * uncheckedGetField turns those into SQL NULLs. Charge the actual
+ * separator width (1 byte for the single-byte fast path, delim.length
+ * for the multi-byte MultiDelimit path) so both paths share the same
+ * downstream code.
+ */
+ if (fieldId == fieldCount || fieldByteEnd == end) {
+ Arrays.fill(startPositions, fieldId, startPositions.length,
+ fieldByteEnd + topLevelSeparatorLen);
+ }
+
+ isEndOfInputReached = (fieldByteEnd == end);
+ }
+
+ /*
+ * Parse-stop cursor written by the three parseXxx helpers and consumed by
+ * topLevelParse() for its sentinel/EOF handling. Using fields instead of a
+ * multi-value return avoids per-row allocation on the hot path.
+ */
+ private int parsedFieldCount;
+ private int parsedFieldByteEnd;
+ /**
+ * Single-byte top-level FIELD_DELIM, no escape — the hot path on every row
+ * of a LazySimple table.
+ */
+ private void parseSingleByteNoEscape() {
final byte separator = this.separators[0];
final int fieldCount = this.fieldCount;
Review Comment:
same thing
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
