Jackie-Jiang commented on code in PR #12697:
URL: https://github.com/apache/pinot/pull/12697#discussion_r1550689745
##########
pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java:
##########
@@ -591,67 +600,52 @@ private boolean processStreamEvents(MessageBatch
messagesAndOffsets, long idlePi
_numRowsErrored++;
// when exception happens we prefer abandoning the whole batch and
not partially indexing some rows
reusedResult.getTransformedRows().clear();
- String errorMessage = String.format("Caught exception while
transforming the record at offset: %s , row: %s",
- messageOffset, decodedRow.getResult());
+ String errorMessage =
+ String.format("Caught exception while transforming the record at
offset: %s , row: %s", offset,
+ decodedRow.getResult());
_segmentLogger.error(errorMessage, e);
_realtimeTableDataManager.addSegmentError(_segmentNameStr, new
SegmentErrorInfo(now(), errorMessage, e));
}
if (reusedResult.getSkippedRowCount() > 0) {
- realtimeRowsDroppedMeter =
- _serverMetrics.addMeteredTableValue(_clientId,
ServerMeter.REALTIME_ROWS_FILTERED,
- reusedResult.getSkippedRowCount(), realtimeRowsDroppedMeter);
+ realtimeRowsDroppedMeter =
_serverMetrics.addMeteredTableValue(_clientId,
ServerMeter.REALTIME_ROWS_FILTERED,
+ reusedResult.getSkippedRowCount(), realtimeRowsDroppedMeter);
}
if (reusedResult.getIncompleteRowCount() > 0) {
realtimeIncompleteRowsConsumedMeter =
_serverMetrics.addMeteredTableValue(_clientId,
ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED,
reusedResult.getIncompleteRowCount(),
realtimeIncompleteRowsConsumedMeter);
}
List<GenericRow> transformedRows = reusedResult.getTransformedRows();
- if (transformedRows.size() > 0) {
- hasTransformedRows = true;
- }
for (GenericRow transformedRow : transformedRows) {
try {
- canTakeMore = _realtimeSegment.index(transformedRow, msgMetadata);
+ canTakeMore = _realtimeSegment.index(transformedRow, metadata);
indexedMessageCount++;
- _lastRowMetadata = msgMetadata;
+ _lastRowMetadata = metadata;
_lastConsumedTimestampMs = System.currentTimeMillis();
realtimeRowsConsumedMeter =
_serverMetrics.addMeteredTableValue(_clientId,
ServerMeter.REALTIME_ROWS_CONSUMED, 1,
realtimeRowsConsumedMeter);
_serverMetrics.addMeteredGlobalValue(ServerMeter.REALTIME_ROWS_CONSUMED, 1L);
} catch (Exception e) {
_numRowsErrored++;
- String errorMessage = String.format("Caught exception while
indexing the record at offset: %s , row: %s",
- messageOffset, transformedRow);
+ String errorMessage =
+ String.format("Caught exception while indexing the record at
offset: %s , row: %s", offset,
+ transformedRow);
_segmentLogger.error(errorMessage, e);
_realtimeTableDataManager.addSegmentError(_segmentNameStr, new
SegmentErrorInfo(now(), errorMessage, e));
}
}
}
- _currentOffset = messageOffset;
+ _currentOffset = nextOffset;
_numRowsIndexed = _realtimeSegment.getNumDocsIndexed();
_numRowsConsumed++;
streamMessageCount++;
}
Review Comment:
The logic was added in #10418, and unfortunately seems like there is no test
added.
Since we already provided a default implementation of
`getLastMessageMetadata()`, it should be safe to simplify this logic.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
