This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 083f2937380 Meter failed ERROR-segment repairs per table (#20018)
083f2937380 is described below
commit 083f293738090b43e9f4b9425654c9810acfbda5
Author: Xiang Fu <[email protected]>
AuthorDate: Fri Oct 23 18:37:02 2026 -0700
Meter failed ERROR-segment repairs per table (#20018)
Since #19909 the periodic ERROR-segment repair resets only the replicas
still in ERROR and processes every segment before failing, but a
non-pauseless table surfaces a failed reset only through the generic
periodic-task error meter. Count each reset that could not be sent and
each failed re-ingestion trigger, once per segment per pass, in the new
ControllerMeter.LLC_ERROR_SEGMENT_REPAIR_FAILURES for every table, and
cover it for pauseless and non-pauseless tables.
---
.../pinot/common/metrics/ControllerMeter.java | 5 +++
.../realtime/PinotLLCRealtimeSegmentManager.java | 9 ++++-
.../PinotLLCRealtimeSegmentManagerTest.java | 42 ++++++++++++++++++++++
3 files changed, 55 insertions(+), 1 deletion(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
index 08c88e8eef5..004e4609301 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
@@ -59,6 +59,11 @@ public enum ControllerMeter implements AbstractMetrics.Meter
{
LLC_SEGMENTS_DEEP_STORE_UPLOAD_RETRY_SUCCESS("LLCSegmentDeepStoreUploadRetrySuccess",
false),
LLC_SEGMENTS_DEEP_STORE_UPLOAD_RETRY_ERROR("LLCSegmentDeepStoreUploadRetryError",
false),
SEGMENT_MISSING_DEEP_STORE_LINK("RealtimeSegmentMissingDeepStoreLink",
false),
+ // Number of ERROR-segment repairs the periodic repair pass failed to
initiate: a reset that could not be sent, or a
+ // re-ingestion trigger that failed. Counted once per affected segment per
pass, so a segment that fails again on a
+ // later pass is counted again. Repairs skipped by policy, and repairs that
fail asynchronously after they were sent,
+ // are not counted.
+ LLC_ERROR_SEGMENT_REPAIR_FAILURES("failures", false),
DELETED_TMP_SEGMENT_COUNT("DeletedTmpSegmentCount", false),
TABLE_REBALANCE_FAILURE_DETECTED("TableRebalanceFailureDetected", false),
TABLE_REBALANCE_RETRY("TableRebalanceRetry", false),
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
index d2339fd5ad1..e1a7df47e42 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
@@ -3151,7 +3151,8 @@ public class PinotLLCRealtimeSegmentManager implements
PinotClusterConfigChangeL
/// Segments counted as unrecoverable are those this pass cannot repair or
cannot issue a repair for: skipped by the
/// policy above, with no replica left consuming, with nothing but a
possible local copy to load from, with no server
/// to re-ingest on, or whose re-ingestion trigger or reset fails. A failed
reset fails the call once every segment
- /// is processed.
+ /// is processed. Each reset that could not be sent and each failed
re-ingestion trigger is also counted, once per
+ /// segment per pass and for every table, in
[ControllerMeter#LLC_ERROR_SEGMENT_REPAIR_FAILURES].
///
/// @param tableConfig The table config
public void repairSegmentsInErrorState(TableConfig tableConfig,
@@ -3282,6 +3283,8 @@ public class PinotLLCRealtimeSegmentManager implements
PinotClusterConfigChangeL
} catch (Exception e) {
unrecoverableSegments.add(segmentName);
LOGGER.error("Failed to trigger re-ingestion for segment: {} on
server: {}", segmentName, aliveServer, e);
+ _controllerMetrics.addMeteredTableValue(realtimeTableName,
ControllerMeter.LLC_ERROR_SEGMENT_REPAIR_FAILURES,
+ 1L);
}
continue;
}
@@ -3337,6 +3340,10 @@ public class PinotLLCRealtimeSegmentManager implements
PinotClusterConfigChangeL
segmentsToReset.isEmpty() ? Set.of() :
_helixResourceManager.resetErrorReplicas(realtimeTableName,
segmentsToReset);
unrecoverableSegments.addAll(segmentsFailedToReset);
+ if (!segmentsFailedToReset.isEmpty()) {
+ _controllerMetrics.addMeteredTableValue(realtimeTableName,
ControllerMeter.LLC_ERROR_SEGMENT_REPAIR_FAILURES,
+ segmentsFailedToReset.size());
+ }
if (isPauselessTable) {
_controllerMetrics.setOrUpdateTableGauge(realtimeTableName,
ControllerGauge.PAUSELESS_SEGMENTS_IN_UNRECOVERABLE_ERROR_COUNT,
unrecoverableSegments.size());
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
index 0af3d8d542c..00e2b51a28b 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
@@ -3180,6 +3180,48 @@ public class PinotLLCRealtimeSegmentManagerTest {
assertTrue(exception.getMessage().contains(donePartialErrorSegment),
exception.getMessage());
verify(mockControllerMetrics).setOrUpdateTableGauge(REALTIME_TABLE_NAME,
ControllerGauge.PAUSELESS_SEGMENTS_IN_UNRECOVERABLE_ERROR_COUNT, 2L);
+ // The failed re-ingestion trigger and the failed reset are each metered
+ verify(mockControllerMetrics,
times(2)).addMeteredTableValue(REALTIME_TABLE_NAME,
+ ControllerMeter.LLC_ERROR_SEGMENT_REPAIR_FAILURES, 1L);
+ }
+
+ @Test
+ public void
testRepairSegmentsInErrorStateMetersFailedResetsOnNonPauselessTable() {
+ PinotHelixResourceManager mockHelixResourceManager =
mock(PinotHelixResourceManager.class);
+ ControllerMetrics mockControllerMetrics = mock(ControllerMetrics.class);
+ FakePinotLLCRealtimeSegmentManager segmentManager =
+ new FakePinotLLCRealtimeSegmentManager(mockHelixResourceManager,
mockControllerMetrics);
+ IdealState idealState = new IdealState(REALTIME_TABLE_NAME);
+ ExternalView externalView = new ExternalView(REALTIME_TABLE_NAME);
+ String failedSegment = addSegmentForErrorRepair(mockHelixResourceManager,
idealState, externalView, 0, 0,
+ Status.DONE, SegmentStateModel.ONLINE, SegmentStateModel.ERROR);
+ String otherFailedSegment =
addSegmentForErrorRepair(mockHelixResourceManager, idealState, externalView, 1,
0,
+ Status.DONE, SegmentStateModel.ERROR, SegmentStateModel.ONLINE);
+ String repairedSegment =
addSegmentForErrorRepair(mockHelixResourceManager, idealState, externalView, 2,
0,
+ Status.DONE, SegmentStateModel.ERROR, SegmentStateModel.ONLINE);
+ segmentManager._idealState = idealState;
+
when(mockHelixResourceManager.getTableExternalView(REALTIME_TABLE_NAME)).thenReturn(externalView);
+ Set<String> segmentsToReset = Set.of(failedSegment, otherFailedSegment,
repairedSegment);
+ Set<String> failedSegments = Set.of(failedSegment, otherFailedSegment);
+ when(mockHelixResourceManager.resetErrorReplicas(REALTIME_TABLE_NAME,
segmentsToReset)).thenReturn(
+ failedSegments);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME)
+ .setNumReplicas(2)
+
.setStreamConfigs(FakeStreamConfigUtils.getDefaultLowLevelStreamConfigs().getStreamConfigsMap())
+ .build();
+
+ // The three segments are reset in one batch. The two failed ones are
metered by their count, which a
+ // non-pauseless table has no unrecoverable gauge for, and fail the pass
as before.
+ RuntimeException exception = expectThrows(RuntimeException.class,
+ () -> segmentManager.repairSegmentsInErrorState(tableConfig, false));
+ for (String segmentName : failedSegments) {
+ assertTrue(exception.getMessage().contains(segmentName),
exception.getMessage());
+ }
+ verify(mockHelixResourceManager).resetErrorReplicas(REALTIME_TABLE_NAME,
segmentsToReset);
+ verify(mockControllerMetrics).addMeteredTableValue(REALTIME_TABLE_NAME,
+ ControllerMeter.LLC_ERROR_SEGMENT_REPAIR_FAILURES, 2L);
+ verify(mockControllerMetrics,
never()).setOrUpdateTableGauge(eq(REALTIME_TABLE_NAME),
+ eq(ControllerGauge.PAUSELESS_SEGMENTS_IN_UNRECOVERABLE_ERROR_COUNT),
anyLong());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
