This is an automated email from the ASF dual-hosted git repository. jlli pushed a commit to branch clean-up-non-leader-metrics in repository https://gitbox.apache.org/repos/asf/pinot.git
commit ecffaf7b9243dcce7c6a881821ef5ab850c36820 Author: Jack Li(Analytics Engineering) <[email protected]> AuthorDate: Mon Oct 11 13:07:53 2021 -0700 Clean up controller-table related metrics in ControllerPeriodicTask --- .../pinot/common/metrics/ValidationMetrics.java | 57 ++++++++++++++++++++++ .../controller/helix/SegmentStatusChecker.java | 16 ++++++ .../core/periodictask/ControllerPeriodicTask.java | 14 ++++++ .../validation/OfflineSegmentIntervalChecker.java | 14 ++++++ .../RealtimeSegmentValidationManager.java | 10 ++++ 5 files changed, 111 insertions(+) diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java index efe3c82..e9208f8 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java @@ -141,6 +141,16 @@ public class ValidationMetrics { } /** + * Cleans up the missing segment count gauge. + * + * @param resource The resource for which the gauge is removed + */ + public void cleanupMissingSegmentCountGauge(final String resource) { + final String fullGaugeName = makeGaugeName(resource, "missingSegmentCount"); + removeGauge(fullGaugeName); + } + + /** * Updates the offline segment delay gauge. * * @param resource The resource for which the gauge is updated @@ -154,6 +164,16 @@ public class ValidationMetrics { } /** + * Cleans up offline segment delay gauge. + * + * @param resource The resource for which the gauge is removed + */ + public void cleanupOfflineSegmentDelayGauge(final String resource) { + final String fullGaugeNameHours = makeGaugeName(resource, "offlineSegmentDelayHours"); + removeGauge(fullGaugeNameHours); + } + + /** * Updates the last push time gauge. * * @param resource The resource for which the gauge is updated @@ -167,6 +187,16 @@ public class ValidationMetrics { } /** + * Cleans up the last push time gauge. + * + * @param resource The resource for which the gauge is removed + */ + public void cleanupLastPushTimeGauge(final String resource) { + final String fullGaugeNameHours = makeGaugeName(resource, "lastPushTimeDelayHours"); + removeGauge(fullGaugeNameHours); + } + + /** * Updates the total document count gauge. * * @param resource The resource for which the gauge is updated @@ -178,6 +208,16 @@ public class ValidationMetrics { } /** + * Cleans up the total document count gauge. + * + * @param resource The resource for which the gauge is removed + */ + public void cleanupTotalDocumentCountGauge(final String resource) { + final String fullGaugeName = makeGaugeName(resource, "TotalDocumentCount"); + removeGauge(fullGaugeName); + } + + /** * Updates the non consuming partition count metric. * * @param resource The resource for which the gauge is updated @@ -199,6 +239,16 @@ public class ValidationMetrics { makeGauge(fullGaugeName, makeMetricName(fullGaugeName), _storedValueGaugeFactory, segmentCount); } + /** + * Cleans up the segment count gauge. + * + * @param resource The resource for which the gauge is removed + */ + public void cleanupSegmentCountGauge(final String resource) { + final String fullGaugeName = makeGaugeName(resource, "SegmentCount"); + removeGauge(fullGaugeName); + } + @VisibleForTesting public static String makeGaugeName(final String resource, final String gaugeName) { return "pinot.controller." + resource + "." + gaugeName; @@ -219,6 +269,13 @@ public class ValidationMetrics { } } + private void removeGauge(final String gaugeName) { + PinotMetricName pinotMetricName = makeMetricName(gaugeName); + PinotMetricUtils.removeMetric(_metricsRegistry, pinotMetricName); + _metricNames.remove(pinotMetricName); + _gaugeValues.remove(gaugeName); + } + /** * Unregisters all validation metrics. */ diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java index d7d74f9..93350fe 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java @@ -240,6 +240,22 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh } } + @Override + protected void nonLeaderCleanup(List<String> tableNamesWithType) { + for (String tableNameWithType : tableNamesWithType) { + _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.NUMBER_OF_REPLICAS); + _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.PERCENT_OF_REPLICAS); + _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE); + + _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.IDEALSTATE_ZNODE_SIZE); + _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.IDEALSTATE_ZNODE_BYTE_SIZE); + _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.SEGMENT_COUNT); + + _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.SEGMENTS_IN_ERROR_STATE); + _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE); + } + } + private void setStatusToDefault() { List<String> allTableNames = _pinotHelixResourceManager.getAllTables(); diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java index 9d7a676..439f8be 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java @@ -65,11 +65,14 @@ public abstract class ControllerPeriodicTask<C> extends BasePeriodicTask { // Process the tables that are managed by this controller List<String> tablesToProcess = new ArrayList<>(); + List<String> nonLeaderForTables = new ArrayList<>(); if (propTableNameWithType == null) { // Table name is not available, so task should run on all tables for which this controller is the lead. for (String tableNameWithType : _pinotHelixResourceManager.getAllTables()) { if (_leadControllerManager.isLeaderForTable(tableNameWithType)) { tablesToProcess.add(tableNameWithType); + } else { + nonLeaderForTables.add(tableNameWithType); } } } else { @@ -82,6 +85,9 @@ public abstract class ControllerPeriodicTask<C> extends BasePeriodicTask { if (!tablesToProcess.isEmpty()) { processTables(tablesToProcess); } + if (!nonLeaderForTables.isEmpty()) { + nonLeaderCleanup(nonLeaderForTables); + } } catch (Exception e) { LOGGER.error("Caught exception while running task: {}", _taskName, e); _controllerMetrics.addMeteredTableValue(_taskName, ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR, 1L); @@ -156,4 +162,12 @@ public abstract class ControllerPeriodicTask<C> extends BasePeriodicTask { */ protected void postprocess() { } + + /** + * Can be overridden to perform cleanups for tables that the current controller isn't the leader. + * + * @param tableNamesWithType the table names that the current controller isn't the leader for + */ + protected void nonLeaderCleanup(List<String> tableNamesWithType) { + } } diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java index 1e11d1e..7a88d36 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java @@ -136,6 +136,20 @@ public class OfflineSegmentIntervalChecker extends ControllerPeriodicTask<Void> _validationMetrics.updateSegmentCountGauge(offlineTableName, numSegments); } + @Override + protected void nonLeaderCleanup(List<String> tableNamesWithType) { + for (String tableNameWithType : tableNamesWithType) { + TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType); + if (tableType == TableType.OFFLINE) { + _validationMetrics.cleanupMissingSegmentCountGauge(tableNameWithType); + _validationMetrics.cleanupOfflineSegmentDelayGauge(tableNameWithType); + _validationMetrics.cleanupLastPushTimeGauge(tableNameWithType); + _validationMetrics.cleanupTotalDocumentCountGauge(tableNameWithType); + _validationMetrics.cleanupSegmentCountGauge(tableNameWithType); + } + } + } + /** * Computes the number of missing segments based on the given existing segment intervals and the expected frequency * of the intervals. diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java index 53291b3..237924a 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java @@ -124,6 +124,16 @@ public class RealtimeSegmentValidationManager extends ControllerPeriodicTask<Rea } } + @Override + protected void nonLeaderCleanup(List<String> tableNamesWithType) { + for (String tableNameWithType : tableNamesWithType) { + TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType); + if (tableType == TableType.REALTIME) { + _validationMetrics.cleanupTotalDocumentCountGauge(tableNameWithType); + } + } + } + @VisibleForTesting static long computeRealtimeTotalDocumentInSegments(List<SegmentZKMetadata> segmentsZKMetadata, boolean countHLCSegments) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
