This is an automated email from the ASF dual-hosted git repository.

snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 02c4998  Add mergeRollupTask delay metrics (#7368)
02c4998 is described below

commit 02c4998c9e1cac35e7102def23fc17d6645e8232
Author: Seunghyun Lee <[email protected]>
AuthorDate: Mon Oct 4 15:07:30 2021 -0700

    Add mergeRollupTask delay metrics (#7368)
    
    * Add mergeRollupTask delay metrics
    
    When operating merge/roll-up tasks in production, we should have
    a metrics that indicates the progress or delay of merge/roll-up
    task. mergeRollupTaskDelayInNumTimeBuckets is computed as follows:
    
    (current_time - watermark - bufferTime) / bucketTime
    
    * Address the non-leader case
---
 .../pinot/common/metrics/AbstractMetrics.java      |  12 +-
 .../helix/core/minion/ClusterInfoAccessor.java     |  27 ++-
 .../helix/core/minion/PinotTaskManager.java        |   4 +-
 .../MergeRollupMinionClusterIntegrationTest.java   |  43 +++--
 .../mergerollup/MergeRollupTaskGenerator.java      | 186 +++++++++++++++++++--
 5 files changed, 241 insertions(+), 31 deletions(-)

diff --git 
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
 
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
index 533bcf2..9cf33de 100644
--- 
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
+++ 
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
@@ -494,7 +494,7 @@ public abstract class AbstractMetrics<QP extends 
AbstractMetrics.QueryPhase, M e
    * Remove gauge from Pinot metrics.
    * @param gaugeName gauge name
    */
-  private void removeGauge(final String gaugeName) {
+  public void removeGauge(final String gaugeName) {
     if (_gaugeValues.remove(gaugeName) != null) {
       removeCallbackGauge(gaugeName);
     }
@@ -518,4 +518,14 @@ public abstract class AbstractMetrics<QP extends 
AbstractMetrics.QueryPhase, M e
   protected String getTableName(String tableName) {
     return _isTableLevelMetricsEnabled || _allowedTables.contains(tableName) ? 
tableName : "allTables";
   }
+
+  /**
+   * Check if the metric name appears in the gauge value map.
+   * @param metricName metric name
+   * @return True if the metric name appears on the gauge value map. False 
otherwise.
+   */
+  @VisibleForTesting
+  public boolean containsGauge(String metricName) {
+    return _gaugeValues.containsKey(metricName);
+  }
 }
diff --git 
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/ClusterInfoAccessor.java
 
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/ClusterInfoAccessor.java
index e22376b..7ea31fb 100644
--- 
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/ClusterInfoAccessor.java
+++ 
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/ClusterInfoAccessor.java
@@ -30,10 +30,12 @@ import org.apache.pinot.common.lineage.SegmentLineage;
 import org.apache.pinot.common.lineage.SegmentLineageAccessHelper;
 import org.apache.pinot.common.metadata.ZKMetadataProvider;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metrics.ControllerMetrics;
 import org.apache.pinot.common.minion.MergeRollupTaskMetadata;
 import org.apache.pinot.common.minion.MinionTaskMetadataUtils;
 import org.apache.pinot.common.minion.RealtimeToOfflineSegmentsTaskMetadata;
 import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.controller.LeadControllerManager;
 import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
 import org.apache.pinot.core.common.MinionConstants;
 import org.apache.pinot.core.minion.PinotTaskConfig;
@@ -49,12 +51,17 @@ public class ClusterInfoAccessor {
   private final PinotHelixResourceManager _pinotHelixResourceManager;
   private final PinotHelixTaskResourceManager _pinotHelixTaskResourceManager;
   private final ControllerConf _controllerConf;
+  private final ControllerMetrics _controllerMetrics;
+  private final LeadControllerManager _leadControllerManager;
 
   public ClusterInfoAccessor(PinotHelixResourceManager 
pinotHelixResourceManager,
-      PinotHelixTaskResourceManager pinotHelixTaskResourceManager, 
ControllerConf controllerConf) {
+      PinotHelixTaskResourceManager pinotHelixTaskResourceManager, 
ControllerConf controllerConf,
+      ControllerMetrics controllerMetrics, LeadControllerManager 
leadControllerManager) {
     _pinotHelixResourceManager = pinotHelixResourceManager;
     _pinotHelixTaskResourceManager = pinotHelixTaskResourceManager;
     _controllerConf = controllerConf;
+    _controllerMetrics = controllerMetrics;
+    _leadControllerManager = leadControllerManager;
   }
 
   /**
@@ -181,4 +188,22 @@ public class ClusterInfoAccessor {
         _pinotHelixResourceManager.getHelixAdmin().getConfig(helixConfigScope, 
Collections.singletonList(configName));
     return configMap != null ? configMap.get(configName) : null;
   }
+
+  /**
+   * Get the controller metrics.
+   *
+   * @return controller metrics
+   */
+  public ControllerMetrics getControllerMetrics() {
+    return _controllerMetrics;
+  }
+
+  /**
+   * Get the leader controller manager.
+   *
+   * @return leader controller manager
+   */
+  public LeadControllerManager getLeaderControllerManager() {
+    return _leadControllerManager;
+  }
 }
diff --git 
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
 
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
index 016a8cd..5bafa0a 100644
--- 
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
+++ 
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
@@ -95,7 +95,9 @@ public class PinotTaskManager extends 
ControllerPeriodicTask<Void> {
         controllerConf.getPinotTaskManagerInitialDelaySeconds(), 
helixResourceManager, leadControllerManager,
         controllerMetrics);
     _helixTaskResourceManager = helixTaskResourceManager;
-    _clusterInfoAccessor = new ClusterInfoAccessor(helixResourceManager, 
helixTaskResourceManager, controllerConf);
+    _clusterInfoAccessor =
+        new ClusterInfoAccessor(helixResourceManager, 
helixTaskResourceManager, controllerConf, controllerMetrics,
+            leadControllerManager);
     _taskGeneratorRegistry = new TaskGeneratorRegistry(_clusterInfoAccessor);
 
     if (controllerConf.isPinotTaskManagerSchedulerEnabled()) {
diff --git 
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MergeRollupMinionClusterIntegrationTest.java
 
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MergeRollupMinionClusterIntegrationTest.java
index 3708ad0..d98dc7e 100644
--- 
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MergeRollupMinionClusterIntegrationTest.java
+++ 
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MergeRollupMinionClusterIntegrationTest.java
@@ -50,13 +50,13 @@ import org.apache.pinot.spi.data.Schema;
 import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
 import org.apache.pinot.spi.utils.builder.TableNameBuilder;
 import org.apache.pinot.util.TestUtils;
-import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 
 
@@ -259,7 +259,7 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
     //    {merged_100days_T4_1_16400_16404_1, myTable1_16405_16435_2}
     //      -> {merged_100days_T5_0_myTable1_16400_16435_0}
 
-    String sqlQuery = "SELECT count(*) FROM mytable1"; // 115545 rows for the 
test table
+    String sqlQuery = "SELECT count(*) FROM myTable1"; // 115545 rows for the 
test table
     JsonNode expectedJson = postSqlQuery(sqlQuery, _brokerBaseApiUrl);
     int[] expectedNumSubTasks = {1, 2, 2, 2, 1};
     int[] expectedNumSegmentsQueried = {13, 12, 13, 13, 12};
@@ -269,11 +269,11 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
     for (String tasks = 
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE);
         tasks != null; tasks =
         
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE),
 numTasks++) {
-      
Assert.assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(), 
expectedNumSubTasks[numTasks]);
-      Assert.assertTrue(_helixTaskResourceManager.getTaskQueues()
+      assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(), 
expectedNumSubTasks[numTasks]);
+      assertTrue(_helixTaskResourceManager.getTaskQueues()
           
.contains(PinotHelixTaskResourceManager.getHelixJobQueueName(MinionConstants.MergeRollupTask.TASK_TYPE)));
       // Will not schedule task if there's incomplete task
-      Assert.assertNull(
+      assertNull(
           
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.RealtimeToOfflineSegmentsTask.TASK_TYPE));
       waitForTaskToComplete();
 
@@ -281,7 +281,7 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
       MergeRollupTaskMetadata minionTaskMetadata = MergeRollupTaskMetadata
           
.fromZNRecord(_taskManager.getClusterInfoAccessor().getMinionMergeRollupTaskZNRecord(offlineTableName));
       assertNotNull(minionTaskMetadata);
-      Assert.assertEquals((long) 
minionTaskMetadata.getWatermarkMap().get("100days"), expectedWatermark);
+      assertEquals((long) minionTaskMetadata.getWatermarkMap().get("100days"), 
expectedWatermark);
       expectedWatermark += 100 * 86_400_000L;
 
       // Check metadata of merged segments
@@ -306,6 +306,9 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
     // Check total tasks
     assertEquals(numTasks, 5);
 
+    assertTrue(_controllerStarter.getControllerMetrics()
+        
.containsGauge("mergeRollupTaskDelayInNumBuckets.myTable1_OFFLINE.100days"));
+
     // Drop the table
     dropOfflineTable(SINGLE_LEVEL_CONCAT_TEST_TABLE);
 
@@ -361,11 +364,11 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
     for (String tasks = 
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE);
         tasks != null; tasks =
         
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE),
 numTasks++) {
-      
Assert.assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(), 1);
-      Assert.assertTrue(_helixTaskResourceManager.getTaskQueues()
+      assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(), 1);
+      assertTrue(_helixTaskResourceManager.getTaskQueues()
           
.contains(PinotHelixTaskResourceManager.getHelixJobQueueName(MinionConstants.MergeRollupTask.TASK_TYPE)));
       // Will not schedule task if there's incomplete task
-      Assert.assertNull(
+      assertNull(
           
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.RealtimeToOfflineSegmentsTask.TASK_TYPE));
       waitForTaskToComplete();
 
@@ -373,7 +376,7 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
       MergeRollupTaskMetadata minionTaskMetadata = MergeRollupTaskMetadata
           
.fromZNRecord(_taskManager.getClusterInfoAccessor().getMinionMergeRollupTaskZNRecord(offlineTableName));
       assertNotNull(minionTaskMetadata);
-      Assert.assertEquals((long) 
minionTaskMetadata.getWatermarkMap().get("150days"), expectedWatermark);
+      assertEquals((long) minionTaskMetadata.getWatermarkMap().get("150days"), 
expectedWatermark);
       expectedWatermark += 150 * 86_400_000L;
 
       // Check metadata of merged segments
@@ -407,10 +410,13 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
         postSqlQuery("SELECT count(*), DaysSinceEpoch FROM myTable2 GROUP BY 
DaysSinceEpoch ORDER BY DaysSinceEpoch");
     for (int i = 0; i < responseJson.get("resultTable").get("rows").size(); 
i++) {
       int daysSinceEpoch = 
responseJson.get("resultTable").get("rows").get(i).get(1).asInt();
-      Assert.assertTrue(daysSinceEpoch % 7 == 0);
+      assertTrue(daysSinceEpoch % 7 == 0);
     }
     // Check total tasks
     assertEquals(numTasks, 3);
+
+    assertTrue(_controllerStarter.getControllerMetrics()
+        
.containsGauge("mergeRollupTaskDelayInNumBuckets.myTable2_OFFLINE.150days"));
   }
 
   /**
@@ -497,11 +503,11 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
     for (String tasks = 
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE);
         tasks != null; tasks =
         
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE),
 numTasks++) {
-      
Assert.assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(), 
expectedNumSubTasks[numTasks]);
-      Assert.assertTrue(_helixTaskResourceManager.getTaskQueues()
+      assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(), 
expectedNumSubTasks[numTasks]);
+      assertTrue(_helixTaskResourceManager.getTaskQueues()
           
.contains(PinotHelixTaskResourceManager.getHelixJobQueueName(MinionConstants.MergeRollupTask.TASK_TYPE)));
       // Will not schedule task if there's incomplete task
-      Assert.assertNull(
+      assertNull(
           
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.RealtimeToOfflineSegmentsTask.TASK_TYPE));
       waitForTaskToComplete();
 
@@ -509,8 +515,8 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
       MergeRollupTaskMetadata minionTaskMetadata = MergeRollupTaskMetadata
           
.fromZNRecord(_taskManager.getClusterInfoAccessor().getMinionMergeRollupTaskZNRecord(offlineTableName));
       assertNotNull(minionTaskMetadata);
-      Assert.assertEquals(minionTaskMetadata.getWatermarkMap().get("45days"), 
expectedWatermarks45Days[numTasks]);
-      Assert.assertEquals(minionTaskMetadata.getWatermarkMap().get("90days"), 
expectedWatermarks90Days[numTasks]);
+      assertEquals(minionTaskMetadata.getWatermarkMap().get("45days"), 
expectedWatermarks45Days[numTasks]);
+      assertEquals(minionTaskMetadata.getWatermarkMap().get("90days"), 
expectedWatermarks90Days[numTasks]);
 
       // Check metadata of merged segments
       for (SegmentZKMetadata metadata : 
_pinotHelixResourceManager.getSegmentsZKMetadata(offlineTableName)) {
@@ -539,6 +545,11 @@ public class MergeRollupMinionClusterIntegrationTest 
extends BaseClusterIntegrat
     }
     // Check total tasks
     assertEquals(numTasks, 8);
+
+    assertTrue(_controllerStarter.getControllerMetrics()
+        
.containsGauge("mergeRollupTaskDelayInNumBuckets.myTable3_OFFLINE.45days"));
+    assertTrue(_controllerStarter.getControllerMetrics()
+        
.containsGauge("mergeRollupTaskDelayInNumBuckets.myTable3_OFFLINE.90days"));
   }
 
   protected void verifyTableDelete(String tableNameWithType) {
diff --git 
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupTaskGenerator.java
 
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupTaskGenerator.java
index 031585f..5c8ec8f 100644
--- 
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupTaskGenerator.java
+++ 
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupTaskGenerator.java
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
 import org.I0Itec.zkclient.exception.ZkException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.helix.ZNRecord;
@@ -35,6 +36,7 @@ import org.apache.pinot.common.lineage.SegmentLineage;
 import org.apache.pinot.common.lineage.SegmentLineageUtils;
 import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metrics.ControllerMetrics;
 import org.apache.pinot.common.minion.MergeRollupTaskMetadata;
 import org.apache.pinot.controller.helix.core.minion.ClusterInfoAccessor;
 import 
org.apache.pinot.controller.helix.core.minion.generator.PinotTaskGenerator;
@@ -90,16 +92,31 @@ import org.slf4j.LoggerFactory;
  */
 @TaskGenerator
 public class MergeRollupTaskGenerator implements PinotTaskGenerator {
+  private static final Logger LOGGER = 
LoggerFactory.getLogger(MergeRollupTaskGenerator.class);
+
   public static final int END_REPLACE_SEGMENTS_SOCKET_TIMEOUT_MS = 30 * 60 * 
1000; // 30 mins
   private static final int DEFAULT_MAX_NUM_RECORDS_PER_TASK = 50_000_000;
   private static final String REFRESH = "REFRESH";
-  private static final Logger LOGGER = 
LoggerFactory.getLogger(MergeRollupTaskGenerator.class);
 
+  // This is the metric that keeps track of the task delay in the number of 
time buckets. For example, if we see this
+  // number to be 7 and merge task is configured with "bucketTimePeriod = 1d", 
this means that we have 7 days of
+  // delay. When operating merge/roll-up task in production, we should set the 
alert on this metrics to find out the
+  // delay. Setting the alert on 7 time buckets of delay would be a good 
starting point.
+  //
+  // NOTE: Based on the current scheduler logic, we are bumping up the 
watermark with some delay. (the current round
+  // will bump up the watermark for the window that got processed from the 
previous round). Due to this, we will
+  // correctly report the delay with one edge case. When we processed all 
available time windows, the watermark
+  // will not get bumped up until we schedule some task for the table. Due to 
this, we will always see the delay >= 1.
+  private static final String MERGE_ROLLUP_TASK_DELAY_IN_NUM_BUCKETS = 
"mergeRollupTaskDelayInNumBuckets";
+
+  // tableNameWithType -> mergeLevel -> watermarkMs
+  private Map<String, Map<String, Long>> _mergeRollupWatermarks;
   private ClusterInfoAccessor _clusterInfoAccessor;
 
   @Override
   public void init(ClusterInfoAccessor clusterInfoAccessor) {
     _clusterInfoAccessor = clusterInfoAccessor;
+    _mergeRollupWatermarks = new HashMap<>();
   }
 
   @Override
@@ -111,7 +128,6 @@ public class MergeRollupTaskGenerator implements 
PinotTaskGenerator {
   public List<PinotTaskConfig> generateTasks(List<TableConfig> tableConfigs) {
     String taskType = MergeRollupTask.TASK_TYPE;
     List<PinotTaskConfig> pinotTaskConfigs = new ArrayList<>();
-
     for (TableConfig tableConfig : tableConfigs) {
       if (!validate(tableConfig, taskType)) {
         continue;
@@ -138,8 +154,11 @@ public class MergeRollupTaskGenerator implements 
PinotTaskGenerator {
       }
 
       if (preSelectedSegments.isEmpty()) {
+        // Reset the watermark time if no segment found. This covers the case 
where the table is newly created or
+        // all segments for the existing table got deleted.
+        resetDelayMetrics(offlineTableName);
         LOGGER
-            .info("Skip generating task: {} for table: {}, no segment is found 
to merge.", taskType, offlineTableName);
+            .info("Skip generating task: {} for table: {}, no segment is 
found.", taskType, offlineTableName);
         continue;
       }
 
@@ -197,6 +216,11 @@ public class MergeRollupTaskGenerator implements 
PinotTaskGenerator {
         // Get the bucket size and buffer
         long bucketMs =
             
TimeUtils.convertPeriodToMillis(mergeConfigs.get(MinionConstants.MergeTask.BUCKET_TIME_PERIOD_KEY));
+        if (bucketMs <= 0) {
+          LOGGER.error("Bucket time period (table : {}, mergeLevel : {}) must 
be larger than 0", offlineTableName,
+              mergeLevel);
+          continue;
+        }
         long bufferMs =
             
TimeUtils.convertPeriodToMillis(mergeConfigs.get(MinionConstants.MergeTask.BUFFER_TIME_PERIOD_KEY));
 
@@ -208,10 +232,8 @@ public class MergeRollupTaskGenerator implements 
PinotTaskGenerator {
         long windowEndMs = windowStartMs + bucketMs;
 
         if (!isValidMergeWindowEndTime(windowEndMs, bufferMs, lowerMergeLevel, 
mergeRollupTaskMetadata)) {
-          LOGGER.info(
-              "Window with start: {} and end: {} of mergeLevel: {} is not a 
valid merge window, Skipping task "
-                  + "generation: {}",
-              windowStartMs, windowEndMs, mergeLevel, taskType);
+          LOGGER.info("Window with start: {} and end: {} of mergeLevel: {} is 
not a valid merge window, Skipping task "
+              + "generation: {}", windowStartMs, windowEndMs, mergeLevel, 
taskType);
           continue;
         }
 
@@ -264,10 +286,8 @@ public class MergeRollupTaskGenerator implements 
PinotTaskGenerator {
         }
 
         if (!isValidMergeWindow) {
-          LOGGER.info(
-              "Window with start: {} and end: {} of mergeLevel: {} is not a 
valid merge window, Skipping task "
-                  + "generation: {}",
-              windowStartMs, windowEndMs, mergeLevel, taskType);
+          LOGGER.info("Window with start: {} and end: {} of mergeLevel: {} is 
not a valid merge window, Skipping task "
+              + "generation: {}", windowStartMs, windowEndMs, mergeLevel, 
taskType);
           continue;
         }
 
@@ -279,7 +299,10 @@ public class MergeRollupTaskGenerator implements 
PinotTaskGenerator {
 
         Long prevWatermarkMs = 
mergeRollupTaskMetadata.getWatermarkMap().put(mergeLevel, windowStartMs);
         LOGGER.info("Update watermark of mergeLevel: {} for table: {} from: {} 
to: {}", mergeLevel, offlineTableName,
-            prevWatermarkMs, waterMarkMs);
+            prevWatermarkMs, windowStartMs);
+
+        // Update the delay metrics
+        updateDelayMetrics(offlineTableName, mergeLevel, windowStartMs, 
bufferMs, bucketMs);
 
         // Create task configs
         int maxNumRecordsPerTask = 
mergeConfigs.get(MergeRollupTask.MAX_NUM_RECORDS_PER_TASK_KEY) != null ? Integer
@@ -340,6 +363,10 @@ public class MergeRollupTaskGenerator implements 
PinotTaskGenerator {
           .info("Finished generating task configs for table: {} for task: {}, 
numTasks: {}", offlineTableName, taskType,
               pinotTaskConfigsForTable.size());
     }
+
+    // Clean up metrics
+    cleanUpDelayMetrics(tableConfigs);
+
     return pinotTaskConfigs;
   }
 
@@ -466,4 +493,139 @@ public class MergeRollupTaskGenerator implements 
PinotTaskGenerator {
 
     return pinotTaskConfigs;
   }
+
+  private long getMergeRollupTaskDelayInNumTimeBuckets(long watermarkMs, long 
bufferTimeMs, long bucketTimeMs) {
+    if (watermarkMs == -1) {
+      return 0;
+    }
+    return (System.currentTimeMillis() - watermarkMs - bufferTimeMs) / 
bucketTimeMs;
+  }
+
+  /**
+   * Update the delay metrics for the given table and merge level. We create 
the new gauge metric if the metric is not
+   * available.
+   *
+   * @param tableNameWithType table name with type
+   * @param mergeLevel merge level
+   * @param watermarkMs current watermark value
+   * @param bufferTimeMs buffer time
+   * @param bucketTimeMs bucket time
+   */
+  private void updateDelayMetrics(String tableNameWithType, String mergeLevel, 
long watermarkMs, long bufferTimeMs,
+      long bucketTimeMs) {
+    ControllerMetrics controllerMetrics = 
_clusterInfoAccessor.getControllerMetrics();
+    if (controllerMetrics == null) {
+      return;
+    }
+
+    // Update gauge value that indicates the delay in terms of the number of 
time buckets.
+    Map<String, Long> watermarkForTable =
+        _mergeRollupWatermarks.computeIfAbsent(tableNameWithType, k -> new 
ConcurrentHashMap<>());
+    watermarkForTable.compute(mergeLevel, (k, v) -> {
+      if (v == null) {
+        LOGGER.info(
+            "Creating the gauge metric for tracking the merge/roll-up task 
delay for table: {} and mergeLevel: {}."
+                + "(watermarkMs={}, bufferTimeMs={}, bucketTimeMs={}, 
taskDelayInNumTimeBuckets={})", tableNameWithType,
+            mergeLevel, watermarkMs, bucketTimeMs, bucketTimeMs,
+            getMergeRollupTaskDelayInNumTimeBuckets(watermarkMs, bufferTimeMs, 
bucketTimeMs));
+        
controllerMetrics.addCallbackGaugeIfNeeded(getMetricNameForTaskDelay(tableNameWithType,
 mergeLevel),
+            (() -> 
getMergeRollupTaskDelayInNumTimeBuckets(watermarkForTable.getOrDefault(k, -1L), 
bufferTimeMs,
+                bucketTimeMs)));
+      }
+      return watermarkMs;
+    });
+  }
+
+  /**
+   * Reset the delay metrics for the given table name.
+   *
+   * @param tableNameWithType a table name with type
+   */
+  private void resetDelayMetrics(String tableNameWithType) {
+    ControllerMetrics controllerMetrics = 
_clusterInfoAccessor.getControllerMetrics();
+    if (controllerMetrics == null) {
+      return;
+    }
+
+    // Delete all the watermarks associated with the given table name
+    Map<String, Long> watermarksForTable = 
_mergeRollupWatermarks.remove(tableNameWithType);
+    if (watermarksForTable != null) {
+      for (String mergeLevel : watermarksForTable.keySet()) {
+        
controllerMetrics.removeGauge(getMetricNameForTaskDelay(tableNameWithType, 
mergeLevel));
+      }
+    }
+  }
+
+  /**
+   * Reset the delay metrics for the given table name and merge level.
+   *
+   * @param tableNameWithType table name with type
+   * @param mergeLevel merge level
+   */
+  private void resetDelayMetrics(String tableNameWithType, String mergeLevel) {
+    ControllerMetrics controllerMetrics = 
_clusterInfoAccessor.getControllerMetrics();
+    if (controllerMetrics == null) {
+      return;
+    }
+
+    // Delete all the watermarks associated with the given the table name and 
the merge level.
+    Map<String, Long> watermarksForTable = 
_mergeRollupWatermarks.get(tableNameWithType);
+    if (watermarksForTable != null) {
+      if (watermarksForTable.remove(mergeLevel) != null) {
+        
controllerMetrics.removeGauge(getMetricNameForTaskDelay(tableNameWithType, 
mergeLevel));
+      }
+    }
+  }
+
+  /**
+   * Clean up the metrics that no longer need to be emitted.
+   *
+   * We clean up the metrics for the following cases:
+   *   1. Table got deleted.
+   *   2. The current controller is no longer the leader for a table.
+   *   3. Merge task config got deleted.
+   *   4. Merge task config got modified and some merge levels got deleted.
+   *
+   * TODO: Current code will remove all metrics in case we invoke the ad-hoc 
task scheduling on a single table.
+   * We will file the follow-up PR to address this issue. We need to separate 
out APIs for ad-hoc scheduling and
+   * periodic scheduling. We will only enable metrics for periodic case.
+   *
+   * @param tableConfigs list of tables
+   */
+  private void cleanUpDelayMetrics(List<TableConfig> tableConfigs) {
+    Map<String, TableConfig> tableConfigMap = new HashMap<>();
+    for (TableConfig tableConfig : tableConfigs) {
+      tableConfigMap.put(tableConfig.getTableName(), tableConfig);
+    }
+
+    for (String tableNameWithType : new 
ArrayList<>(_mergeRollupWatermarks.keySet())) {
+      TableConfig currentTableConfig = tableConfigMap.get(tableNameWithType);
+      // Table does not exist in the cluster or merge task config is removed
+      if (currentTableConfig == null) {
+        resetDelayMetrics(tableNameWithType);
+        continue;
+      }
+
+      // The current controller is no longer leader for this table
+      if 
(!_clusterInfoAccessor.getLeaderControllerManager().isLeaderForTable(tableNameWithType))
 {
+        resetDelayMetrics(tableNameWithType);
+        continue;
+      }
+
+      // Task config is modified and some merge level got removed
+      Map<String, String> taskConfigs = 
currentTableConfig.getTaskConfig().getConfigsForTaskType(getTaskType());
+      Map<String, Map<String, String>> mergeLevelToConfigs = 
MergeRollupTaskUtils.getLevelToConfigMap(taskConfigs);
+      Map<String, Long> tableToWatermark = 
_mergeRollupWatermarks.get(tableNameWithType);
+      for (String mergeLevel : tableToWatermark.keySet()) {
+        if (!mergeLevelToConfigs.containsKey(mergeLevel)) {
+          resetDelayMetrics(tableNameWithType, mergeLevel);
+        }
+      }
+    }
+  }
+
+  private String getMetricNameForTaskDelay(String tableNameWithType, String 
mergeLevel) {
+    // e.g. mergeRollupTaskDelayInNumBuckets.myTable_OFFLINE.daily
+    return MERGE_ROLLUP_TASK_DELAY_IN_NUM_BUCKETS + "." + tableNameWithType + 
"." + mergeLevel;
+  }
 }

---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to