This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1ae53fe Adding cron scheduler metrics reporting (#6502)
1ae53fe is described below
commit 1ae53fe01bb7ca504c174c62dae44fce6cb9084c
Author: Xiang Fu <[email protected]>
AuthorDate: Fri Jan 29 12:32:42 2021 -0800
Adding cron scheduler metrics reporting (#6502)
---
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 15 +++++++++++++++
.../pinot/common/metrics/ControllerGauge.java | 5 ++++-
.../pinot/common/metrics/ControllerMeter.java | 3 ++-
.../pinot/common/metrics/ControllerTimer.java | 2 +-
.../helix/core/minion/CronJobScheduleJob.java | 9 +++++++++
.../helix/core/minion/PinotTaskManager.java | 22 +++++++++++++---------
.../core/periodictask/ControllerPeriodicTask.java | 4 ++++
7 files changed, 48 insertions(+), 12 deletions(-)
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
index 312512c..0221fde 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
@@ -52,6 +52,21 @@ rules:
name: "pinot_controller_validateion_$2_$3"
labels:
table: "$1"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.cronSchedulerJobScheduled.(\\w+)\\.(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_cronSchedulerJobScheduled_$3"
+ labels:
+ table: "$1"
+ taskType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.(\\w+)\\.(\\w+).cronSchedulerTriggered\"><>(\\w+)"
+ name: "pinot_controller_cronSchedulerTriggered_$3"
+ labels:
+ table: "$1"
+ taskType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.(\\w+)\\.(\\w+).cronSchedulerJobExecutionTimeMs\"><>(\\w+)"
+ name: "pinot_controller_cronSchedulerJobExecutionTimeMs_$3"
+ labels:
+ table: "$1"
+ taskType: "$2"
# Pinot Broker
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\",
name=\"pinot.broker.(\\w+).authorization\"><>(\\w+)"
name: "pinot_broker_authorization_$2"
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
index 0eb1fd6..3f69f45 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
@@ -64,7 +64,10 @@ public enum ControllerGauge implements AbstractMetrics.Gauge
{
TABLE_STORAGE_QUOTA_UTILIZATION("TableStorageQuotaUtilization", false),
// Percentage of segments we failed to get size for
-
TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT("TableStorageEstMissingSegmentPercent",
false);
+
TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT("TableStorageEstMissingSegmentPercent",
false),
+
+ // Number of scheduled Cron jobs
+ CRON_SCHEDULER_JOB_SCHEDULED("cronSchedulerJobScheduled", false);
private final String gaugeName;
private final String unit;
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
index 1e7fc46..8c3012d 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java
@@ -48,7 +48,8 @@ public enum ControllerMeter implements AbstractMetrics.Meter {
CONTROLLER_PERIODIC_TASK_ERROR("periodicTaskError", false),
NUMBER_TIMES_SCHEDULE_TASKS_CALLED("tasks", true),
NUMBER_TASKS_SUBMITTED("tasks", false),
- NUMBER_SEGMENT_UPLOAD_TIMEOUT_EXCEEDED("SegmentUploadTimeouts", true);
+ NUMBER_SEGMENT_UPLOAD_TIMEOUT_EXCEEDED("SegmentUploadTimeouts", true),
+ CRON_SCHEDULER_JOB_TRIGGERED("cronSchedulerJobTriggered", false);
private final String brokerMeterName;
private final String unit;
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerTimer.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerTimer.java
index 7617f5a..40ad9d5 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerTimer.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerTimer.java
@@ -26,7 +26,7 @@ import org.apache.pinot.common.Utils;
*
*/
public enum ControllerTimer implements AbstractMetrics.Timer {
- ;
+ CRON_SCHEDULER_JOB_EXECUTION_TIME_MS("cronSchedulerJobExecutionTimeMs",
false);
private final String timerName;
private final boolean global;
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/CronJobScheduleJob.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/CronJobScheduleJob.java
index e5653f4..1a250dc 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/CronJobScheduleJob.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/CronJobScheduleJob.java
@@ -18,6 +18,9 @@
*/
package org.apache.pinot.controller.helix.core.minion;
+import java.util.concurrent.TimeUnit;
+import org.apache.pinot.common.metrics.ControllerMeter;
+import org.apache.pinot.common.metrics.ControllerTimer;
import org.apache.pinot.controller.LeadControllerManager;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
@@ -42,11 +45,17 @@ public class CronJobScheduleJob implements Job {
.get(PinotTaskManager.LEAD_CONTROLLER_MANAGER_KEY);
String table = jobExecutionContext.getJobDetail().getKey().getName();
String taskType = jobExecutionContext.getJobDetail().getKey().getGroup();
+
pinotTaskManager.getControllerMetrics().addMeteredTableValue(PinotTaskManager.getCronJobName(table,
taskType),
+ ControllerMeter.CRON_SCHEDULER_JOB_TRIGGERED, 1L);
if (leadControllerManager.isLeaderForTable(table)) {
+ long jobStartTime = System.currentTimeMillis();
LOGGER.info("Execute CronJob: table - {}, task - {} at {}", table,
taskType, jobExecutionContext.getFireTime());
pinotTaskManager.scheduleTask(taskType, table);
LOGGER.info("Finished CronJob: table - {}, task - {}, next runtime is
{}", table, taskType,
jobExecutionContext.getNextFireTime());
+
pinotTaskManager.getControllerMetrics().addTimedTableValue(PinotTaskManager.getCronJobName(table,
taskType),
+ ControllerTimer.CRON_SCHEDULER_JOB_EXECUTION_TIME_MS,
(System.currentTimeMillis() - jobStartTime),
+ TimeUnit.MILLISECONDS);
} else {
LOGGER.info("Not Lead, skip processing CronJob: table - {}, task - {}",
table, taskType);
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
index 436f10f..0a899c5 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
@@ -29,6 +29,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
+import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.metrics.ControllerMeter;
import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.controller.ControllerConf;
@@ -124,14 +125,6 @@ public class PinotTaskManager extends
ControllerPeriodicTask<Void> {
return TABLE_CONFIG_PATH_PREFIX + tableWithType;
}
- protected synchronized void cleanupCronTaskScheduler() {
- try {
- _scheduledExecutorService.clear();
- } catch (SchedulerException e) {
- LOGGER.error("Failed to clear all tasks in scheduler", e);
- }
- }
-
public synchronized void cleanUpCronTaskSchedulerForTable(String
tableWithType) {
LOGGER.info("Cleaning up task in scheduler for table {}", tableWithType);
TableTaskSchedulerUpdater tableTaskSchedulerUpdater =
_tableTaskSchedulerUpdaterMap.get(tableWithType);
@@ -158,6 +151,8 @@ public class PinotTaskManager extends
ControllerPeriodicTask<Void> {
if (jobKey.getName().equals(tableWithType)) {
try {
_scheduledExecutorService.deleteJob(jobKey);
+
_controllerMetrics.addValueToTableGauge(getCronJobName(tableWithType,
jobKey.getGroup()),
+ ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED, -1L);
} catch (SchedulerException e) {
LOGGER.error("Got exception when deleting the scheduled job - {}",
jobKey, e);
}
@@ -166,6 +161,10 @@ public class PinotTaskManager extends
ControllerPeriodicTask<Void> {
_tableTaskTypeToCronExpressionMap.remove(tableWithType);
}
+ public static String getCronJobName(String tableWithType, String taskType) {
+ return String.format("%s.%s", tableWithType, taskType);
+ }
+
public synchronized void subscribeTableConfigChanges(String tableWithType) {
if (_tableTaskSchedulerUpdaterMap.containsKey(tableWithType)) {
return;
@@ -220,6 +219,8 @@ public class PinotTaskManager extends
ControllerPeriodicTask<Void> {
if (!taskToCronExpressionMap.containsKey(existingTaskType)) {
try {
_scheduledExecutorService.deleteJob(JobKey.jobKey(tableWithType,
existingTaskType));
+
_controllerMetrics.addValueToTableGauge(getCronJobName(tableWithType,
existingTaskType),
+ ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED, -1L);
} catch (SchedulerException e) {
LOGGER.error("Failed to delete scheduled job for table {}, task
type {}", tableWithType,
existingScheduledTasks, e);
@@ -279,7 +280,7 @@ public class PinotTaskManager extends
ControllerPeriodicTask<Void> {
}
if (!exists) {
LOGGER
- .info("Trying to put cron expression: {} for table {}, task type:
{}", cronExprStr, tableWithType, taskType);
+ .info("Trying to schedule a job with cron expression: {} for table
{}, task type: {}", cronExprStr, tableWithType, taskType);
Trigger trigger =
TriggerBuilder.newTrigger().withIdentity(TriggerKey.triggerKey(tableWithType,
taskType))
.withSchedule(CronScheduleBuilder.cronSchedule(cronExprStr)).build();
JobDataMap jobDataMap = new JobDataMap();
@@ -290,6 +291,9 @@ public class PinotTaskManager extends
ControllerPeriodicTask<Void> {
.build();
try {
_scheduledExecutorService.scheduleJob(jobDetail, trigger);
+ _controllerMetrics
+ .addValueToTableGauge(getCronJobName(tableWithType, taskType),
ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED,
+ 1L);
} catch (Exception e) {
LOGGER.error("Failed to parse Cron expression - " + cronExprStr, e);
throw e;
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
index d2c2b13..9139f1d 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
@@ -72,6 +72,10 @@ public abstract class ControllerPeriodicTask<C> extends
BasePeriodicTask {
}
}
+ public final ControllerMetrics getControllerMetrics() {
+ return _controllerMetrics;
+ }
+
/**
* Processes the given list of tables, and returns the number of tables
processed.
* <p>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
