This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 702e94321 [AMORO-3707] Add group tag to table metrics (#3708)
702e94321 is described below
commit 702e9432183ad7406150ee5172c8c2b206827a2b
Author: Marig_Weizhi <[email protected]>
AuthorDate: Tue Aug 5 19:36:48 2025 +0800
[AMORO-3707] Add group tag to table metrics (#3708)
* Add group tag to table metrics
* add group tag for TableOptimizingMetrics
---
.../amoro/server/table/DefaultOptimizingState.java | 5 +-
.../amoro/server/table/TableOptimizingMetrics.java | 84 +++++++++++++++-------
.../table/TableOrphanFilesCleaningMetrics.java | 2 +-
docs/user-guides/metrics.md | 46 ++++++------
4 files changed, 87 insertions(+), 50 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultOptimizingState.java
b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultOptimizingState.java
index 1953049a6..c4b2993fd 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultOptimizingState.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultOptimizingState.java
@@ -113,7 +113,7 @@ public class DefaultOptimizingState extends
StatedPersistentBase implements Proc
this.tableConfiguration = TableConfigurations.parseTableConfig(properties);
this.optimizerGroup =
tableConfiguration.getOptimizingConfig().getOptimizerGroup();
persistTableRuntime();
- optimizingMetrics = new TableOptimizingMetrics(tableIdentifier);
+ optimizingMetrics = new TableOptimizingMetrics(tableIdentifier,
optimizerGroup);
orphanFilesCleaningMetrics = new
TableOrphanFilesCleaningMetrics(tableIdentifier);
tableSummaryMetrics = new TableSummaryMetrics(tableIdentifier);
}
@@ -151,7 +151,7 @@ public class DefaultOptimizingState extends
StatedPersistentBase implements Proc
: tableRuntimeMeta.getTableStatus();
this.pendingInput = tableRuntimeMeta.getPendingInput();
this.tableSummary = tableRuntimeMeta.getTableSummary();
- optimizingMetrics = new TableOptimizingMetrics(tableIdentifier);
+ optimizingMetrics = new TableOptimizingMetrics(tableIdentifier,
optimizerGroup);
optimizingMetrics.statusChanged(optimizingStatus,
this.currentStatusStartTime);
optimizingMetrics.lastOptimizingTime(OptimizingType.MINOR,
this.lastMinorOptimizingTime);
optimizingMetrics.lastOptimizingTime(OptimizingType.MAJOR,
this.lastMajorOptimizingTime);
@@ -439,6 +439,7 @@ public class DefaultOptimizingState extends
StatedPersistentBase implements Proc
optimizingProcess.close();
}
this.optimizerGroup =
newTableConfig.getOptimizingConfig().getOptimizerGroup();
+ this.optimizingMetrics.optimizerGroupChanged(optimizerGroup);
}
this.tableConfiguration = newTableConfig;
return true;
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableOptimizingMetrics.java
b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableOptimizingMetrics.java
index 4e4452eba..e92685e07 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableOptimizingMetrics.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableOptimizingMetrics.java
@@ -24,12 +24,15 @@ import static
org.apache.amoro.metrics.MetricDefine.defineGauge;
import org.apache.amoro.ServerTableIdentifier;
import org.apache.amoro.metrics.Counter;
import org.apache.amoro.metrics.Gauge;
+import org.apache.amoro.metrics.Metric;
import org.apache.amoro.metrics.MetricDefine;
+import org.apache.amoro.metrics.MetricKey;
import org.apache.amoro.optimizing.OptimizingType;
import org.apache.amoro.server.AmoroServiceConstants;
import org.apache.amoro.server.metrics.MetricRegistry;
import org.apache.amoro.server.optimizing.OptimizingStatus;
import org.apache.amoro.server.optimizing.maintainer.IcebergTableMaintainer;
+import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap;
import org.apache.amoro.shade.guava32.com.google.common.primitives.Longs;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.SnapshotSummary;
@@ -55,142 +58,142 @@ public class TableOptimizingMetrics extends
AbstractTableMetrics {
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IDLE_DURATION =
defineGauge("table_optimizing_status_idle_duration_mills")
.withDescription("Duration in milliseconds after table be in idle
status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_PENDING_DURATION =
defineGauge("table_optimizing_status_pending_duration_mills")
.withDescription("Duration in milliseconds after table be in pending
status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_PLANNING_DURATION =
defineGauge("table_optimizing_status_planning_duration_mills")
.withDescription("Duration in milliseconds after table be in
planning status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_EXECUTING_DURATION =
defineGauge("table_optimizing_status_executing_duration_mills")
.withDescription("Duration in milliseconds after table be in
executing status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_COMMITTING_DURATION
=
defineGauge("table_optimizing_status_committing_duration_mills")
.withDescription("Duration in milliseconds after table be in
committing status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
// table optimizing process count metrics
public static final MetricDefine TABLE_OPTIMIZING_PROCESS_TOTAL_COUNT =
defineCounter("table_optimizing_process_total_count")
.withDescription("Count of all optimizing process since ams started")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_PROCESS_FAILED_COUNT =
defineCounter("table_optimizing_process_failed_count")
.withDescription("Count of failed optimizing process since ams
started")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_MINOR_TOTAL_COUNT =
defineCounter("table_optimizing_minor_total_count")
.withDescription("Count of minor optimizing process since ams
started")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_MINOR_FAILED_COUNT =
defineCounter("table_optimizing_minor_failed_count")
.withDescription("Count of failed minor optimizing process since ams
started")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_MAJOR_TOTAL_COUNT =
defineCounter("table_optimizing_major_total_count")
.withDescription("Count of major optimizing process since ams
started")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_MAJOR_FAILED_COUNT =
defineCounter("table_optimizing_major_failed_count")
.withDescription("Count of failed major optimizing process since ams
started")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_FULL_TOTAL_COUNT =
defineCounter("table_optimizing_full_total_count")
.withDescription("Count of full optimizing process since ams
started")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_FULL_FAILED_COUNT =
defineCounter("table_optimizing_full_failed_count")
.withDescription("Count of failed full optimizing process since ams
started")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
// table optimizing process status metrics
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_IDLE =
defineGauge("table_optimizing_status_in_idle")
.withDescription("If currently table is in idle status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_PENDING =
defineGauge("table_optimizing_status_in_pending")
.withDescription("If currently table is in pending status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_PLANNING =
defineGauge("table_optimizing_status_in_planning")
.withDescription("If currently table is in planning status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_EXECUTING =
defineGauge("table_optimizing_status_in_executing")
.withDescription("If currently table is in executing status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_COMMITTING =
defineGauge("table_optimizing_status_in_committing")
.withDescription("If currently table is in committing status")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine
TABLE_OPTIMIZING_SINCE_LAST_MINOR_OPTIMIZATION =
defineGauge("table_optimizing_since_last_minor_optimization_mills")
.withDescription("Duration in milliseconds since last successful
minor optimization")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine
TABLE_OPTIMIZING_SINCE_LAST_MAJOR_OPTIMIZATION =
defineGauge("table_optimizing_since_last_major_optimization_mills")
.withDescription("Duration in milliseconds since last successful
major optimization")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine
TABLE_OPTIMIZING_SINCE_LAST_FULL_OPTIMIZATION =
defineGauge("table_optimizing_since_last_full_optimization_mills")
.withDescription("Duration in milliseconds since last successful
full optimization")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_SINCE_LAST_OPTIMIZATION =
defineGauge("table_optimizing_since_last_optimization_mills")
.withDescription("Duration in milliseconds since last successful
optimization")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
public static final MetricDefine TABLE_OPTIMIZING_LAG_DURATION =
defineGauge("table_optimizing_lag_duration_mills")
.withDescription(
"Duration in milliseconds between last self-optimizing snapshot
and refreshed snapshot")
- .withTags("catalog", "database", "table")
+ .withTags("catalog", "database", "table", "group")
.build();
private final Counter processTotalCount = new Counter();
@@ -208,8 +211,29 @@ public class TableOptimizingMetrics extends
AbstractTableMetrics {
private long lastNonMaintainedTime = AmoroServiceConstants.INVALID_TIME;
private long lastOptimizingTime = AmoroServiceConstants.INVALID_TIME;
- public TableOptimizingMetrics(ServerTableIdentifier identifier) {
+ private String optimizerGroup;
+
+ public TableOptimizingMetrics(ServerTableIdentifier identifier, String
optimizerGroup) {
super(identifier);
+ this.optimizerGroup = optimizerGroup;
+ }
+
+ @Override
+ protected void registerMetric(MetricRegistry registry, MetricDefine define,
Metric metric) {
+ MetricKey key =
+ registry.register(
+ define,
+ ImmutableMap.of(
+ "catalog",
+ identifier.getCatalog(),
+ "database",
+ identifier.getDatabase(),
+ "table",
+ identifier.getTableName(),
+ "group",
+ optimizerGroup),
+ metric);
+ registeredMetricKeys.add(key);
}
@Override
@@ -288,6 +312,18 @@ public class TableOptimizingMetrics extends
AbstractTableMetrics {
this.statusSetTimestamp = statusSetTimestamp;
}
+ /**
+ * Handle table optimizer group change event.
+ *
+ * @param optimizerGroup new optimizer group name
+ */
+ public void optimizerGroupChanged(String optimizerGroup) {
+ MetricRegistry metricRegistry = globalRegistry;
+ unregister();
+ this.optimizerGroup = optimizerGroup;
+ register(metricRegistry);
+ }
+
/**
* Handle table self optimizing process complete event.
*
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableOrphanFilesCleaningMetrics.java
b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableOrphanFilesCleaningMetrics.java
index cc6b0894b..a307eff68 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableOrphanFilesCleaningMetrics.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableOrphanFilesCleaningMetrics.java
@@ -81,7 +81,7 @@ public class TableOrphanFilesCleaningMetrics extends
AbstractTableMetrics {
}
public void completeOrphanDataFiles(int expected, int cleaned) {
- expectedOrphanMetadataFilesCount.inc(expected);
+ expectedOrphanDataFilesCount.inc(expected);
orphanDataFilesCount.inc(cleaned);
}
diff --git a/docs/user-guides/metrics.md b/docs/user-guides/metrics.md
index 22855f947..ee6c11e21 100644
--- a/docs/user-guides/metrics.md
+++ b/docs/user-guides/metrics.md
@@ -39,29 +39,29 @@ Amoro has supported built-in metrics to measure status of
table self-optimizing
| Metric Name | Type | Tags
| Description
|
|------------------------------------------------------|---------|--------------------------|----------------------------------------------------------------------------------------|
-| table_optimizing_status_idle_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in idle status
|
-| table_optimizing_status_pending_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in pending status
|
-| table_optimizing_status_planning_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in planning status
|
-| table_optimizing_status_executing_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in executing status
|
-| table_optimizing_status_committing_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in committing status
|
-| table_optimizing_process_total_count | Counter | catalog,
database, table | Count of all optimizing process since ams started
|
-| table_optimizing_process_failed_count | Counter | catalog,
database, table | Count of failed optimizing process since ams started
|
-| table_optimizing_minor_total_count | Counter | catalog,
database, table | Count of minor optimizing process since ams started
|
-| table_optimizing_minor_failed_count | Counter | catalog,
database, table | Count of failed minor optimizing process since ams started
|
-| table_optimizing_major_total_count | Counter | catalog,
database, table | Count of major optimizing process since ams started
|
-| table_optimizing_major_failed_count | Counter | catalog,
database, table | Count of failed major optimizing process since ams started
|
-| table_optimizing_full_total_count | Counter | catalog,
database, table | Count of full optimizing process since ams started
|
-| table_optimizing_full_failed_count | Counter | catalog,
database, table | Count of failed full optimizing process since ams started
|
-| table_optimizing_status_in_idle | Gauge | catalog,
database, table | If currently table is in idle status
|
-| table_optimizing_status_in_pending | Gauge | catalog,
database, table | If currently table is in pending status
|
-| table_optimizing_status_in_planning | Gauge | catalog,
database, table | If currently table is in planning status
|
-| table_optimizing_status_in_executing | Gauge | catalog,
database, table | If currently table is in executing status
|
-| table_optimizing_status_in_committing | Gauge | catalog,
database, table | If currently table is in committing status
|
-| table_optimizing_since_last_minor_optimization_mills | Gauge | catalog,
database, table | Duration in milliseconds since last successful minor
optimization |
-| table_optimizing_since_last_major_optimization_mills | Gauge | catalog,
database, table | Duration in milliseconds since last successful major
optimization |
-| table_optimizing_since_last_full_optimization_mills | Gauge | catalog,
database, table | Duration in milliseconds since last successful full
optimization |
-| table_optimizing_since_last_optimization_mills | Gauge | catalog,
database, table | Duration in milliseconds since last successful optimization
|
-| table_optimizing_lag_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds between last self-optimizing
snapshot and refreshed snapshot |
+| table_optimizing_status_idle_duration_mills | Gauge | catalog,
database, table, group | Duration in milliseconds after table be in idle status
|
+| table_optimizing_status_pending_duration_mills | Gauge | catalog,
database, table, group | Duration in milliseconds after table be in pending
status |
+| table_optimizing_status_planning_duration_mills | Gauge | catalog,
database, table, group | Duration in milliseconds after table be in planning
status |
+| table_optimizing_status_executing_duration_mills | Gauge | catalog,
database, table, group | Duration in milliseconds after table be in executing
status |
+| table_optimizing_status_committing_duration_mills | Gauge | catalog,
database, table, group | Duration in milliseconds after table be in committing
status |
+| table_optimizing_process_total_count | Counter | catalog,
database, table, group | Count of all optimizing process since ams started
|
+| table_optimizing_process_failed_count | Counter | catalog,
database, table, group | Count of failed optimizing process since ams started
|
+| table_optimizing_minor_total_count | Counter | catalog,
database, table, group | Count of minor optimizing process since ams started
|
+| table_optimizing_minor_failed_count | Counter | catalog,
database, table, group | Count of failed minor optimizing process since ams
started |
+| table_optimizing_major_total_count | Counter | catalog,
database, table, group | Count of major optimizing process since ams started
|
+| table_optimizing_major_failed_count | Counter | catalog,
database, table, group | Count of failed major optimizing process since ams
started |
+| table_optimizing_full_total_count | Counter | catalog,
database, table, group | Count of full optimizing process since ams started
|
+| table_optimizing_full_failed_count | Counter | catalog,
database, table, group | Count of failed full optimizing process since ams
started |
+| table_optimizing_status_in_idle | Gauge | catalog,
database, table, group | If currently table is in idle status
|
+| table_optimizing_status_in_pending | Gauge | catalog,
database, table, group | If currently table is in pending status
|
+| table_optimizing_status_in_planning | Gauge | catalog,
database, table, group | If currently table is in planning status
|
+| table_optimizing_status_in_executing | Gauge | catalog,
database, table, group | If currently table is in executing status
|
+| table_optimizing_status_in_committing | Gauge | catalog,
database, table, group | If currently table is in committing status
|
+| table_optimizing_since_last_minor_optimization_mills | Gauge | catalog,
database, table, group | Duration in milliseconds since last successful minor
optimization |
+| table_optimizing_since_last_major_optimization_mills | Gauge | catalog,
database, table, group | Duration in milliseconds since last successful major
optimization |
+| table_optimizing_since_last_full_optimization_mills | Gauge | catalog,
database, table, group | Duration in milliseconds since last successful full
optimization |
+| table_optimizing_since_last_optimization_mills | Gauge | catalog,
database, table, group | Duration in milliseconds since last successful
optimization |
+| table_optimizing_lag_duration_mills | Gauge | catalog,
database, table, group | Duration in milliseconds between last self-optimizing
snapshot and refreshed snapshot |
## Optimizer Group metrics
