This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 8e78f3441 [AMORO-1528] Add metrics document (#2859)
8e78f3441 is described below
commit 8e78f3441c5e0dae7809bbe52824345c955789b6
Author: ZhouJinsong <[email protected]>
AuthorDate: Thu May 23 17:31:47 2024 +0800
[AMORO-1528] Add metrics document (#2859)
* Add metrics document
* Change some title in docs
* Update docs/user-guides/using-tables.md
Co-authored-by: Xavier Bai <[email protected]>
---------
Co-authored-by: Xavier Bai <[email protected]>
---
.../amoro/server/table/TableOptimizingMetrics.java | 62 +++++++++++-----------
docs/user-guides/configurations.md | 2 +-
docs/user-guides/metrics.md | 56 ++++++++++++-------
docs/user-guides/using-tables.md | 34 ++++++++++++
4 files changed, 103 insertions(+), 51 deletions(-)
diff --git
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableOptimizingMetrics.java
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableOptimizingMetrics.java
index 6ee04c5cc..7c08d7c51 100644
---
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableOptimizingMetrics.java
+++
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableOptimizingMetrics.java
@@ -55,111 +55,111 @@ public class TableOptimizingMetrics {
// table optimizing status duration metrics
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IDLE_DURATION =
defineGauge("table_optimizing_status_idle_duration_mills")
- .withDescription("Duration in seconds after table be in idle status")
+ .withDescription("Duration in milliseconds after table be in idle
status")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_PENDING_DURATION =
defineGauge("table_optimizing_status_pending_duration_mills")
- .withDescription("Duration in seconds after table be in pending
status")
+ .withDescription("Duration in milliseconds after table be in pending
status")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_PLANNING_DURATION =
defineGauge("table_optimizing_status_planning_duration_mills")
- .withDescription("Duration in seconds after table be in planning
status")
+ .withDescription("Duration in milliseconds after table be in
planning status")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_EXECUTING_DURATION =
defineGauge("table_optimizing_status_executing_duration_mills")
- .withDescription("Duration in seconds after table be in executing
status")
+ .withDescription("Duration in milliseconds after table be in
executing status")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_COMMITTING_DURATION
=
defineGauge("table_optimizing_status_committing_duration_mills")
- .withDescription("Duration in seconds after table be in committing
status")
+ .withDescription("Duration in milliseconds after table be in
committing status")
.withTags("catalog", "database", "table")
.build();
// table optimizing process count metrics
public static final MetricDefine TABLE_OPTIMIZING_PROCESS_TOTAL_COUNT =
defineCounter("table_optimizing_process_total_count")
- .withDescription("Count of all process since ams started")
+ .withDescription("Count of all optimizing process since ams started")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_PROCESS_FAILED_COUNT =
defineCounter("table_optimizing_process_failed_count")
- .withDescription("Count of failed process since ams started")
+ .withDescription("Count of failed optimizing process since ams
started")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_MINOR_TOTAL_COUNT =
defineCounter("table_optimizing_minor_total_count")
- .withDescription("Count of minor process since ams started")
+ .withDescription("Count of minor optimizing process since ams
started")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_MINOR_FAILED_COUNT =
defineCounter("table_optimizing_minor_failed_count")
- .withDescription("Count of failed minor process since ams started")
+ .withDescription("Count of failed minor optimizing process since ams
started")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_MAJOR_TOTAL_COUNT =
defineCounter("table_optimizing_major_total_count")
- .withDescription("Count of major process since ams started")
+ .withDescription("Count of major optimizing process since ams
started")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_MAJOR_FAILED_COUNT =
defineCounter("table_optimizing_major_failed_count")
- .withDescription("Count of failed major process since ams started")
+ .withDescription("Count of failed major optimizing process since ams
started")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_FULL_TOTAL_COUNT =
defineCounter("table_optimizing_full_total_count")
- .withDescription("Count of full process since ams started")
+ .withDescription("Count of full optimizing process since ams
started")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_FULL_FAILED_COUNT =
defineCounter("table_optimizing_full_failed_count")
- .withDescription("Count of failed full process since ams started")
+ .withDescription("Count of failed full optimizing process since ams
started")
.withTags("catalog", "database", "table")
.build();
// table optimizing process status metrics
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_IDLE =
defineGauge("table_optimizing_status_in_idle")
- .withDescription("If currently table is in status idle")
+ .withDescription("If currently table is in idle status")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_PENDING =
defineGauge("table_optimizing_status_in_pending")
- .withDescription("If currently table is in status pending")
+ .withDescription("If currently table is in pending status")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_PLANNING =
defineGauge("table_optimizing_status_in_planning")
- .withDescription("If currently table is in status planning")
+ .withDescription("If currently table is in planning status")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_EXECUTING =
defineGauge("table_optimizing_status_in_executing")
- .withDescription("If currently table is in status executing")
+ .withDescription("If currently table is in executing status")
.withTags("catalog", "database", "table")
.build();
public static final MetricDefine TABLE_OPTIMIZING_STATUS_IN_COMMITTING =
defineGauge("table_optimizing_status_in_committing")
- .withDescription("If currently table is in status committing")
+ .withDescription("If currently table is in committing status")
.withTags("catalog", "database", "table")
.build();
@@ -175,7 +175,7 @@ public class TableOptimizingMetrics {
private final ServerTableIdentifier identifier;
private OptimizingStatus optimizingStatus;
- private long stateSetTimestamp = System.currentTimeMillis();
+ private long statusSetTimestamp = System.currentTimeMillis();
private final List<MetricKey> registeredMetricKeys = Lists.newArrayList();
private MetricRegistry globalRegistry;
@@ -200,7 +200,7 @@ public class TableOptimizingMetrics {
public void register(MetricRegistry registry) {
if (globalRegistry == null) {
- // register state duration metrics
+ // register status duration metrics
registerMetric(
registry, TABLE_OPTIMIZING_STATUS_IDLE_DURATION, new
StatusDurationGauge(STATUS_IDLE));
registerMetric(
@@ -252,14 +252,14 @@ public class TableOptimizingMetrics {
}
/**
- * Handle table self optimizing state change event.
+ * Handle table self optimizing status change event.
*
* @param optimizingStatus new optimizing status
- * @param stateSetTimestamp timestamp of status changed.
+ * @param statusSetTimestamp timestamp of status changed.
*/
- public void statusChanged(OptimizingStatus optimizingStatus, long
stateSetTimestamp) {
+ public void statusChanged(OptimizingStatus optimizingStatus, long
statusSetTimestamp) {
this.optimizingStatus = optimizingStatus;
- this.stateSetTimestamp = stateSetTimestamp;
+ this.statusSetTimestamp = statusSetTimestamp;
}
/**
@@ -322,15 +322,15 @@ public class TableOptimizingMetrics {
@Override
public Long getValue() {
- String state = getOptimizingStatusDesc(optimizingStatus);
- if (targetStatus.equals(state)) {
- return stateDuration();
+ String status = getOptimizingStatusDesc(optimizingStatus);
+ if (targetStatus.equals(status)) {
+ return statusDuration();
}
return 0L;
}
- private Long stateDuration() {
- return System.currentTimeMillis() - stateSetTimestamp;
+ private Long statusDuration() {
+ return System.currentTimeMillis() - statusSetTimestamp;
}
}
@@ -343,8 +343,8 @@ public class TableOptimizingMetrics {
@Override
public Long getValue() {
- String state = getOptimizingStatusDesc(optimizingStatus);
- if (targetStatus.equals(state)) {
+ String status = getOptimizingStatusDesc(optimizingStatus);
+ if (targetStatus.equals(status)) {
return 1L;
}
return 0L;
diff --git a/docs/user-guides/configurations.md
b/docs/user-guides/configurations.md
index 8a02c1d27..cc24f1ce1 100644
--- a/docs/user-guides/configurations.md
+++ b/docs/user-guides/configurations.md
@@ -38,7 +38,7 @@ Self-optimizing configurations are applicable to both Iceberg
Format and Mixed s
| self-optimizing.max-task-size-bytes | 134217728(128MB) | Maximum
file size bytes in a single task for splitting tasks
|
| self-optimizing.fragment-ratio | 8 | The
fragment file size threshold. We could divide self-optimizing.target-size by
this ratio to get the actual fragment file size |
| self-optimizing.min-target-size-ratio | 0.75 | The
undersized segment file size threshold. Segment files under this threshold will
be considered for rewriting |
-| self-optimizing.minor.trigger.file-count | 12 | The
minimum number of files to trigger minor optimizing is determined by the sum of
fragment file count and equality delete file count |
+| self-optimizing.minor.trigger.file-count | 12 | The
minimum number of files to trigger minor optimizing is determined by the sum of
fragment file count and equality delete file count |
| self-optimizing.minor.trigger.interval | 3600000(1 hour) | The time
interval in milliseconds to trigger minor optimizing
|
| self-optimizing.major.trigger.duplicate-ratio | 0.1 | The ratio
of duplicate data of segment files to trigger major optimizing
|
| self-optimizing.full.trigger.interval | -1(closed) | The time
interval in milliseconds to trigger full optimizing
|
diff --git a/docs/user-guides/metrics.md b/docs/user-guides/metrics.md
index 8ec133cdb..cf1b4e6b8 100644
--- a/docs/user-guides/metrics.md
+++ b/docs/user-guides/metrics.md
@@ -9,30 +9,48 @@ menu:
weight: 500
---
# Metrics
-Amoro provides both table-level and platform-level metrics to help users
understand the runtime status of the current table.
-## Table metrics
-The Amoro Tables details page provides multiple tabs to display the status of
the table from various dimensions, mainly including:
-| **Tab Name** | **Description**
|
-| ------------ | ------------------------------------------------------------ |
-| Details | Display the table's schema, primary key configuration,
partition configuration, properties; as well as the metric information of the
files stored in ChangeStore and BaseStore, including the number of files and
average file size, as well as the latest submission time of the files. |
-| Transactions | The displayed transaction list does not include snapshots
generated by Self-optimizing. |
-| Optimized | Display all the historical Optimize records of the table,
each record shows the number and average size of files before and after
Optimize, as well as the execution time of each Optimize. |
-| Operations | Display the current table's DDL historical change records. |
+Amoro build a metrics system to measure the behaviours of table management
processes, like how long has it been since a table last performed
self-optimizing process, and how much resources does a optimizer group
currently has?
-
+There are two types of metrics provided in the Amoro metric system: Gauge and
Counter.
-
+Gauge: Provides a value of any type at a point in time.
+Counter: Used to count values by incrementing and decrementing.
+
+Amoro has supported built-in metrics to measure status of table
self-optimizing processes and optimizer resources, which can be [reported to
external metric system like Prometheus
etc](../deployment/#configure-metric-reporter).
## Self-optimizing metrics
-The Optimizers page displays platform-level Self-optimizing information,
including the total amount and usage of resources under different groups, as
well as the status and resource usage of all tables that have executed
Self-optimizing under the group.
-
+| Metric Name | Type | Tags
| Description |
+|---------------------------------------------------|---------|--------------------------|--------------------------------------------------------------|
+| table_optimizing_status_idle_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in idle status |
+| table_optimizing_status_pending_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in pending status |
+| table_optimizing_status_planning_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in planning status |
+| table_optimizing_status_executing_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in executing status |
+| table_optimizing_status_committing_duration_mills | Gauge | catalog,
database, table | Duration in milliseconds after table be in committing status |
+| table_optimizing_process_total_count | Counter | catalog,
database, table | Count of all optimizing process since ams started |
+| table_optimizing_process_failed_count | Counter | catalog,
database, table | Count of failed optimizing process since ams started |
+| table_optimizing_minor_total_count | Counter | catalog,
database, table | Count of minor optimizing process since ams started |
+| table_optimizing_minor_failed_count | Counter | catalog,
database, table | Count of failed minor optimizing process since ams started |
+| table_optimizing_major_total_count | Counter | catalog,
database, table | Count of major optimizing process since ams started |
+| table_optimizing_major_failed_count | Counter | catalog,
database, table | Count of failed major optimizing process since ams started |
+| table_optimizing_full_total_count | Counter | catalog,
database, table | Count of full optimizing rocess since ams started |
+| table_optimizing_full_failed_count | Counter | catalog,
database, table | Count of failed full optimizing process since ams started |
+| table_optimizing_status_in_idle | Gauge | catalog,
database, table | If currently table is in idle status |
+| table_optimizing_status_in_pending | Gauge | catalog,
database, table | If currently table is in pending status |
+| table_optimizing_status_in_planning | Gauge | catalog,
database, table | If currently table is in planning status |
+| table_optimizing_status_in_executing | Gauge | catalog,
database, table | If currently table is in executing status |
+| table_optimizing_status_in_committing | Gauge | catalog,
database, table | If currently table is in committing status |
-- **Optimizing Status**: The current optimizing status of the table, including
minor, major, full, idle, and pending.
-- **Duration**: The duration of the current status.
-- **File Count**: The total number of files involved in the current
Self-optimizing, including base, insert, eq-delete, and pos-delete file types.
-- **File Size**: The total size of files involved in the current
Self-optimizing.
-- **Quota**: The proportion of self-optimizing execution time executed per
unit time.
-- **Quota Occupation**: The actual Quota used for self-optimizing during
execution of the table in the last hour. When optimizer resources are
sufficient and the table requires more resources for self-optimizing, this
value will be greater than 100%. When resources are scarce or the table
requires fewer resources for self-optimizing, this value will be less than 100%.
+## Optimizer Group metrics
+| Metric Name | Type | Tags | Description
|
+|----------------------------------------|--------|-------|--------------------------------------------------|
+| optimizer_group_pending_tasks | Gauge | group | Number of pending
tasks in optimizer group |
+| optimizer_group_executing_tasks | Gauge | group | Number of
executing tasks in optimizer group |
+| optimizer_group_planing_tables | Gauge | group | Number of planing
tables in optimizer group |
+| optimizer_group_pending_tables | Gauge | group | Number of pending
tables in optimizer group |
+| optimizer_group_executing_tables | Gauge | group | Number of
executing tables in optimizer group |
+| optimizer_group_optimizer_instances | Gauge | group | Number of
optimizer instances in optimizer group |
+| optimizer_group_memory_bytes_allocated | Gauge | group | Memory bytes
allocated in optimizer group |
+| optimizer_group_threads | Gauge | group | Number of total
threads in optimizer group |
diff --git a/docs/user-guides/using-tables.md b/docs/user-guides/using-tables.md
index f8f0a4604..7915f91be 100644
--- a/docs/user-guides/using-tables.md
+++ b/docs/user-guides/using-tables.md
@@ -216,3 +216,37 @@ DROP TABLE test_db.test_log_store;
```
The current terminal is using the Spark engine to execute SQL. For more
information about deleting tables, you can refer to [Spark
DDL](../spark-ddl/#drop-table).
+
+## Explore table details
+The Amoro Tables details page provides multiple tabs to display the status of
the table from various dimensions, mainly including:
+
+| **Tab Name** | **Description**
|
+|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Details | Display the table's schema, primary key configuration,
partition configuration, properties; as well as the metric information of the
files stored in ChangeStore and BaseStore, including the number of files and
average file size, as well as the latest submission time of the files. |
+| Files | Display all partitions and files of the table.
|
+| Snapshots | Display all snapshots of the table, which can be filtered by
branch and tag.
|
+| Optimizing | Display all the self-optimizing processes of the table, each
record shows the number and average size of files before and after Optimize, as
well as the execution time of each process.
|
+| Operations | Display the current table's DDL historical change records.
|
+
+
+
+
+
+## Explore self-optimizing status
+The Optimizing page displays self-optimizing status of all tables.
+
+
+
+- **Optimizing Status**: The current optimizing status of the table, including
idle, pending, planning, minor, major, full, committing.
+ - idle: means that self-optimizing is not required on the table.
+ - pending: means that self-optimizing is required on the table and is
waiting for resources.
+ - planning: means that self-optimizing process is being planed.
+ - minor: means that minor optimizing is being executed on the table.
+ - major: means that major optimizing is being executed on the table.
+ - full: means that full optimizing is being executed on the table.
+ - committing: means that self-optimizing process is being committed.
+- **Duration**: The duration of the current status.
+- **File Count**: The total number of files involved in the current
Self-optimizing, including base, insert, eq-delete, and pos-delete file types.
+- **File Size**: The total size of files involved in the current
self-optimizing.
+- **Quota**: The proportion of self-optimizing execution time executed per
unit time.
+- **Quota Occupation**: The actual Quota used for self-optimizing during
execution of the table in the last hour. When optimizer resources are
sufficient and the table requires more resources for self-optimizing, this
value will be greater than 100%. When resources are scarce or the table
requires fewer resources for self-optimizing, this value will be less than 100%.
\ No newline at end of file
