This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 923d81bee [AMORO-2875] Added metric for orphan file action (#3004)
923d81bee is described below
commit 923d81beeddab24daaa3e01a9c60caaac8ee659b
Author: big face cat <[email protected]>
AuthorDate: Fri Aug 16 10:16:38 2024 +0800
[AMORO-2875] Added metric for orphan file action (#3004)
* [AMORO-2875] Added metric for orphan file action
* fix test
* spotless
* fix name
---------
Co-authored-by: huyuanfeng <[email protected]>
---
.../maintainer/IcebergTableMaintainer.java | 63 ++++++-----
.../maintainer/MixedTableMaintainer.java | 15 ++-
.../table/TableOrphanFilesCleaningMetrics.java | 124 +++++++++++++++++++++
.../apache/amoro/server/table/TableRuntime.java | 9 ++
.../optimizing/maintainer/TestOrphanFileClean.java | 63 +++++++++--
.../maintainer/TestOrphanFileCleanHive.java | 13 ++-
docs/user-guides/metrics.md | 11 +-
7 files changed, 256 insertions(+), 42 deletions(-)
diff --git
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
index bcc52595f..a9968ef34 100644
---
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
+++
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
@@ -27,6 +27,7 @@ import org.apache.amoro.io.AuthenticatedFileIO;
import org.apache.amoro.io.PathInfo;
import org.apache.amoro.io.SupportsFileSystemOperations;
import org.apache.amoro.server.AmoroServiceConstants;
+import org.apache.amoro.server.table.TableOrphanFilesCleaningMetrics;
import org.apache.amoro.server.table.TableRuntime;
import org.apache.amoro.server.utils.IcebergTableUtil;
import
org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting;
@@ -118,6 +119,8 @@ public class IcebergTableMaintainer implements
TableMaintainer {
@Override
public void cleanOrphanFiles(TableRuntime tableRuntime) {
TableConfiguration tableConfiguration =
tableRuntime.getTableConfiguration();
+ TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics =
+ tableRuntime.getOrphanFilesCleaningMetrics();
if (!tableConfiguration.isCleanOrphanEnabled()) {
return;
@@ -125,13 +128,13 @@ public class IcebergTableMaintainer implements
TableMaintainer {
long keepTime = tableConfiguration.getOrphanExistingMinutes() * 60 * 1000;
- cleanContentFiles(System.currentTimeMillis() - keepTime);
+ cleanContentFiles(System.currentTimeMillis() - keepTime,
orphanFilesCleaningMetrics);
// refresh
table.refresh();
// clear metadata files
- cleanMetadata(System.currentTimeMillis() - keepTime);
+ cleanMetadata(System.currentTimeMillis() - keepTime,
orphanFilesCleaningMetrics);
}
@Override
@@ -299,18 +302,20 @@ public class IcebergTableMaintainer implements
TableMaintainer {
.execute();
}
- protected void cleanContentFiles(long lastTime) {
+ protected void cleanContentFiles(
+ long lastTime, TableOrphanFilesCleaningMetrics
orphanFilesCleaningMetrics) {
// For clean data files, should getRuntime valid files in the base store
and the change store,
// so acquire in advance
// to prevent repeated acquisition
Set<String> validFiles = orphanFileCleanNeedToExcludeFiles();
LOG.info("{} start cleaning orphan files in content", table.name());
- clearInternalTableContentsFiles(lastTime, validFiles);
+ clearInternalTableContentsFiles(lastTime, validFiles,
orphanFilesCleaningMetrics);
}
- protected void cleanMetadata(long lastTime) {
+ protected void cleanMetadata(
+ long lastTime, TableOrphanFilesCleaningMetrics
orphanFilesCleaningMetrics) {
LOG.info("{} start clean metadata files", table.name());
- clearInternalTableMetadata(lastTime);
+ clearInternalTableMetadata(lastTime, orphanFilesCleaningMetrics);
}
protected void cleanDanglingDeleteFiles() {
@@ -353,9 +358,12 @@ public class IcebergTableMaintainer implements
TableMaintainer {
return (AuthenticatedFileIO) table.io();
}
- private void clearInternalTableContentsFiles(long lastTime, Set<String>
exclude) {
+ private void clearInternalTableContentsFiles(
+ long lastTime,
+ Set<String> exclude,
+ TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics) {
String dataLocation = table.location() + File.separator + DATA_FOLDER_NAME;
- int slated = 0, deleted = 0;
+ int expected = 0, deleted = 0;
try (AuthenticatedFileIO io = fileIO()) {
// listPrefix will not return the directory and the orphan file clean
should clean the empty
@@ -365,7 +373,7 @@ public class IcebergTableMaintainer implements
TableMaintainer {
Set<PathInfo> directories = new HashSet<>();
Set<String> filesToDelete =
deleteInvalidFilesInFs(fio, dataLocation, lastTime, exclude,
directories);
- slated = filesToDelete.size();
+ expected = filesToDelete.size();
deleted = TableFileUtil.deleteFiles(io, filesToDelete);
/* delete empty directories */
deleteEmptyDirectories(fio, directories, lastTime, exclude);
@@ -373,7 +381,7 @@ public class IcebergTableMaintainer implements
TableMaintainer {
SupportsPrefixOperations pio = io.asPrefixFileIO();
Set<String> filesToDelete =
deleteInvalidFilesByPrefix(pio, dataLocation, lastTime, exclude);
- slated = filesToDelete.size();
+ expected = filesToDelete.size();
deleted = TableFileUtil.deleteFiles(io, filesToDelete);
} else {
LOG.warn(
@@ -383,19 +391,22 @@ public class IcebergTableMaintainer implements
TableMaintainer {
}
}
- final int finalCandidate = slated;
+ final int finalExpected = expected;
final int finalDeleted = deleted;
runWithCondition(
- slated > 0,
- () ->
- LOG.info(
- "{}: There were {} files slated for deletion and {} files were
successfully deleted",
- table.name(),
- finalCandidate,
- finalDeleted));
+ expected > 0,
+ () -> {
+ LOG.info(
+ "{}: There were {} files expected for deletion and {} files were
successfully deleted",
+ table.name(),
+ finalExpected,
+ finalDeleted);
+ orphanFilesCleaningMetrics.completeOrphanDataFiles(finalExpected,
finalDeleted);
+ });
}
- private void clearInternalTableMetadata(long lastTime) {
+ private void clearInternalTableMetadata(
+ long lastTime, TableOrphanFilesCleaningMetrics
orphanFilesCleaningMetrics) {
Set<String> validFiles = getValidMetadataFiles(table);
LOG.info("{} table getRuntime {} valid files", table.name(),
validFiles.size());
Pattern excludeFileNameRegex = getExcludeFileNameRegex(table);
@@ -414,12 +425,14 @@ public class IcebergTableMaintainer implements
TableMaintainer {
runWithCondition(
!filesToDelete.isEmpty(),
- () ->
- LOG.info(
- "{}: There were {} metadata files to be deleted and {}
metadata files were successfully deleted",
- table.name(),
- filesToDelete.size(),
- deleted));
+ () -> {
+ LOG.info(
+ "{}: There were {} metadata files to be deleted and {}
metadata files were successfully deleted",
+ table.name(),
+ filesToDelete.size(),
+ deleted);
+
orphanFilesCleaningMetrics.completeOrphanMetadataFiles(filesToDelete.size(),
deleted);
+ });
} else {
LOG.warn(
String.format(
diff --git
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/MixedTableMaintainer.java
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/MixedTableMaintainer.java
index 124a96fee..cc9f113ca 100644
---
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/MixedTableMaintainer.java
+++
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/optimizing/maintainer/MixedTableMaintainer.java
@@ -26,6 +26,7 @@ import org.apache.amoro.TableFormat;
import org.apache.amoro.api.config.DataExpirationConfig;
import org.apache.amoro.data.FileNameRules;
import org.apache.amoro.scan.TableEntriesScan;
+import org.apache.amoro.server.table.TableOrphanFilesCleaningMetrics;
import org.apache.amoro.server.table.TableRuntime;
import org.apache.amoro.server.utils.HiveLocationUtil;
import
org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting;
@@ -244,18 +245,20 @@ public class MixedTableMaintainer implements
TableMaintainer {
throw new UnsupportedOperationException("Mixed table doesn't support auto
create tags");
}
- protected void cleanContentFiles(long lastTime) {
+ protected void cleanContentFiles(
+ long lastTime, TableOrphanFilesCleaningMetrics
orphanFilesCleaningMetrics) {
if (changeMaintainer != null) {
- changeMaintainer.cleanContentFiles(lastTime);
+ changeMaintainer.cleanContentFiles(lastTime, orphanFilesCleaningMetrics);
}
- baseMaintainer.cleanContentFiles(lastTime);
+ baseMaintainer.cleanContentFiles(lastTime, orphanFilesCleaningMetrics);
}
- protected void cleanMetadata(long lastTime) {
+ protected void cleanMetadata(
+ long lastTime, TableOrphanFilesCleaningMetrics
orphanFilesCleaningMetrics) {
if (changeMaintainer != null) {
- changeMaintainer.cleanMetadata(lastTime);
+ changeMaintainer.cleanMetadata(lastTime, orphanFilesCleaningMetrics);
}
- baseMaintainer.cleanMetadata(lastTime);
+ baseMaintainer.cleanMetadata(lastTime, orphanFilesCleaningMetrics);
}
protected void cleanDanglingDeleteFiles() {
diff --git
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableOrphanFilesCleaningMetrics.java
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableOrphanFilesCleaningMetrics.java
new file mode 100644
index 000000000..ef4f6b62a
--- /dev/null
+++
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableOrphanFilesCleaningMetrics.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.table;
+
+import static org.apache.amoro.api.metrics.MetricDefine.defineCounter;
+
+import org.apache.amoro.api.ServerTableIdentifier;
+import org.apache.amoro.api.metrics.Counter;
+import org.apache.amoro.api.metrics.Metric;
+import org.apache.amoro.api.metrics.MetricDefine;
+import org.apache.amoro.api.metrics.MetricKey;
+import org.apache.amoro.server.metrics.MetricRegistry;
+import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
+
+import java.util.List;
+
+/** Table Orphan Files Cleaning metrics. */
+public class TableOrphanFilesCleaningMetrics {
+
+ private final Counter orphanDataFilesCount = new Counter();
+ private final Counter expectedOrphanDataFilesCount = new Counter();
+
+ private final Counter orphanMetadataFilesCount = new Counter();
+ private final Counter expectedOrphanMetadataFilesCount = new Counter();
+
+ private final ServerTableIdentifier identifier;
+
+ public TableOrphanFilesCleaningMetrics(ServerTableIdentifier identifier) {
+ this.identifier = identifier;
+ }
+
+ public static final MetricDefine TABLE_ORPHAN_CONTENT_FILE_CLEANING_COUNT =
+ defineCounter("table_orphan_content_file_cleaning_count")
+ .withDescription("Count of orphan content files cleaned in the table
since ams started")
+ .withTags("catalog", "database", "table")
+ .build();
+
+ public static final MetricDefine TABLE_ORPHAN_METADATA_FILE_CLEANING_COUNT =
+ defineCounter("table_orphan_metadata_file_cleaning_count")
+ .withDescription("Count of orphan metadata files cleaned in the
table since ams started")
+ .withTags("catalog", "database", "table")
+ .build();
+
+ public static final MetricDefine
TABLE_EXPECTED_ORPHAN_CONTENT_FILE_CLEANING_COUNT =
+ defineCounter("table_expected_orphan_content_file_cleaning_count")
+ .withDescription(
+ "Expected count of orphan content files cleaned in the table
since ams started")
+ .withTags("catalog", "database", "table")
+ .build();
+
+ public static final MetricDefine
TABLE_EXPECTED_ORPHAN_METADATA_FILE_CLEANING_COUNT =
+ defineCounter("table_expected_orphan_metadata_file_cleaning_count")
+ .withDescription(
+ "Expected count of orphan metadata files cleaned in the table
since ams started")
+ .withTags("catalog", "database", "table")
+ .build();
+
+ private final List<MetricKey> registeredMetricKeys = Lists.newArrayList();
+ private MetricRegistry globalRegistry;
+
+ private void registerMetric(MetricRegistry registry, MetricDefine define,
Metric metric) {
+ MetricKey key =
+ registry.register(
+ define,
+ ImmutableMap.of(
+ "catalog",
+ identifier.getCatalog(),
+ "database",
+ identifier.getDatabase(),
+ "table",
+ identifier.getTableName()),
+ metric);
+ registeredMetricKeys.add(key);
+ }
+
+ public void register(MetricRegistry registry) {
+ if (globalRegistry == null) {
+ registerMetric(registry, TABLE_ORPHAN_CONTENT_FILE_CLEANING_COUNT,
orphanDataFilesCount);
+ registerMetric(registry, TABLE_ORPHAN_METADATA_FILE_CLEANING_COUNT,
orphanMetadataFilesCount);
+ registerMetric(
+ registry,
+ TABLE_EXPECTED_ORPHAN_CONTENT_FILE_CLEANING_COUNT,
+ expectedOrphanDataFilesCount);
+ registerMetric(
+ registry,
+ TABLE_EXPECTED_ORPHAN_METADATA_FILE_CLEANING_COUNT,
+ expectedOrphanMetadataFilesCount);
+ globalRegistry = registry;
+ }
+ }
+
+ public void completeOrphanDataFiles(int expected, int cleaned) {
+ expectedOrphanMetadataFilesCount.inc(expected);
+ orphanDataFilesCount.inc(cleaned);
+ }
+
+ public void completeOrphanMetadataFiles(int expected, int cleaned) {
+ expectedOrphanMetadataFilesCount.inc(expected);
+ orphanMetadataFilesCount.inc(cleaned);
+ }
+
+ public void unregister() {
+ registeredMetricKeys.forEach(globalRegistry::unregister);
+ registeredMetricKeys.clear();
+ globalRegistry = null;
+ }
+}
diff --git
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableRuntime.java
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableRuntime.java
index 95c68e08d..9cf17b4fc 100644
---
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableRuntime.java
+++
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/table/TableRuntime.java
@@ -87,6 +87,7 @@ public class TableRuntime extends StatedPersistentBase {
@StateField private volatile OptimizingEvaluator.PendingInput pendingInput;
private volatile long lastPlanTime;
private final TableOptimizingMetrics optimizingMetrics;
+ private final TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics;
protected TableRuntime(
ServerTableIdentifier tableIdentifier,
@@ -100,6 +101,7 @@ public class TableRuntime extends StatedPersistentBase {
this.optimizerGroup =
tableConfiguration.getOptimizingConfig().getOptimizerGroup();
persistTableRuntime();
optimizingMetrics = new TableOptimizingMetrics(tableIdentifier);
+ orphanFilesCleaningMetrics = new
TableOrphanFilesCleaningMetrics(tableIdentifier);
}
protected TableRuntime(TableRuntimeMeta tableRuntimeMeta,
TableRuntimeHandler tableHandler) {
@@ -131,6 +133,7 @@ public class TableRuntime extends StatedPersistentBase {
this.pendingInput = tableRuntimeMeta.getPendingInput();
optimizingMetrics = new TableOptimizingMetrics(tableIdentifier);
optimizingMetrics.statusChanged(optimizingStatus,
this.currentStatusStartTime);
+ orphanFilesCleaningMetrics = new
TableOrphanFilesCleaningMetrics(tableIdentifier);
}
public void recover(OptimizingProcess optimizingProcess) {
@@ -143,6 +146,7 @@ public class TableRuntime extends StatedPersistentBase {
public void registerMetric(MetricRegistry metricRegistry) {
this.optimizingMetrics.register(metricRegistry);
+ this.orphanFilesCleaningMetrics.register(metricRegistry);
}
public void dispose() {
@@ -156,6 +160,7 @@ public class TableRuntime extends StatedPersistentBase {
mapper ->
mapper.deleteOptimizingRuntime(tableIdentifier.getId())));
});
optimizingMetrics.unregister();
+ orphanFilesCleaningMetrics.unregister();
}
public void beginPlanning() {
@@ -445,6 +450,10 @@ public class TableRuntime extends StatedPersistentBase {
return optimizerGroup;
}
+ public TableOrphanFilesCleaningMetrics getOrphanFilesCleaningMetrics() {
+ return orphanFilesCleaningMetrics;
+ }
+
public void setCurrentChangeSnapshotId(long currentChangeSnapshotId) {
this.currentChangeSnapshotId = currentChangeSnapshotId;
}
diff --git
a/amoro-ams/amoro-ams-server/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestOrphanFileClean.java
b/amoro-ams/amoro-ams-server/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestOrphanFileClean.java
index bf464ca58..4f945b5ac 100644
---
a/amoro-ams/amoro-ams-server/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestOrphanFileClean.java
+++
b/amoro-ams/amoro-ams-server/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestOrphanFileClean.java
@@ -29,8 +29,10 @@ import org.apache.amoro.api.config.TableConfiguration;
import org.apache.amoro.catalog.BasicCatalogTestHelper;
import org.apache.amoro.catalog.CatalogTestHelper;
import org.apache.amoro.server.dashboard.utils.AmsUtil;
+import org.apache.amoro.server.table.TableOrphanFilesCleaningMetrics;
import org.apache.amoro.server.table.TableRuntime;
import org.apache.amoro.server.table.executor.ExecutorTestBase;
+import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.table.TableProperties;
import org.apache.amoro.table.UnkeyedTable;
import org.apache.iceberg.AppendFiles;
@@ -110,14 +112,23 @@ public class TestOrphanFileClean extends ExecutorTestBase
{
changeOrphanDataFile.createOrOverwrite().close();
Assert.assertTrue(getMixedTable().io().exists(changeOrphanFilePath));
}
-
+ TableIdentifier tableIdentifier = getMixedTable().id();
+ TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics =
+ new TableOrphanFilesCleaningMetrics(
+ ServerTableIdentifier.of(
+ tableIdentifier.getCatalog(),
+ tableIdentifier.getDatabase(),
+ tableIdentifier.getTableName(),
+ getTestFormat()));
MixedTableMaintainer maintainer = new
MixedTableMaintainer(getMixedTable());
maintainer.cleanContentFiles(
System.currentTimeMillis()
- - TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT * 60 *
1000);
+ - TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT * 60 *
1000,
+ orphanFilesCleaningMetrics);
maintainer.cleanMetadata(
System.currentTimeMillis()
- - TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT * 60 *
1000);
+ - TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT * 60 *
1000,
+ orphanFilesCleaningMetrics);
Assert.assertTrue(getMixedTable().io().exists(baseOrphanFileDir));
Assert.assertTrue(getMixedTable().io().exists(baseOrphanFilePath));
@@ -126,8 +137,8 @@ public class TestOrphanFileClean extends ExecutorTestBase {
Assert.assertTrue(getMixedTable().io().exists(changeOrphanFilePath));
}
- maintainer.cleanContentFiles(System.currentTimeMillis());
- maintainer.cleanMetadata(System.currentTimeMillis());
+ maintainer.cleanContentFiles(System.currentTimeMillis(),
orphanFilesCleaningMetrics);
+ maintainer.cleanMetadata(System.currentTimeMillis(),
orphanFilesCleaningMetrics);
Assert.assertFalse(getMixedTable().io().exists(baseOrphanFileDir));
Assert.assertFalse(getMixedTable().io().exists(baseOrphanFilePath));
@@ -194,7 +205,15 @@ public class TestOrphanFileClean extends ExecutorTestBase {
}
MixedTableMaintainer maintainer = new
MixedTableMaintainer(getMixedTable());
- maintainer.cleanMetadata(System.currentTimeMillis());
+ TableIdentifier tableIdentifier = getMixedTable().id();
+ TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics =
+ new TableOrphanFilesCleaningMetrics(
+ ServerTableIdentifier.of(
+ tableIdentifier.getCatalog(),
+ tableIdentifier.getDatabase(),
+ tableIdentifier.getTableName(),
+ getTestFormat()));
+ maintainer.cleanMetadata(System.currentTimeMillis(),
orphanFilesCleaningMetrics);
Assert.assertFalse(getMixedTable().io().exists(baseOrphanFilePath));
if (isKeyedTable()) {
@@ -279,7 +298,16 @@ public class TestOrphanFileClean extends ExecutorTestBase {
}
MixedTableMaintainer tableMaintainer = new
MixedTableMaintainer(getMixedTable());
- tableMaintainer.cleanMetadata(System.currentTimeMillis());
+ TableIdentifier tableIdentifier = getMixedTable().id();
+ TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics =
+ new TableOrphanFilesCleaningMetrics(
+ ServerTableIdentifier.of(
+ tableIdentifier.getCatalog(),
+ tableIdentifier.getDatabase(),
+ tableIdentifier.getTableName(),
+ getTestFormat()));
+
+ tableMaintainer.cleanMetadata(System.currentTimeMillis(),
orphanFilesCleaningMetrics);
Assert.assertFalse(getMixedTable().io().exists(baseOrphanFilePath));
if (isKeyedTable()) {
// files whose file name starts with flink.job-id should not be deleted
@@ -306,11 +334,22 @@ public class TestOrphanFileClean extends ExecutorTestBase
{
StatisticsFile file3 =
commitStatisticsFile(unkeyedTable, unkeyedTable.location() +
"/data/puffin/test3.puffin");
+ TableIdentifier tableIdentifier = getMixedTable().id();
+ TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics =
+ new TableOrphanFilesCleaningMetrics(
+ ServerTableIdentifier.of(
+ tableIdentifier.getCatalog(),
+ tableIdentifier.getDatabase(),
+ tableIdentifier.getTableName(),
+ getTestFormat()));
+
Assert.assertTrue(unkeyedTable.io().exists(file1.path()));
Assert.assertTrue(unkeyedTable.io().exists(file2.path()));
Assert.assertTrue(unkeyedTable.io().exists(file3.path()));
- new
MixedTableMaintainer(getMixedTable()).cleanContentFiles(System.currentTimeMillis()
+ 1);
- new
MixedTableMaintainer(getMixedTable()).cleanMetadata(System.currentTimeMillis()
+ 1);
+ new MixedTableMaintainer(getMixedTable())
+ .cleanContentFiles(System.currentTimeMillis() + 1,
orphanFilesCleaningMetrics);
+ new MixedTableMaintainer(getMixedTable())
+ .cleanMetadata(System.currentTimeMillis() + 1,
orphanFilesCleaningMetrics);
Assert.assertTrue(unkeyedTable.io().exists(file1.path()));
Assert.assertTrue(unkeyedTable.io().exists(file2.path()));
Assert.assertTrue(unkeyedTable.io().exists(file3.path()));
@@ -364,6 +403,12 @@ public class TestOrphanFileClean extends ExecutorTestBase {
Mockito.when(tableRuntime.getTableConfiguration())
.thenReturn(TableConfiguration.parseConfig(baseTable.properties()));
+ Mockito.when(tableRuntime.getOrphanFilesCleaningMetrics())
+ .thenReturn(
+ new TableOrphanFilesCleaningMetrics(
+ ServerTableIdentifier.of(
+ AmsUtil.toTableIdentifier(baseTable.id()),
getTestFormat())));
+
MixedTableMaintainer maintainer = new
MixedTableMaintainer(getMixedTable());
maintainer.cleanOrphanFiles(tableRuntime);
diff --git
a/amoro-ams/amoro-ams-server/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestOrphanFileCleanHive.java
b/amoro-ams/amoro-ams-server/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestOrphanFileCleanHive.java
index 08410862d..73bf2d6e8 100644
---
a/amoro-ams/amoro-ams-server/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestOrphanFileCleanHive.java
+++
b/amoro-ams/amoro-ams-server/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestOrphanFileCleanHive.java
@@ -22,11 +22,14 @@ import static
org.apache.amoro.server.optimizing.maintainer.IcebergTableMaintain
import org.apache.amoro.TableFormat;
import org.apache.amoro.TableTestHelper;
+import org.apache.amoro.api.ServerTableIdentifier;
import org.apache.amoro.catalog.CatalogTestHelper;
import org.apache.amoro.hive.TestHMS;
import org.apache.amoro.hive.catalog.HiveCatalogTestHelper;
import org.apache.amoro.hive.catalog.HiveTableTestHelper;
import org.apache.amoro.hive.table.SupportHive;
+import org.apache.amoro.server.table.TableOrphanFilesCleaningMetrics;
+import org.apache.amoro.table.TableIdentifier;
import org.apache.iceberg.io.OutputFile;
import org.junit.Assert;
import org.junit.ClassRule;
@@ -82,7 +85,15 @@ public class TestOrphanFileCleanHive extends
TestOrphanFileClean {
Assert.assertTrue(getMixedTable().io().exists(hiveOrphanFilePath));
MixedTableMaintainer maintainer = new
MixedTableMaintainer(getMixedTable());
- maintainer.cleanContentFiles(System.currentTimeMillis());
+ TableIdentifier tableIdentifier = getMixedTable().id();
+ TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics =
+ new TableOrphanFilesCleaningMetrics(
+ ServerTableIdentifier.of(
+ tableIdentifier.getCatalog(),
+ tableIdentifier.getDatabase(),
+ tableIdentifier.getTableName(),
+ getTestFormat()));
+ maintainer.cleanContentFiles(System.currentTimeMillis(),
orphanFilesCleaningMetrics);
Assert.assertTrue(getMixedTable().io().exists(hiveOrphanFilePath));
}
}
diff --git a/docs/user-guides/metrics.md b/docs/user-guides/metrics.md
index a84567b79..83a4010a4 100644
--- a/docs/user-guides/metrics.md
+++ b/docs/user-guides/metrics.md
@@ -71,6 +71,15 @@ Amoro has supported built-in metrics to measure status of
table self-optimizing
| optimizer_group_memory_bytes_allocated | Gauge | group | Memory bytes
allocated in optimizer group |
| optimizer_group_threads | Gauge | group | Number of total
threads in optimizer group |
+## Orphan Files Cleaning metrics
+
+| Metric Name | Type | Tags
| Description
|
+|-------------------------------------------|---------|--------------------------|--------------------------------------------------------------------------------|
+| table_orphan_content_file_cleaning_count | Counter | catalog, database,
table | Count of orphan content files cleaned in the table since ams started
|
+| table_orphan_metadata_file_cleaning_count | Counter | catalog, database,
table | Count of orphan metadata files cleaned in the table since ams started
|
+| table_expected_orphan_content_file_cleaning_count | Counter | catalog,
database, table | Expected Count of orphan content files cleaned in the table
since ams started |
+| table_expected_orphan_metadata_file_cleaning_count | Counter | catalog,
database, table | Expected Count of orphan metadata files cleaned in the table
since ams started |
+
## Ams service metrics
| Metric Name | Type | Tags
| Description |
@@ -82,4 +91,4 @@ Amoro has supported built-in metrics to measure status of
table self-optimizing
| ams_jvm_memory_heap_max | Gauge |
| The maximum heap memory (bytes), set by -Xmx JVM argument |
| ams_jvm_threads_count | Gauge |
| The total number of live threads used by the AMS |
| ams_jvm_garbage_collector_count | Gauge
|garbage_collector| The count of the JVM's Garbage Collector, such as G1 Young
|
-| ams_jvm_garbage_collector_time | Gauge
|garbage_collector| The time spent by the JVM's Garbage Collector, such as G1
Young |
\ No newline at end of file
+| ams_jvm_garbage_collector_time | Gauge
|garbage_collector| The time spent by the JVM's Garbage Collector, such as G1
Young |
