This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new d7d653446 [AMORO-3515] Improve: format table maintainer log
information (#3519)
d7d653446 is described below
commit d7d653446c76d58ab84afd12ce5490680e1192d0
Author: LiJie20190102 <[email protected]>
AuthorDate: Mon Apr 21 14:50:25 2025 +0800
[AMORO-3515] Improve: format table maintainer log information (#3519)
* Improve: format table maintainer log information
* Improve: format table maintainer log information
* Improve: format table maintainer log information
---------
Co-authored-by: lijie <[email protected]>
---
.../maintainer/IcebergTableMaintainer.java | 52 +++++++++++-----------
1 file changed, 27 insertions(+), 25 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
index b4f59d706..5adcff3e0 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
@@ -162,7 +162,7 @@ public class IcebergTableMaintainer implements
TableMaintainer {
cleanDanglingDeleteFiles();
} else {
LOG.debug(
- "Table {} does not have any delete files, so there is no need to
clean dangling delete file",
+ "There are no delete files here, so there is no need to clean
dangling delete file for table {}",
table.name());
}
}
@@ -189,7 +189,7 @@ public class IcebergTableMaintainer implements
TableMaintainer {
private void expireSnapshots(long olderThan, int minCount, Set<String>
exclude) {
LOG.debug(
- "{}:start expire snapshots older than {} and retain last {} snapshots,
the exclude is {}",
+ "Starting snapshots expiration for table {}, expiring snapshots older
than {} and retain last {} snapshots, excluding {}",
table.name(),
olderThan,
minCount,
@@ -230,7 +230,7 @@ public class IcebergTableMaintainer implements
TableMaintainer {
TableFileUtil.deleteEmptyDirectory(fileIO(), parent, exclude);
} catch (Exception e) {
// Ignore exceptions to remove as many directories as possible
- LOG.warn("{}:fail to delete empty directory {}", table.name(),
parent, e);
+ LOG.warn("Failed to delete empty directory {} for table {}",
parent, table.name(), e);
}
});
@@ -238,10 +238,10 @@ public class IcebergTableMaintainer implements
TableMaintainer {
toDeleteFiles.get() > 0,
() ->
LOG.info(
- "To delete {} files in {}, success delete {} files",
+ "Deleted {}/{} files for table {}",
+ deletedFiles,
toDeleteFiles.get(),
- getTable().name(),
- deletedFiles));
+ getTable().name()));
}
@Override
@@ -321,24 +321,26 @@ public class IcebergTableMaintainer implements
TableMaintainer {
// so acquire in advance
// to prevent repeated acquisition
Set<String> validFiles = orphanFileCleanNeedToExcludeFiles();
- LOG.info("{} start cleaning orphan files in content", table.name());
+ LOG.info("Starting cleaning orphan content files for table {}",
table.name());
clearInternalTableContentsFiles(lastTime, validFiles,
orphanFilesCleaningMetrics);
}
protected void cleanMetadata(
long lastTime, TableOrphanFilesCleaningMetrics
orphanFilesCleaningMetrics) {
- LOG.info("{} start clean metadata files", table.name());
+ LOG.info("Starting cleaning metadata files for table {}", table.name());
clearInternalTableMetadata(lastTime, orphanFilesCleaningMetrics);
}
protected void cleanDanglingDeleteFiles() {
- LOG.info("{} start delete dangling delete files", table.name());
+ LOG.info("Starting cleaning dangling delete files for table {}",
table.name());
int danglingDeleteFilesCnt = clearInternalTableDanglingDeleteFiles();
runWithCondition(
danglingDeleteFilesCnt > 0,
() ->
LOG.info(
- "{} total delete {} dangling delete files", table.name(),
danglingDeleteFilesCnt));
+ "Deleted {} dangling delete files for table {}",
+ danglingDeleteFilesCnt,
+ table.name()));
}
protected long mustOlderThan(TableRuntime tableRuntime, long now) {
@@ -410,10 +412,10 @@ public class IcebergTableMaintainer implements
TableMaintainer {
expected > 0,
() -> {
LOG.info(
- "{}: There were {} files expected for deletion and {} files were
successfully deleted",
- table.name(),
+ "Deleted {}/{} orphan content files for table {}",
+ finalDeleted,
finalExpected,
- finalDeleted);
+ table.name());
orphanFilesCleaningMetrics.completeOrphanDataFiles(finalExpected,
finalDeleted);
});
}
@@ -421,10 +423,12 @@ public class IcebergTableMaintainer implements
TableMaintainer {
private void clearInternalTableMetadata(
long lastTime, TableOrphanFilesCleaningMetrics
orphanFilesCleaningMetrics) {
Set<String> validFiles = getValidMetadataFiles(table);
- LOG.info("{} table getRuntime {} valid files", table.name(),
validFiles.size());
+ LOG.info("Found {} valid metadata files for table {}", validFiles.size(),
table.name());
Pattern excludeFileNameRegex = getExcludeFileNameRegex(table);
LOG.info(
- "{} table getRuntime exclude file name pattern {}", table.name(),
excludeFileNameRegex);
+ "Exclude metadata files with name pattern {} for table {}",
+ excludeFileNameRegex,
+ table.name());
String metadataLocation = table.location() + File.separator +
METADATA_FOLDER_NAME;
LOG.info("start orphan files clean in {}", metadataLocation);
@@ -440,10 +444,10 @@ public class IcebergTableMaintainer implements
TableMaintainer {
!filesToDelete.isEmpty(),
() -> {
LOG.info(
- "{}: There were {} metadata files to be deleted and {}
metadata files were successfully deleted",
- table.name(),
+ "Deleted {}/{} metadata files for table {}",
+ deleted,
filesToDelete.size(),
- deleted);
+ table.name());
orphanFilesCleaningMetrics.completeOrphanMetadataFiles(filesToDelete.size(),
deleted);
});
} else {
@@ -473,9 +477,7 @@ public class IcebergTableMaintainer implements
TableMaintainer {
rewriteFiles.commit();
} catch (ValidationException e) {
LOG.warn(
- "{}:iceberg rewriteFiles commit failed on clear danglingDeleteFiles,
but ignore",
- table.name(),
- e);
+ "Failed to commit dangling delete file for table {}, but ignore it",
table.name(), e);
return 0;
}
return danglingDeleteFiles.size();
@@ -602,7 +604,7 @@ public class IcebergTableMaintainer implements
TableMaintainer {
Set<String> validFiles = new HashSet<>();
Iterable<Snapshot> snapshots = internalTable.snapshots();
int size = Iterables.size(snapshots);
- LOG.info("{} getRuntime {} snapshots to scan", tableName, size);
+ LOG.info("Found {} snapshots to scan for table {}", size, tableName);
for (Snapshot snapshot : snapshots) {
String manifestListLocation = snapshot.manifestListLocation();
validFiles.add(TableFileUtil.getUriPath(manifestListLocation));
@@ -828,13 +830,13 @@ public class IcebergTableMaintainer implements
TableMaintainer {
// sequenceNumber) and expireTimestamp...
LOG.info(
- "Expired {} files older than {}, {} data files[{}] and {} delete
files[{}]",
- table.name(),
+ "Expired files older than {}, {} data files[{}] and {} delete
files[{}] for table {}",
expireTimestamp,
dataFiles.size(),
dataFiles.stream().map(ContentFile::path).collect(Collectors.joining(",")),
deleteFiles.size(),
-
deleteFiles.stream().map(ContentFile::path).collect(Collectors.joining(",")));
+
deleteFiles.stream().map(ContentFile::path).collect(Collectors.joining(",")),
+ table.name());
}
public static class ExpireFiles {
