This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 1159e4d01 [AMORO-3692] Remove deprecated metrics and update related
logic in MetricsSummary (#3693)
1159e4d01 is described below
commit 1159e4d01c0f3a8278e90719632ba258b261b385
Author: Qishang Zhong <[email protected]>
AuthorDate: Thu Jul 31 16:10:14 2025 +0800
[AMORO-3692] Remove deprecated metrics and update related logic in
MetricsSummary (#3693)
[Hotfix] Remove deprecated metrics and update related logic in
MetricsSummary
---
.../server/dashboard/utils/OptimizingUtil.java | 30 +--------
.../TestIcebergServerTableDescriptor.java | 4 +-
.../apache/amoro/optimizing/MetricsSummary.java | 75 +++-------------------
.../apache/amoro/optimizing/RewriteStageTask.java | 2 -
4 files changed, 13 insertions(+), 98 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/utils/OptimizingUtil.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/utils/OptimizingUtil.java
index a3b042a01..148ac2e76 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/utils/OptimizingUtil.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/utils/OptimizingUtil.java
@@ -28,7 +28,6 @@ import org.apache.amoro.server.optimizing.OptimizingTaskMeta;
import org.apache.amoro.server.optimizing.TaskRuntime;
import org.apache.amoro.server.persistence.TableRuntimeMeta;
import org.apache.amoro.table.descriptor.FilesStatistics;
-import org.apache.iceberg.ContentFile;
import java.util.List;
import java.util.stream.Collectors;
@@ -131,35 +130,8 @@ public class OptimizingUtil {
}
return FilesStatistics.builder()
.addFiles(metricsSummary.getEqualityDeleteSize(),
metricsSummary.getEqDeleteFileCnt())
- .addFiles(
- Math.max(
- metricsSummary.getPositionalDeleteSize(),
metricsSummary.getPositionDeleteSize()),
- metricsSummary.getPosDeleteFileCnt())
+ .addFiles(metricsSummary.getPositionDeleteSize(),
metricsSummary.getPosDeleteFileCnt())
.addFiles(metricsSummary.getRewriteDataSize(),
metricsSummary.getRewriteDataFileCnt())
.build();
}
-
- public static long getFileSize(ContentFile<?>[] contentFiles) {
- long size = 0;
- if (contentFiles != null) {
- for (ContentFile<?> contentFile : contentFiles) {
- size += contentFile.fileSizeInBytes();
- }
- }
- return size;
- }
-
- public static int getFileCount(ContentFile<?>[] contentFiles) {
- return contentFiles == null ? 0 : contentFiles.length;
- }
-
- public static long getRecordCnt(ContentFile<?>[] contentFiles) {
- int recordCnt = 0;
- if (contentFiles != null) {
- for (ContentFile<?> contentFile : contentFiles) {
- recordCnt += contentFile.recordCount();
- }
- }
- return recordCnt;
- }
}
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestIcebergServerTableDescriptor.java
b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestIcebergServerTableDescriptor.java
index 5992f9107..6a88748ad 100644
---
a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestIcebergServerTableDescriptor.java
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestIcebergServerTableDescriptor.java
@@ -88,8 +88,8 @@ public class TestIcebergServerTableDescriptor extends
TestServerTableDescriptor
descriptor.insertTable(identifier);
MetricsSummary dummySummery = new MetricsSummary();
dummySummery.setNewDeleteFileCnt(1);
- dummySummery.setNewFileCnt(1);
- dummySummery.setNewFileSize(1);
+ dummySummery.setNewDataFileCnt(1);
+ dummySummery.setNewDataSize(1);
dummySummery.setNewDeleteFileCnt(1);
dummySummery.setNewDeleteSize(1);
descriptor.insertOptimizingProcess(
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/MetricsSummary.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/MetricsSummary.java
index 6554168f9..55cad9784 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/MetricsSummary.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/MetricsSummary.java
@@ -56,24 +56,15 @@ public class MetricsSummary {
private long rewriteDataRecordCnt = 0;
private long rewritePosDataSize = 0;
private int rewritePosDataFileCnt = 0;
- /** @deprecated since 0.7.0, will be removed in 0.8.0 */
- @Deprecated private int reRowDeletedDataFileCnt = 0;
private long rewritePosDataRecordCnt = 0;
private long equalityDeleteSize = 0;
private int eqDeleteFileCnt = 0;
private long eqDeleteRecordCnt = 0;
private long positionDeleteSize = 0;
- /** @deprecated since 0.7.0, will be removed in 0.8.0 */
- @Deprecated private long positionalDeleteSize = 0;
private int posDeleteFileCnt = 0;
private long posDeleteRecordCnt = 0;
-
- /** @deprecated since 0.7.0, will be removed in 0.8.0 */
- @Deprecated private long newFileSize = 0;
-
- @Deprecated private int newFileCnt = 0;
private long newDataSize = 0;
private int newDataFileCnt = 0;
private long newDataRecordCnt = 0;
@@ -88,7 +79,6 @@ public class MetricsSummary {
protected MetricsSummary(RewriteFilesInput input) {
rewriteDataFileCnt = input.rewrittenDataFiles().length;
rewritePosDataFileCnt = input.rePosDeletedDataFiles().length;
- reRowDeletedDataFileCnt = input.rePosDeletedDataFiles().length;
for (DataFile rewriteFile : input.rewrittenDataFiles()) {
rewriteDataSize += rewriteFile.fileSizeInBytes();
rewriteDataRecordCnt += rewriteFile.recordCount();
@@ -100,7 +90,6 @@ public class MetricsSummary {
for (ContentFile<?> delete : input.deleteFiles()) {
if (delete.content() == FileContent.POSITION_DELETES) {
positionDeleteSize += delete.fileSizeInBytes();
- positionalDeleteSize += delete.fileSizeInBytes();
posDeleteRecordCnt += delete.recordCount();
posDeleteFileCnt++;
} else {
@@ -121,26 +110,17 @@ public class MetricsSummary {
newDeleteFileCnt += metrics.getNewDeleteFileCnt();
newDeleteRecordCnt += metrics.getNewDeleteRecordCnt();
rewriteDataFileCnt += metrics.getRewriteDataFileCnt();
- // to be compatible with old metrics name when calculating total
metrics of all tasks
- reRowDeletedDataFileCnt +=
- Math.max(metrics.getReRowDeletedDataFileCnt(),
metrics.getRewritePosDataFileCnt());
- rewritePosDataFileCnt +=
- Math.max(metrics.getReRowDeletedDataFileCnt(),
metrics.getRewritePosDataFileCnt());
+ rewritePosDataFileCnt += metrics.getRewritePosDataFileCnt();
rewriteDataSize += metrics.getRewriteDataSize();
rewritePosDataSize += metrics.getRewritePosDataSize();
posDeleteFileCnt += metrics.getPosDeleteFileCnt();
- positionalDeleteSize +=
- Math.max(metrics.getPositionalDeleteSize(),
metrics.getPositionDeleteSize());
- positionDeleteSize +=
- Math.max(metrics.getPositionalDeleteSize(),
metrics.getPositionDeleteSize());
+ positionDeleteSize += metrics.getPositionDeleteSize();
eqDeleteFileCnt += metrics.getEqDeleteFileCnt();
equalityDeleteSize += metrics.getEqualityDeleteSize();
rewriteDataRecordCnt += metrics.getRewriteDataRecordCnt();
rewritePosDataRecordCnt += metrics.getRewritePosDataRecordCnt();
eqDeleteRecordCnt += metrics.getEqDeleteRecordCnt();
posDeleteRecordCnt += metrics.getPosDeleteRecordCnt();
- newFileCnt += metrics.getNewFileCnt();
- newFileSize += metrics.getNewFileSize();
});
}
@@ -149,21 +129,14 @@ public class MetricsSummary {
put(summary, INPUT_DATA_FILES, rewriteDataFileCnt);
put(summary, INPUT_DATA_SIZE, rewriteDataSize, humanReadable);
put(summary, INPUT_DATA_RECORDS, rewriteDataRecordCnt);
- putIfPositive(
- summary,
- INPUT_READ_ONLY_DATA_FILES,
- Math.max(reRowDeletedDataFileCnt, rewritePosDataFileCnt));
+ putIfPositive(summary, INPUT_READ_ONLY_DATA_FILES, rewritePosDataFileCnt);
putIfPositive(summary, INPUT_READ_ONLY_DATA_SIZE, rewritePosDataSize,
humanReadable);
putIfPositive(summary, INPUT_READ_ONLY_DATA_RECORDS,
rewritePosDataRecordCnt);
putIfPositive(summary, INPUT_EQ_DELETE_FILES, eqDeleteFileCnt);
putIfPositive(summary, INPUT_EQ_DELETE_SIZE, equalityDeleteSize,
humanReadable);
putIfPositive(summary, INPUT_EQ_DELETE_RECORDS, eqDeleteRecordCnt);
putIfPositive(summary, INPUT_POS_DELETE_FILES, posDeleteFileCnt);
- putIfPositive(
- summary,
- INPUT_POS_DELETE_SIZE,
- Math.max(positionalDeleteSize, positionDeleteSize),
- humanReadable);
+ putIfPositive(summary, INPUT_POS_DELETE_SIZE, positionDeleteSize,
humanReadable);
putIfPositive(summary, INPUT_POS_DELETE_RECORDS, posDeleteRecordCnt);
put(summary, OUTPUT_DATA_FILES, newDataFileCnt);
put(summary, OUTPUT_DATA_SIZE, newDataSize, humanReadable);
@@ -196,32 +169,24 @@ public class MetricsSummary {
public FilesStatistics getInputFilesStatistics() {
FilesStatisticsBuilder inputBuilder = new FilesStatisticsBuilder();
inputBuilder.addFiles(equalityDeleteSize, eqDeleteFileCnt);
- inputBuilder.addFiles(Math.max(positionDeleteSize, positionalDeleteSize),
posDeleteFileCnt);
+ inputBuilder.addFiles(positionDeleteSize, posDeleteFileCnt);
inputBuilder.addFiles(rewriteDataSize, rewriteDataFileCnt);
- inputBuilder.addFiles(
- rewritePosDataSize, Math.max(reRowDeletedDataFileCnt,
rewritePosDataFileCnt));
+ inputBuilder.addFiles(rewritePosDataSize, rewritePosDataFileCnt);
return inputBuilder.build();
}
public FilesStatistics getOutputFilesStatistics() {
FilesStatisticsBuilder outputBuilder = new FilesStatisticsBuilder();
- outputBuilder.addFiles(newFileSize, newFileCnt);
+ outputBuilder.addFiles(newDataSize, newDataFileCnt);
+ outputBuilder.addFiles(newDeleteSize, newDeleteFileCnt);
return outputBuilder.build();
}
- public long getNewFileSize() {
- return newFileSize;
- }
-
- public int getNewFileCnt() {
- return newFileCnt;
- }
-
public long getNewDataSize() {
return newDataSize;
}
- protected void setNewDataSize(long newDataSize) {
+ public void setNewDataSize(long newDataSize) {
this.newDataSize = newDataSize;
}
@@ -229,7 +194,7 @@ public class MetricsSummary {
return newDataFileCnt;
}
- protected void setNewDataFileCnt(int newDataFileCnt) {
+ public void setNewDataFileCnt(int newDataFileCnt) {
this.newDataFileCnt = newDataFileCnt;
}
@@ -265,14 +230,6 @@ public class MetricsSummary {
this.newDeleteRecordCnt = newDeleteRecordCnt;
}
- public void setNewFileSize(long newFileSize) {
- this.newFileSize = newFileSize;
- }
-
- public void setNewFileCnt(int newFileCnt) {
- this.newFileCnt = newFileCnt;
- }
-
public long getRewriteDataSize() {
return rewriteDataSize;
}
@@ -285,10 +242,6 @@ public class MetricsSummary {
return equalityDeleteSize;
}
- public long getPositionalDeleteSize() {
- return positionalDeleteSize;
- }
-
public long getPositionDeleteSize() {
return positionDeleteSize;
}
@@ -297,10 +250,6 @@ public class MetricsSummary {
return rewriteDataFileCnt;
}
- public int getReRowDeletedDataFileCnt() {
- return reRowDeletedDataFileCnt;
- }
-
public int getRewritePosDataFileCnt() {
return rewritePosDataFileCnt;
}
@@ -337,17 +286,13 @@ public class MetricsSummary {
.add("rewriteDataRecordCnt", rewriteDataRecordCnt)
.add("rewritePosDataSize", rewritePosDataSize)
.add("rewritePosDataFileCnt", rewritePosDataFileCnt)
- .add("reRowDeletedDataFileCnt", reRowDeletedDataFileCnt)
.add("rewritePosDataRecordCnt", rewritePosDataRecordCnt)
.add("equalityDeleteSize", equalityDeleteSize)
.add("eqDeleteFileCnt", eqDeleteFileCnt)
.add("eqDeleteRecordCnt", eqDeleteRecordCnt)
.add("positionDeleteSize", positionDeleteSize)
- .add("positionalDeleteSize", positionalDeleteSize)
.add("posDeleteFileCnt", posDeleteFileCnt)
.add("posDeleteRecordCnt", posDeleteRecordCnt)
- .add("newFileSize", newFileSize)
- .add("newFileCnt", newFileCnt)
.add("newDataSize", newDataSize)
.add("newDataFileCnt", newDataFileCnt)
.add("newDataRecordCnt", newDataRecordCnt)
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/RewriteStageTask.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/RewriteStageTask.java
index 60c5fba77..4a7f71f26 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/RewriteStageTask.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/RewriteStageTask.java
@@ -54,8 +54,6 @@ public class RewriteStageTask
summary.setNewDeleteFileCnt(getFileCount(output.getDeleteFiles()));
summary.setNewDeleteSize(getFileSize(output.getDeleteFiles()));
summary.setNewDeleteRecordCnt(getRecordCnt(output.getDeleteFiles()));
- summary.setNewFileSize(summary.getNewDataSize() +
summary.getNewDeleteSize());
- summary.setNewFileCnt(summary.getNewDataFileCnt() +
summary.getNewDeleteFileCnt());
}
}
}
