This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new f00d3094a1 Spark 3.4: Display more read metrics on Spark SQL UI (#9009)
f00d3094a1 is described below
commit f00d3094a1728b60dba96fc251707dd08d8a6310
Author: Karuppayya <[email protected]>
AuthorDate: Wed Nov 8 16:31:28 2023 -0800
Spark 3.4: Display more read metrics on Spark SQL UI (#9009)
This change cherry-picks PR #8717 to Spark 3.4.
---
.../org/apache/iceberg/spark/source/SparkScan.java | 83 +++++++++++++---
...TotalFileSize.java => EqualityDeleteFiles.java} | 6 +-
...{TotalFileSize.java => IndexedDeleteFiles.java} | 6 +-
...talFileSize.java => PositionalDeleteFiles.java} | 6 +-
...{ScannedDataFiles.java => ResultDataFiles.java} | 6 +-
.../{TotalFileSize.java => ResultDeleteFiles.java} | 6 +-
...alFileSize.java => ScannedDeleteManifests.java} | 6 +-
...{TotalFileSize.java => SkippedDeleteFiles.java} | 6 +-
...alFileSize.java => SkippedDeleteManifests.java} | 6 +-
...DataFiles.java => TaskEqualityDeleteFiles.java} | 12 +--
...dDataFiles.java => TaskIndexedDeleteFiles.java} | 12 +--
...taFiles.java => TaskPositionalDeleteFiles.java} | 12 +--
...nnedDataFiles.java => TaskResultDataFiles.java} | 10 +-
...talFileSize.java => TaskResultDeleteFiles.java} | 13 ++-
...aFiles.java => TaskScannedDeleteManifests.java} | 12 +--
...dDataFiles.java => TaskSkippedDeleteFiles.java} | 12 +--
...aFiles.java => TaskSkippedDeleteManifests.java} | 12 +--
...talFileSize.java => TaskTotalDataFileSize.java} | 10 +-
...dDataFiles.java => TaskTotalDataManifests.java} | 12 +--
...DataFiles.java => TaskTotalDeleteFileSize.java} | 13 +--
...ataFiles.java => TaskTotalDeleteManifests.java} | 12 +--
.../{TotalFileSize.java => TotalDataFileSize.java} | 6 +-
...{TotalFileSize.java => TotalDataManifests.java} | 6 +-
...TotalFileSize.java => TotalDeleteFileSize.java} | 6 +-
...otalFileSize.java => TotalDeleteManifests.java} | 6 +-
.../iceberg/spark/source/TestSparkReadMetrics.java | 106 +++++++++++++++++++--
26 files changed, 277 insertions(+), 126 deletions(-)
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
index 6c3b2db143..6efe8a080b 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
@@ -35,19 +35,39 @@ import
org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.Spark3Util;
import org.apache.iceberg.spark.SparkReadConf;
import org.apache.iceberg.spark.SparkSchemaUtil;
+import org.apache.iceberg.spark.source.metrics.EqualityDeleteFiles;
+import org.apache.iceberg.spark.source.metrics.IndexedDeleteFiles;
import org.apache.iceberg.spark.source.metrics.NumDeletes;
import org.apache.iceberg.spark.source.metrics.NumSplits;
-import org.apache.iceberg.spark.source.metrics.ScannedDataFiles;
+import org.apache.iceberg.spark.source.metrics.PositionalDeleteFiles;
+import org.apache.iceberg.spark.source.metrics.ResultDataFiles;
+import org.apache.iceberg.spark.source.metrics.ResultDeleteFiles;
import org.apache.iceberg.spark.source.metrics.ScannedDataManifests;
+import org.apache.iceberg.spark.source.metrics.ScannedDeleteManifests;
import org.apache.iceberg.spark.source.metrics.SkippedDataFiles;
import org.apache.iceberg.spark.source.metrics.SkippedDataManifests;
-import org.apache.iceberg.spark.source.metrics.TaskScannedDataFiles;
+import org.apache.iceberg.spark.source.metrics.SkippedDeleteFiles;
+import org.apache.iceberg.spark.source.metrics.SkippedDeleteManifests;
+import org.apache.iceberg.spark.source.metrics.TaskEqualityDeleteFiles;
+import org.apache.iceberg.spark.source.metrics.TaskIndexedDeleteFiles;
+import org.apache.iceberg.spark.source.metrics.TaskPositionalDeleteFiles;
+import org.apache.iceberg.spark.source.metrics.TaskResultDataFiles;
+import org.apache.iceberg.spark.source.metrics.TaskResultDeleteFiles;
import org.apache.iceberg.spark.source.metrics.TaskScannedDataManifests;
+import org.apache.iceberg.spark.source.metrics.TaskScannedDeleteManifests;
import org.apache.iceberg.spark.source.metrics.TaskSkippedDataFiles;
import org.apache.iceberg.spark.source.metrics.TaskSkippedDataManifests;
-import org.apache.iceberg.spark.source.metrics.TaskTotalFileSize;
+import org.apache.iceberg.spark.source.metrics.TaskSkippedDeleteFiles;
+import org.apache.iceberg.spark.source.metrics.TaskSkippedDeleteManifests;
+import org.apache.iceberg.spark.source.metrics.TaskTotalDataFileSize;
+import org.apache.iceberg.spark.source.metrics.TaskTotalDataManifests;
+import org.apache.iceberg.spark.source.metrics.TaskTotalDeleteFileSize;
+import org.apache.iceberg.spark.source.metrics.TaskTotalDeleteManifests;
import org.apache.iceberg.spark.source.metrics.TaskTotalPlanningDuration;
-import org.apache.iceberg.spark.source.metrics.TotalFileSize;
+import org.apache.iceberg.spark.source.metrics.TotalDataFileSize;
+import org.apache.iceberg.spark.source.metrics.TotalDataManifests;
+import org.apache.iceberg.spark.source.metrics.TotalDeleteFileSize;
+import org.apache.iceberg.spark.source.metrics.TotalDeleteManifests;
import org.apache.iceberg.spark.source.metrics.TotalPlanningDuration;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.PropertyUtil;
@@ -200,12 +220,32 @@ abstract class SparkScan implements Scan,
SupportsReportStatistics {
}
List<CustomTaskMetric> driverMetrics = Lists.newArrayList();
- driverMetrics.add(TaskTotalFileSize.from(scanReport));
+
+ // common
driverMetrics.add(TaskTotalPlanningDuration.from(scanReport));
- driverMetrics.add(TaskSkippedDataFiles.from(scanReport));
- driverMetrics.add(TaskScannedDataFiles.from(scanReport));
- driverMetrics.add(TaskSkippedDataManifests.from(scanReport));
+
+ // data manifests
+ driverMetrics.add(TaskTotalDataManifests.from(scanReport));
driverMetrics.add(TaskScannedDataManifests.from(scanReport));
+ driverMetrics.add(TaskSkippedDataManifests.from(scanReport));
+
+ // data files
+ driverMetrics.add(TaskResultDataFiles.from(scanReport));
+ driverMetrics.add(TaskSkippedDataFiles.from(scanReport));
+ driverMetrics.add(TaskTotalDataFileSize.from(scanReport));
+
+ // delete manifests
+ driverMetrics.add(TaskTotalDeleteManifests.from(scanReport));
+ driverMetrics.add(TaskScannedDeleteManifests.from(scanReport));
+ driverMetrics.add(TaskSkippedDeleteManifests.from(scanReport));
+
+ // delete files
+ driverMetrics.add(TaskTotalDeleteFileSize.from(scanReport));
+ driverMetrics.add(TaskResultDeleteFiles.from(scanReport));
+ driverMetrics.add(TaskEqualityDeleteFiles.from(scanReport));
+ driverMetrics.add(TaskIndexedDeleteFiles.from(scanReport));
+ driverMetrics.add(TaskPositionalDeleteFiles.from(scanReport));
+ driverMetrics.add(TaskSkippedDeleteFiles.from(scanReport));
return driverMetrics.toArray(new CustomTaskMetric[0]);
}
@@ -213,14 +253,35 @@ abstract class SparkScan implements Scan,
SupportsReportStatistics {
@Override
public CustomMetric[] supportedCustomMetrics() {
return new CustomMetric[] {
+ // task metrics
new NumSplits(),
new NumDeletes(),
- new TotalFileSize(),
+
+ // common
new TotalPlanningDuration(),
+
+ // data manifests
+ new TotalDataManifests(),
new ScannedDataManifests(),
new SkippedDataManifests(),
- new ScannedDataFiles(),
- new SkippedDataFiles()
+
+ // data files
+ new ResultDataFiles(),
+ new SkippedDataFiles(),
+ new TotalDataFileSize(),
+
+ // delete manifests
+ new TotalDeleteManifests(),
+ new ScannedDeleteManifests(),
+ new SkippedDeleteManifests(),
+
+ // delete files
+ new TotalDeleteFileSize(),
+ new ResultDeleteFiles(),
+ new EqualityDeleteFiles(),
+ new IndexedDeleteFiles(),
+ new PositionalDeleteFiles(),
+ new SkippedDeleteFiles()
};
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/EqualityDeleteFiles.java
similarity index 86%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/EqualityDeleteFiles.java
index 994626e54f..754145f7d2 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/EqualityDeleteFiles.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class EqualityDeleteFiles extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "equalityDeleteFiles";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "number of equality delete files";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/IndexedDeleteFiles.java
similarity index 87%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/IndexedDeleteFiles.java
index 994626e54f..7fc5b9066c 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/IndexedDeleteFiles.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class IndexedDeleteFiles extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "indexedDeleteFiles";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "number of indexed delete files";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/PositionalDeleteFiles.java
similarity index 86%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/PositionalDeleteFiles.java
index 994626e54f..5de75776ea 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/PositionalDeleteFiles.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class PositionalDeleteFiles extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "positionalDeleteFiles";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "number of positional delete files";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ResultDataFiles.java
similarity index 87%
rename from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ScannedDataFiles.java
rename to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ResultDataFiles.java
index f453872fdc..21959cbf6c 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ResultDataFiles.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class ScannedDataFiles extends CustomSumMetric {
+public class ResultDataFiles extends CustomSumMetric {
- static final String NAME = "scannedDataFiles";
+ static final String NAME = "resultDataFiles";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class ScannedDataFiles extends CustomSumMetric {
@Override
public String description() {
- return "number of scanned data files";
+ return "number of result data files";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ResultDeleteFiles.java
similarity index 87%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ResultDeleteFiles.java
index 994626e54f..9c6ad2ca32 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ResultDeleteFiles.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class ResultDeleteFiles extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "resultDeleteFiles";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "number of result delete files";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ScannedDeleteManifests.java
similarity index 86%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ScannedDeleteManifests.java
index 994626e54f..1fa006b7b1 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/ScannedDeleteManifests.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class ScannedDeleteManifests extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "scannedDeleteManifests";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "number of scanned delete manifests";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/SkippedDeleteFiles.java
similarity index 87%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/SkippedDeleteFiles.java
index 994626e54f..70597be671 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/SkippedDeleteFiles.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class SkippedDeleteFiles extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "skippedDeleteFiles";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "number of skipped delete files";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/SkippedDeleteManifests.java
similarity index 86%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/SkippedDeleteManifests.java
index 994626e54f..0336170b45 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/SkippedDeleteManifests.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class SkippedDeleteManifests extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "skippedDeleteManifests";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "number of skipped delete manifest";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskEqualityDeleteFiles.java
similarity index 78%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskEqualityDeleteFiles.java
index d9a527da08..ecd14bcca3 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskEqualityDeleteFiles.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskEqualityDeleteFiles implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskEqualityDeleteFiles(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return EqualityDeleteFiles.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskEqualityDeleteFiles from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().equalityDeleteFiles();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskEqualityDeleteFiles(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskIndexedDeleteFiles.java
similarity index 78%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskIndexedDeleteFiles.java
index d9a527da08..63b6767e95 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskIndexedDeleteFiles.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskIndexedDeleteFiles implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskIndexedDeleteFiles(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return IndexedDeleteFiles.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskIndexedDeleteFiles from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().indexedDeleteFiles();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskIndexedDeleteFiles(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskPositionalDeleteFiles.java
similarity index 77%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskPositionalDeleteFiles.java
index d9a527da08..805f22bf0d 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskPositionalDeleteFiles.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskPositionalDeleteFiles implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskPositionalDeleteFiles(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return PositionalDeleteFiles.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskPositionalDeleteFiles from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().positionalDeleteFiles();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskPositionalDeleteFiles(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskResultDataFiles.java
similarity index 83%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskResultDataFiles.java
index d9a527da08..a271421314 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskResultDataFiles.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskResultDataFiles implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskResultDataFiles(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return ResultDataFiles.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
+ public static TaskResultDataFiles from(ScanReport scanReport) {
CounterResult counter = scanReport.scanMetrics().resultDataFiles();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskResultDataFiles(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskResultDeleteFiles.java
similarity index 78%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskResultDeleteFiles.java
index c300d835e7..aea8ca07dd 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskResultDeleteFiles.java
@@ -22,17 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskTotalFileSize implements CustomTaskMetric {
-
+public class TaskResultDeleteFiles implements CustomTaskMetric {
private final long value;
- private TaskTotalFileSize(long value) {
+ private TaskResultDeleteFiles(long value) {
this.value = value;
}
@Override
public String name() {
- return TotalFileSize.NAME;
+ return ResultDeleteFiles.NAME;
}
@Override
@@ -40,9 +39,9 @@ public class TaskTotalFileSize implements CustomTaskMetric {
return value;
}
- public static TaskTotalFileSize from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().totalFileSizeInBytes();
+ public static TaskResultDeleteFiles from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().resultDeleteFiles();
long value = counter != null ? counter.value() : 0L;
- return new TaskTotalFileSize(value);
+ return new TaskResultDeleteFiles(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDeleteManifests.java
similarity index 77%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDeleteManifests.java
index d9a527da08..1766cf2f68 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDeleteManifests.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskScannedDeleteManifests implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskScannedDeleteManifests(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return ScannedDeleteManifests.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskScannedDeleteManifests from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().scannedDeleteManifests();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskScannedDeleteManifests(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskSkippedDeleteFiles.java
similarity index 78%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskSkippedDeleteFiles.java
index d9a527da08..8757975174 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskSkippedDeleteFiles.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskSkippedDeleteFiles implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskSkippedDeleteFiles(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return SkippedDeleteFiles.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskSkippedDeleteFiles from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().skippedDeleteFiles();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskSkippedDeleteFiles(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskSkippedDeleteManifests.java
similarity index 77%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskSkippedDeleteManifests.java
index d9a527da08..4a9c71e0c1 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskSkippedDeleteManifests.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskSkippedDeleteManifests implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskSkippedDeleteManifests(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return SkippedDeleteManifests.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskSkippedDeleteManifests from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().skippedDeleteManifests();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskSkippedDeleteManifests(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDataFileSize.java
similarity index 83%
rename from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalFileSize.java
rename to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDataFileSize.java
index c300d835e7..3f5a224425 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDataFileSize.java
@@ -22,17 +22,17 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskTotalFileSize implements CustomTaskMetric {
+public class TaskTotalDataFileSize implements CustomTaskMetric {
private final long value;
- private TaskTotalFileSize(long value) {
+ private TaskTotalDataFileSize(long value) {
this.value = value;
}
@Override
public String name() {
- return TotalFileSize.NAME;
+ return TotalDataFileSize.NAME;
}
@Override
@@ -40,9 +40,9 @@ public class TaskTotalFileSize implements CustomTaskMetric {
return value;
}
- public static TaskTotalFileSize from(ScanReport scanReport) {
+ public static TaskTotalDataFileSize from(ScanReport scanReport) {
CounterResult counter = scanReport.scanMetrics().totalFileSizeInBytes();
long value = counter != null ? counter.value() : 0L;
- return new TaskTotalFileSize(value);
+ return new TaskTotalDataFileSize(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDataManifests.java
similarity index 78%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDataManifests.java
index d9a527da08..6d8c3c24e4 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDataManifests.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskTotalDataManifests implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskTotalDataManifests(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return TotalDataManifests.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskTotalDataManifests from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().totalDataManifests();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskTotalDataManifests(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDeleteFileSize.java
similarity index 78%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDeleteFileSize.java
index d9a527da08..17ecec78da 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDeleteFileSize.java
@@ -22,16 +22,17 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskTotalDeleteFileSize implements CustomTaskMetric {
+
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskTotalDeleteFileSize(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return TotalDeleteFileSize.NAME;
}
@Override
@@ -39,9 +40,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskTotalDeleteFileSize from(ScanReport scanReport) {
+ CounterResult counter =
scanReport.scanMetrics().totalDeleteFileSizeInBytes();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskTotalDeleteFileSize(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDeleteManifests.java
similarity index 77%
rename from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
rename to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDeleteManifests.java
index d9a527da08..ff55c1be89 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskScannedDataFiles.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskTotalDeleteManifests.java
@@ -22,16 +22,16 @@ import org.apache.iceberg.metrics.CounterResult;
import org.apache.iceberg.metrics.ScanReport;
import org.apache.spark.sql.connector.metric.CustomTaskMetric;
-public class TaskScannedDataFiles implements CustomTaskMetric {
+public class TaskTotalDeleteManifests implements CustomTaskMetric {
private final long value;
- private TaskScannedDataFiles(long value) {
+ private TaskTotalDeleteManifests(long value) {
this.value = value;
}
@Override
public String name() {
- return ScannedDataFiles.NAME;
+ return TotalDeleteManifests.NAME;
}
@Override
@@ -39,9 +39,9 @@ public class TaskScannedDataFiles implements CustomTaskMetric
{
return value;
}
- public static TaskScannedDataFiles from(ScanReport scanReport) {
- CounterResult counter = scanReport.scanMetrics().resultDataFiles();
+ public static TaskTotalDeleteManifests from(ScanReport scanReport) {
+ CounterResult counter = scanReport.scanMetrics().totalDeleteManifests();
long value = counter != null ? counter.value() : 0L;
- return new TaskScannedDataFiles(value);
+ return new TaskTotalDeleteManifests(value);
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDataFileSize.java
similarity index 87%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDataFileSize.java
index 994626e54f..b1ff8a4636 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDataFileSize.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class TotalDataFileSize extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "totalDataFileSize";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "total data file size (bytes)";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDataManifests.java
similarity index 87%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDataManifests.java
index 994626e54f..de8f04be77 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDataManifests.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class TotalDataManifests extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "totalDataManifest";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "total data manifests";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDeleteFileSize.java
similarity index 86%
copy from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
copy to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDeleteFileSize.java
index 994626e54f..da43033252 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDeleteFileSize.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class TotalDeleteFileSize extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "totalDeleteFileSize";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "total delete file size (bytes)";
}
}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDeleteManifests.java
similarity index 87%
rename from
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
rename to
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDeleteManifests.java
index 994626e54f..7442dfdb6f 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalFileSize.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TotalDeleteManifests.java
@@ -20,9 +20,9 @@ package org.apache.iceberg.spark.source.metrics;
import org.apache.spark.sql.connector.metric.CustomSumMetric;
-public class TotalFileSize extends CustomSumMetric {
+public class TotalDeleteManifests extends CustomSumMetric {
- static final String NAME = "totalFileSize";
+ static final String NAME = "totalDeleteManifests";
@Override
public String name() {
@@ -31,6 +31,6 @@ public class TotalFileSize extends CustomSumMetric {
@Override
public String description() {
- return "total file size (bytes)";
+ return "total delete manifests";
}
}
diff --git
a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadMetrics.java
b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadMetrics.java
index 7b943372d1..fea5a1d3e1 100644
---
a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadMetrics.java
+++
b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadMetrics.java
@@ -43,7 +43,7 @@ public class TestSparkReadMetrics extends
SparkTestBaseWithCatalog {
@Test
public void testReadMetricsForV1Table() throws NoSuchTableException {
sql(
- "CREATE TABLE %s (id BIGINT) USING iceberg TBLPROPERTIES
('format-version'='2')",
+ "CREATE TABLE %s (id BIGINT) USING iceberg TBLPROPERTIES
('format-version'='1')",
tableName);
spark.range(10000).coalesce(1).writeTo(tableName).append();
@@ -56,12 +56,31 @@ public class TestSparkReadMetrics extends
SparkTestBaseWithCatalog {
seqAsJavaListConverter(df.queryExecution().executedPlan().collectLeaves()).asJava();
Map<String, SQLMetric> metricsMap =
JavaConverters.mapAsJavaMapConverter(sparkPlans.get(0).metrics()).asJava();
-
Assertions.assertThat(metricsMap.get("skippedDataFiles").value()).isEqualTo(1);
+ // Common
+
Assertions.assertThat(metricsMap.get("totalPlanningDuration").value()).isNotEqualTo(0);
+
+ // data manifests
+
Assertions.assertThat(metricsMap.get("totalDataManifest").value()).isEqualTo(2);
Assertions.assertThat(metricsMap.get("scannedDataManifests").value()).isEqualTo(2);
-
Assertions.assertThat(metricsMap.get("scannedDataFiles").value()).isEqualTo(1);
Assertions.assertThat(metricsMap.get("skippedDataManifests").value()).isEqualTo(0);
-
Assertions.assertThat(metricsMap.get("totalFileSize").value()).isNotEqualTo(0);
-
Assertions.assertThat(metricsMap.get("totalPlanningDuration").value()).isNotEqualTo(0);
+
+ // data files
+
Assertions.assertThat(metricsMap.get("resultDataFiles").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("skippedDataFiles").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("totalDataFileSize").value()).isNotEqualTo(0);
+
+ // delete manifests
+
Assertions.assertThat(metricsMap.get("totalDeleteManifests").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("scannedDeleteManifests").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("skippedDeleteManifests").value()).isEqualTo(0);
+
+ // delete files
+
Assertions.assertThat(metricsMap.get("totalDeleteFileSize").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("resultDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("equalityDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("indexedDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("positionalDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("skippedDeleteFiles").value()).isEqualTo(0);
}
@Test
@@ -80,11 +99,82 @@ public class TestSparkReadMetrics extends
SparkTestBaseWithCatalog {
seqAsJavaListConverter(df.queryExecution().executedPlan().collectLeaves()).asJava();
Map<String, SQLMetric> metricsMap =
JavaConverters.mapAsJavaMapConverter(sparkPlans.get(0).metrics()).asJava();
-
Assertions.assertThat(metricsMap.get("skippedDataFiles").value()).isEqualTo(1);
+
+ // Common
+
Assertions.assertThat(metricsMap.get("totalPlanningDuration").value()).isNotEqualTo(0);
+
+ // data manifests
+
Assertions.assertThat(metricsMap.get("totalDataManifest").value()).isEqualTo(2);
Assertions.assertThat(metricsMap.get("scannedDataManifests").value()).isEqualTo(2);
-
Assertions.assertThat(metricsMap.get("scannedDataFiles").value()).isEqualTo(1);
Assertions.assertThat(metricsMap.get("skippedDataManifests").value()).isEqualTo(0);
-
Assertions.assertThat(metricsMap.get("totalFileSize").value()).isNotEqualTo(0);
+
+ // data files
+
Assertions.assertThat(metricsMap.get("resultDataFiles").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("skippedDataFiles").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("totalDataFileSize").value()).isNotEqualTo(0);
+
+ // delete manifests
+
Assertions.assertThat(metricsMap.get("totalDeleteManifests").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("scannedDeleteManifests").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("skippedDeleteManifests").value()).isEqualTo(0);
+
+ // delete files
+
Assertions.assertThat(metricsMap.get("totalDeleteFileSize").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("resultDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("equalityDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("indexedDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("positionalDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("skippedDeleteFiles").value()).isEqualTo(0);
+ }
+
+ @Test
+ public void testDeleteMetrics() throws NoSuchTableException {
+ sql(
+ "CREATE TABLE %s (id BIGINT)"
+ + " USING iceberg"
+ + " TBLPROPERTIES (\n"
+ + " 'write.delete.mode'='merge-on-read',\n"
+ + " 'write.update.mode'='merge-on-read',\n"
+ + " 'write.merge.mode'='merge-on-read',\n"
+ + " 'format-version'='2'\n"
+ + " )",
+ tableName);
+
+ spark.range(10000).coalesce(1).writeTo(tableName).append();
+
+ spark.sql(String.format("DELETE FROM %s WHERE id = 1",
tableName)).collect();
+ Dataset<Row> df = spark.sql(String.format("SELECT * FROM %s", tableName));
+ df.collect();
+
+ List<SparkPlan> sparkPlans =
+
seqAsJavaListConverter(df.queryExecution().executedPlan().collectLeaves()).asJava();
+ Map<String, SQLMetric> metricsMap =
+
JavaConverters.mapAsJavaMapConverter(sparkPlans.get(0).metrics()).asJava();
+
+ // Common
Assertions.assertThat(metricsMap.get("totalPlanningDuration").value()).isNotEqualTo(0);
+
+ // data manifests
+
Assertions.assertThat(metricsMap.get("totalDataManifest").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("scannedDataManifests").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("skippedDataManifests").value()).isEqualTo(0);
+
+ // data files
+
Assertions.assertThat(metricsMap.get("resultDataFiles").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("skippedDataFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("totalDataFileSize").value()).isNotEqualTo(0);
+
+ // delete manifests
+
Assertions.assertThat(metricsMap.get("totalDeleteManifests").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("scannedDeleteManifests").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("skippedDeleteManifests").value()).isEqualTo(0);
+
+ // delete files
+
Assertions.assertThat(metricsMap.get("totalDeleteFileSize").value()).isNotEqualTo(0);
+
Assertions.assertThat(metricsMap.get("resultDeleteFiles").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("equalityDeleteFiles").value()).isEqualTo(0);
+
Assertions.assertThat(metricsMap.get("indexedDeleteFiles").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("positionalDeleteFiles").value()).isEqualTo(1);
+
Assertions.assertThat(metricsMap.get("skippedDeleteFiles").value()).isEqualTo(0);
}
}
