This is an automated email from the ASF dual-hosted git repository.
dweeks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 313f49770 Spark: Add compute stats to scan builder also (#5136)
313f49770 is described below
commit 313f49770695fd0201baa8e0158a2f4f41b8b468
Author: Bryan Keller <[email protected]>
AuthorDate: Mon Jun 27 08:55:37 2022 -0700
Spark: Add compute stats to scan builder also (#5136)
---
.../apache/iceberg/spark/source/SparkScanBuilder.java | 16 +++++++++++++++-
.../apache/iceberg/spark/source/SparkScanBuilder.java | 16 +++++++++++++++-
.../apache/iceberg/spark/source/SparkScanBuilder.java | 16 +++++++++++++++-
3 files changed, 45 insertions(+), 3 deletions(-)
diff --git
a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
index 07c85fb8d..a9b82df80 100644
---
a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
+++
b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
@@ -39,14 +39,18 @@ import org.apache.iceberg.types.Types;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.connector.read.Scan;
import org.apache.spark.sql.connector.read.ScanBuilder;
+import org.apache.spark.sql.connector.read.Statistics;
import org.apache.spark.sql.connector.read.SupportsPushDownFilters;
import org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns;
+import org.apache.spark.sql.connector.read.SupportsReportStatistics;
import org.apache.spark.sql.sources.Filter;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-public class SparkScanBuilder implements ScanBuilder, SupportsPushDownFilters,
SupportsPushDownRequiredColumns {
+public class SparkScanBuilder implements ScanBuilder, SupportsPushDownFilters,
SupportsPushDownRequiredColumns,
+ SupportsReportStatistics {
+
private static final Filter[] NO_FILTERS = new Filter[0];
private final SparkSession spark;
@@ -167,4 +171,14 @@ public class SparkScanBuilder implements ScanBuilder,
SupportsPushDownFilters, S
spark, table, readConf, caseSensitive, ignoreResiduals,
schemaWithMetadataColumns(), filterExpressions, options);
}
+
+ @Override
+ public Statistics estimateStatistics() {
+ return ((SparkBatchScan) build()).estimateStatistics();
+ }
+
+ @Override
+ public StructType readSchema() {
+ return build().readSchema();
+ }
}
diff --git
a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
index 07c85fb8d..a9b82df80 100644
---
a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
+++
b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
@@ -39,14 +39,18 @@ import org.apache.iceberg.types.Types;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.connector.read.Scan;
import org.apache.spark.sql.connector.read.ScanBuilder;
+import org.apache.spark.sql.connector.read.Statistics;
import org.apache.spark.sql.connector.read.SupportsPushDownFilters;
import org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns;
+import org.apache.spark.sql.connector.read.SupportsReportStatistics;
import org.apache.spark.sql.sources.Filter;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-public class SparkScanBuilder implements ScanBuilder, SupportsPushDownFilters,
SupportsPushDownRequiredColumns {
+public class SparkScanBuilder implements ScanBuilder, SupportsPushDownFilters,
SupportsPushDownRequiredColumns,
+ SupportsReportStatistics {
+
private static final Filter[] NO_FILTERS = new Filter[0];
private final SparkSession spark;
@@ -167,4 +171,14 @@ public class SparkScanBuilder implements ScanBuilder,
SupportsPushDownFilters, S
spark, table, readConf, caseSensitive, ignoreResiduals,
schemaWithMetadataColumns(), filterExpressions, options);
}
+
+ @Override
+ public Statistics estimateStatistics() {
+ return ((SparkBatchScan) build()).estimateStatistics();
+ }
+
+ @Override
+ public StructType readSchema() {
+ return build().readSchema();
+ }
}
diff --git
a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
index 30c033f29..aa2214bc7 100644
---
a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
+++
b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java
@@ -44,14 +44,18 @@ import org.apache.iceberg.types.Types;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.connector.read.Scan;
import org.apache.spark.sql.connector.read.ScanBuilder;
+import org.apache.spark.sql.connector.read.Statistics;
import org.apache.spark.sql.connector.read.SupportsPushDownFilters;
import org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns;
+import org.apache.spark.sql.connector.read.SupportsReportStatistics;
import org.apache.spark.sql.sources.Filter;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-public class SparkScanBuilder implements ScanBuilder, SupportsPushDownFilters,
SupportsPushDownRequiredColumns {
+public class SparkScanBuilder implements ScanBuilder, SupportsPushDownFilters,
SupportsPushDownRequiredColumns,
+ SupportsReportStatistics {
+
private static final Filter[] NO_FILTERS = new Filter[0];
private final SparkSession spark;
@@ -278,4 +282,14 @@ public class SparkScanBuilder implements ScanBuilder,
SupportsPushDownFilters, S
return configuredScan;
}
+
+ @Override
+ public Statistics estimateStatistics() {
+ return ((SparkScan) build()).estimateStatistics();
+ }
+
+ @Override
+ public StructType readSchema() {
+ return build().readSchema();
+ }
}
