aokolnychyi commented on a change in pull request #2256:
URL: https://github.com/apache/iceberg/pull/2256#discussion_r579515625
##########
File path: spark/src/main/java/org/apache/iceberg/actions/BaseSparkAction.java
##########
@@ -101,47 +117,43 @@
return allManifests.flatMap(new ReadManifest(ioBroadcast),
Encoders.STRING()).toDF("file_path");
}
- protected Dataset<Row> buildManifestFileDF(SparkSession spark, String
tableName) {
- return loadMetadataTable(spark, tableName, table().location(),
ALL_MANIFESTS).selectExpr("path as file_path");
+ protected Dataset<Row> buildManifestFileDF(Table table) {
+ return loadMetadataTable(table, ALL_MANIFESTS).selectExpr("path as
file_path");
}
- protected Dataset<Row> buildManifestListDF(SparkSession spark, Table table) {
+ protected Dataset<Row> buildManifestListDF(Table table) {
List<String> manifestLists = getManifestListPaths(table.snapshots());
return spark.createDataset(manifestLists,
Encoders.STRING()).toDF("file_path");
}
- protected Dataset<Row> buildManifestListDF(SparkSession spark, String
metadataFileLocation) {
- StaticTableOperations ops = new
StaticTableOperations(metadataFileLocation, table().io());
- return buildManifestListDF(spark, new BaseTable(ops, table().name()));
- }
-
- protected Dataset<Row> buildOtherMetadataFileDF(SparkSession spark,
TableOperations ops) {
+ protected Dataset<Row> buildOtherMetadataFileDF(TableOperations ops) {
List<String> otherMetadataFiles = getOtherMetadataFilePaths(ops);
return spark.createDataset(otherMetadataFiles,
Encoders.STRING()).toDF("file_path");
}
- protected Dataset<Row> buildValidMetadataFileDF(SparkSession spark, Table
table, TableOperations ops) {
- Dataset<Row> manifestDF = buildManifestFileDF(spark, table.name());
- Dataset<Row> manifestListDF = buildManifestListDF(spark, table);
- Dataset<Row> otherMetadataFileDF = buildOtherMetadataFileDF(spark, ops);
+ protected Dataset<Row> buildValidMetadataFileDF(Table table, TableOperations
ops) {
+ Dataset<Row> manifestDF = buildManifestFileDF(table);
+ Dataset<Row> manifestListDF = buildManifestListDF(table);
+ Dataset<Row> otherMetadataFileDF = buildOtherMetadataFileDF(ops);
return manifestDF.union(otherMetadataFileDF).union(manifestListDF);
}
// Attempt to use Spark3 Catalog resolution if available on the path
private static final DynMethods.UnboundMethod LOAD_CATALOG =
DynMethods.builder("loadCatalogMetadataTable")
- .hiddenImpl("org.apache.iceberg.spark.Spark3Util",
- SparkSession.class, String.class, MetadataTableType.class)
+ .hiddenImpl("org.apache.iceberg.spark.Spark3Util", SparkSession.class,
String.class, MetadataTableType.class)
.orNoop()
.build();
- private static Dataset<Row> loadCatalogMetadataTable(SparkSession spark,
String tableName, MetadataTableType type) {
Review comment:
This does not have to be static anymore.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
