JonasJ-ap opened a new pull request, #5644:
URL: https://github.com/apache/iceberg/pull/5644
Compiled iceberg-spark-runtime-3.3 and tested on Glue 3.0. The job succeed
with dualstack enabled.
My Spark job used during the testing is listed below:
```
import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession
import org.apache.iceberg.Table
import org.apache.iceberg.aws.glue.GlueCatalog
import org.apache.iceberg.catalog.Catalog
import org.apache.iceberg.catalog.TableIdentifier
import org.apache.iceberg.spark.actions.SparkActions
import scala.jdk.CollectionConverters._
object IcebergSparkCompaction {
def main(sysArgs: Array[String]) {
val sparkContext: SparkContext = new SparkContext()
val spark: SparkSession = SparkSession.builder.
config("spark.sql.catalog.demo",
"org.apache.iceberg.spark.SparkCatalog").
config("spark.sql.catalog.demo.warehouse",
"s3://yzhaoqin-healthlake-us-west-2/glueiceberg").
config("spark.sql.catalog.demo.catalog-impl",
"org.apache.iceberg.aws.glue.GlueCatalog").
config("spark.sql.catalog.demo.s3.dualstack-enabled", "true").
getOrCreate()
val catalog: Catalog = new GlueCatalog();
catalog.initialize("demo", Map("warehouse" ->
"s3://aws-lake-house-iceberg-blog-demo").asJava);
val table: Table = catalog.loadTable(TableIdentifier.of("reviews",
"book_reviews"));
val actions: SparkActions = SparkActions.get(spark)
actions.rewriteDataFiles(table).option("min-input-files",
"1").execute();
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
