This is an automated email from the ASF dual-hosted git repository.
yzheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 7c9073166 Additional refs update for iceberg 1.9.0 (#1491)
7c9073166 is described below
commit 7c90731660e6889e9b94bf178af12ae991c35521
Author: MonkeyCanCode <yongzheng0...@gmail.com>
AuthorDate: Mon May 5 10:44:08 2025 -0500
Additional refs update for iceberg 1.9.0 (#1491)
* Additional refs update for iceberg 1.9.0
* Additional refs update for iceberg 1.9.0
* Additional refs update for iceberg 1.9.0
---
getting-started/eclipselink/docker-compose.yml | 2 +-
getting-started/jdbc/docker-compose.yml | 2 +-
getting-started/spark/notebooks/SparkPolaris.ipynb | 2 +-
plugins/pluginlibs.versions.toml | 2 +-
plugins/spark/README.md | 14 +++++++-------
plugins/spark/v3.5/spark/build.gradle.kts | 2 ++
.../in-dev/unreleased/getting-started/using-polaris.md | 2 +-
7 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/getting-started/eclipselink/docker-compose.yml
b/getting-started/eclipselink/docker-compose.yml
index 0c2ca17a8..5589dfb3a 100644
--- a/getting-started/eclipselink/docker-compose.yml
+++ b/getting-started/eclipselink/docker-compose.yml
@@ -73,7 +73,7 @@ services:
retries: 15
command: [
/opt/spark/bin/spark-sql,
- --packages,
"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.7.0,org.apache.iceberg:iceberg-azure-bundle:1.7.0",
+ --packages,
"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.9.0,org.apache.iceberg:iceberg-azure-bundle:1.9.0",
--conf,
"spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
--conf,
"spark.sql.catalog.quickstart_catalog=org.apache.iceberg.spark.SparkCatalog",
--conf, "spark.sql.catalog.quickstart_catalog.type=rest",
diff --git a/getting-started/jdbc/docker-compose.yml
b/getting-started/jdbc/docker-compose.yml
index 4c85226c2..924f20daf 100644
--- a/getting-started/jdbc/docker-compose.yml
+++ b/getting-started/jdbc/docker-compose.yml
@@ -73,7 +73,7 @@ services:
retries: 15
command: [
/opt/spark/bin/spark-sql,
- --packages,
"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.7.0,org.apache.iceberg:iceberg-azure-bundle:1.7.0",
+ --packages,
"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.9.0,org.apache.iceberg:iceberg-azure-bundle:1.9.0",
--conf,
"spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
--conf,
"spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog",
--conf, "spark.sql.catalog.polaris.type=rest",
diff --git a/getting-started/spark/notebooks/SparkPolaris.ipynb
b/getting-started/spark/notebooks/SparkPolaris.ipynb
index 48fff32ac..42d56a87e 100644
--- a/getting-started/spark/notebooks/SparkPolaris.ipynb
+++ b/getting-started/spark/notebooks/SparkPolaris.ipynb
@@ -256,7 +256,7 @@
"\n",
"spark = (SparkSession.builder\n",
" .config(\"spark.sql.catalog.spark_catalog\",
\"org.apache.iceberg.spark.SparkSessionCatalog\")\n",
- " .config(\"spark.jars.packages\",
\"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n",
+ " .config(\"spark.jars.packages\",
\"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n",
" .config('spark.sql.iceberg.vectorization.enabled', 'false')\n",
" \n",
" # Configure the 'polaris' catalog as an Iceberg rest catalog\n",
diff --git a/plugins/pluginlibs.versions.toml b/plugins/pluginlibs.versions.toml
index 4f7288ff6..0ca3ba5d3 100644
--- a/plugins/pluginlibs.versions.toml
+++ b/plugins/pluginlibs.versions.toml
@@ -18,7 +18,7 @@
#
[versions]
-iceberg = "1.8.1"
+iceberg = "1.9.0"
spark35 = "3.5.5"
scala212 = "2.12.19"
scala213 = "2.13.15"
diff --git a/plugins/spark/README.md b/plugins/spark/README.md
index 66d4c2983..07fa8ed86 100644
--- a/plugins/spark/README.md
+++ b/plugins/spark/README.md
@@ -6,9 +6,9 @@
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,13 +20,13 @@
# Polaris Spark Plugin
The Polaris Spark plugin provides a SparkCatalog class, which communicates
with the Polaris
-REST endpoints, and provides implementations for Apache Spark's
-[TableCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java),
-[SupportsNamespaces](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java),
+REST endpoints, and provides implementations for Apache Spark's
+[TableCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java),
+[SupportsNamespaces](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java),
[ViewCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/ViewCatalog.java)
classes.
Right now, the plugin only provides support for Spark 3.5, Scala version 2.12
and 2.13,
-and depends on iceberg-spark-runtime 1.8.1.
+and depends on iceberg-spark-runtime 1.9.0.
# Build Plugin Jar
A task createPolarisSparkJar is added to build a jar for the Polaris Spark
plugin, the jar is named as:
@@ -87,7 +87,7 @@ bin/spark-shell \
```
# Limitations
-The Polaris Spark client supports catalog management for both Iceberg and
Delta tables, it routes all Iceberg table
+The Polaris Spark client supports catalog management for both Iceberg and
Delta tables, it routes all Iceberg table
requests to the Iceberg REST endpoints, and routes all Delta table requests to
the Generic Table REST endpoints.
The Spark Client requires at least delta 3.2.1 to work with Delta tables,
which requires at least Apache Spark 3.5.3.
diff --git a/plugins/spark/v3.5/spark/build.gradle.kts
b/plugins/spark/v3.5/spark/build.gradle.kts
index 5ce7e73c0..600d150f4 100644
--- a/plugins/spark/v3.5/spark/build.gradle.kts
+++ b/plugins/spark/v3.5/spark/build.gradle.kts
@@ -167,9 +167,11 @@ tasks.register<ShadowJar>("createPolarisSparkJar") {
minimize {
exclude(dependency("org.apache.iceberg:iceberg-spark-runtime-*.*"))
exclude(dependency("org.apache.iceberg:iceberg-core*.*"))
+ exclude(dependency("org.apache.avro:avro*.*"))
}
relocate("com.fasterxml", "org.apache.polaris.shaded.com.fasterxml.jackson")
+ relocate("org.apache.avro", "org.apache.polaris.shaded.org.apache.avro")
}
tasks.withType(Jar::class).named("sourcesJar") {
dependsOn("createPolarisSparkJar") }
diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md
b/site/content/in-dev/unreleased/getting-started/using-polaris.md
index ff83e8748..75004e3ea 100644
--- a/site/content/in-dev/unreleased/getting-started/using-polaris.md
+++ b/site/content/in-dev/unreleased/getting-started/using-polaris.md
@@ -147,7 +147,7 @@ _Note: the credentials provided here are those for our
principal, not the root c
```shell
bin/spark-sql \
---packages
org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0
\
+--packages
org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.hadoop:hadoop-aws:3.4.0
\
--conf
spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
\
--conf spark.sql.catalog.quickstart_catalog.warehouse=quickstart_catalog \
--conf
spark.sql.catalog.quickstart_catalog.header.X-Iceberg-Access-Delegation=vended-credentials
\
