This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new df66d9b7e3 Spark 4.0: Rename SparkCatalogConfig#SPARK to
SparkCatalogConfig#SPARK_SESSION (#13607)
df66d9b7e3 is described below
commit df66d9b7e36a401de4689c88e3a6e37b1bb321d7
Author: Manu Zhang <[email protected]>
AuthorDate: Tue Jul 22 02:51:35 2025 +0800
Spark 4.0: Rename SparkCatalogConfig#SPARK to
SparkCatalogConfig#SPARK_SESSION (#13607)
---
.../spark/extensions/TestAddFilesProcedure.java | 8 +++---
.../extensions/TestAlterTablePartitionFields.java | 6 ++---
.../iceberg/spark/extensions/TestBranchDDL.java | 6 ++---
.../spark/extensions/TestChangelogTable.java | 6 ++---
.../spark/extensions/TestMetadataTables.java | 12 ++++-----
.../spark/extensions/TestReplaceBranch.java | 6 ++---
.../extensions/TestRewriteDataFilesProcedure.java | 4 +--
.../iceberg/spark/extensions/TestTagDDL.java | 6 ++---
.../org/apache/iceberg/spark/CatalogTestBase.java | 6 ++---
.../apache/iceberg/spark/SparkCatalogConfig.java | 2 +-
.../iceberg/spark/TestSparkCatalogOperations.java | 4 +--
.../spark/source/TestCompressionSettings.java | 30 +++++++++++-----------
.../iceberg/spark/sql/TestCreateTableAsSelect.java | 6 ++---
.../apache/iceberg/spark/sql/TestNamespaceSQL.java | 6 ++---
.../apache/iceberg/spark/sql/TestRefreshTable.java | 2 +-
.../org/apache/iceberg/spark/sql/TestSelect.java | 6 ++---
16 files changed, 58 insertions(+), 58 deletions(-)
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
index fcbfc80af3..1cfdbdaf7c 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
@@ -85,9 +85,9 @@ public class TestAddFilesProcedure extends ExtensionsTestBase
{
2
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
2
}
};
@@ -770,7 +770,7 @@ public class TestAddFilesProcedure extends
ExtensionsTestBase {
sql("ALTER TABLE %s ADD PARTITION FIELD subdept", tableName);
String fullTableName = tableName;
- if (implementation.equals(SparkCatalogConfig.SPARK.implementation())) {
+ if
(implementation.equals(SparkCatalogConfig.SPARK_SESSION.implementation())) {
fullTableName = String.format("%s.%s", catalogName, tableName);
}
assertThatThrownBy(
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
index dd49d8a254..57e4dec402 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
@@ -49,9 +49,9 @@ public class TestAlterTablePartitionFields extends
ExtensionsTestBase {
1
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
2
}
};
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
index f70f2d8190..c49def204f 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
@@ -57,9 +57,9 @@ public class TestBranchDDL extends ExtensionsTestBase {
public static Object[][] parameters() {
return new Object[][] {
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties()
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties()
}
};
}
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestChangelogTable.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestChangelogTable.java
index 5491f01e94..742235935e 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestChangelogTable.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestChangelogTable.java
@@ -48,9 +48,9 @@ public class TestChangelogTable extends ExtensionsTestBase {
public static Object[][] parameters() {
return new Object[][] {
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
1
},
{
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
index 82a065073d..ac528d1c47 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
@@ -91,15 +91,15 @@ public class TestMetadataTables extends ExtensionsTestBase {
2
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
2
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
3
},
{
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestReplaceBranch.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestReplaceBranch.java
index 414d5abc87..078df9d8aa 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestReplaceBranch.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestReplaceBranch.java
@@ -46,9 +46,9 @@ public class TestReplaceBranch extends ExtensionsTestBase {
public static Object[][] parameters() {
return new Object[][] {
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties()
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties()
}
};
}
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteDataFilesProcedure.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteDataFilesProcedure.java
index ea45024c83..75610f06d0 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteDataFilesProcedure.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteDataFilesProcedure.java
@@ -509,7 +509,7 @@ public class TestRewriteDataFilesProcedure extends
ExtensionsTestBase {
public void testRewriteDataFilesWithFilterOnOnBucketExpression() {
// currently spark session catalog only resolve to v1 functions instead of
desired v2 functions
//
https://github.com/apache/spark/blob/branch-3.4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala#L2070-L2083
-
assumeThat(catalogName).isNotEqualTo(SparkCatalogConfig.SPARK.catalogName());
+
assumeThat(catalogName).isNotEqualTo(SparkCatalogConfig.SPARK_SESSION.catalogName());
createBucketPartitionTable();
// create 5 files for each partition (c2 = 'foo' and c2 = 'bar')
insertData(10);
@@ -636,7 +636,7 @@ public class TestRewriteDataFilesProcedure extends
ExtensionsTestBase {
public void testRewriteDataFilesWithPossibleV2Filters() {
// currently spark session catalog only resolve to v1 functions instead of
desired v2 functions
//
https://github.com/apache/spark/blob/branch-3.4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala#L2070-L2083
-
assumeThat(catalogName).isNotEqualTo(SparkCatalogConfig.SPARK.catalogName());
+
assumeThat(catalogName).isNotEqualTo(SparkCatalogConfig.SPARK_SESSION.catalogName());
SystemFunctionPushDownHelper.createPartitionedTable(spark, tableName,
"id");
sql(
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java
index f8711cd7ee..79d5b0ab41 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java
@@ -49,9 +49,9 @@ public class TestTagDDL extends ExtensionsTestBase {
public static Object[][] parameters() {
return new Object[][] {
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties()
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties()
}
};
}
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/CatalogTestBase.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/CatalogTestBase.java
index 6cc100097c..fe4d7bc01b 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/CatalogTestBase.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/CatalogTestBase.java
@@ -42,9 +42,9 @@ public abstract class CatalogTestBase extends
TestBaseWithCatalog {
SparkCatalogConfig.HADOOP.properties()
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties()
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties()
},
{
SparkCatalogConfig.REST.catalogName(),
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java
index ef6c49db57..1d96d6f939 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java
@@ -38,7 +38,7 @@ public enum SparkCatalogConfig {
"testrest",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "rest", "cache-enabled", "false")),
- SPARK(
+ SPARK_SESSION(
"spark_catalog",
SparkSessionCatalog.class.getName(),
ImmutableMap.of(
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkCatalogOperations.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkCatalogOperations.java
index d7ead52880..c71fc5954c 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkCatalogOperations.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkCatalogOperations.java
@@ -68,8 +68,8 @@ public class TestSparkCatalogOperations extends
CatalogTestBase {
Boolean.toString(USE_NULLABLE_QUERY_SCHEMA))
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
ImmutableMap.of(
"type",
"hive",
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
index 28d66776f2..19ec6d13dd 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
@@ -105,37 +105,37 @@ public class TestCompressionSettings extends
CatalogTestBase {
public static Object[][] parameters() {
return new Object[][] {
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
PARQUET,
ImmutableMap.of(COMPRESSION_CODEC, "zstd", COMPRESSION_LEVEL, "1")
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
PARQUET,
ImmutableMap.of(COMPRESSION_CODEC, "gzip")
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
ORC,
ImmutableMap.of(COMPRESSION_CODEC, "zstd", COMPRESSION_STRATEGY,
"speed")
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
ORC,
ImmutableMap.of(COMPRESSION_CODEC, "zstd", COMPRESSION_STRATEGY,
"compression")
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
AVRO,
ImmutableMap.of(COMPRESSION_CODEC, "snappy", COMPRESSION_LEVEL, "3")
}
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java
index 1e04eba0b9..6cb0ca5a66 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java
@@ -59,9 +59,9 @@ public class TestCreateTableAsSelect extends CatalogTestBase {
SparkCatalogConfig.HADOOP.catalogName() + ".default.source"
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
"default.source"
}
};
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java
index dd17e7c417..d702f65e64 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java
@@ -69,9 +69,9 @@ public class TestNamespaceSQL extends CatalogTestBase {
true
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
NS.toString(),
false
}
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java
index 75708a2a08..6becc07b2d 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java
@@ -51,7 +51,7 @@ public class TestRefreshTable extends CatalogTestBase {
// We are not allowed to change the session catalog after it has been
initialized, so build a
// new one
if (Set.of(
- SparkCatalogConfig.SPARK.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
SparkCatalogConfig.HADOOP.catalogName(),
SparkCatalogConfig.REST.catalogName())
.contains(catalogName)) {
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java
index db5c6216a0..240622750f 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java
@@ -71,9 +71,9 @@ public class TestSelect extends CatalogTestBase {
SparkCatalogConfig.HADOOP.catalogName() + ".default.binary_table"
},
{
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
"default.binary_table"
}
};
