This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch release-0.8 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 0b63cb8a965b6aeec27ef9fb5f033da6755755f7 Author: Zouxxyy <[email protected]> AuthorDate: Mon May 20 10:52:53 2024 +0800 [spark] Fix drop database cascade when database with view with SparkGenericCatalog (#3347) --- .../main/java/org/apache/paimon/spark/SparkGenericCatalog.java | 9 ++++++++- .../test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala | 5 +++++ .../org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala | 8 ++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java index b947523fe..9576379f8 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java @@ -134,7 +134,14 @@ public class SparkGenericCatalog extends SparkBaseCatalog implements CatalogExte throws NoSuchNamespaceException, NonEmptyNamespaceException { if (namespace.length == 1 && namespaceExists(namespace) && cascade) { for (Identifier table : listTables(namespace)) { - dropTable(table); + try { + dropTable(table); + } catch (Exception e) { + LOG.warn( + "Failed to drop {}, fallback to use sessionCatalog to drop, for {}", + table, + e.getMessage()); + } } } return asNamespaceCatalog().dropNamespace(namespace, cascade); diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala index 6ccf08016..814001bc9 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala @@ -84,6 +84,11 @@ class PaimonSparkTestBase } } + protected def reset(): Unit = { + afterAll() + beforeAll() + } + /** Default is paimon catalog */ override protected def beforeEach(): Unit = { super.beforeAll() diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala index c03f61bf6..c542633dc 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala @@ -99,7 +99,9 @@ abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase { } } + // Since we created a new sparkContext, we need to stop it and reset the default sparkContext reusedSpark.stop() + reset() } test("Paimon DDL with hive catalog: drop database cascade which contains paimon table") { @@ -111,6 +113,12 @@ abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase { spark.sql(s"CREATE DATABASE paimon_db") spark.sql(s"USE paimon_db") spark.sql(s"CREATE TABLE paimon_tbl (id int, name string, dt string) using paimon") + // Currently, only spark_catalog supports create other table or view + if (catalogName.equals(sparkCatalogName)) { + spark.sql(s"CREATE TABLE parquet_tbl (id int, name string, dt string) using parquet") + spark.sql(s"CREATE VIEW parquet_tbl_view AS SELECT * FROM parquet_tbl") + spark.sql(s"CREATE VIEW paimon_tbl_view AS SELECT * FROM paimon_tbl") + } spark.sql(s"USE default") spark.sql(s"DROP DATABASE paimon_db CASCADE") }
