This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new d5a9d34edc Spark: Add support for describing/showing views (#9513)
d5a9d34edc is described below
commit d5a9d34edc5fd944f5c45b9667ddcd8eab33fe37
Author: Eduard Tudenhoefner <[email protected]>
AuthorDate: Wed Jan 31 10:04:08 2024 +0100
Spark: Add support for describing/showing views (#9513)
This adds support for:
* `DESCRIBE <viewName>` / `DESCRIBE EXTENDED <viewName>`
* `SHOW VIEWS` / `SHOW VIEWS LIKE <pattern>`
* `SHOW TBLPROPERTIES <viewName>`
* `SHOW CREATE TABLE <viewName>`
---
.../catalyst/analysis/RewriteViewCommands.scala | 10 ++
.../plans/logical/views/ShowIcebergViews.scala | 36 +++++
.../datasources/v2/DescribeV2ViewExec.scala | 77 ++++++++++
.../v2/ExtendedDataSourceV2Strategy.scala | 17 +++
.../datasources/v2/ShowCreateV2ViewExec.scala | 79 ++++++++++
.../datasources/v2/ShowV2ViewPropertiesExec.scala | 55 +++++++
.../execution/datasources/v2/ShowV2ViewsExec.scala | 66 +++++++++
.../apache/iceberg/spark/extensions/TestViews.java | 164 ++++++++++++++++++++-
.../org/apache/iceberg/spark/SparkCatalog.java | 9 +-
9 files changed, 509 insertions(+), 4 deletions(-)
diff --git
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
index 066ba59394..b746d0951a 100644
---
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
+++
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
@@ -25,9 +25,12 @@ import
org.apache.spark.sql.catalyst.expressions.SubqueryExpression
import org.apache.spark.sql.catalyst.plans.logical.CreateView
import org.apache.spark.sql.catalyst.plans.logical.DropView
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.ShowViews
import org.apache.spark.sql.catalyst.plans.logical.View
import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
import org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView
+import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
+import org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.connector.catalog.CatalogManager
import org.apache.spark.sql.connector.catalog.CatalogPlugin
@@ -60,6 +63,13 @@ case class RewriteViewCommands(spark: SparkSession) extends
Rule[LogicalPlan] wi
properties = properties,
allowExisting = allowExisting,
replace = replace)
+
+ case ShowViews(UnresolvedNamespace(Seq()), pattern, output) if
isViewCatalog(catalogManager.currentCatalog) =>
+ ShowIcebergViews(ResolvedNamespace(catalogManager.currentCatalog,
Seq.empty), pattern, output)
+
+ case ShowViews(UnresolvedNamespace(CatalogAndNamespace(catalog, ns)),
pattern, output)
+ if isViewCatalog(catalog) =>
+ ShowIcebergViews(ResolvedNamespace(catalog, ns), pattern, output)
}
private def isTempView(nameParts: Seq[String]): Boolean = {
diff --git
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
new file mode 100644
index 0000000000..b09c27acdc
--- /dev/null
+++
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.spark.sql.catalyst.plans.logical.views
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.ShowViews
+import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
+
+case class ShowIcebergViews(
+ namespace: LogicalPlan,
+ pattern: Option[String],
+ override val output: Seq[Attribute] = ShowViews.getOutputAttrs) extends
UnaryCommand {
+ override def child: LogicalPlan = namespace
+
+ override protected def withNewChildInternal(newChild: LogicalPlan):
ShowIcebergViews =
+ copy(namespace = newChild)
+}
diff --git
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
new file mode 100644
index 0000000000..bb08fb18b2
--- /dev/null
+++
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.util.escapeSingleQuotedString
+import org.apache.spark.sql.connector.catalog.View
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.LeafExecNode
+import scala.collection.JavaConverters._
+
+case class DescribeV2ViewExec(
+ output: Seq[Attribute],
+ view: View,
+ isExtended: Boolean) extends V2CommandExec with LeafExecNode {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override protected def run(): Seq[InternalRow] = {
+ if (isExtended) {
+ (describeSchema :+ emptyRow) ++ describeExtended
+ } else {
+ describeSchema
+ }
+ }
+
+ private def describeSchema: Seq[InternalRow] =
+ view.schema().map { column =>
+ toCatalystRow(
+ column.name,
+ column.dataType.simpleString,
+ column.getComment().getOrElse(""))
+ }
+
+ private def emptyRow: InternalRow = toCatalystRow("", "", "")
+
+ private def describeExtended: Seq[InternalRow] = {
+ val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
+ val properties: Map[String, String] = view.properties.asScala.toMap --
ViewCatalog.RESERVED_PROPERTIES.asScala
+ val viewCatalogAndNamespace: Seq[String] = view.currentCatalog +:
view.currentNamespace.toSeq
+ val viewProperties = properties.toSeq.sortBy(_._1).map {
+ case (key, value) =>
+ s"'${escapeSingleQuotedString(key)}' =
'${escapeSingleQuotedString(value)}'"
+ }.mkString("[", ", ", "]")
+
+
+ toCatalystRow("# Detailed View Information", "", "") ::
+ toCatalystRow("Comment",
view.properties.getOrDefault(ViewCatalog.PROP_COMMENT, ""), "") ::
+ toCatalystRow("View Catalog and Namespace",
viewCatalogAndNamespace.quoted, "") ::
+ toCatalystRow("View Query Output Columns", outputColumns, "") ::
+ toCatalystRow("View Properties", viewProperties, "") ::
+ toCatalystRow("Created By",
view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""), "") ::
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DescribeV2ViewExec"
+ }
+}
diff --git
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
index 0505fe4e30..b24cf67488 100644
---
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
+++
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.Strategy
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier
+import org.apache.spark.sql.catalyst.analysis.ResolvedNamespace
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.spark.sql.catalyst.expressions.PredicateHelper
@@ -34,6 +35,7 @@ import
org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
import org.apache.spark.sql.catalyst.plans.logical.Call
import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch
import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag
+import org.apache.spark.sql.catalyst.plans.logical.DescribeRelation
import org.apache.spark.sql.catalyst.plans.logical.DropBranch
import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
@@ -44,9 +46,12 @@ import
org.apache.spark.sql.catalyst.plans.logical.RenameTable
import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
import
org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
+import org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable
+import org.apache.spark.sql.catalyst.plans.logical.ShowTableProperties
import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
import org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView
import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
+import org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
import org.apache.spark.sql.connector.catalog.ViewCatalog
@@ -123,6 +128,18 @@ case class ExtendedDataSourceV2Strategy(spark:
SparkSession) extends Strategy wi
allowExisting = allowExisting,
replace = replace) :: Nil
+ case DescribeRelation(ResolvedV2View(catalog, ident), _, isExtended,
output) =>
+ DescribeV2ViewExec(output, catalog.loadView(ident), isExtended) :: Nil
+
+ case ShowTableProperties(ResolvedV2View(catalog, ident), propertyKey,
output) =>
+ ShowV2ViewPropertiesExec(output, catalog.loadView(ident), propertyKey)
:: Nil
+
+ case ShowIcebergViews(ResolvedNamespace(catalog: ViewCatalog, namespace),
pattern, output) =>
+ ShowV2ViewsExec(output, catalog, namespace, pattern) :: Nil
+
+ case ShowCreateTable(ResolvedV2View(catalog, ident), _, output) =>
+ ShowCreateV2ViewExec(output, catalog.loadView(ident)) :: Nil
+
case _ => Nil
}
diff --git
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
new file mode 100644
index 0000000000..3be0f15031
--- /dev/null
+++
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.util.escapeSingleQuotedString
+import org.apache.spark.sql.connector.catalog.View
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.LeafExecNode
+import scala.collection.JavaConverters._
+
+case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
+ extends V2CommandExec with LeafExecNode {
+
+ override protected def run(): Seq[InternalRow] = {
+ val builder = new StringBuilder
+ builder ++= s"CREATE VIEW ${view.name} "
+ showColumns(view, builder)
+ showComment(view, builder)
+ showProperties(view, builder)
+ builder ++= s"AS\n${view.query}\n"
+
+ Seq(toCatalystRow(builder.toString))
+ }
+
+ private def showColumns(view: View, builder: StringBuilder): Unit = {
+ val columns = concatByMultiLines(
+ view.schema().fields
+ .map(x => s"${x.name}${x.getComment().map(c => s" COMMENT
'$c'").getOrElse("")}"))
+ builder ++= columns
+ }
+
+ private def showComment(view: View, builder: StringBuilder): Unit = {
+ Option(view.properties.get(ViewCatalog.PROP_COMMENT))
+ .map("COMMENT '" + escapeSingleQuotedString(_) + "'\n")
+ .foreach(builder.append)
+ }
+
+ private def showProperties(
+ view: View,
+ builder: StringBuilder): Unit = {
+ val showProps = view.properties.asScala.toMap --
ViewCatalog.RESERVED_PROPERTIES.asScala
+ if (showProps.nonEmpty) {
+ val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map {
+ case (key, value) =>
+ s"'${escapeSingleQuotedString(key)}' =
'${escapeSingleQuotedString(value)}'"
+ }
+
+ builder ++= "TBLPROPERTIES "
+ builder ++= concatByMultiLines(props)
+ }
+ }
+
+ private def concatByMultiLines(iter: Iterable[String]): String = {
+ iter.mkString("(\n ", ",\n ", ")\n")
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"ShowCreateV2ViewExec"
+ }
+}
diff --git
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
new file mode 100644
index 0000000000..89fafe99ef
--- /dev/null
+++
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.View
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.LeafExecNode
+import scala.collection.JavaConverters._
+
+case class ShowV2ViewPropertiesExec(
+ output: Seq[Attribute],
+ view: View,
+ propertyKey: Option[String]) extends V2CommandExec with LeafExecNode {
+
+ override protected def run(): Seq[InternalRow] = {
+ propertyKey match {
+ case Some(p) =>
+ val propValue = properties.getOrElse(p,
+ s"View ${view.name()} does not have property: $p")
+ Seq(toCatalystRow(p, propValue))
+ case None =>
+ properties.map {
+ case (k, v) => toCatalystRow(k, v)
+ }.toSeq
+ }
+ }
+
+
+ private def properties = {
+ view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"ShowV2ViewPropertiesExec"
+ }
+}
diff --git
a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
new file mode 100644
index 0000000000..3aa85c3db5
--- /dev/null
+++
b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.LeafExecNode
+import scala.collection.mutable.ArrayBuffer
+
+case class ShowV2ViewsExec(
+ output: Seq[Attribute],
+ catalog: ViewCatalog,
+ namespace: Seq[String],
+ pattern: Option[String]) extends V2CommandExec with LeafExecNode {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override protected def run(): Seq[InternalRow] = {
+ val rows = new ArrayBuffer[InternalRow]()
+
+ // handle GLOBAL VIEWS
+ val globalTemp =
session.sessionState.catalog.globalTempViewManager.database
+ if (namespace.nonEmpty && globalTemp == namespace.head) {
+ pattern.map(p =>
session.sessionState.catalog.globalTempViewManager.listViewNames(p))
+
.getOrElse(session.sessionState.catalog.globalTempViewManager.listViewNames("*"))
+ .map(name => rows += toCatalystRow(globalTemp, name, true))
+ } else {
+ val views = catalog.listViews(namespace: _*)
+ views.map { view =>
+ if (pattern.map(StringUtils.filterPattern(Seq(view.name()),
_).nonEmpty).getOrElse(true)) {
+ rows += toCatalystRow(view.namespace().quoted, view.name(), false)
+ }
+ }
+ }
+
+ // include TEMP VIEWS
+ pattern.map(p => session.sessionState.catalog.listLocalTempViews(p))
+ .getOrElse(session.sessionState.catalog.listLocalTempViews("*"))
+ .map(v => rows += toCatalystRow(v.database.toArray.quoted, v.table,
true))
+
+ rows.toSeq
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"ShowV2ViewsExec"
+ }
+}
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
index 1b9c403f14..b05666f3c6 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
@@ -1172,7 +1172,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(String.format("The table or view `%s` cannot be
found", tableName));
// the underlying SQL in the View should be rewritten to have catalog &
namespace
- assertThat(sql("SELECT * FROM %s.%s.%s", catalogName, "default", viewName))
+ assertThat(sql("SELECT * FROM %s.%s.%s", catalogName, NAMESPACE, viewName))
.hasSize(1)
.containsExactly(row(5));
}
@@ -1197,11 +1197,171 @@ public class TestViews extends SparkExtensionsTestBase
{
.hasMessageContaining(String.format("The table or view `%s` cannot be
found", tableName));
// the underlying SQL in the View should be rewritten to have catalog &
namespace
- assertThat(sql("SELECT * FROM %s.%s.%s", catalogName, "default", viewName))
+ assertThat(sql("SELECT * FROM %s.%s.%s", catalogName, NAMESPACE, viewName))
.hasSize(3)
.containsExactly(row(3), row(3), row(3));
}
+ @Test
+ public void describeView() {
+ String viewName = "describeView";
+
+ sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id <= 3", viewName,
tableName);
+ assertThat(sql("DESCRIBE %s", viewName))
+ .containsExactly(row("id", "int", ""), row("data", "string", ""));
+ }
+
+ @Test
+ public void describeExtendedView() {
+ String viewName = "describeExtendedView";
+ String sql = String.format("SELECT id, data FROM %s WHERE id <= 3",
tableName);
+
+ sql(
+ "CREATE VIEW %s (new_id COMMENT 'ID', new_data COMMENT 'DATA') COMMENT
'view comment' AS %s",
+ viewName, sql);
+ assertThat(sql("DESCRIBE EXTENDED %s", viewName))
+ .contains(
+ row("new_id", "int", "ID"),
+ row("new_data", "string", "DATA"),
+ row("", "", ""),
+ row("# Detailed View Information", "", ""),
+ row("Comment", "view comment", ""),
+ row("View Catalog and Namespace", String.format("%s.%s",
catalogName, NAMESPACE), ""),
+ row("View Query Output Columns", "[id, data]", ""),
+ row(
+ "View Properties",
+ String.format(
+ "['format-version' = '1', 'location' = '/%s/%s',
'provider' = 'iceberg']",
+ NAMESPACE, viewName),
+ ""));
+ }
+
+ @Test
+ public void showViewProperties() {
+ String viewName = "showViewProps";
+
+ sql(
+ "CREATE VIEW %s TBLPROPERTIES ('key1'='val1', 'key2'='val2') AS SELECT
id, data FROM %s WHERE id <= 3",
+ viewName, tableName);
+ assertThat(sql("SHOW TBLPROPERTIES %s", viewName))
+ .contains(row("key1", "val1"), row("key2", "val2"));
+ }
+
+ @Test
+ public void showViewPropertiesByKey() {
+ String viewName = "showViewPropsByKey";
+
+ sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id <= 3", viewName,
tableName);
+ assertThat(sql("SHOW TBLPROPERTIES %s",
viewName)).contains(row("provider", "iceberg"));
+
+ assertThat(sql("SHOW TBLPROPERTIES %s (provider)", viewName))
+ .contains(row("provider", "iceberg"));
+
+ assertThat(sql("SHOW TBLPROPERTIES %s (non.existing)", viewName))
+ .contains(
+ row(
+ "non.existing",
+ String.format(
+ "View %s.%s.%s does not have property: non.existing",
+ catalogName, NAMESPACE, viewName)));
+ }
+
+ @Test
+ public void showViews() throws NoSuchTableException {
+ insertRows(6);
+ String sql = String.format("SELECT * from %s", tableName);
+ sql("CREATE VIEW v1 AS %s", sql);
+ sql("CREATE VIEW prefixV2 AS %s", sql);
+ sql("CREATE VIEW prefixV3 AS %s", sql);
+ sql("CREATE GLOBAL TEMPORARY VIEW globalViewForListing AS %s", sql);
+ sql("CREATE TEMPORARY VIEW tempViewForListing AS %s", sql);
+
+ // spark stores temp views case-insensitive by default
+ Object[] tempView = row("", "tempviewforlisting", true);
+ assertThat(sql("SHOW VIEWS"))
+ .contains(
+ row(NAMESPACE.toString(), "prefixV2", false),
+ row(NAMESPACE.toString(), "prefixV3", false),
+ row(NAMESPACE.toString(), "v1", false),
+ tempView);
+
+ assertThat(sql("SHOW VIEWS IN %s", catalogName))
+ .contains(
+ row(NAMESPACE.toString(), "prefixV2", false),
+ row(NAMESPACE.toString(), "prefixV3", false),
+ row(NAMESPACE.toString(), "v1", false),
+ tempView);
+
+ assertThat(sql("SHOW VIEWS IN %s.%s", catalogName, NAMESPACE))
+ .contains(
+ row(NAMESPACE.toString(), "prefixV2", false),
+ row(NAMESPACE.toString(), "prefixV3", false),
+ row(NAMESPACE.toString(), "v1", false),
+ tempView);
+
+ assertThat(sql("SHOW VIEWS LIKE 'pref*'"))
+ .contains(
+ row(NAMESPACE.toString(), "prefixV2", false),
+ row(NAMESPACE.toString(), "prefixV3", false));
+
+ assertThat(sql("SHOW VIEWS LIKE 'non-existing'")).isEmpty();
+
+ assertThat(sql("SHOW VIEWS IN spark_catalog.default")).contains(tempView);
+
+ assertThat(sql("SHOW VIEWS IN global_temp"))
+ .contains(
+ // spark stores temp views case-insensitive by default
+ row("global_temp", "globalviewforlisting", true), tempView);
+ }
+
+ @Test
+ public void showCreateSimpleView() {
+ String viewName = "showCreateSimpleView";
+ String sql = String.format("SELECT id, data FROM %s WHERE id <= 3",
tableName);
+
+ sql("CREATE VIEW %s AS %s", viewName, sql);
+
+ String expected =
+ String.format(
+ "CREATE VIEW %s.%s.%s (\n"
+ + " id,\n"
+ + " data)\n"
+ + "TBLPROPERTIES (\n"
+ + " 'format-version' = '1',\n"
+ + " 'location' = '/%s/%s',\n"
+ + " 'provider' = 'iceberg')\n"
+ + "AS\n%s\n",
+ catalogName, NAMESPACE, viewName, NAMESPACE, viewName, sql);
+ assertThat(sql("SHOW CREATE TABLE %s",
viewName)).containsExactly(row(expected));
+ }
+
+ @Test
+ public void showCreateComplexView() {
+ String viewName = "showCreateComplexView";
+ String sql = String.format("SELECT id, data FROM %s WHERE id <= 3",
tableName);
+
+ sql(
+ "CREATE VIEW %s (new_id COMMENT 'ID', new_data COMMENT 'DATA')"
+ + "COMMENT 'view comment' TBLPROPERTIES ('key1'='val1',
'key2'='val2') AS %s",
+ viewName, sql);
+
+ String expected =
+ String.format(
+ "CREATE VIEW %s.%s.%s (\n"
+ + " new_id COMMENT 'ID',\n"
+ + " new_data COMMENT 'DATA')\n"
+ + "COMMENT 'view comment'\n"
+ + "TBLPROPERTIES (\n"
+ + " 'format-version' = '1',\n"
+ + " 'key1' = 'val1',\n"
+ + " 'key2' = 'val2',\n"
+ + " 'location' = '/%s/%s',\n"
+ + " 'provider' = 'iceberg')\n"
+ + "AS\n%s\n",
+ catalogName, NAMESPACE, viewName, NAMESPACE, viewName, sql);
+ assertThat(sql("SHOW CREATE TABLE %s",
viewName)).containsExactly(row(expected));
+ }
+
private void insertRows(int numRows) throws NoSuchTableException {
List<SimpleRecord> records = Lists.newArrayListWithCapacity(numRows);
for (int i = 1; i <= numRows; i++) {
diff --git
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
index 37e7387d69..cf5e8da15c 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
@@ -539,8 +539,13 @@ public class SparkCatalog extends BaseCatalog
@Override
public Identifier[] listViews(String... namespace) {
- throw new UnsupportedOperationException(
- "Listing views is not supported by catalog: " + catalogName);
+ if (null != asViewCatalog) {
+ return asViewCatalog.listViews(Namespace.of(namespace)).stream()
+ .map(ident -> Identifier.of(ident.namespace().levels(),
ident.name()))
+ .toArray(Identifier[]::new);
+ }
+
+ return new Identifier[0];
}
@Override
