cloud-fan commented on code in PR #37287:
URL: https://github.com/apache/spark/pull/37287#discussion_r933091882
##########
sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala:
##########
@@ -110,74 +106,93 @@ class CatalogImpl(sparkSession: SparkSession) extends
Catalog {
override def listTables(dbName: String): Dataset[Table] = {
// `dbName` could be either a single database name (behavior in Spark 3.3
and prior) or
// a qualified namespace with catalog name. We assume it's a single
database name
- // and check if we can find the dbName in sessionCatalog. If so we
listTables under
- // that database. Otherwise we try 3-part name parsing and locate the
database.
- if (sessionCatalog.databaseExists(dbName) ||
sessionCatalog.isGlobalTempViewDB(dbName)) {
- val tables = sessionCatalog.listTables(dbName).map(makeTable)
- CatalogImpl.makeDataset(tables, sparkSession)
+ // and check if we can find it in the sessionCatalog. If so we list tables
under
+ // that database. Otherwise we will resolve the catalog/namespace and list
tables there.
+ val namespace = if (sessionCatalog.databaseExists(dbName)) {
+ Seq(CatalogManager.SESSION_CATALOG_NAME, dbName)
} else {
- val ident =
sparkSession.sessionState.sqlParser.parseMultipartIdentifier(dbName)
- val plan = ShowTables(UnresolvedNamespace(ident), None)
- val ret = sparkSession.sessionState.executePlan(plan).toRdd.collect()
- val tables = ret
- .map(row => ident ++ Seq(row.getString(1)))
- .map(makeTable)
- CatalogImpl.makeDataset(tables, sparkSession)
+ parseIdent(dbName)
}
- }
+ val plan = ShowTables(UnresolvedNamespace(namespace), None)
+ val qe = sparkSession.sessionState.executePlan(plan)
+ val catalog = qe.analyzed.collectFirst {
+ case ShowTables(r: ResolvedNamespace, _, _) => r.catalog
+ case _: ShowTablesCommand =>
+ sparkSession.sessionState.catalogManager.v2SessionCatalog
+ }.get
+ val tables = qe.toRdd.collect().map { row =>
+ val tableName = row.getString(1)
+ val namespaceName = row.getString(0)
+ val isTemp = row.getBoolean(2)
+ if (isTemp) {
+ // Temp views do not belong to any catalog. We shouldn't prepend the
catalog name here.
+ val ns = if (namespaceName.isEmpty) Nil else Seq(namespaceName)
+ makeTable(ns :+ tableName)
+ } else {
+ val ns = parseIdent(namespaceName)
+ makeTable(catalog.name() +: ns :+ tableName)
+ }
+ }
+ CatalogImpl.makeDataset(tables, sparkSession)
+ }
+
+ private def makeTable(nameParts: Seq[String]): Table = {
+ sessionCatalog.lookupLocalOrGlobalRawTempView(nameParts).map { tempView =>
+ new Table(
+ name = tempView.tableMeta.identifier.table,
+ catalog = null,
+ namespace = tempView.tableMeta.identifier.database.toArray,
+ description = tempView.tableMeta.comment.orNull,
+ tableType = "TEMPORARY",
+ isTemporary = true)
+ }.getOrElse {
+ val plan = UnresolvedIdentifier(nameParts)
+ sparkSession.sessionState.executePlan(plan).analyzed match {
+ case ResolvedIdentifier(catalog: TableCatalog, ident) =>
+ val tableOpt = try {
+ loadTable(catalog, ident)
+ } catch {
+ // Even if the table exits, error may still happen. For example,
Spark can't read Hive's
+ // index table. We return a Table without description and
tableType in this case.
Review Comment:
This is never documented before and is a hidden assumption. To cover this
case I have to call `TableCatalog.loadTable` manually instead of running the v2
command.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
