Repository: spark
Updated Branches:
refs/heads/master e52691398 -> eb10b481c
[SPARK-15286][SQL] Make the output readable for EXPLAIN CREATE TABLE and DESC
EXTENDED
#### What changes were proposed in this pull request?
Before this PR, the output of EXPLAIN of following SQL is like
```SQL
CREATE EXTERNAL TABLE extTable_with_partitions (key INT, value STRING)
PARTITIONED BY (ds STRING, hr STRING)
LOCATION
'/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-b39a6185-8981-403b-a4aa-36fb2f4ca8a9'
```
``ExecutedCommand CreateTableCommand
CatalogTable(`extTable_with_partitions`,CatalogTableType(EXTERNAL),CatalogStorageFormat(Some(/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-dd234718-e85d-4c5a-8353-8f1834ac0323),Some(org.apache.hadoop.mapred.TextInputFormat),Some(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat),None,false,Map()),List(CatalogColumn(key,int,true,None),
CatalogColumn(value,string,true,None), CatalogColumn(ds,string,true,None),
CatalogColumn(hr,string,true,None)),List(ds,
hr),List(),List(),-1,,1463026413544,-1,Map(),None,None,None), false``
After this PR, the output is like
```
ExecutedCommand
: +- CreateTableCommand CatalogTable(
Table:`extTable_with_partitions`
Created:Thu Jun 02 21:30:54 PDT 2016
Last Access:Wed Dec 31 15:59:59 PST 1969
Type:EXTERNAL
Schema:[`key` int, `value` string, `ds` string, `hr` string]
Partition Columns:[`ds`, `hr`]
Storage(Location:/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-a06083b8-8e88-4d07-9ff0-d6bd8d943ad3,
InputFormat:org.apache.hadoop.mapred.TextInputFormat,
OutputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat)), false
```
This is also applicable to `DESC EXTENDED`. However, this does not have special
handling for Data Source Tables. If needed, we need to move the logics of
`DDLUtil`. Let me know if we should do it in this PR. Thanks! rxin liancheng
#### How was this patch tested?
Manual testing
Author: gatorsmile <[email protected]>
Closes #13070 from gatorsmile/betterExplainCatalogTable.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eb10b481
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eb10b481
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eb10b481
Branch: refs/heads/master
Commit: eb10b481ca51a3e010c168a1fb0baf1a94dcadfa
Parents: e526913
Author: gatorsmile <[email protected]>
Authored: Fri Jun 3 13:56:22 2016 -0700
Committer: Wenchen Fan <[email protected]>
Committed: Fri Jun 3 13:56:22 2016 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/catalog/interface.scala | 60 +++++++++++++++++++-
1 file changed, 58 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/eb10b481/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index b591957..6197aca 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.catalyst.catalog
+import java.util.Date
import javax.annotation.Nullable
import org.apache.spark.sql.AnalysisException
@@ -48,7 +49,26 @@ case class CatalogStorageFormat(
outputFormat: Option[String],
serde: Option[String],
compressed: Boolean,
- serdeProperties: Map[String, String])
+ serdeProperties: Map[String, String]) {
+
+ override def toString: String = {
+ val serdePropsToString =
+ if (serdeProperties.nonEmpty) {
+ s"Properties: " + serdeProperties.map(p => p._1 + "=" +
p._2).mkString("[", ", ", "]")
+ } else {
+ ""
+ }
+ val output =
+ Seq(locationUri.map("Location: " + _).getOrElse(""),
+ inputFormat.map("InputFormat: " + _).getOrElse(""),
+ outputFormat.map("OutputFormat: " + _).getOrElse(""),
+ if (compressed) "Compressed" else "",
+ serde.map("Serde: " + _).getOrElse(""),
+ serdePropsToString)
+ output.filter(_.nonEmpty).mkString("Storage(", ", ", ")")
+ }
+
+}
object CatalogStorageFormat {
/** Empty storage format for default values and copies. */
@@ -65,8 +85,18 @@ case class CatalogColumn(
// as a string due to issues in converting Hive varchars to and from
SparkSQL strings.
@Nullable dataType: String,
nullable: Boolean = true,
- comment: Option[String] = None)
+ comment: Option[String] = None) {
+
+ override def toString: String = {
+ val output =
+ Seq(s"`$name`",
+ dataType,
+ if (!nullable) "NOT NULL" else "",
+ comment.map("(" + _ + ")").getOrElse(""))
+ output.filter(_.nonEmpty).mkString(" ")
+ }
+}
/**
* A partition (Hive style) defined in the catalog.
@@ -140,6 +170,32 @@ case class CatalogTable(
locationUri, inputFormat, outputFormat, serde, compressed,
serdeProperties))
}
+ override def toString: String = {
+ val tableProperties = properties.map(p => p._1 + "=" + p._2).mkString("[",
", ", "]")
+ val partitionColumns = partitionColumnNames.map("`" + _ +
"`").mkString("[", ", ", "]")
+ val sortColumns = sortColumnNames.map("`" + _ + "`").mkString("[", ", ",
"]")
+ val bucketColumns = bucketColumnNames.map("`" + _ + "`").mkString("[", ",
", "]")
+
+ val output =
+ Seq(s"Table: ${identifier.quotedString}",
+ if (owner.nonEmpty) s"Owner: $owner" else "",
+ s"Created: ${new Date(createTime).toString}",
+ s"Last Access: ${new Date(lastAccessTime).toString}",
+ s"Type: ${tableType.name}",
+ if (schema.nonEmpty) s"Schema: ${schema.mkString("[", ", ", "]")}"
else "",
+ if (partitionColumnNames.nonEmpty) s"Partition Columns:
$partitionColumns" else "",
+ if (numBuckets != -1) s"Num Buckets: $numBuckets" else "",
+ if (bucketColumnNames.nonEmpty) s"Bucket Columns: $bucketColumns" else
"",
+ if (sortColumnNames.nonEmpty) s"Sort Columns: $sortColumns" else "",
+ viewOriginalText.map("Original View: " + _).getOrElse(""),
+ viewText.map("View: " + _).getOrElse(""),
+ comment.map("Comment: " + _).getOrElse(""),
+ if (properties.nonEmpty) s"Properties: $tableProperties" else "",
+ s"$storage")
+
+ output.filter(_.nonEmpty).mkString("CatalogTable(\n\t", "\n\t", ")")
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
