Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/16422#discussion_r122517212
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala ---
@@ -619,6 +620,104 @@ case class DescribeTableCommand(
}
}
+/**
+ * A command to list the info for a column, including name, data type,
column stats and comment.
+ * This function creates a [[DescribeColumnCommand]] logical plan.
+ *
+ * The syntax of using this command in SQL is:
+ * {{{
+ * DESCRIBE [EXTENDED|FORMATTED] table_name column_name;
+ * }}}
+ */
+case class DescribeColumnCommand(
+ table: TableIdentifier,
+ column: String,
+ isFormatted: Boolean)
+ extends RunnableCommand {
+
+ override val output: Seq[Attribute] = {
+ // The displayed names are based on Hive.
+ // (Link for the corresponding Hive Jira:
https://issues.apache.org/jira/browse/HIVE-7050)
+ if (isFormatted) {
+ Seq(
+ AttributeReference("col_name", StringType, nullable = false,
+ new MetadataBuilder().putString("comment", "name of the
column").build())(),
+ AttributeReference("data_type", StringType, nullable = false,
+ new MetadataBuilder().putString("comment", "data type of the
column").build())(),
+ AttributeReference("min", StringType, nullable = true,
+ new MetadataBuilder().putString("comment", "min value of the
column").build())(),
+ AttributeReference("max", StringType, nullable = true,
+ new MetadataBuilder().putString("comment", "max value of the
column").build())(),
+ AttributeReference("num_nulls", StringType, nullable = true,
+ new MetadataBuilder().putString("comment", "number of nulls of
the column").build())(),
+ AttributeReference("distinct_count", StringType, nullable = true,
+ new MetadataBuilder().putString("comment", "distinct count of
the column").build())(),
+ AttributeReference("avg_col_len", StringType, nullable = true,
+ new MetadataBuilder().putString("comment",
+ "average length of the values of the column").build())(),
+ AttributeReference("max_col_len", StringType, nullable = true,
+ new MetadataBuilder().putString("comment",
+ "maximum length of the values of the column").build())(),
+ AttributeReference("comment", StringType, nullable = true,
+ new MetadataBuilder().putString("comment", "comment of the
column").build())())
+ } else {
+ Seq(
+ AttributeReference("col_name", StringType, nullable = false,
+ new MetadataBuilder().putString("comment", "name of the
column").build())(),
+ AttributeReference("data_type", StringType, nullable = false,
+ new MetadataBuilder().putString("comment", "data type of the
column").build())(),
+ AttributeReference("comment", StringType, nullable = true,
+ new MetadataBuilder().putString("comment", "comment of the
column").build())())
+ }
+ }
+
+ override def run(sparkSession: SparkSession): Seq[Row] = {
+ val catalog = sparkSession.sessionState.catalog
+ val resolver = sparkSession.sessionState.conf.resolver
+ val catalogTable = catalog.getTempViewOrPermanentTableMetadata(table)
+ val attribute = {
+ val field = catalogTable.schema.find(f => resolver(f.name, column))
+ field.getOrElse {
+ if (column.contains(".")) {
+ throw new AnalysisException(
+ s"DESC TABLE COLUMN is not supported for nested column:
$column")
+ } else {
+ throw new AnalysisException(s"Column $column does not exist.")
+ }
+ }
+ }
+
+ val colStats = catalogTable.stats.map(_.colStats).getOrElse(Map.empty)
+ val cs = colStats.get(attribute.name)
+
+ val comment = if (attribute.metadata.contains("comment")) {
+ Option(attribute.metadata.getString("comment"))
+ } else {
+ None
+ }
+
+ val result = if (isFormatted) {
+ // Show column stats only when formatted is specified.
+ Row(
+ attribute.name,
+ attribute.dataType.simpleString,
+ cs.flatMap(_.min.map(_.toString)).orNull,
+ cs.flatMap(_.max.map(_.toString)).orNull,
+ cs.map(_.nullCount.toString).orNull,
+ cs.map(_.distinctCount.toString).orNull,
+ cs.map(_.avgLen.toString).orNull,
+ cs.map(_.maxLen.toString).orNull,
+ comment.orNull)
+ } else {
+ Row(
+ attribute.name,
+ attribute.dataType.simpleString,
--- End diff --
`simpleString` -> `catalogString`. We do not want to see the truncated
string.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
