dongjoon-hyun commented on code in PR #38823:
URL: https://github.com/apache/spark/pull/38823#discussion_r1116513313
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala:
##########
@@ -471,43 +473,63 @@ private[sql] object CatalogV2Util {
/**
* Converts a StructType to DS v2 columns, which decodes the StructField
metadata to v2 column
- * comment and default value. This is mainly used to generate DS v2 columns
from table schema in
- * DDL commands, so that Spark can pass DS v2 columns to DS v2 createTable
and related APIs.
+ * comment and default value or generation expression. This is mainly used
to generate DS v2
+ * columns from table schema in DDL commands, so that Spark can pass DS v2
columns to DS v2
+ * createTable and related APIs.
*/
def structTypeToV2Columns(schema: StructType): Array[Column] = {
schema.fields.map(structFieldToV2Column)
}
private def structFieldToV2Column(f: StructField): Column = {
- def createV2Column(defaultValue: ColumnDefaultValue, metadata: Metadata):
Column = {
- val metadataJSON = if (metadata == Metadata.empty) {
+ def metadataAsJson(metadata: Metadata): String = {
+ if (metadata == Metadata.empty) {
null
} else {
metadata.json
}
- Column.create(
- f.name, f.dataType, f.nullable, f.getComment().orNull, defaultValue,
metadataJSON)
}
- if (f.getCurrentDefaultValue().isDefined &&
f.getExistenceDefaultValue().isDefined) {
+ def metadataWithKeysRemoved(keys: Seq[String]): Metadata = {
+ keys.foldLeft(new MetadataBuilder().withMetadata(f.metadata)) {
+ (builder, key) => builder.remove(key)
+ }.build()
+ }
+
+ val isDefaultColumn = f.getCurrentDefaultValue().isDefined &&
+ f.getExistenceDefaultValue().isDefined
+ val isGeneratedColumn = GeneratedColumn.isGeneratedColumn(f)
+ if (isDefaultColumn && isGeneratedColumn) {
+ // todo: should we throw this error earlier? (i.e. on parsing)
+ throw new AnalysisException(
Review Comment:
It seems that we missed to define a new error function. I thought we need to
define like `generatedColumnsUnsupported` and use it here via `throw
QueryCompilationErrors.generatedColumnsUnsupported`. Is there a reason not to
define that?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
