imback82 commented on code in PR #42577:
URL: https://github.com/apache/spark/pull/42577#discussion_r1386788291
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala:
##########
@@ -170,6 +174,55 @@ case class CatalogTablePartition(
}
}
+/**
+ * A container for clustering information.
+ *
+ * @param columnNames the names of the columns used for clustering.
+ */
+case class ClusterBySpec(columnNames: Seq[NamedReference]) {
+ override def toString: String = toJson
+
+ def toJson: String =
ClusterBySpec.mapper.writeValueAsString(columnNames.map(_.fieldNames))
+}
+
+object ClusterBySpec {
+ private val mapper = {
+ val ret = new ObjectMapper() with ClassTagExtensions
+ ret.setSerializationInclusion(Include.NON_ABSENT)
+ ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+ ret.registerModule(DefaultScalaModule)
+ ret
+ }
+
+ def fromProperty(columns: String): ClusterBySpec = {
+
ClusterBySpec(mapper.readValue[Seq[Seq[String]]](columns).map(FieldReference(_)))
+ }
+
+ def toProperty(
+ schema: StructType,
+ clusterBySpec: ClusterBySpec,
+ resolver: Resolver): (String, String) = {
+ CatalogTable.PROP_CLUSTERING_COLUMNS ->
+ normalizeClusterBySpec(schema, clusterBySpec, resolver).toJson
+ }
+
+ private def normalizeClusterBySpec(
+ schema: StructType,
+ clusterBySpec: ClusterBySpec,
+ resolver: Resolver): ClusterBySpec = {
+ val normalizedColumns = clusterBySpec.columnNames.map { columnName =>
+ val position = SchemaUtils.findColumnPosition(
Review Comment:
it's happening here for v2 path:
https://github.com/apache/spark/blob/6abc4a1a58ef4e5d896717b10b2314dae2af78af/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala#L273-L279
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
