xiarixiaoyao commented on a change in pull request #4308:
URL: https://github.com/apache/hudi/pull/4308#discussion_r770175565
##########
File path:
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
##########
@@ -125,6 +123,25 @@ case class HoodieFileIndex(
}
}
+ /**
+ * This method traverses StructType recursively to build map of columnName
-> StructField
+ * Note : If there is nesting of columns like ["a.b.c.d", "a.b.c.e"] ->
final map will have keys corresponding
+ * only to ["a.b.c.d", "a.b.c.e"] and not for subsets like ["a.b.c", "a.b"]
+ * @param structField
+ * @return map of ( columns names -> StructField )
+ */
+ private def convertStructToMap(structField : Either[StructField,StructType])
: Map[String,StructField] = {
+ structField match {
+ case Right(field) => field.fields.map(f =>
convertStructToMap(Left(f))).flatten.toMap
+ case Left(field) => field.dataType match {
+ case struct: StructType => convertStructToMap(Right(struct)).map {
+ case (key :String , sf:StructField) => (field.name + "." + key , sf
)
+ }
+ case _ => Map(field.name -> field)
+ }
+ }
+ }
+
Review comment:
pls notice the code style of scalaļ¼
structField : Either[StructField,StructType] => structField:
Either[StructField, StructType]
Map[String,StructField] => Map[String, StructField]
case (key :String , sf:StructField) => (field.name + "." + key , sf ) =>
case (key: String, sf: StructField) => (field.name + "." + key, sf )
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
