rahil-c commented on code in PR #17904:
URL: https://github.com/apache/hudi/pull/17904#discussion_r2734525504
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/lance/SparkLanceReaderBase.scala:
##########
@@ -84,9 +85,21 @@ class SparkLanceReaderBase(enableVectorizedReader: Boolean)
extends SparkColumna
// Open Lance file reader
val lanceReader = LanceFileReader.open(filePath, allocator)
- // Extract column names from required schema for projection
- val columnNames: java.util.List[String] = if (requiredSchema.nonEmpty)
{
- requiredSchema.fields.map(_.name).toList.asJava
+ // Get schema from Lance file
+ val arrowSchema = lanceReader.schema()
+ val fileSchema = LanceArrowUtils.fromArrowSchema(arrowSchema)
+
+ // Create lance schema evolution helper
+ val evolution = new LanceBasicSchemaEvolution(
+ fileSchema,
+ requiredSchema,
Review Comment:
have removed this LanceBasicSchemaEvolution class to reuse existing schema
utils
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
