xiarixiaoyao commented on a change in pull request #3668:
URL: https://github.com/apache/hudi/pull/3668#discussion_r717299317
##########
File path:
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/MergeOnReadSnapshotRelation.scala
##########
@@ -52,26 +54,33 @@ case class HoodieMergeOnReadTableState(tableStructSchema:
StructType,
requiredAvroSchema: String,
hoodieRealtimeFileSplits:
List[HoodieMergeOnReadFileSplit],
preCombineField: Option[String],
- recordKeyFieldOpt: Option[String])
+ recordKeyFieldOpt: Option[String],
+ internalSchema: Option[InternalSchema]
= None,
+ requiredInternalSchema:
Option[InternalSchema] = None)
class MergeOnReadSnapshotRelation(val sqlContext: SQLContext,
val optParams: Map[String, String],
val userSchema: StructType,
val globPaths: Option[Seq[Path]],
val metaClient: HoodieTableMetaClient)
- extends BaseRelation with PrunedFilteredScan with Logging {
+ extends BaseRelation with PrunedFilteredScan with Logging with
SparkAdapterSupport {
private val conf = sqlContext.sparkContext.hadoopConfiguration
private val jobConf = new JobConf(conf)
// use schema from latest metadata, if not present, read schema from the
data file
private val schemaUtil = new TableSchemaResolver(metaClient)
- private lazy val tableAvroSchema = {
+ private lazy val (tableAvroSchema, internalSchema) = {
try {
- schemaUtil.getTableAvroSchema
+ val internalSchemaOpt =
schemaUtil.getTableInternalSchemaFromCommitMetadata
+ if (internalSchemaOpt.isPresent) {
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
