jonvex commented on code in PR #10957:
URL: https://github.com/apache/hudi/pull/10957#discussion_r1571150439
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala:
##########
@@ -71,24 +64,36 @@ class
HoodieFileGroupReaderBasedParquetFileFormat(tableState: HoodieTableState,
isIncremental: Boolean,
shouldUseRecordPosition:
Boolean,
requiredFilters: Seq[Filter]
- ) extends ParquetFileFormat with
SparkAdapterSupport with HoodieFormatTrait {
+ ) extends ParquetFileFormat
with SparkAdapterSupport with HoodieFormatTrait {
def getRequiredFilters: Seq[Filter] = requiredFilters
+ private val sanitizedTableName =
AvroSchemaUtils.getAvroRecordQualifiedName(tableName)
+
/**
* Support batch needs to remain consistent, even if one side of a bootstrap
merge can support
* while the other side can't
*/
- private var supportBatchCalled = false
- private var supportBatchResult = false
+ /*
+private var supportBatchCalled = false
+private var supportBatchResult = false
+
+override def supportBatch(sparkSession: SparkSession, schema: StructType):
Boolean = {
+ if (!supportBatchCalled || supportBatchResult) {
+ supportBatchCalled = true
+ supportBatchResult = tableSchema.internalSchema.isEmpty && !isMOR &&
!isIncremental && !isBootstrap && super.supportBatch(sparkSession, schema)
+ }
+ supportBatchResult
+}
+ */
+ override def supportBatch(sparkSession: SparkSession, schema: StructType):
Boolean = false
Review Comment:
I couldn't get batch working right, but it is still something that we need
to get fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
