codope commented on code in PR #6098:
URL: https://github.com/apache/hudi/pull/6098#discussion_r926493385
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala:
##########
@@ -84,20 +96,62 @@ class HoodieStreamingSink(sqlContext: SQLContext,
var updatedOptions = options.updated(HoodieWriteConfig.MARKERS_TYPE.key(),
MarkerType.DIRECT.name())
// we need auto adjustment enabled for streaming sink since async table
services are feasible within the same JVM.
updatedOptions =
updatedOptions.updated(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key, "true")
+ // disable row writer bulk insert of write stream
+ if (options.getOrDefault(OPERATION.key,
UPSERT_OPERATION_OPT_VAL).equalsIgnoreCase(BULK_INSERT_OPERATION_OPT_VAL)) {
+ updatedOptions = updatedOptions.updated(ENABLE_ROW_WRITER.key, "false")
+ }
+
+ val queryId =
sqlContext.sparkContext.getLocalProperty(StreamExecution.QUERY_ID_KEY)
+ assert(queryId != null)
+ log.warn(s"Query id: $queryId")
+
+ if (metaClient != null) {
+ val lastCommit =
metaClient.getActiveTimeline.getCommitsTimeline.filterCompletedInstants().lastInstant()
+ if (lastCommit.isPresent) {
+ val commitMetadata =
HoodieCommitMetadata.fromBytes(metaClient.getActiveTimeline.getInstantDetails(lastCommit.get()).get(),
classOf[HoodieCommitMetadata])
+ val lastCheckpoint =
commitMetadata.getMetadata(HoodieWriteConfig.DELTASTREAMER_CHECKPOINT_KEY)
Review Comment:
yes this is not going to work w/ multi-writer. We need a better incremental
checkpoint management around this. Filed HUDI-4432 to track.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
