danny0405 commented on code in PR #20516:
URL: https://github.com/apache/hudi/pull/20516#discussion_r4533509810
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala:
##########
@@ -651,6 +651,130 @@ class TestStreamingSource extends StreamTest {
)
}
+ Seq(COPY_ON_WRITE, MERGE_ON_READ).foreach { tableType =>
+ Seq(false, true).foreach { completionTime =>
+ test(s"guarded source rejects resets and recreation: $tableType
completionTime=$completionTime") {
+ withTempDir { dir =>
+ val tablePath = s"${dir.getCanonicalPath}/guarded"
+ val conf =
HadoopFSUtils.getStorageConf(spark.sessionState.newHadoopConf())
+ def initialize(): HoodieTableMetaClient =
HoodieTableMetaClient.newTableBuilder()
+
.setTableType(tableType).setTableName("guarded").setRecordKeyFields("id")
+ .setOrderingFields("ts").initTable(conf, tablePath)
+ val metaClient = initialize()
+ addData(tablePath, Seq(("1", "old", "10", "001")))
+ def source(enabled: Option[Boolean] = Some(true)):
org.apache.spark.sql.execution.streaming.Source = {
+ val options = commonOptions + ("path" -> tablePath) ++ enabled.map
{ value =>
+ DataSourceReadOptions.STREAMING_READ_FAIL_ON_DATA_LOSS.key() ->
value.toString
+ }
+ val checkpoint = s"${dir.getCanonicalPath}/checkpoint"
+ if (completionTime) {
+ new
org.apache.spark.sql.hudi.streaming.HoodieStreamSourceV2(spark.sqlContext,
+ metaClient, checkpoint, None, options,
+
org.apache.spark.sql.hudi.streaming.HoodieEarliestOffsetRangeLimit,
+ HoodieTableVersion.current())
+ } else {
+ new
org.apache.spark.sql.hudi.streaming.HoodieStreamSourceV1(spark.sqlContext,
+ metaClient, checkpoint, None, options,
+
org.apache.spark.sql.hudi.streaming.HoodieEarliestOffsetRangeLimit,
+ HoodieTableVersion.current())
+ }
+ }
+ val reader = source()
+ val saved = reader.getOffset.get
+ addData(tablePath, Seq(("2", "new", "20", "002")),
enableInlineCluster = true)
+ assert(reader.getBatch(Some(saved),
reader.getOffset.get).isStreaming)
+
assert(metaClient.reloadActiveTimeline().getCompletedReplaceTimeline.countInstants()
> 0)
+ val beforeResetEnd = reader.getOffset.get
+ Seq(("3", "replacement", "30", "003")).toDF(columns:
_*).write.format("hudi")
+ .options(commonOptions).option(TBL_NAME.key, "guarded")
+ .option("hoodie.datasource.write.operation",
"insert_overwrite_table")
+ .mode(SaveMode.Append).save(tablePath)
+ val resetEnd = reader.getOffset.get
+ val resetError =
intercept[org.apache.hudi.exception.HoodieException] {
+ reader.getBatch(Some(saved), resetEnd)
+ }
+ assert(resetError.getMessage.contains("rebuild required"))
+ assert(resetError.getMessage.contains("logical reset"))
+ val concurrentResetError =
intercept[org.apache.hudi.exception.HoodieException] {
+ reader.getBatch(Some(saved), beforeResetEnd)
+ }
+ assert(concurrentResetError.getMessage.contains("logical reset"))
+ val sessionReader = withSQLConf(
Review Comment:
The earlier fix assigns the source to a var inside withSQLConf, avoiding
Spark 3.5's Unit return type. Revalidated with the November 11 changes in
57883c3ccfe: Spark 3.5 / Scala 2.12 / JDK 17 reactor build and the full source
suite passed (24 tests, 2 expected variant skips).
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV2.scala:
##########
@@ -51,6 +51,10 @@ class HoodieStreamSourceV2(sqlContext: SQLContext,
writeTableVersion: HoodieTableVersion)
extends Source with Logging with Serializable with SparkAdapterSupport {
+ // Capture flow-scoped SQLConf once; explicit reader options take precedence.
+ private val recoveryStrategy = HoodieStreamingSourceSafety.strategy(
+ DataSourceOptionsHelper.collectSparkHoodieConfs(sqlContext, parameters))
Review Comment:
Fixed in 57883c3ccfe: both sources now use collectHoodieAndSparkHoodieConfs.
Tests exercise bare hoodie.* and spark.hoodie.* SQLConf, construct the source
inside the scope, execute outside it, and verify explicit reader options still
win.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]