yihua commented on code in PR #11943:
URL: https://github.com/apache/hudi/pull/11943#discussion_r1812187657
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala:
##########
@@ -167,9 +166,21 @@ object HoodieWriterUtils {
if (!isOverWriteMode) {
val resolver = spark.sessionState.conf.resolver
val diffConfigs = StringBuilder.newBuilder
+ val payloadIsExpressionPayload =
params.getOrElse(PAYLOAD_CLASS_NAME.key(),
"").equals("org.apache.spark.sql.hudi.command.payload.ExpressionPayload")
params.foreach { case (key, value) =>
+ var ignoreConfig = false
// Base file format can change between writes, so ignore it.
- if (!HoodieTableConfig.BASE_FILE_FORMAT.key.equals(key)) {
+ ignoreConfig = ignoreConfig ||
HoodieTableConfig.BASE_FILE_FORMAT.key.equals(key)
+
+ //expression payload will never be the table config so skip validation
of merge configs
+ ignoreConfig = ignoreConfig || (payloadIsExpressionPayload &&
(key.equals(PAYLOAD_CLASS_NAME.key())
+ || key.equals(HoodieTableConfig.PAYLOAD_CLASS_NAME.key()) ||
key.equals(RECORD_MERGE_MODE.key())
+ || key.equals(RECORD_MERGER_STRATEGY_ID.key())))
+
+ //don't validate the payload only in the case that insert into is
using fallback to some legacy configs
+ ignoreConfig = ignoreConfig || (key.equals(PAYLOAD_CLASS_NAME.key())
&&
value.equals("org.apache.spark.sql.hudi.command.ValidateDuplicateKeyPayload"))
Review Comment:
HUDI-8426 as a minor follow-up.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
