codope commented on code in PR #7668:
URL: https://github.com/apache/hudi/pull/7668#discussion_r1085250796
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala:
##########
@@ -185,7 +191,9 @@ object HoodieWriterUtils {
/**
* Detects conflicts between datasourceKeyGen and existing table
configuration keyGen
*/
- def validateKeyGeneratorConfig(datasourceKeyGen: String, tableConfig:
HoodieConfig): Unit = {
+ def validateAndSetKeyGeneratorConfigs(datasourceKeyGen: String, log: Logger,
hoodieConfig: HoodieConfig,
Review Comment:
better to define a separate log for this object than passing from the caller.
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala:
##########
@@ -203,6 +211,44 @@ object HoodieWriterUtils {
diffConfigs.insert(0, "\nConfig conflict(key\tcurrent value\texisting
value):\n")
throw new HoodieException(diffConfigs.toString.trim)
}
+
+ val parameters = mutable.Map() ++ inputParams
+ if
(hoodieConfig.getBoolean(KeyGeneratorOptions.AUTO_GENERATE_RECORD_KEYS)) {
+ val autoGenerateRecordKey =
KeyGeneratorOptions.AUTO_GENERATE_RECORD_KEYS.key()
+ if (hoodieConfig.getBoolean(HoodieWriteConfig.COMBINE_BEFORE_INSERT)) {
+ throw new HoodieKeyGeneratorException(s"Config $autoGenerateRecordKey
can not be used when " +
+ s"${HoodieWriteConfig.COMBINE_BEFORE_INSERT.key()} is enabled")
+ }
+ if
(!hoodieConfig.getBoolean(HoodieWriteConfig.MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE))
{
+
hoodieConfig.setValue(HoodieWriteConfig.MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE,
"true")
+ parameters +=
(HoodieWriteConfig.MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE.key() -> "true")
+ log.warn(s"Enabling config
{${HoodieWriteConfig.MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE.key()}} when
$autoGenerateRecordKey is used")
+ }
+ if (hoodieConfig.getString(DataSourceWriteOptions.TABLE_TYPE) ==
MOR_TABLE_TYPE_OPT_VAL) {
+ throw new HoodieKeyGeneratorException(s"Config
${DataSourceWriteOptions.TABLE_TYPE.key()} should be set to " +
+ s"COW_TABLE_TYPE_OPT_VAL when $autoGenerateRecordKey is used")
+ }
+ // If OPERATION is explicitly set as UPSERT by the user, throw an
exception. If user is using default value then
+ // operation is overridden to INSERT_OPERATION_OPT_VAL
+ if (parameters.getOrElse(OPERATION.key(), StringUtils.EMPTY_STRING) ==
UPSERT_OPERATION_OPT_VAL) {
+ throw new HoodieKeyGeneratorException(s"Config ${OPERATION.key()}
should not be set to $UPSERT_OPERATION_OPT_VAL" +
+ s" when $autoGenerateRecordKey is used")
+ } else if (hoodieConfig.getString(OPERATION) ==
UPSERT_OPERATION_OPT_VAL) {
+ hoodieConfig.setValue(OPERATION.key(), INSERT_OPERATION_OPT_VAL)
+ parameters += (OPERATION.key() -> INSERT_OPERATION_OPT_VAL)
+ log.warn(s"Setting config ${OPERATION.key()} to
$INSERT_OPERATION_OPT_VAL when $autoGenerateRecordKey is used")
+ }
+ if (hoodieConfig.getString(DataSourceWriteOptions.PRECOMBINE_FIELD) !=
DataSourceWriteOptions.PRECOMBINE_FIELD.defaultValue()) {
Review Comment:
Shouldn't we check for precombine field in `parameters`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
