cshuo commented on code in PR #19205:
URL: https://github.com/apache/hudi/pull/19205#discussion_r3643621674
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java:
##########
@@ -3883,6 +3919,33 @@ private void validate() {
checkArgument(ttlStatsMaxParallelism > 0,
String.format("%s must be positive, but was %d",
HoodieTTLConfig.STATS_MAX_PARALLELISM.key(),
ttlStatsMaxParallelism));
+
+ // hoodie.meta.fields.mode is an additive opt-in on top of
populate.meta.fields=false. Setting
+ // populate.meta.fields=true together with a non-ALL mode is ambiguous
(the mode has no effect
+ // when all meta fields are already populated) so reject it explicitly
rather than silently
+ // ignore. MetaFieldsMode.fromConfig also throws on unrecognized on-disk
values.
+ MetaFieldsMode metaFieldsMode = writeConfig.getMetaFieldsMode();
+ boolean populateMetaFields = writeConfig.populateMetaFields();
+ String rawMode =
writeConfig.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE);
+ checkArgument(!(populateMetaFields && rawMode != null &&
!rawMode.isEmpty()),
+ String.format("%s must be empty when %s=true. Disable
populate.meta.fields or clear the mode.",
+ HoodieTableConfig.META_FIELDS_MODE.key(),
+ HoodieTableConfig.POPULATE_META_FIELDS.key()));
+ // Selective meta-field modes are CoW-only in this release. MoR
log-write path does not yet
+ // respect the mode, which would silently produce log records with null
meta columns.
+ boolean isSelective = metaFieldsMode != MetaFieldsMode.ALL &&
metaFieldsMode != MetaFieldsMode.NONE;
+ checkArgument(!(writeConfig.getTableType() ==
HoodieTableType.MERGE_ON_READ && isSelective),
+ String.format("%s=%s is currently supported for COPY_ON_WRITE tables
only. MoR support is a follow-up. "
+ + "For MoR either keep %s=true or use NONE mode.",
+ HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode,
+ HoodieTableConfig.POPULATE_META_FIELDS.key()));
+ // Selective meta-field modes are wired only for the Spark writer path
in this release. Flink
+ // RowData / Java-client writers ignore the mode and would silently
produce NONE-mode output.
+ checkArgument(!(engineType != EngineType.SPARK && isSelective),
Review Comment:
HIGH: Only Parquet receives MetaFieldsMode, while Spark CoW ORC and other
formats can pass validation. Evidence: HoodieAvroFileWriterFactory.java:132-141
creates the ORC writer without passing MetaFieldsMode. This can make the table
configuration disagree with the physical data. Either support these formats or
restrict selective modes to Parquet.
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java:
##########
@@ -478,6 +478,8 @@ HoodieTableMetaClient
initializeEmptyTable(HoodieTableMetaClient.TableBuilder ta
.setRecordKeyFields(props.getProperty(DataSourceWriteOptions.RECORDKEY_FIELD().key()))
.setPopulateMetaFields(props.getBoolean(HoodieTableConfig.POPULATE_META_FIELDS.key(),
HoodieTableConfig.POPULATE_META_FIELDS.defaultValue()))
+
.setMetaFieldsModeFromString(props.getString(HoodieTableConfig.META_FIELDS_MODE.key(),
Review Comment:
HIGH: For an existing COMMIT_TIME_ONLY table, restarting Streamer with only
hoodie.populate.meta.fields=false resolves the write mode to NONE. New records
may have null commit times, causing incremental queries to miss data.
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java:
##########
@@ -158,10 +159,45 @@ public HoodieRowCreateHandle(HoodieTable table,
* @throws IOException
*/
public void write(InternalRow row) throws IOException {
- if (populateMetaFields) {
- writeRow(row);
- } else {
- writeRowNoMetaFields(row);
+ switch (metaFieldsMode) {
+ case ALL:
+ writeRow(row);
+ break;
+ case NONE:
+ writeRowNoMetaFields(row);
+ break;
+ default:
+ writeRowSelectiveMetaFields(row);
+ break;
+ }
+ }
+
+ /**
+ * Selective meta-field write path: populate only the meta columns opted in
via
+ * {@code hoodie.meta.fields.mode} — {@code _hoodie_commit_time} and/or
{@code _hoodie_file_name}.
+ * The other meta columns stay null on disk. Record key is never populated
in this path, so the
+ * record key is not registered with the write support (bloom filter / RLI
hooks are meaningless
+ * without the record-key column).
+ */
+ private void writeRowSelectiveMetaFields(InternalRow row) {
+ try {
+ UTF8String[] metaFields = new UTF8String[5];
+ if (metaFieldsMode.isCommitTimePopulated()) {
+ metaFields[HoodieRecord.COMMIT_TIME_METADATA_FIELD_ORD] =
shouldPreserveHoodieMetadata
+ ? row.getUTF8String(HoodieRecord.COMMIT_TIME_METADATA_FIELD_ORD) :
commitTime;
+ }
+ if (metaFieldsMode.isFileNamePopulated()) {
+ metaFields[HoodieRecord.FILENAME_META_FIELD_ORD] =
shouldPreserveHoodieMetadata
Review Comment:
HIGH: Row-writer clustering copies the old _hoodie_file_name when preserving
metadata. Rewritten records then reference replaced files. The file name should
always be updated to the new output file.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
