nsivabalan commented on a change in pull request #3315:
URL: https://github.com/apache/hudi/pull/3315#discussion_r678682342
##########
File path:
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java
##########
@@ -206,16 +206,28 @@ private static Configuration
addProjectionField(Configuration conf, String field
public static void addRequiredProjectionFields(Configuration configuration) {
// Need this to do merge records in HoodieRealtimeRecordReader
- addProjectionField(configuration, HoodieRecord.RECORD_KEY_METADATA_FIELD,
HoodieInputFormatUtils.HOODIE_RECORD_KEY_COL_POS);
- addProjectionField(configuration, HoodieRecord.COMMIT_TIME_METADATA_FIELD,
HoodieInputFormatUtils.HOODIE_COMMIT_TIME_COL_POS);
- addProjectionField(configuration,
HoodieRecord.PARTITION_PATH_METADATA_FIELD,
HoodieInputFormatUtils.HOODIE_PARTITION_PATH_COL_POS);
+ if (configuration.get(HoodieInputFormatUtils.HOODIE_USE_META_FIELDS) ==
null || configuration.get(HoodieInputFormatUtils.HOODIE_USE_META_FIELDS)
+ .equals(HoodieInputFormatUtils.DEFAULT_HOODIE_USE_META_FIELDS)) {
+ addProjectionField(configuration,
HoodieRecord.RECORD_KEY_METADATA_FIELD,
HoodieInputFormatUtils.HOODIE_RECORD_KEY_COL_POS);
+ addProjectionField(configuration,
HoodieRecord.COMMIT_TIME_METADATA_FIELD,
HoodieInputFormatUtils.HOODIE_COMMIT_TIME_COL_POS);
+ addProjectionField(configuration,
HoodieRecord.PARTITION_PATH_METADATA_FIELD,
HoodieInputFormatUtils.HOODIE_PARTITION_PATH_COL_POS);
+ } else {
+ addProjectionField(configuration,
configuration.get(HoodieInputFormatUtils.RECORD_KEY_FIELD),
Integer.parseInt(configuration.get(HoodieInputFormatUtils.RECORD_KEY_FIELD_INDEX)));
+ addProjectionField(configuration,
configuration.get(HoodieInputFormatUtils.PARTITION_PATH_FIELD),
Integer.parseInt(configuration.get(HoodieInputFormatUtils.PARTITION_PATH_FIELD_INDEX)));
+ }
}
public static boolean requiredProjectionFieldsExistInConf(Configuration
configuration) {
String readColNames =
configuration.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, "");
- return readColNames.contains(HoodieRecord.RECORD_KEY_METADATA_FIELD)
- && readColNames.contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD)
- && readColNames.contains(HoodieRecord.PARTITION_PATH_METADATA_FIELD);
+ if (configuration.get(HoodieInputFormatUtils.HOODIE_USE_META_FIELDS) ==
null || configuration.get(HoodieInputFormatUtils.HOODIE_USE_META_FIELDS)
+ .equals(HoodieInputFormatUtils.DEFAULT_HOODIE_USE_META_FIELDS)) {
+ return readColNames.contains(HoodieRecord.RECORD_KEY_METADATA_FIELD)
+ && readColNames.contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD)
+ && readColNames.contains(HoodieRecord.PARTITION_PATH_METADATA_FIELD);
+ } else {
+ return readColNames.contains(HoodieInputFormatUtils.RECORD_KEY_FIELD)
Review comment:
when meta fields are disabled we don't project the commit time field.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
