yihua commented on code in PR #12843:
URL: https://github.com/apache/hudi/pull/12843#discussion_r2010931567
##########
hudi-common/src/main/java/org/apache/hudi/common/table/read/PositionBasedFileGroupRecordBuffer.java:
##########
@@ -202,6 +203,15 @@ public void processDeleteBlock(HoodieDeleteBlock
deleteBlock) throws IOException
}
}
+ protected void processCustomDeleteRecord(T record, Map<String, Object>
metadata, long recordPosition) {
+ DeleteRecord deleteRecord = DeleteRecord.create(
+ new HoodieKey(
+ (String) metadata.get(INTERNAL_META_RECORD_KEY), null),
Review Comment:
similar on documenting the `null` here.
##########
hudi-common/src/main/java/org/apache/hudi/common/table/read/FileGroupRecordBuffer.java:
##########
@@ -143,6 +149,69 @@ public
HoodieBaseFileGroupRecordBuffer(HoodieReaderContext<T> readerContext,
} catch (IOException e) {
throw new HoodieIOException("IOException when creating
ExternalSpillableMap at " + spillableMapBasePath, e);
}
+ // Initialize custom delete marker related variables.
+ initCustomDeleteConfigs();
+ }
+
+ protected void initCustomDeleteConfigs() {
+ this.shouldCheckCustomDeleteMarker = hasCustomDeleteConfigs(props,
readerSchema);
+ if (shouldCheckCustomDeleteMarker) {
+ this.customDeleteMarkerKey = props.getProperty(DELETE_KEY);
+ this.customDeleteMarkerValue = props.getProperty(DELETE_MARKER);
+ }
+ this.shouldCheckHoodieDeleteMarker = hasHoodieDeleteField(readerSchema);
+ }
+
+ /**
+ * Check if
+ * 1. delete key and delete markers are set properly.
+ * 2. schema contains delete key column.
+ */
+ static boolean hasCustomDeleteConfigs(TypedProperties props, Schema schema) {
+ String deleteKey = props.getProperty(DELETE_KEY);
+ String deleteMarker = props.getProperty(DELETE_MARKER);
+ boolean deleteKeyExists = !StringUtils.isNullOrEmpty(deleteKey);
+ boolean deleteMarkerExists = !StringUtils.isNullOrEmpty(deleteMarker);
+
+ // DELETE_KEY and DELETE_MARKER both should be set.
+ if (deleteKeyExists && deleteMarkerExists) {
+ // DELETE_KEY field exists in the schema.
+ return schema.getField(deleteKey) != null;
+ } else if (!deleteKeyExists && !deleteMarkerExists) {
+ // Normal case.
+ return false;
+ } else {
+ throw new RuntimeException("Either custom delete key or marker is not
specified");
Review Comment:
Use `IllegalArgumentException` instead of `RuntimeException`
##########
hudi-common/src/main/java/org/apache/hudi/common/table/read/KeyBasedFileGroupRecordBuffer.java:
##########
@@ -123,6 +132,18 @@ public void processNextDeletedRecord(DeleteRecord
deleteRecord, Serializable rec
}
}
+ protected void processCustomDeleteRecord(T record, Map<String, Object>
metadata) {
+ DeleteRecord deleteRecord = DeleteRecord.create(
+ new HoodieKey(
+ (String) metadata.get(INTERNAL_META_RECORD_KEY),
+ null),
Review Comment:
nit: add a comment on the `null` here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
