This is an automated email from the ASF dual-hosted git repository. sivabalan pushed a commit to branch release-0.10.1-rc1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c4673b228d2da65804272d07809748fa7e61b5f8 Author: Sagar Sumit <[email protected]> AuthorDate: Wed Jan 5 18:39:58 2022 +0530 [HUDI-3170] Do not preserve filename when preserveCommitMetadata enabled (#4512) --- .../src/main/java/org/apache/hudi/io/HoodieCreateHandle.java | 2 ++ .../hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java index 01ad453..a110cd9 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java @@ -139,6 +139,8 @@ public class HoodieCreateHandle<T extends HoodieRecordPayload, I, K, O> extends // Convert GenericRecord to GenericRecord with hoodie commit metadata in schema IndexedRecord recordWithMetadataInSchema = rewriteRecord((GenericRecord) avroRecord.get()); if (preserveHoodieMetadata) { + // do not preserve FILENAME_METADATA_FIELD + recordWithMetadataInSchema.put(HoodieRecord.HOODIE_META_COLUMNS_NAME_TO_POS.get(HoodieRecord.FILENAME_METADATA_FIELD), path.getName()); fileWriter.writeAvro(record.getRecordKey(), recordWithMetadataInSchema); } else { fileWriter.writeAvroWithMetadata(recordWithMetadataInSchema, record); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index 566db22..aa3ead4 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -1836,9 +1836,12 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { Map<String, List<GenericRecord>> recordsByCommitTime = records.stream() .collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString())); assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet())); + Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet()); for (GenericRecord record : records) { String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString(); assertTrue(expectedKeys.contains(recordKey)); + String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString(); + assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName))); } }
