yihua commented on code in PR #13602:
URL: https://github.com/apache/hudi/pull/13602#discussion_r2231849527
##########
hudi-common/src/test/java/org/apache/hudi/common/table/read/TestFileGroupReaderSchemaHandler.java:
##########
@@ -111,4 +123,127 @@ FileGroupReaderSchemaHandler
createSchemaHandler(HoodieReaderContext<String> rea
return new FileGroupReaderSchemaHandler(readerContext, dataSchema,
requestedSchema,
Option.empty(), hoodieTableConfig, new TypedProperties());
}
+
+ @ParameterizedTest
+ @CsvSource({
+ "true, true, true, EVENT_TIME_ORDERING, false, EIGHT,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5",
+ "true, false, false, EVENT_TIME_ORDERING, false, EIGHT,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5",
+ "false, true, false, EVENT_TIME_ORDERING, false, EIGHT,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5",
+ "false, false, true, EVENT_TIME_ORDERING, false, EIGHT,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5",
+ "true, true, true, COMMIT_TIME_ORDERING, false, EIGHT,
ce9acb64-bde0-424c-9b91-f6ebba25356d",
+ "true, false, false, COMMIT_TIME_ORDERING, false, EIGHT,
ce9acb64-bde0-424c-9b91-f6ebba25356d",
+ "false, true, false, COMMIT_TIME_ORDERING, false, EIGHT,
ce9acb64-bde0-424c-9b91-f6ebba25356d",
+ "false, false, true, COMMIT_TIME_ORDERING, false, EIGHT,
ce9acb64-bde0-424c-9b91-f6ebba25356d",
+ "true, true, true, CUSTOM, false, EIGHT,
00000000-0000-0000-0000-000000000000",
+ "true, false, false, CUSTOM, false, EIGHT,
00000000-0000-0000-0000-000000000000",
+ "false, true, false, CUSTOM, false, EIGHT,
00000000-0000-0000-0000-000000000000",
+ "false, false, true, CUSTOM, false, EIGHT,
00000000-0000-0000-0000-000000000000",
+ "true, true, true, , false, EIGHT, 00000000-0000-0000-0000-000000000000",
+ "true, false, false, , false, EIGHT,
00000000-0000-0000-0000-000000000000",
+ "false, true, false, , false, EIGHT,
00000000-0000-0000-0000-000000000000",
+ "false, false, true, , false, EIGHT,
00000000-0000-0000-0000-000000000000",
+ "true, true, true, EVENT_TIME_ORDERING, false, SIX,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5",
+ "true, false, false, EVENT_TIME_ORDERING, false, SIX,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5",
+ "false, true, false, EVENT_TIME_ORDERING, false, SIX,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5",
+ "false, false, true, EVENT_TIME_ORDERING, false, SIX,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5",
+ "true, true, true, COMMIT_TIME_ORDERING, false, SIX,
ce9acb64-bde0-424c-9b91-f6ebba25356d",
+ "true, false, false, COMMIT_TIME_ORDERING, false, SIX,
ce9acb64-bde0-424c-9b91-f6ebba25356d",
+ "false, true, false, COMMIT_TIME_ORDERING, false, SIX,
ce9acb64-bde0-424c-9b91-f6ebba25356d",
+ "false, false, true, COMMIT_TIME_ORDERING, false, SIX,
ce9acb64-bde0-424c-9b91-f6ebba25356d",
+ "true, true, true, CUSTOM, false, SIX,
00000000-0000-0000-0000-000000000000",
+ "true, false, false, CUSTOM, false, SIX,
00000000-0000-0000-0000-000000000000",
+ "false, true, false, CUSTOM, false, SIX,
00000000-0000-0000-0000-000000000000",
+ "false, false, true, CUSTOM, false, SIX,
00000000-0000-0000-0000-000000000000",
+ "true, true, true, , false, SIX, 00000000-0000-0000-0000-000000000000",
+ "true, false, false, , false, SIX, 00000000-0000-0000-0000-000000000000",
+ "false, true, false, , false, SIX, 00000000-0000-0000-0000-000000000000",
+ "false, false, true, , false, SIX, 00000000-0000-0000-0000-000000000000",
+ "true, true, true, COMMIT_TIME_ORDERING, true, SIX,
eeb8d96f-b1e4-49fd-bbf8-28ac514178e5", /// with table version 6, commit time
based merge mode can have event time based merge strategy id.
+ })
+ public void testSchemaForMandatoryFields(boolean setPrecombine,
+ boolean addHoodieIsDeleted,
+ boolean addCustomDeleteMarker,
+ RecordMergeMode mergeMode,
+ boolean isProjectionCompatible,
+ HoodieTableVersion tableVersion,
+ String mergeStrategyId) {
+ HoodieReaderContext readerContext = mock(HoodieReaderContext.class);
+ when(readerContext.getInstantRange()).thenReturn(Option.empty());
+ when(readerContext.getHasBootstrapBaseFile()).thenReturn(false);
+ when(readerContext.getHasLogFiles()).thenReturn(true);
+ HoodieRecordMerger recordMerger = mock(HoodieRecordMerger.class);
+ when(readerContext.getRecordMerger()).thenReturn(Option.of(recordMerger));
+
when(recordMerger.isProjectionCompatible()).thenReturn(isProjectionCompatible);
+
+ String preCombineField = "ts";
+ String customDeleteKey = "colC";
+ String customDeleteValue = "D";
+ List<String> dataSchemaFields = new ArrayList<>();
+ dataSchemaFields.addAll(Arrays.asList(
+ HoodieRecord.RECORD_KEY_METADATA_FIELD,
HoodieRecord.PARTITION_PATH_METADATA_FIELD, preCombineField,
+ "colA", "colB", "colC", "colD"));
+ if (addHoodieIsDeleted) {
+ dataSchemaFields.add(HoodieRecord.HOODIE_IS_DELETED_FIELD);
+ }
+
+ Schema dataSchema = getSchema(dataSchemaFields);
+ Schema requestedSchema =
getSchema(Arrays.asList(HoodieRecord.RECORD_KEY_METADATA_FIELD,
HoodieRecord.PARTITION_PATH_METADATA_FIELD));
+
+ HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getRecordMergeMode()).thenReturn(mergeMode);
+ when(tableConfig.populateMetaFields()).thenReturn(true);
+ when(tableConfig.getPreCombineField()).thenReturn(setPrecombine ?
preCombineField : StringUtils.EMPTY_STRING);
+ when(tableConfig.getTableVersion()).thenReturn(tableVersion);
+ if (tableConfig.getTableVersion() == HoodieTableVersion.SIX) {
+ if (mergeMode == RecordMergeMode.EVENT_TIME_ORDERING) {
+
when(tableConfig.getPayloadClass()).thenReturn(DefaultHoodieRecordPayload.class.getName());
+ } else if (mergeMode == RecordMergeMode.COMMIT_TIME_ORDERING) {
+
when(tableConfig.getPayloadClass()).thenReturn(OverwriteWithLatestAvroPayload.class.getName());
+ } else {
+
when(tableConfig.getPayloadClass()).thenReturn(OverwriteNonDefaultsWithLatestAvroPayload.class.getName());
+ }
+ }
+ if (mergeMode != null) {
+ when(tableConfig.getRecordMergeStrategyId()).thenReturn(mergeStrategyId);
+ }
+
+ TypedProperties props = new TypedProperties();
+ if (addCustomDeleteMarker) {
+ props.setProperty(DELETE_KEY, customDeleteKey);
+ props.setProperty(DELETE_MARKER, customDeleteValue);
+ }
+
+ List<String> expectedFields = new ArrayList();
+ expectedFields.add(HoodieRecord.RECORD_KEY_METADATA_FIELD);
+ expectedFields.add(HoodieRecord.PARTITION_PATH_METADATA_FIELD);
+ if (addCustomDeleteMarker) {
+ expectedFields.add(customDeleteKey);
+ }
+ if (setPrecombine && mergeMode != RecordMergeMode.COMMIT_TIME_ORDERING) {
// commit time ordering does not project ordering field.
+ expectedFields.add(preCombineField);
+ }
+ if (addHoodieIsDeleted) {
+ expectedFields.add(HoodieRecord.HOODIE_IS_DELETED_FIELD);
+ }
+ Schema expectedSchema = ((mergeMode == RecordMergeMode.CUSTOM) &&
!isProjectionCompatible) ? dataSchema : getSchema(expectedFields);
+ when(recordMerger.getMandatoryFieldsForMerging(dataSchema, tableConfig,
props)).thenReturn(expectedFields.toArray(new String[0]));
+
+ FileGroupReaderSchemaHandler fileGroupReaderSchemaHandler = new
FileGroupReaderSchemaHandler(readerContext,
+ dataSchema, requestedSchema, Option.empty(), tableConfig, props);
+ Schema actualSchema =
fileGroupReaderSchemaHandler.generateRequiredSchema();
+ assertEquals(expectedSchema, actualSchema);
+ assertEquals(addHoodieIsDeleted,
fileGroupReaderSchemaHandler.hasBuiltInDelete());
+ assertEquals(addCustomDeleteMarker
+ ? Option.of(Pair.of(customDeleteKey, customDeleteValue)) :
Option.empty(),
+ fileGroupReaderSchemaHandler.getCustomDeleteMarkerKeyValue());
+ }
+
+ private Schema getSchema(List<String> fields) {
Review Comment:
nit: should this be a common test util since the same method exists in
`TestFileGroupRecordBuffer`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
