yihua commented on code in PR #12984:
URL: https://github.com/apache/hudi/pull/12984#discussion_r2001950535
##########
hudi-common/src/main/java/org/apache/hudi/metadata/RecordIndexRecordKeyParsingUtils.java:
##########
@@ -33,20 +33,17 @@
import org.apache.hadoop.fs.Path;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Stream;
import static java.util.stream.Collectors.toList;
-public class BaseFileRecordParsingUtils {
+public class RecordIndexRecordKeyParsingUtils {
Review Comment:
nit: rename to `RecordIndexUtils`
##########
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java:
##########
@@ -281,9 +282,16 @@ public void testRecordGenerationAPIsForMOR() throws
IOException {
assertTrue(compactionInstantOpt.isPresent());
HoodieWriteMetadata compactionWriteMetadata =
client.compact(compactionInstantOpt.get());
HoodieCommitMetadata compactionCommitMetadata = (HoodieCommitMetadata)
compactionWriteMetadata.getCommitMetadata().get();
- // no RLI records should be generated for compaction operation.
- assertTrue(convertMetadataToRecordIndexRecords(context,
compactionCommitMetadata, writeConfig.getMetadataConfig(),
- metaClient, writeConfig.getWritesFileIdEncoding(),
compactionInstantOpt.get(), EngineType.SPARK).isEmpty());
+
+ HoodieBackedTableMetadata tableMetadata = new
HoodieBackedTableMetadata(engineContext, metaClient.getStorage(),
writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true);
+ HoodieTableFileSystemView fsView = new
HoodieTableFileSystemView(tableMetadata, metaClient,
metaClient.getActiveTimeline());
+ try {
Review Comment:
try with resources for both `tableMetadata` and `fsView`?
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -1087,20 +1088,27 @@ engineContext, dataWriteConfig, commitMetadata,
instantTime, dataMetaClient, get
getMetadataPartitionsToUpdate(),
dataWriteConfig.getBloomFilterType(),
dataWriteConfig.getBloomIndexParallelism(),
dataWriteConfig.getWritesFileIdEncoding(), getEngineType(),
Option.of(dataWriteConfig.getRecordMerger().getRecordType()));
-
- // Updates for record index are created by parsing the WriteStatus which
is a hudi-client object. Hence, we cannot yet move this code
- // to the HoodieTableMetadataUtil class in hudi-common.
- if
(getMetadataPartitionsToUpdate().contains(RECORD_INDEX.getPartitionPath())) {
- HoodieData<HoodieRecord> additionalUpdates =
getRecordIndexAdditionalUpserts(partitionToRecordMap.get(RECORD_INDEX.getPartitionPath()),
commitMetadata);
- partitionToRecordMap.put(RECORD_INDEX.getPartitionPath(),
partitionToRecordMap.get(RECORD_INDEX.getPartitionPath()).union(additionalUpdates));
- }
+ updateRecordIndexRecordsIfPresent(commitMetadata, instantTime,
partitionToRecordMap);
updateExpressionIndexIfPresent(commitMetadata, instantTime,
partitionToRecordMap);
updateSecondaryIndexIfPresent(commitMetadata, partitionToRecordMap,
instantTime);
return partitionToRecordMap;
});
closeInternal();
}
+ private void updateRecordIndexRecordsIfPresent(HoodieCommitMetadata
commitMetadata, String instantTime, Map<String, HoodieData<HoodieRecord>>
partitionToRecordMap) {
+ if (!RECORD_INDEX.isMetadataPartitionAvailable(dataMetaClient)) {
Review Comment:
Should this still follow the same check as before:
`getMetadataPartitionsToUpdate().contains(RECORD_INDEX.getPartitionPath())`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
