Davis-Zhang-Onehouse commented on code in PR #13523:
URL: https://github.com/apache/hudi/pull/13523#discussion_r2229203636
##########
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java:
##########
@@ -631,6 +603,28 @@ private <T> ClosableIterator<T> lookupRecords(List<String>
sortedKeys,
}
}
+ static Predicate buildPredicate(String partitionName, List<String>
sortedKeys, boolean isFullKey) {
+ if (MetadataPartitionType.fromPartitionPath(partitionName)
+ .equals(MetadataPartitionType.SECONDARY_INDEX)) {
+ // For secondary index, always use prefix matching
Review Comment:
secondary index must make an exception because we reuse fullKey lookup API
for sec key lookup. We need to make exceptions to check partition path not
isFullKey in this case.
when fullkey is false - in SI v1 we have separate prefix index lookup and we
pass fullkey false.
##########
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java:
##########
@@ -563,63 +536,62 @@ private HoodieFileGroupReader<IndexedRecord>
buildFileGroupReader(List<String> s
private HoodiePairData<String, HoodieRecord<HoodieMetadataPayload>>
lookupKeyRecordPairs(String partitionName,
List<String> sortedKeys,
-
FileSlice fileSlice,
-
Boolean isFullKey) {
+
FileSlice fileSlice) {
Map<String, List<HoodieRecord<HoodieMetadataPayload>>> map = new
HashMap<>();
try (ClosableIterator<Pair<String, HoodieRecord<HoodieMetadataPayload>>>
iterator =
- lookupKeyRecordPairsItr(partitionName, sortedKeys, fileSlice,
isFullKey)) {
+ lookupKeyRecordPairsItr(partitionName, sortedKeys, fileSlice)) {
iterator.forEachRemaining(entry -> map.put(entry.getKey(),
Collections.singletonList(entry.getValue())));
}
return HoodieListPairData.eager(map);
}
private HoodieData<HoodieRecord<HoodieMetadataPayload>> lookupRecords(String
partitionName,
List<String> sortedKeys,
-
FileSlice fileSlice,
-
Boolean isFullKey) {
+
FileSlice fileSlice) {
List<HoodieRecord<HoodieMetadataPayload>> res = new ArrayList<>();
- try (ClosableIterator<HoodieRecord<HoodieMetadataPayload>> iterator =
lookupRecordsItr(partitionName, sortedKeys, fileSlice, isFullKey)) {
+ try (ClosableIterator<HoodieRecord<HoodieMetadataPayload>> iterator =
lookupRecordsItr(partitionName, sortedKeys, fileSlice)) {
iterator.forEachRemaining(res::add);
}
return HoodieListData.lazy(res);
}
private ClosableIterator<Pair<String, HoodieRecord<HoodieMetadataPayload>>>
lookupKeyRecordPairsItr(String partitionName,
List<String> sortedKeys,
-
FileSlice fileSlice,
-
Boolean isFullKey) {
- return lookupRecords(sortedKeys, fileSlice, isFullKey, metadataRecord -> {
+
FileSlice fileSlice) {
+ return lookupRecords(partitionName, sortedKeys, fileSlice, metadataRecord
-> {
Review Comment:
will do in a follow up
##########
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java:
##########
@@ -563,63 +536,62 @@ private HoodieFileGroupReader<IndexedRecord>
buildFileGroupReader(List<String> s
private HoodiePairData<String, HoodieRecord<HoodieMetadataPayload>>
lookupKeyRecordPairs(String partitionName,
List<String> sortedKeys,
-
FileSlice fileSlice,
-
Boolean isFullKey) {
+
FileSlice fileSlice) {
Map<String, List<HoodieRecord<HoodieMetadataPayload>>> map = new
HashMap<>();
try (ClosableIterator<Pair<String, HoodieRecord<HoodieMetadataPayload>>>
iterator =
- lookupKeyRecordPairsItr(partitionName, sortedKeys, fileSlice,
isFullKey)) {
+ lookupKeyRecordPairsItr(partitionName, sortedKeys, fileSlice)) {
iterator.forEachRemaining(entry -> map.put(entry.getKey(),
Collections.singletonList(entry.getValue())));
}
return HoodieListPairData.eager(map);
}
private HoodieData<HoodieRecord<HoodieMetadataPayload>> lookupRecords(String
partitionName,
List<String> sortedKeys,
-
FileSlice fileSlice,
-
Boolean isFullKey) {
+
FileSlice fileSlice) {
List<HoodieRecord<HoodieMetadataPayload>> res = new ArrayList<>();
- try (ClosableIterator<HoodieRecord<HoodieMetadataPayload>> iterator =
lookupRecordsItr(partitionName, sortedKeys, fileSlice, isFullKey)) {
+ try (ClosableIterator<HoodieRecord<HoodieMetadataPayload>> iterator =
lookupRecordsItr(partitionName, sortedKeys, fileSlice)) {
iterator.forEachRemaining(res::add);
}
return HoodieListData.lazy(res);
}
private ClosableIterator<Pair<String, HoodieRecord<HoodieMetadataPayload>>>
lookupKeyRecordPairsItr(String partitionName,
List<String> sortedKeys,
-
FileSlice fileSlice,
-
Boolean isFullKey) {
- return lookupRecords(sortedKeys, fileSlice, isFullKey, metadataRecord -> {
+
FileSlice fileSlice) {
+ return lookupRecords(partitionName, sortedKeys, fileSlice, metadataRecord
-> {
HoodieMetadataPayload payload = new
HoodieMetadataPayload(Option.of(metadataRecord));
String rowKey = payload.key != null ? payload.key :
metadataRecord.get(KEY_FIELD_NAME).toString();
HoodieKey hoodieKey = new HoodieKey(rowKey, partitionName);
return Pair.of(rowKey, new HoodieAvroRecord<>(hoodieKey, payload));
- });
+ }, true);
}
private ClosableIterator<HoodieRecord<HoodieMetadataPayload>>
lookupRecordsItr(String partitionName,
List<String> keys,
-
FileSlice fileSlice,
-
Boolean isFullKey) {
- return lookupRecords(keys, fileSlice, isFullKey,
+
FileSlice fileSlice) {
Review Comment:
will do in a follow up
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
