manojpec commented on a change in pull request #4352:
URL: https://github.com/apache/hudi/pull/4352#discussion_r786388149
##########
File path:
hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
##########
@@ -146,6 +155,97 @@ protected BaseTableMetadata(HoodieEngineContext
engineContext, HoodieMetadataCon
.getAllFilesInPartitions(partitions);
}
+ @Override
+ public Option<ByteBuffer> getBloomFilter(final PartitionIndexID
partitionIndexID, final FileIndexID fileIndexID)
+ throws HoodieMetadataException {
+ if (!isMetaIndexBloomFilterEnabled) {
+ LOG.error("Meta bloom filter index is disabled!");
+ return Option.empty();
+ }
+
+ HoodieTimer timer = new HoodieTimer().startTimer();
+ final String bloomIndexKey =
partitionIndexID.asBase64EncodedString().concat(fileIndexID.asBase64EncodedString());
+ Option<HoodieRecord<HoodieMetadataPayload>> hoodieRecord =
getRecordByKey(bloomIndexKey,
+ MetadataPartitionType.BLOOM_FILTERS.partitionPath());
+ metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_METADATA_STR,
timer.endTimer()));
+
+ if (!hoodieRecord.isPresent()) {
+ LOG.error("Meta bloom filter index: lookup failed for partition: " +
partitionIndexID.getName() + ", file: "
+ + fileIndexID.getName());
+ return Option.empty();
+ }
+
+ final Option<HoodieMetadataBloomFilter> fileBloomFilter =
hoodieRecord.get().getData().getBloomFilterMetadata();
+ if (!fileBloomFilter.isPresent()) {
+ LOG.error("Meta bloom filter index: bloom filter missing for partition:
" + partitionIndexID.getName()
+ + ", file: " + fileIndexID.getName());
+ return Option.empty();
+ }
+
+ return Option.of(fileBloomFilter.get().getBloomFilter());
+ }
+
+ @Override
+ public Map<String, ByteBuffer> getBloomFilters(final
List<Pair<PartitionIndexID, FileIndexID>> partitionFileIndexIDList)
+ throws HoodieMetadataException {
+ if (!isMetaIndexBloomFilterEnabled) {
+ LOG.error("Meta bloom filter index is disabled!");
+ return Collections.emptyMap();
+ }
+
+ HoodieTimer timer = new HoodieTimer().startTimer();
+ Set<String> partitionIDFileIDSortedStrings = new TreeSet<>();
+ partitionFileIndexIDList.forEach(partitionIDFileIDPair -> {
+ final String bloomKey =
partitionIDFileIDPair.getLeft().asBase64EncodedString()
+
.concat(partitionIDFileIDPair.getRight().asBase64EncodedString());
+ partitionIDFileIDSortedStrings.add(bloomKey);
+ }
+ );
+ List<String> partitionIDFileIDStrings = new
ArrayList<>(partitionIDFileIDSortedStrings);
+
+ List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>>
hoodieRecordList =
+ getRecordsByKeys(partitionIDFileIDStrings,
MetadataPartitionType.BLOOM_FILTERS.partitionPath());
+ metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_METADATA_STR,
timer.endTimer()));
+
+ Map<String, ByteBuffer> fileToBloomFilterMap = new HashMap<>();
+ for (final Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>> entry
: hoodieRecordList) {
+ if (entry.getRight().isPresent()) {
+ final Option<HoodieMetadataBloomFilter> optionalBloomFilterMetadata =
+ entry.getRight().get().getData().getBloomFilterMetadata();
+ if (optionalBloomFilterMetadata.isPresent()) {
+ fileToBloomFilterMap.put(entry.getLeft(),
optionalBloomFilterMetadata.get().getBloomFilter());
+ }
+ }
+ }
+ return fileToBloomFilterMap;
+ }
+
+ @Override
+ public Map<String, HoodieColumnStats> getColumnStats(List<String> keySet)
throws HoodieMetadataException {
+ if (!isMetaIndexColumnStatsEnabled) {
+ LOG.error("Meta column range index is disabled!");
+ return Collections.emptyMap();
+ }
+
+ HoodieTimer timer = new HoodieTimer().startTimer();
+ List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>>
hoodieRecordList =
+ getRecordsByKeys(keySet,
MetadataPartitionType.COLUMN_STATS.partitionPath());
+ metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR,
timer.endTimer()));
+
+ Map<String, HoodieColumnStats> columnToRangeMap = new HashMap<>();
+ for (final Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>> entry
: hoodieRecordList) {
+ if (entry.getRight().isPresent()) {
+ final Option<HoodieColumnStats> optionalColumnStatPayload =
+ entry.getRight().get().getData().getColumnStatMetadata();
+ if (optionalColumnStatPayload.isPresent()) {
+
ValidationUtils.checkState(!columnToRangeMap.containsKey(entry.getLeft()));
+ columnToRangeMap.put(entry.getLeft(),
optionalColumnStatPayload.get());
+ }
+ }
Review comment:
error logging added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
