manojpec commented on a change in pull request #4352:
URL: https://github.com/apache/hudi/pull/4352#discussion_r796262469
##########
File path:
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
##########
@@ -79,14 +98,53 @@ public static void deleteMetadataTable(String basePath,
HoodieEngineContext cont
}
}
+ /**
+ * Convert commit action to metadata records for the enabled partition types.
+ *
+ * @param commitMetadata - Commit action metadata
+ * @param dataMetaClient - Meta client for the data
table
+ * @param isMetaIndexColumnStatsForAllColumns - Do all columns need meta
indexing?
+ * @param instantTime - Action instant time
+ * @return Map of partition to metadata records for the commit action
+ */
+ public static Map<MetadataPartitionType, HoodieData<HoodieRecord>>
convertMetadataToRecords(
+ HoodieEngineContext context, List<MetadataPartitionType>
enabledPartitionTypes,
+ HoodieCommitMetadata commitMetadata, HoodieTableMetaClient
dataMetaClient,
+ boolean isMetaIndexColumnStatsForAllColumns, String instantTime) {
+ final Map<MetadataPartitionType, HoodieData<HoodieRecord>>
partitionToRecordsMap = new HashMap<>();
+ final HoodieData<HoodieRecord> filesPartitionRecordsRDD =
context.parallelize(
+ convertMetadataToFilesPartitionRecords(commitMetadata, instantTime),
1);
+ partitionToRecordsMap.put(MetadataPartitionType.FILES,
filesPartitionRecordsRDD);
+
+ if (enabledPartitionTypes.contains(MetadataPartitionType.BLOOM_FILTERS)) {
+ final List<HoodieRecord> metadataBloomFilterRecords =
convertMetadataToBloomFilterRecords(commitMetadata,
Review comment:
HUDI-3356 is addressing this issue.
##########
File path:
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
##########
@@ -79,14 +98,53 @@ public static void deleteMetadataTable(String basePath,
HoodieEngineContext cont
}
}
+ /**
+ * Convert commit action to metadata records for the enabled partition types.
+ *
+ * @param commitMetadata - Commit action metadata
+ * @param dataMetaClient - Meta client for the data
table
+ * @param isMetaIndexColumnStatsForAllColumns - Do all columns need meta
indexing?
+ * @param instantTime - Action instant time
+ * @return Map of partition to metadata records for the commit action
+ */
+ public static Map<MetadataPartitionType, HoodieData<HoodieRecord>>
convertMetadataToRecords(
+ HoodieEngineContext context, List<MetadataPartitionType>
enabledPartitionTypes,
+ HoodieCommitMetadata commitMetadata, HoodieTableMetaClient
dataMetaClient,
+ boolean isMetaIndexColumnStatsForAllColumns, String instantTime) {
+ final Map<MetadataPartitionType, HoodieData<HoodieRecord>>
partitionToRecordsMap = new HashMap<>();
+ final HoodieData<HoodieRecord> filesPartitionRecordsRDD =
context.parallelize(
+ convertMetadataToFilesPartitionRecords(commitMetadata, instantTime),
1);
+ partitionToRecordsMap.put(MetadataPartitionType.FILES,
filesPartitionRecordsRDD);
+
+ if (enabledPartitionTypes.contains(MetadataPartitionType.BLOOM_FILTERS)) {
+ final List<HoodieRecord> metadataBloomFilterRecords =
convertMetadataToBloomFilterRecords(commitMetadata,
+ dataMetaClient, instantTime);
+ if (!metadataBloomFilterRecords.isEmpty()) {
+ final HoodieData<HoodieRecord> metadataBloomFilterRecordsRDD =
context.parallelize(metadataBloomFilterRecords, 1);
+ partitionToRecordsMap.put(MetadataPartitionType.BLOOM_FILTERS,
metadataBloomFilterRecordsRDD);
+ }
+ }
+
+ if (enabledPartitionTypes.contains(MetadataPartitionType.COLUMN_STATS)) {
+ final List<HoodieRecord> metadataColumnStats =
convertMetadataToColumnStatsRecords(commitMetadata, context,
Review comment:
HUDI-3356 is addressing this issue.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
