prashantwason commented on code in PR #8684:
URL: https://github.com/apache/hudi/pull/8684#discussion_r1192180200
##########
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java:
##########
@@ -1378,6 +1339,206 @@ public static Set<String>
getInflightAndCompletedMetadataPartitions(HoodieTableC
*/
public static boolean isIndexingCommit(String instantTime) {
return instantTime.length() == MILLIS_INSTANT_ID_LENGTH +
METADATA_INDEXER_TIME_SUFFIX.length()
- && instantTime.endsWith(METADATA_INDEXER_TIME_SUFFIX);
+ && instantTime.endsWith(METADATA_INDEXER_TIME_SUFFIX);
+ }
+
+ /**
+ * Delete the metadata table for the dataset and backup if required.
+ *
+ * @param dataMetaClient {@code HoodieTableMetaClient} of the dataset for
which metadata table is to be deleted
+ * @param context instance of {@link HoodieEngineContext}.
+ * @param backup Whether metadata table should be backed up before
deletion. If true, the table is backed up to the
+ * directory with name metadata_<current_timestamp>.
+ * @return The backup directory if backup was requested
+ */
+ public static String deleteMetadataTable(HoodieTableMetaClient
dataMetaClient, HoodieEngineContext context, boolean backup) {
+ final Path metadataTablePath =
HoodieTableMetadata.getMetadataTableBasePath(dataMetaClient.getBasePathV2());
+ FileSystem fs = FSUtils.getFs(metadataTablePath.toString(),
context.getHadoopConf().get());
+ setMetadataPartitionState(dataMetaClient, MetadataPartitionType.FILES,
false);
+ try {
+ if (!fs.exists(metadataTablePath)) {
+ return null;
+ }
+ } catch (FileNotFoundException e) {
+ // Ignoring exception as metadata table already does not exist
+ return null;
+ } catch (IOException e) {
+ throw new HoodieMetadataException("Failed to check metadata table
existence", e);
+ }
+
+ if (backup) {
+ final Path metadataBackupPath = new Path(metadataTablePath.getParent(),
".metadata_" + HoodieActiveTimeline.createNewInstantTime());
+ LOG.info("Backing up metadata directory to " + metadataBackupPath + "
before deletion");
+ try {
+ if (fs.rename(metadataTablePath, metadataBackupPath)) {
+ return metadataBackupPath.toString();
+ }
+ } catch (Exception e) {
+ // If rename fails, we will ignore the backup and still delete the MDT
+ LOG.error("Failed to backup metadata table using rename", e);
+ }
+ }
+
+ LOG.info("Deleting metadata table from " + metadataTablePath);
+ try {
+ fs.delete(metadataTablePath, true);
+ } catch (Exception e) {
+ throw new HoodieMetadataException("Failed to delete metadata table from
path " + metadataTablePath, e);
+ }
+
+ return null;
+ }
+
+ public static HoodieTableMetaClient
setMetadataPartitionState(HoodieTableMetaClient dataMetaClient,
MetadataPartitionType partitionType, boolean enabled) {
+ dataMetaClient.getTableConfig().setMetadataPartitionState(partitionType,
enabled);
+ HoodieTableConfig.update(dataMetaClient.getFs(), new
Path(dataMetaClient.getMetaPath()), dataMetaClient.getTableConfig().getProps());
+ dataMetaClient = HoodieTableMetaClient.reload(dataMetaClient);
+
ValidationUtils.checkState(dataMetaClient.getTableConfig().isMetadataPartitionEnabled(partitionType)
== enabled,
+ "Metadata table state change should be persisted");
+
+ LOG.info(String.format("Metadata table %s partition %s has been %s",
dataMetaClient.getBasePathV2(), partitionType,
+ enabled ? "enabled" : "disabled"));
+ return dataMetaClient;
+ }
+
+ /**
+ * Delete a partition within the metadata table.
+ * <p>
+ * This can be used to delete a partition so that it can be re-bootstrapped.
+ *
+ * @param dataMetaClient {@code HoodieTableMetaClient} of the dataset for
which metadata table is to be deleted
+ * @param context instance of {@code HoodieEngineContext}.
+ * @param backup Whether metadata table should be backed up before
deletion. If true, the table is backed up to the
+ * directory with name metadata_<current_timestamp>.
+ * @param partitionType The partition to delete
+ * @return The backup directory if backup was requested, null otherwise
+ */
+ public static String deleteMetadataTablePartition(HoodieTableMetaClient
dataMetaClient, HoodieEngineContext context,
+ MetadataPartitionType
partitionType, boolean backup) {
+ if (partitionType.equals(MetadataPartitionType.FILES)) {
+ return deleteMetadataTable(dataMetaClient, context, backup);
+ }
+
+ final Path metadataTablePartitionPath = new
Path(HoodieTableMetadata.getMetadataTableBasePath(dataMetaClient.getBasePath()),
partitionType.getPartitionPath());
+ FileSystem fs = FSUtils.getFs(metadataTablePartitionPath.toString(),
context.getHadoopConf().get());
+ setMetadataPartitionState(dataMetaClient, partitionType, false);
+ try {
+ if (!fs.exists(metadataTablePartitionPath)) {
+ return null;
+ }
+ } catch (FileNotFoundException e) {
+ // Ignoring exception as metadata table already does not exist
+ LOG.debug("Metadata table partition " + partitionType + " not found at
path " + metadataTablePartitionPath);
+ return null;
+ } catch (Exception e) {
+ throw new HoodieMetadataException("Failed to check metadata table
partition existence", e);
+ }
+
+ if (backup) {
+ final Path metadataPartitionBackupPath = new
Path(metadataTablePartitionPath.getParent().getParent(),
+ String.format(".metadata_%s_%s",
partitionType.getPartitionPath(), HoodieActiveTimeline.createNewInstantTime()));
+ LOG.info(String.format("Backing up MDT partition %s to %s before
deletion", partitionType, metadataPartitionBackupPath));
+ try {
+ if (fs.rename(metadataTablePartitionPath,
metadataPartitionBackupPath)) {
+ return metadataPartitionBackupPath.toString();
+ }
+ } catch (Exception e) {
+ // If rename fails, we will try to delete the table instead
+ LOG.error(String.format("Failed to backup MDT partition %s using
rename", partitionType), e);
+ }
+ } else {
+ LOG.info("Deleting metadata table partition from " +
metadataTablePartitionPath);
+ try {
+ fs.delete(metadataTablePartitionPath, true);
+ } catch (Exception e) {
+ throw new HoodieMetadataException("Failed to delete metadata table
partition from path " + metadataTablePartitionPath, e);
+ }
+ }
+
+ return null;
+ }
+
+ public static HoodieTableMetaClient
setMetadataInflightPartitions(HoodieTableMetaClient dataMetaClient,
List<MetadataPartitionType> partitionTypes) {
+
dataMetaClient.getTableConfig().setMetadataPartitionsAsInflight(partitionTypes);
+ HoodieTableConfig.update(dataMetaClient.getFs(), new
Path(dataMetaClient.getMetaPath()), dataMetaClient.getTableConfig().getProps());
+ dataMetaClient = HoodieTableMetaClient.reload(dataMetaClient);
+
+ LOG.info(String.format("Metadata table %s partitions %s hasvebeen set to
inflight", dataMetaClient.getBasePathV2(), partitionTypes));
+ return dataMetaClient;
+ }
+
+ /**
+ * Return the complete fileID for a file group within a MDT partition.
+ * <p>
+ * MDT fileGroups have the format <fileIDPrefix>-<index>. The fileIDPrefix
is hardcoded for each MDT partition and index is an integer.
+ *
+ * @param partitionType The type of the MDT partition
+ * @param index Index of the file group within the partition
+ * @return The fileID
+ */
+ public static String getFileIDForFileGroup(MetadataPartitionType
partitionType, int index) {
+ return String.format("%s%04d", partitionType.getFileIdPrefix(), index);
+ }
+
+ /**
+ * Extract the index from the fileID of a file group in the MDT partition.
See {@code getFileIDForFileGroup} for the format of the fileID.
+ *
+ * @param fileId fileID of a file group.
+ * @return The index of file group
+ */
+ public static int getFileGroupIndexFromFileId(String fileId) {
Review Comment:
Added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
