This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6b8b844dfa HDDS-12009. Merge FSORepairTool and FSORepairCLI (#7639)
6b8b844dfa is described below
commit 6b8b844dfa7eb30df29ab849e834e0a00d3bc97e
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Jan 7 07:42:06 2025 +0100
HDDS-12009. Merge FSORepairTool and FSORepairCLI (#7639)
---
.../hadoop/ozone/repair/om/TestFSORepairTool.java | 25 +-
.../hadoop/ozone/repair/om/FSORepairCLI.java | 78 ---
.../hadoop/ozone/repair/om/FSORepairTool.java | 752 +++++++++++----------
.../apache/hadoop/ozone/repair/om/OMRepair.java | 2 +-
4 files changed, 404 insertions(+), 453 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/repair/om/TestFSORepairTool.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/repair/om/TestFSORepairTool.java
index d37f8ce57f..fb6472d7bc 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/repair/om/TestFSORepairTool.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/repair/om/TestFSORepairTool.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.repair.OzoneRepair;
import org.apache.ozone.test.GenericTestUtils;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
@@ -89,8 +88,8 @@ public class TestFSORepairTool {
private static FSORepairTool.Report fullReport;
private static FSORepairTool.Report emptyReport;
- private GenericTestUtils.PrintStreamCapturer out;
- private GenericTestUtils.PrintStreamCapturer err;
+ private static GenericTestUtils.PrintStreamCapturer out;
+ private static GenericTestUtils.PrintStreamCapturer err;
@BeforeAll
public static void setup() throws Exception {
@@ -103,6 +102,8 @@ public class TestFSORepairTool {
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
fs = FileSystem.get(conf);
+ out = GenericTestUtils.captureOut();
+ err = GenericTestUtils.captureErr();
cmd = new OzoneRepair().getCmd();
dbPath = new File(OMStorage.getOmDbDir(conf) + "/" + OM_DB_NAME).getPath();
@@ -147,19 +148,13 @@ public class TestFSORepairTool {
@BeforeEach
public void init() throws Exception {
- out = GenericTestUtils.captureOut();
- err = GenericTestUtils.captureErr();
- }
-
- @AfterEach
- public void clean() throws Exception {
- // reset stream after each unit test
- IOUtils.closeQuietly(out, err);
+ out.reset();
+ err.reset();
}
@AfterAll
public static void reset() throws IOException {
- IOUtils.closeQuietly(fs, client, cluster);
+ IOUtils.closeQuietly(fs, client, cluster, out, err);
}
/**
@@ -239,7 +234,7 @@ public class TestFSORepairTool {
// When a non-existent bucket filter is passed
int exitCode = dryRun("--volume", "/vol1", "--bucket", "bucket3");
assertEquals(0, exitCode);
- String cliOutput = out.getOutput();
+ String cliOutput = err.getOutput();
assertThat(cliOutput).contains("Bucket 'bucket3' does not exist in volume
'/vol1'.");
}
@@ -249,7 +244,7 @@ public class TestFSORepairTool {
// When a non-existent volume filter is passed
int exitCode = dryRun("--volume", "/vol5");
assertEquals(0, exitCode);
- String cliOutput = out.getOutput();
+ String cliOutput = err.getOutput();
assertThat(cliOutput).contains("Volume '/vol5' does not exist.");
}
@@ -259,7 +254,7 @@ public class TestFSORepairTool {
// When bucket filter is passed without the volume filter.
int exitCode = dryRun("--bucket", "bucket1");
assertEquals(0, exitCode);
- String cliOutput = out.getOutput();
+ String cliOutput = err.getOutput();
assertThat(cliOutput).contains("--bucket flag cannot be used without
specifying --volume.");
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairCLI.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairCLI.java
deleted file mode 100644
index fd6d75c713..0000000000
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairCLI.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.repair.om;
-
-import org.apache.hadoop.ozone.repair.RepairTool;
-import picocli.CommandLine;
-
-/**
- * Parser for scm.db file.
- */
[email protected](
- name = "fso-tree",
- description = "Identify and repair a disconnected FSO tree by marking
unreferenced entries for deletion. " +
- "OM should be stopped while this tool is run."
-)
-public class FSORepairCLI extends RepairTool {
-
- @CommandLine.Option(names = {"--db"},
- required = true,
- description = "Path to OM RocksDB")
- private String dbPath;
-
- @CommandLine.Option(names = {"-r", "--repair"},
- defaultValue = "false",
- description = "Run in repair mode to move unreferenced files and
directories to deleted tables.")
- private boolean repair;
-
- @CommandLine.Option(names = {"-v", "--volume"},
- description = "Filter by volume name. Add '/' before the volume name.")
- private String volume;
-
- @CommandLine.Option(names = {"-b", "--bucket"},
- description = "Filter by bucket name")
- private String bucket;
-
- @CommandLine.Option(names = {"--verbose"},
- description = "Verbose output. Show all intermediate steps and deleted
keys info.")
- private boolean verbose;
-
- @Override
- public void execute() throws Exception {
- if (checkIfServiceIsRunning("OM")) {
- return;
- }
- if (repair) {
- info("FSO Repair Tool is running in repair mode");
- } else {
- info("FSO Repair Tool is running in debug mode");
- }
- try {
- FSORepairTool
- repairTool = new FSORepairTool(dbPath, repair, volume, bucket,
verbose);
- repairTool.run();
- } catch (Exception ex) {
- throw new IllegalArgumentException("FSO repair failed: " +
ex.getMessage());
- }
-
- if (verbose) {
- info("FSO repair finished.");
- }
- }
-}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java
index 7e0fb23f5a..a4068415db 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java
@@ -36,9 +36,11 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.WithObjectID;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
+import org.apache.hadoop.ozone.repair.RepairTool;
import org.apache.ratis.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
import java.io.File;
import java.io.IOException;
@@ -69,402 +71,471 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
* The tool is idempotent. reachable.db will not be deleted automatically when
the tool finishes,
* in case users want to manually inspect it. It can be safely deleted once
the tool finishes.
*/
-public class FSORepairTool {
[email protected](
+ name = "fso-tree",
+ description = "Identify and repair a disconnected FSO tree by marking
unreferenced entries for deletion. " +
+ "OM should be stopped while this tool is run."
+)
+public class FSORepairTool extends RepairTool {
public static final Logger LOG =
LoggerFactory.getLogger(FSORepairTool.class);
-
- private final String omDBPath;
- private final DBStore store;
- private final Table<String, OmVolumeArgs> volumeTable;
- private final Table<String, OmBucketInfo> bucketTable;
- private final Table<String, OmDirectoryInfo> directoryTable;
- private final Table<String, OmKeyInfo> fileTable;
- private final Table<String, OmKeyInfo> deletedDirectoryTable;
- private final Table<String, RepeatedOmKeyInfo> deletedTable;
- private final Table<String, SnapshotInfo> snapshotInfoTable;
- private final String volumeFilter;
- private final String bucketFilter;
private static final String REACHABLE_TABLE = "reachable";
- private DBStore reachableDB;
- private final ReportStatistics reachableStats;
- private final ReportStatistics unreachableStats;
- private final ReportStatistics unreferencedStats;
- private final boolean repair;
- private final boolean verbose;
-
- public FSORepairTool(String dbPath, boolean repair, String volume, String
bucket, boolean verbose)
- throws IOException {
- this(getStoreFromPath(dbPath), dbPath, repair, volume, bucket, verbose);
- }
- /**
- * Allows passing RocksDB instance from a MiniOzoneCluster directly to this
class for testing.
- */
- public FSORepairTool(DBStore dbStore, String dbPath, boolean repair, String
volume, String bucket, boolean verbose)
- throws IOException {
- this.reachableStats = new ReportStatistics(0, 0, 0);
- this.unreachableStats = new ReportStatistics(0, 0, 0);
- this.unreferencedStats = new ReportStatistics(0, 0, 0);
-
- this.store = dbStore;
- this.omDBPath = dbPath;
- this.repair = repair;
- this.volumeFilter = volume;
- this.bucketFilter = bucket;
- this.verbose = verbose;
- volumeTable = store.getTable(OmMetadataManagerImpl.VOLUME_TABLE,
- String.class,
- OmVolumeArgs.class);
- bucketTable = store.getTable(OmMetadataManagerImpl.BUCKET_TABLE,
- String.class,
- OmBucketInfo.class);
- directoryTable = store.getTable(OmMetadataManagerImpl.DIRECTORY_TABLE,
- String.class,
- OmDirectoryInfo.class);
- fileTable = store.getTable(OmMetadataManagerImpl.FILE_TABLE,
- String.class,
- OmKeyInfo.class);
- deletedDirectoryTable =
store.getTable(OmMetadataManagerImpl.DELETED_DIR_TABLE,
- String.class,
- OmKeyInfo.class);
- deletedTable = store.getTable(OmMetadataManagerImpl.DELETED_TABLE,
- String.class,
- RepeatedOmKeyInfo.class);
- snapshotInfoTable =
store.getTable(OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE,
- String.class,
- SnapshotInfo.class);
- }
+ @CommandLine.Option(names = {"--db"},
+ required = true,
+ description = "Path to OM RocksDB")
+ private String omDBPath;
- protected static DBStore getStoreFromPath(String dbPath) throws IOException {
- File omDBFile = new File(dbPath);
- if (!omDBFile.exists() || !omDBFile.isDirectory()) {
- throw new IOException(String.format("Specified OM DB instance %s does " +
- "not exist or is not a RocksDB directory.", dbPath));
- }
- // Load RocksDB and tables needed.
- return OmMetadataManagerImpl.loadDB(new OzoneConfiguration(), new
File(dbPath).getParentFile(), -1);
- }
+ @CommandLine.Option(names = {"-r", "--repair"},
+ defaultValue = "false",
+ description = "Run in repair mode to move unreferenced files and
directories to deleted tables.")
+ private boolean repair;
+
+ @CommandLine.Option(names = {"-v", "--volume"},
+ description = "Filter by volume name. Add '/' before the volume name.")
+ private String volumeFilter;
- public FSORepairTool.Report run() throws Exception {
+ @CommandLine.Option(names = {"-b", "--bucket"},
+ description = "Filter by bucket name")
+ private String bucketFilter;
+
+ @CommandLine.Option(names = {"--verbose"},
+ description = "Verbose output. Show all intermediate steps and deleted
keys info.")
+ private boolean verbose;
+
+ @Override
+ public void execute() throws Exception {
+ if (checkIfServiceIsRunning("OM")) {
+ return;
+ }
+ if (repair) {
+ info("FSO Repair Tool is running in repair mode");
+ } else {
+ info("FSO Repair Tool is running in debug mode");
+ }
try {
- if (bucketFilter != null && volumeFilter == null) {
- System.out.println("--bucket flag cannot be used without specifying
--volume.");
- return null;
- }
+ Impl repairTool = new Impl();
+ repairTool.run();
+ } catch (Exception ex) {
+ throw new IllegalArgumentException("FSO repair failed: " +
ex.getMessage());
+ }
- if (volumeFilter != null) {
- OmVolumeArgs volumeArgs = volumeTable.getIfExist(volumeFilter);
- if (volumeArgs == null) {
- System.out.println("Volume '" + volumeFilter + "' does not exist.");
+ if (verbose) {
+ info("FSO repair finished.");
+ }
+ }
+
+ private class Impl {
+
+ private final DBStore store;
+ private final Table<String, OmVolumeArgs> volumeTable;
+ private final Table<String, OmBucketInfo> bucketTable;
+ private final Table<String, OmDirectoryInfo> directoryTable;
+ private final Table<String, OmKeyInfo> fileTable;
+ private final Table<String, OmKeyInfo> deletedDirectoryTable;
+ private final Table<String, RepeatedOmKeyInfo> deletedTable;
+ private final Table<String, SnapshotInfo> snapshotInfoTable;
+ private DBStore reachableDB;
+ private final ReportStatistics reachableStats;
+ private final ReportStatistics unreachableStats;
+ private final ReportStatistics unreferencedStats;
+
+ Impl() throws IOException {
+ this.reachableStats = new ReportStatistics(0, 0, 0);
+ this.unreachableStats = new ReportStatistics(0, 0, 0);
+ this.unreferencedStats = new ReportStatistics(0, 0, 0);
+
+ this.store = getStoreFromPath(omDBPath);
+ volumeTable = store.getTable(OmMetadataManagerImpl.VOLUME_TABLE,
+ String.class,
+ OmVolumeArgs.class);
+ bucketTable = store.getTable(OmMetadataManagerImpl.BUCKET_TABLE,
+ String.class,
+ OmBucketInfo.class);
+ directoryTable = store.getTable(OmMetadataManagerImpl.DIRECTORY_TABLE,
+ String.class,
+ OmDirectoryInfo.class);
+ fileTable = store.getTable(OmMetadataManagerImpl.FILE_TABLE,
+ String.class,
+ OmKeyInfo.class);
+ deletedDirectoryTable =
store.getTable(OmMetadataManagerImpl.DELETED_DIR_TABLE,
+ String.class,
+ OmKeyInfo.class);
+ deletedTable = store.getTable(OmMetadataManagerImpl.DELETED_TABLE,
+ String.class,
+ RepeatedOmKeyInfo.class);
+ snapshotInfoTable =
store.getTable(OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE,
+ String.class,
+ SnapshotInfo.class);
+ }
+
+ public Report run() throws Exception {
+ try {
+ if (bucketFilter != null && volumeFilter == null) {
+ error("--bucket flag cannot be used without specifying --volume.");
return null;
}
- }
- // Iterate all volumes or a specific volume if specified
- try (TableIterator<String, ? extends Table.KeyValue<String,
OmVolumeArgs>>
- volumeIterator = volumeTable.iterator()) {
- try {
- openReachableDB();
- } catch (IOException e) {
- System.out.println("Failed to open reachable database: " +
e.getMessage());
- throw e;
+ if (volumeFilter != null) {
+ OmVolumeArgs volumeArgs = volumeTable.getIfExist(volumeFilter);
+ if (volumeArgs == null) {
+ error("Volume '" + volumeFilter + "' does not exist.");
+ return null;
+ }
}
- while (volumeIterator.hasNext()) {
- Table.KeyValue<String, OmVolumeArgs> volumeEntry =
volumeIterator.next();
- String volumeKey = volumeEntry.getKey();
- if (volumeFilter != null && !volumeFilter.equals(volumeKey)) {
- continue;
+ // Iterate all volumes or a specific volume if specified
+ try (TableIterator<String, ? extends Table.KeyValue<String,
OmVolumeArgs>>
+ volumeIterator = volumeTable.iterator()) {
+ try {
+ openReachableDB();
+ } catch (IOException e) {
+ error("Failed to open reachable database: " + e.getMessage());
+ throw e;
}
+ while (volumeIterator.hasNext()) {
+ Table.KeyValue<String, OmVolumeArgs> volumeEntry =
volumeIterator.next();
+ String volumeKey = volumeEntry.getKey();
- System.out.println("Processing volume: " + volumeKey);
-
- if (bucketFilter != null) {
- OmBucketInfo bucketInfo = bucketTable.getIfExist(volumeKey + "/" +
bucketFilter);
- if (bucketInfo == null) {
- //Bucket does not exist in the volume
- System.out.println("Bucket '" + bucketFilter + "' does not exist
in volume '" + volumeKey + "'.");
- return null;
- }
-
- if (bucketInfo.getBucketLayout() !=
BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- System.out.println("Skipping non-FSO bucket " + bucketFilter);
+ if (volumeFilter != null && !volumeFilter.equals(volumeKey)) {
continue;
}
- processBucket(volumeEntry.getValue(), bucketInfo);
- } else {
+ info("Processing volume: " + volumeKey);
- // Iterate all buckets in the volume.
- try (TableIterator<String, ? extends Table.KeyValue<String,
OmBucketInfo>>
- bucketIterator = bucketTable.iterator()) {
- bucketIterator.seek(volumeKey);
- while (bucketIterator.hasNext()) {
- Table.KeyValue<String, OmBucketInfo> bucketEntry =
bucketIterator.next();
- String bucketKey = bucketEntry.getKey();
- OmBucketInfo bucketInfo = bucketEntry.getValue();
-
- if (bucketInfo.getBucketLayout() !=
BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- System.out.println("Skipping non-FSO bucket " + bucketKey);
- continue;
- }
+ if (bucketFilter != null) {
+ OmBucketInfo bucketInfo = bucketTable.getIfExist(volumeKey + "/"
+ bucketFilter);
+ if (bucketInfo == null) {
+ //Bucket does not exist in the volume
+ error("Bucket '" + bucketFilter + "' does not exist in volume
'" + volumeKey + "'.");
+ return null;
+ }
- // Stop this loop once we have seen all buckets in the current
- // volume.
- if (!bucketKey.startsWith(volumeKey)) {
- break;
- }
+ if (bucketInfo.getBucketLayout() !=
BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ info("Skipping non-FSO bucket " + bucketFilter);
+ continue;
+ }
- processBucket(volumeEntry.getValue(), bucketInfo);
+ processBucket(volumeEntry.getValue(), bucketInfo);
+ } else {
+
+ // Iterate all buckets in the volume.
+ try (TableIterator<String, ? extends Table.KeyValue<String,
OmBucketInfo>>
+ bucketIterator = bucketTable.iterator()) {
+ bucketIterator.seek(volumeKey);
+ while (bucketIterator.hasNext()) {
+ Table.KeyValue<String, OmBucketInfo> bucketEntry =
bucketIterator.next();
+ String bucketKey = bucketEntry.getKey();
+ OmBucketInfo bucketInfo = bucketEntry.getValue();
+
+ if (bucketInfo.getBucketLayout() !=
BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ info("Skipping non-FSO bucket " + bucketKey);
+ continue;
+ }
+
+ // Stop this loop once we have seen all buckets in the
current
+ // volume.
+ if (!bucketKey.startsWith(volumeKey)) {
+ break;
+ }
+
+ processBucket(volumeEntry.getValue(), bucketInfo);
+ }
}
}
}
}
+ } catch (IOException e) {
+ error("An error occurred while processing" + e.getMessage());
+ throw e;
+ } finally {
+ closeReachableDB();
+ store.close();
}
- } catch (IOException e) {
- System.out.println("An error occurred while processing" +
e.getMessage());
- throw e;
- } finally {
- closeReachableDB();
- store.close();
+
+ return buildReportAndLog();
}
- return buildReportAndLog();
- }
+ private boolean checkIfSnapshotExistsForBucket(String volumeName, String
bucketName) throws IOException {
+ if (snapshotInfoTable == null) {
+ return false;
+ }
- private boolean checkIfSnapshotExistsForBucket(String volumeName, String
bucketName) throws IOException {
- if (snapshotInfoTable == null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String,
SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ while (iterator.hasNext()) {
+ SnapshotInfo snapshotInfo = iterator.next().getValue();
+ String snapshotPath = (volumeName + "/" +
bucketName).replaceFirst("^/", "");
+ if (snapshotInfo.getSnapshotPath().equals(snapshotPath)) {
+ return true;
+ }
+ }
+ }
return false;
}
- try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>>
iterator =
- snapshotInfoTable.iterator()) {
- while (iterator.hasNext()) {
- SnapshotInfo snapshotInfo = iterator.next().getValue();
- String snapshotPath = (volumeName + "/" +
bucketName).replaceFirst("^/", "");
- if (snapshotInfo.getSnapshotPath().equals(snapshotPath)) {
- return true;
+ private void processBucket(OmVolumeArgs volume, OmBucketInfo bucketInfo)
throws IOException {
+ info("Processing bucket: " + volume.getVolume() + "/" +
bucketInfo.getBucketName());
+ if (checkIfSnapshotExistsForBucket(volume.getVolume(),
bucketInfo.getBucketName())) {
+ if (!repair) {
+ info(
+ "Snapshot detected in bucket '" + volume.getVolume() + "/" +
bucketInfo.getBucketName() + "'. ");
+ } else {
+ info(
+ "Skipping repair for bucket '" + volume.getVolume() + "/" +
bucketInfo.getBucketName() + "' " +
+ "due to snapshot presence.");
+ return;
}
}
+ markReachableObjectsInBucket(volume, bucketInfo);
+ handleUnreachableAndUnreferencedObjects(volume, bucketInfo);
}
- return false;
- }
- private void processBucket(OmVolumeArgs volume, OmBucketInfo bucketInfo)
throws IOException {
- System.out.println("Processing bucket: " + volume.getVolume() + "/" +
bucketInfo.getBucketName());
- if (checkIfSnapshotExistsForBucket(volume.getVolume(),
bucketInfo.getBucketName())) {
- if (!repair) {
- System.out.println(
- "Snapshot detected in bucket '" + volume.getVolume() + "/" +
bucketInfo.getBucketName() + "'. ");
- } else {
- System.out.println(
- "Skipping repair for bucket '" + volume.getVolume() + "/" +
bucketInfo.getBucketName() + "' " +
- "due to snapshot presence.");
- return;
- }
+ private Report buildReportAndLog() {
+ Report report = new Report.Builder()
+ .setReachable(reachableStats)
+ .setUnreachable(unreachableStats)
+ .setUnreferenced(unreferencedStats)
+ .build();
+
+ info("\n" + report);
+ return report;
}
- markReachableObjectsInBucket(volume, bucketInfo);
- handleUnreachableAndUnreferencedObjects(volume, bucketInfo);
- }
- private Report buildReportAndLog() {
- Report report = new Report.Builder()
- .setReachable(reachableStats)
- .setUnreachable(unreachableStats)
- .setUnreferenced(unreferencedStats)
- .build();
+ private void markReachableObjectsInBucket(OmVolumeArgs volume,
OmBucketInfo bucket) throws IOException {
+ // Only put directories in the stack.
+ // Directory keys should have the form /volumeID/bucketID/parentID/name.
+ Stack<String> dirKeyStack = new Stack<>();
- System.out.println("\n" + report);
- return report;
- }
+ // Since the tool uses parent directories to check for reachability, add
+ // a reachable entry for the bucket as well.
+ addReachableEntry(volume, bucket, bucket);
+ // Initialize the stack with all immediate child directories of the
+ // bucket, and mark them all as reachable.
+ Collection<String> childDirs =
getChildDirectoriesAndMarkAsReachable(volume, bucket, bucket);
+ dirKeyStack.addAll(childDirs);
+
+ while (!dirKeyStack.isEmpty()) {
+ // Get one directory and process its immediate children.
+ String currentDirKey = dirKeyStack.pop();
+ OmDirectoryInfo currentDir = directoryTable.get(currentDirKey);
+ if (currentDir == null) {
+ info("Directory key" + currentDirKey + "to be processed was not
found in the directory table.");
+ continue;
+ }
- private void markReachableObjectsInBucket(OmVolumeArgs volume, OmBucketInfo
bucket) throws IOException {
- // Only put directories in the stack.
- // Directory keys should have the form /volumeID/bucketID/parentID/name.
- Stack<String> dirKeyStack = new Stack<>();
-
- // Since the tool uses parent directories to check for reachability, add
- // a reachable entry for the bucket as well.
- addReachableEntry(volume, bucket, bucket);
- // Initialize the stack with all immediate child directories of the
- // bucket, and mark them all as reachable.
- Collection<String> childDirs =
getChildDirectoriesAndMarkAsReachable(volume, bucket, bucket);
- dirKeyStack.addAll(childDirs);
-
- while (!dirKeyStack.isEmpty()) {
- // Get one directory and process its immediate children.
- String currentDirKey = dirKeyStack.pop();
- OmDirectoryInfo currentDir = directoryTable.get(currentDirKey);
- if (currentDir == null) {
- System.out.println("Directory key" + currentDirKey + "to be processed
was not found in the directory table.");
- continue;
+ // TODO revisit this for a more memory efficient implementation,
+ // possibly making better use of RocksDB iterators.
+ childDirs = getChildDirectoriesAndMarkAsReachable(volume, bucket,
currentDir);
+ dirKeyStack.addAll(childDirs);
}
+ }
- // TODO revisit this for a more memory efficient implementation,
- // possibly making better use of RocksDB iterators.
- childDirs = getChildDirectoriesAndMarkAsReachable(volume, bucket,
currentDir);
- dirKeyStack.addAll(childDirs);
+ private boolean isDirectoryInDeletedDirTable(String dirKey) throws
IOException {
+ return deletedDirectoryTable.isExist(dirKey);
}
- }
- private boolean isDirectoryInDeletedDirTable(String dirKey) throws
IOException {
- return deletedDirectoryTable.isExist(dirKey);
- }
+ private boolean isFileKeyInDeletedTable(String fileKey) throws IOException
{
+ return deletedTable.isExist(fileKey);
+ }
- private boolean isFileKeyInDeletedTable(String fileKey) throws IOException {
- return deletedTable.isExist(fileKey);
- }
+ private void handleUnreachableAndUnreferencedObjects(OmVolumeArgs volume,
OmBucketInfo bucket) throws IOException {
+ // Check for unreachable and unreferenced directories in the bucket.
+ String bucketPrefix = OM_KEY_PREFIX +
+ volume.getObjectID() +
+ OM_KEY_PREFIX +
+ bucket.getObjectID();
- private void handleUnreachableAndUnreferencedObjects(OmVolumeArgs volume,
OmBucketInfo bucket) throws IOException {
- // Check for unreachable and unreferenced directories in the bucket.
- String bucketPrefix = OM_KEY_PREFIX +
- volume.getObjectID() +
- OM_KEY_PREFIX +
- bucket.getObjectID();
-
- try (TableIterator<String, ? extends Table.KeyValue<String,
OmDirectoryInfo>> dirIterator =
- directoryTable.iterator()) {
- dirIterator.seek(bucketPrefix);
- while (dirIterator.hasNext()) {
- Table.KeyValue<String, OmDirectoryInfo> dirEntry = dirIterator.next();
- String dirKey = dirEntry.getKey();
-
- // Only search directories in this bucket.
- if (!dirKey.startsWith(bucketPrefix)) {
- break;
- }
+ try (TableIterator<String, ? extends Table.KeyValue<String,
OmDirectoryInfo>> dirIterator =
+ directoryTable.iterator()) {
+ dirIterator.seek(bucketPrefix);
+ while (dirIterator.hasNext()) {
+ Table.KeyValue<String, OmDirectoryInfo> dirEntry =
dirIterator.next();
+ String dirKey = dirEntry.getKey();
+
+ // Only search directories in this bucket.
+ if (!dirKey.startsWith(bucketPrefix)) {
+ break;
+ }
- if (!isReachable(dirKey)) {
- if (!isDirectoryInDeletedDirTable(dirKey)) {
- System.out.println("Found unreferenced directory: " + dirKey);
- unreferencedStats.addDir();
+ if (!isReachable(dirKey)) {
+ if (!isDirectoryInDeletedDirTable(dirKey)) {
+ info("Found unreferenced directory: " + dirKey);
+ unreferencedStats.addDir();
- if (!repair) {
- if (verbose) {
- System.out.println("Marking unreferenced directory " + dirKey
+ " for deletion.");
+ if (!repair) {
+ if (verbose) {
+ info("Marking unreferenced directory " + dirKey + " for
deletion.");
+ }
+ } else {
+ info("Deleting unreferenced directory " + dirKey);
+ OmDirectoryInfo dirInfo = dirEntry.getValue();
+ markDirectoryForDeletion(volume.getVolume(),
bucket.getBucketName(), dirKey, dirInfo);
}
} else {
- System.out.println("Deleting unreferenced directory " + dirKey);
- OmDirectoryInfo dirInfo = dirEntry.getValue();
- markDirectoryForDeletion(volume.getVolume(),
bucket.getBucketName(), dirKey, dirInfo);
+ unreachableStats.addDir();
}
- } else {
- unreachableStats.addDir();
}
}
}
- }
- // Check for unreachable and unreferenced files
- try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
- fileIterator = fileTable.iterator()) {
- fileIterator.seek(bucketPrefix);
- while (fileIterator.hasNext()) {
- Table.KeyValue<String, OmKeyInfo> fileEntry = fileIterator.next();
- String fileKey = fileEntry.getKey();
- // Only search files in this bucket.
- if (!fileKey.startsWith(bucketPrefix)) {
- break;
- }
+ // Check for unreachable and unreferenced files
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ fileIterator = fileTable.iterator()) {
+ fileIterator.seek(bucketPrefix);
+ while (fileIterator.hasNext()) {
+ Table.KeyValue<String, OmKeyInfo> fileEntry = fileIterator.next();
+ String fileKey = fileEntry.getKey();
+ // Only search files in this bucket.
+ if (!fileKey.startsWith(bucketPrefix)) {
+ break;
+ }
- OmKeyInfo fileInfo = fileEntry.getValue();
- if (!isReachable(fileKey)) {
- if (!isFileKeyInDeletedTable(fileKey)) {
- System.out.println("Found unreferenced file: " + fileKey);
- unreferencedStats.addFile(fileInfo.getDataSize());
+ OmKeyInfo fileInfo = fileEntry.getValue();
+ if (!isReachable(fileKey)) {
+ if (!isFileKeyInDeletedTable(fileKey)) {
+ info("Found unreferenced file: " + fileKey);
+ unreferencedStats.addFile(fileInfo.getDataSize());
- if (!repair) {
- if (verbose) {
- System.out.println("Marking unreferenced file " + fileKey + "
for deletion." + fileKey);
+ if (!repair) {
+ if (verbose) {
+ info("Marking unreferenced file " + fileKey + " for
deletion." + fileKey);
+ }
+ } else {
+ info("Deleting unreferenced file " + fileKey);
+ markFileForDeletion(fileKey, fileInfo);
}
} else {
- System.out.println("Deleting unreferenced file " + fileKey);
- markFileForDeletion(fileKey, fileInfo);
+ unreachableStats.addFile(fileInfo.getDataSize());
}
} else {
- unreachableStats.addFile(fileInfo.getDataSize());
+ // NOTE: We are deserializing the proto of every reachable file
+ // just to log it's size. If we don't need this information we
could
+ // save time by skipping this step.
+ reachableStats.addFile(fileInfo.getDataSize());
}
- } else {
- // NOTE: We are deserializing the proto of every reachable file
- // just to log it's size. If we don't need this information we could
- // save time by skipping this step.
- reachableStats.addFile(fileInfo.getDataSize());
}
}
}
- }
- protected void markFileForDeletion(String fileKey, OmKeyInfo fileInfo)
throws IOException {
- try (BatchOperation batch = store.initBatchOperation()) {
- fileTable.deleteWithBatch(batch, fileKey);
-
- RepeatedOmKeyInfo originalRepeatedKeyInfo = deletedTable.get(fileKey);
- RepeatedOmKeyInfo updatedRepeatedOmKeyInfo = OmUtils.prepareKeyForDelete(
- fileInfo, fileInfo.getUpdateID(), true);
- // NOTE: The FSO code seems to write the open key entry with the whole
- // path, using the object's names instead of their ID. This would only
- // be possible when the file is deleted explicitly, and not part of a
- // directory delete. It is also not possible here if the file's parent
- // is gone. The name of the key does not matter so just use IDs.
- deletedTable.putWithBatch(batch, fileKey, updatedRepeatedOmKeyInfo);
- if (verbose) {
- System.out.println("Added entry " + fileKey + " to open key table: " +
updatedRepeatedOmKeyInfo);
+ protected void markFileForDeletion(String fileKey, OmKeyInfo fileInfo)
throws IOException {
+ try (BatchOperation batch = store.initBatchOperation()) {
+ fileTable.deleteWithBatch(batch, fileKey);
+
+ RepeatedOmKeyInfo originalRepeatedKeyInfo = deletedTable.get(fileKey);
+ RepeatedOmKeyInfo updatedRepeatedOmKeyInfo =
OmUtils.prepareKeyForDelete(
+ fileInfo, fileInfo.getUpdateID(), true);
+ // NOTE: The FSO code seems to write the open key entry with the whole
+ // path, using the object's names instead of their ID. This would only
+ // be possible when the file is deleted explicitly, and not part of a
+ // directory delete. It is also not possible here if the file's parent
+ // is gone. The name of the key does not matter so just use IDs.
+ deletedTable.putWithBatch(batch, fileKey, updatedRepeatedOmKeyInfo);
+ if (verbose) {
+ info("Added entry " + fileKey + " to open key table: " +
updatedRepeatedOmKeyInfo);
+ }
+ store.commitBatchOperation(batch);
}
- store.commitBatchOperation(batch);
}
- }
- protected void markDirectoryForDeletion(String volumeName, String bucketName,
- String dirKeyName, OmDirectoryInfo
dirInfo) throws IOException {
- try (BatchOperation batch = store.initBatchOperation()) {
- directoryTable.deleteWithBatch(batch, dirKeyName);
- // HDDS-7592: Make directory entries in deleted dir table unique.
- String deleteDirKeyName = dirKeyName + OM_KEY_PREFIX +
dirInfo.getObjectID();
+ protected void markDirectoryForDeletion(String volumeName, String
bucketName,
+ String dirKeyName, OmDirectoryInfo dirInfo) throws IOException {
+ try (BatchOperation batch = store.initBatchOperation()) {
+ directoryTable.deleteWithBatch(batch, dirKeyName);
+ // HDDS-7592: Make directory entries in deleted dir table unique.
+ String deleteDirKeyName = dirKeyName + OM_KEY_PREFIX +
dirInfo.getObjectID();
- // Convert the directory to OmKeyInfo for deletion.
- OmKeyInfo dirAsKeyInfo = OMFileRequest.getOmKeyInfo(volumeName,
bucketName, dirInfo, dirInfo.getName());
- deletedDirectoryTable.putWithBatch(batch, deleteDirKeyName,
dirAsKeyInfo);
+ // Convert the directory to OmKeyInfo for deletion.
+ OmKeyInfo dirAsKeyInfo = OMFileRequest.getOmKeyInfo(volumeName,
bucketName, dirInfo, dirInfo.getName());
+ deletedDirectoryTable.putWithBatch(batch, deleteDirKeyName,
dirAsKeyInfo);
- store.commitBatchOperation(batch);
+ store.commitBatchOperation(batch);
+ }
}
- }
- private Collection<String>
getChildDirectoriesAndMarkAsReachable(OmVolumeArgs volume, OmBucketInfo bucket,
-
WithObjectID currentDir) throws IOException {
-
- Collection<String> childDirs = new ArrayList<>();
-
- try (TableIterator<String, ? extends Table.KeyValue<String,
OmDirectoryInfo>>
- dirIterator = directoryTable.iterator()) {
- String dirPrefix = buildReachableKey(volume, bucket, currentDir);
- // Start searching the directory table at the current directory's
- // prefix to get its immediate children.
- dirIterator.seek(dirPrefix);
- while (dirIterator.hasNext()) {
- Table.KeyValue<String, OmDirectoryInfo> childDirEntry =
dirIterator.next();
- String childDirKey = childDirEntry.getKey();
- // Stop processing once we have seen all immediate children of this
- // directory.
- if (!childDirKey.startsWith(dirPrefix)) {
- break;
+ private Collection<String>
getChildDirectoriesAndMarkAsReachable(OmVolumeArgs volume, OmBucketInfo bucket,
+ WithObjectID currentDir) throws IOException {
+
+ Collection<String> childDirs = new ArrayList<>();
+
+ try (TableIterator<String, ? extends Table.KeyValue<String,
OmDirectoryInfo>>
+ dirIterator = directoryTable.iterator()) {
+ String dirPrefix = buildReachableKey(volume, bucket, currentDir);
+ // Start searching the directory table at the current directory's
+ // prefix to get its immediate children.
+ dirIterator.seek(dirPrefix);
+ while (dirIterator.hasNext()) {
+ Table.KeyValue<String, OmDirectoryInfo> childDirEntry =
dirIterator.next();
+ String childDirKey = childDirEntry.getKey();
+ // Stop processing once we have seen all immediate children of this
+ // directory.
+ if (!childDirKey.startsWith(dirPrefix)) {
+ break;
+ }
+ // This directory was reached by search.
+ addReachableEntry(volume, bucket, childDirEntry.getValue());
+ childDirs.add(childDirKey);
+ reachableStats.addDir();
}
- // This directory was reached by search.
- addReachableEntry(volume, bucket, childDirEntry.getValue());
- childDirs.add(childDirKey);
- reachableStats.addDir();
}
+
+ return childDirs;
+ }
+
+ /**
+ * Add the specified object to the reachable table, indicating it is part
+ * of the connected FSO tree.
+ */
+ private void addReachableEntry(OmVolumeArgs volume, OmBucketInfo bucket,
WithObjectID object) throws IOException {
+ String reachableKey = buildReachableKey(volume, bucket, object);
+ // No value is needed for this table.
+ reachableDB.getTable(REACHABLE_TABLE, String.class,
byte[].class).put(reachableKey, new byte[]{});
+ }
+
+ /**
+ * @param fileOrDirKey The key of a file or directory in RocksDB.
+ * @return true if the entry's parent is in the reachable table.
+ */
+ protected boolean isReachable(String fileOrDirKey) throws IOException {
+ String reachableParentKey = buildReachableParentKey(fileOrDirKey);
+
+ return reachableDB.getTable(REACHABLE_TABLE, String.class,
byte[].class).get(reachableParentKey) != null;
+ }
+
+ private void openReachableDB() throws IOException {
+ File reachableDBFile = new File(new File(omDBPath).getParentFile(),
"reachable.db");
+ info("Creating database of reachable directories at " + reachableDBFile);
+ // Delete the DB from the last run if it exists.
+ if (reachableDBFile.exists()) {
+ FileUtils.deleteDirectory(reachableDBFile);
+ }
+
+ ConfigurationSource conf = new OzoneConfiguration();
+ reachableDB = DBStoreBuilder.newBuilder(conf)
+ .setName("reachable.db")
+ .setPath(reachableDBFile.getParentFile().toPath())
+ .addTable(REACHABLE_TABLE)
+ .build();
}
- return childDirs;
+ private void closeReachableDB() throws IOException {
+ if (reachableDB != null) {
+ reachableDB.close();
+ }
+ File reachableDBFile = new File(new File(omDBPath).getParentFile(),
"reachable.db");
+ if (reachableDBFile.exists()) {
+ FileUtils.deleteDirectory(reachableDBFile);
+ }
+ }
}
- /**
- * Add the specified object to the reachable table, indicating it is part
- * of the connected FSO tree.
- */
- private void addReachableEntry(OmVolumeArgs volume, OmBucketInfo bucket,
WithObjectID object) throws IOException {
- String reachableKey = buildReachableKey(volume, bucket, object);
- // No value is needed for this table.
- reachableDB.getTable(REACHABLE_TABLE, String.class,
byte[].class).put(reachableKey, new byte[]{});
+ protected static DBStore getStoreFromPath(String dbPath) throws IOException {
+ File omDBFile = new File(dbPath);
+ if (!omDBFile.exists() || !omDBFile.isDirectory()) {
+ throw new IOException(String.format("Specified OM DB instance %s does " +
+ "not exist or is not a RocksDB directory.", dbPath));
+ }
+ // Load RocksDB and tables needed.
+ return OmMetadataManagerImpl.loadDB(new OzoneConfiguration(), new
File(dbPath).getParentFile(), -1);
}
/**
@@ -480,17 +551,6 @@ public class FSORepairTool {
object.getObjectID();
}
- /**
- *
- * @param fileOrDirKey The key of a file or directory in RocksDB.
- * @return true if the entry's parent is in the reachable table.
- */
- protected boolean isReachable(String fileOrDirKey) throws IOException {
- String reachableParentKey = buildReachableParentKey(fileOrDirKey);
-
- return reachableDB.getTable(REACHABLE_TABLE, String.class,
byte[].class).get(reachableParentKey) != null;
- }
-
/**
* Build an entry in the reachable table for the current object's parent
* object. The object could be a file or directory.
@@ -512,32 +572,6 @@ public class FSORepairTool {
parentID;
}
- private void openReachableDB() throws IOException {
- File reachableDBFile = new File(new File(omDBPath).getParentFile(),
"reachable.db");
- System.out.println("Creating database of reachable directories at " +
reachableDBFile);
- // Delete the DB from the last run if it exists.
- if (reachableDBFile.exists()) {
- FileUtils.deleteDirectory(reachableDBFile);
- }
-
- ConfigurationSource conf = new OzoneConfiguration();
- reachableDB = DBStoreBuilder.newBuilder(conf)
- .setName("reachable.db")
- .setPath(reachableDBFile.getParentFile().toPath())
- .addTable(REACHABLE_TABLE)
- .build();
- }
-
- private void closeReachableDB() throws IOException {
- if (reachableDB != null) {
- reachableDB.close();
- }
- File reachableDBFile = new File(new File(omDBPath).getParentFile(),
"reachable.db");
- if (reachableDBFile.exists()) {
- FileUtils.deleteDirectory(reachableDBFile);
- }
- }
-
/**
* Define a Report to be created.
*/
@@ -549,19 +583,19 @@ public class FSORepairTool {
/**
* Builds one report that is the aggregate of multiple others.
*/
- public Report(FSORepairTool.Report... reports) {
+ public Report(Report... reports) {
reachable = new ReportStatistics();
unreachable = new ReportStatistics();
unreferenced = new ReportStatistics();
- for (FSORepairTool.Report report : reports) {
+ for (Report report : reports) {
reachable.add(report.reachable);
unreachable.add(report.unreachable);
unreferenced.add(report.unreferenced);
}
}
- private Report(FSORepairTool.Report.Builder builder) {
+ private Report(Report.Builder builder) {
this.reachable = builder.reachable;
this.unreachable = builder.unreachable;
this.unreferenced = builder.unreferenced;
@@ -591,7 +625,7 @@ public class FSORepairTool {
if (other == null || getClass() != other.getClass()) {
return false;
}
- FSORepairTool.Report report = (FSORepairTool.Report) other;
+ Report report = (Report) other;
// Useful for testing.
System.out.println("Comparing reports\nExpect:\n" + this + "\nActual:\n"
+ report);
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
index 3b880f8754..9e20f6b9d1 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
@@ -27,7 +27,7 @@ import picocli.CommandLine;
*/
@CommandLine.Command(name = "om",
subcommands = {
- FSORepairCLI.class,
+ FSORepairTool.class,
SnapshotRepair.class,
TransactionInfoRepair.class
},
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
