This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 402e459b87e HDDS-16171. Tool to download OM metadata with snapshots.
(#11017)
402e459b87e is described below
commit 402e459b87e06aaecf0b976e0e5110cada94d115
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Tue Aug 25 02:54:35 2026 +0530
HDDS-16171. Tool to download OM metadata with snapshots. (#11017)
Co-authored-by: Wei-Chiu Chuang <[email protected]>
Generated-by: Cursor <[email protected]>
---
.../hadoop/ozone/repair/om/DownloadOMDB.java | 205 ++++++++++++++++
.../apache/hadoop/ozone/repair/om/OMRepair.java | 3 +-
.../ozone/repair/om/TestDownloadOMDBTool.java | 260 +++++++++++++++++++++
3 files changed, 467 insertions(+), 1 deletion(-)
diff --git
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/DownloadOMDB.java
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/DownloadOMDB.java
new file mode 100644
index 00000000000..d9455859bc1
--- /dev/null
+++
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/DownloadOMDB.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.repair.om;
+
+import static
org.apache.hadoop.hdds.utils.HddsServerUtil.OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
+import org.apache.hadoop.ozone.om.ratis_snapshot.OmRatisSnapshotProvider;
+import org.apache.hadoop.ozone.repair.RepairTool;
+import picocli.CommandLine;
+
+/**
+ * Tool to download OM metadata using the follower bootstrap checkpoint flow.
+ */
[email protected](
+ name = "download",
+ description = "Downloads OM metadata (om.db and db.snapshots) from an OM
node using the same "
+ + "checkpoint transfer flow as follower bootstrap.",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class
+)
+public class DownloadOMDB extends RepairTool {
+
+ @CommandLine.Option(
+ names = {"--service-id", "--om-service-id"},
+ description = "Ozone Manager Service ID",
+ required = false
+ )
+ private String omServiceId;
+
+ @CommandLine.Option(
+ names = {"--node-id"},
+ description = "Node ID of the OM to download om.db from. Required when
OM HA is configured.",
+ required = false
+ )
+ private String nodeId;
+
+ @CommandLine.Option(
+ names = {"--output-dir"},
+ description = "Path where the downloaded OM metadata directory will be
written. "
+ + "The output matches follower bootstrap layout: om.db and
db.snapshots.",
+ required = true
+ )
+ private Path outputDir;
+
+ @CommandLine.Option(
+ names = {"--overwrite"},
+ description = "Overwrite output directory if it already exists."
+ )
+ private boolean overwrite;
+
+ @Override
+ public void execute() throws Exception {
+ OzoneConfiguration conf = getOzoneConf();
+ String effectiveServiceId = resolveServiceId(conf);
+
+ boolean outputExists = Files.exists(outputDir);
+ if (outputExists && !overwrite) {
+ fatal("Output directory already exists: %s. Use --overwrite to replace
it.",
+ outputDir.toAbsolutePath());
+ }
+
+ if (outputExists && !Files.isDirectory(outputDir)) {
+ fatal("Output path is not a directory: %s", outputDir.toAbsolutePath());
+ }
+
+ if (isDryRun()) {
+ info("Would download OM metadata at %s (using follower bootstrap flow).",
+ outputDir.toAbsolutePath());
+ return;
+ }
+
+ if (outputExists) {
+ FileUtils.forceDelete(outputDir.toFile());
+ }
+ Files.createDirectories(outputDir);
+
+ // This tool intentionally follows the inode-based follower bootstrap
transfer.
+ conf.setBoolean(OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_KEY, true);
+
+ Path snapshotWorkDir = Files.createTempDirectory(outputDir,
".omdb-bootstrap-");
+ DBCheckpoint checkpoint = null;
+ try {
+ checkpoint = downloadCheckpoint(conf, effectiveServiceId,
snapshotWorkDir);
+ Path checkpointRoot = checkpoint.getCheckpointLocation();
+ Path omDbPath = checkpointRoot.resolve(OM_DB_NAME);
+ if (!Files.isDirectory(omDbPath)) {
+ throw new IOException("Constructed OM DB directory not found in
checkpoint: " + omDbPath);
+ }
+ Path completionMarker =
checkpointRoot.resolve(OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME);
+ if (Files.exists(completionMarker)) {
+ Files.delete(completionMarker);
+ }
+ moveDirectoryContents(checkpointRoot, outputDir);
+ checkpoint = null;
+ info("Successfully downloaded OM metadata at: %s (includes %s and %s if
present on the leader).",
+ outputDir.toAbsolutePath(), OM_DB_NAME, OM_SNAPSHOT_DIR);
+ } finally {
+ if (checkpoint != null) {
+ checkpoint.cleanupCheckpoint();
+ }
+ FileUtils.deleteQuietly(snapshotWorkDir.toFile());
+ }
+ }
+
+ private DBCheckpoint downloadCheckpoint(OzoneConfiguration conf,
+ String serviceId, Path snapshotWorkDir) throws Exception {
+ List<String> nodeIds = getCandidateNodeIds(conf, serviceId);
+ Exception lastFailure = null;
+ for (String candidateNodeId : nodeIds) {
+ OMNodeDetails omNodeDetails =
+ OMNodeDetails.getOMNodeDetailsFromConf(conf, serviceId,
candidateNodeId);
+ if (omNodeDetails == null) {
+ if (nodeId != null) {
+ fatal("Couldn't determine OM node from the given service-id: %s and
node-id: %s.",
+ serviceId, nodeId);
+ }
+ continue;
+ }
+ String providerNodeId = omNodeDetails.getNodeId();
+ if (providerNodeId == null) {
+ providerNodeId = "non-ha";
+ }
+ try (OmRatisSnapshotProvider provider = new OmRatisSnapshotProvider(
+ conf, snapshotWorkDir.toFile(),
+ Collections.singletonMap(providerNodeId, omNodeDetails))) {
+ return provider.downloadDBSnapshotFromLeader(providerNodeId);
+ } catch (Exception ex) {
+ lastFailure = ex;
+ if (nodeId != null) {
+ throw ex;
+ }
+ }
+ }
+ if (lastFailure != null) {
+ throw lastFailure;
+ }
+ fatal("Couldn't determine OM node from the given service-id: %s and
node-id: %s.",
+ serviceId, nodeId);
+ return null;
+ }
+
+ private List<String> getCandidateNodeIds(OzoneConfiguration conf,
+ String serviceId) {
+ if (nodeId != null) {
+ return Collections.singletonList(nodeId);
+ }
+ if (!OmUtils.isServiceIdsDefined(conf)) {
+ return Collections.singletonList(null);
+ }
+ String omNodesKey = ConfUtils.addKeySuffixes(OZONE_OM_NODES_KEY,
serviceId);
+ Collection<String> omNodeIds = conf.getTrimmedStringCollection(omNodesKey);
+ return new ArrayList<>(omNodeIds);
+ }
+
+ private static void moveDirectoryContents(Path sourceDir, Path targetDir)
+ throws IOException {
+ try (java.util.stream.Stream<Path> entries = Files.list(sourceDir)) {
+ for (Path entry : (Iterable<Path>) entries::iterator) {
+ Files.move(entry, targetDir.resolve(entry.getFileName()),
+ StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+ }
+
+ private String resolveServiceId(OzoneConfiguration conf) throws IOException {
+ if (omServiceId != null && !omServiceId.isEmpty()) {
+ return omServiceId;
+ }
+ return OmUtils.getOzoneManagerServiceId(conf);
+ }
+}
diff --git
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
index 7232ccf2aae..6965fcf6b25 100644
---
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
+++
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
@@ -33,7 +33,8 @@
TransactionInfoRepair.class,
QuotaRepair.class,
CompactOMDB.class,
- OMRatisLogRepair.class
+ OMRatisLogRepair.class,
+ DownloadOMDB.class
},
description = "Operational tool to repair OM.")
@MetaInfServices(RepairSubcommand.class)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/repair/om/TestDownloadOMDBTool.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/repair/om/TestDownloadOMDBTool.java
new file mode 100644
index 00000000000..a31e21c2a2a
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/repair/om/TestDownloadOMDBTool.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.repair.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
+import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.ozone.DataTestUtil;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneSnapshot;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.ratis_snapshot.OmRatisSnapshotProvider;
+import org.apache.hadoop.ozone.repair.OzoneRepair;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.GenericTestUtils.LogCapturer;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Integration tests for `ozone repair om download`.
+ */
+public class TestDownloadOMDBTool {
+
+ private static final String OM_SERVICE_ID = "om-service-download-test";
+ private static MiniOzoneHAClusterImpl cluster;
+ private static OzoneConfiguration conf;
+
+ private GenericTestUtils.PrintStreamCapturer out;
+ private GenericTestUtils.PrintStreamCapturer err;
+
+ @BeforeAll
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newHABuilder(conf)
+ .setOMServiceId(OM_SERVICE_ID)
+ .setNumOfOzoneManagers(3)
+ .setNumOfActiveOMs(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ @AfterAll
+ public static void cleanup() {
+ IOUtils.closeQuietly(cluster);
+ }
+
+ @BeforeEach
+ public void setup() {
+ out = GenericTestUtils.captureOut();
+ err = GenericTestUtils.captureErr();
+ }
+
+ @AfterEach
+ public void reset() {
+ IOUtils.closeQuietly(out, err);
+ }
+
+ @Test
+ public void testDownloadConstructsOmDbUsingV2Transfer(@TempDir Path tempDir)
throws Exception {
+ Path outputDir = tempDir.resolve("downloaded-metadata");
+
+ LogCapturer providerLog =
LogCapturer.captureLogs(OmRatisSnapshotProvider.class);
+ try {
+ int exitCode = new OzoneRepair().getCmd().execute(withHAConf(new
String[] {
+ "om", "download",
+ "--service-id", OM_SERVICE_ID,
+ "--output-dir", outputDir.toString()
+ }));
+
+ assertEquals(0, exitCode, err.getOutput());
+ Path omDbDir = outputDir.resolve(OM_DB_NAME);
+ assertTrue(Files.isDirectory(omDbDir), "Expected downloaded om.db
directory to exist.");
+ assertTrue(Files.exists(omDbDir.resolve("CURRENT")), "Expected RocksDB
CURRENT file in downloaded om.db.");
+
assertThat(providerLog.getOutput()).contains(OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2);
+ } finally {
+ providerLog.stopCapturing();
+ }
+ }
+
+ @Test
+ public void testDownloadPreservesDbSnapshots(@TempDir Path tempDir) throws
Exception {
+ String volumeName = uniqueObjectName("vol");
+ String bucketName = uniqueObjectName("buck");
+ String snapshotName = uniqueObjectName("snap");
+ String keyName = uniqueObjectName("key");
+
+ OzoneClient client = cluster.newClient();
+ try {
+ ObjectStore store = client.getObjectStore();
+ OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client,
volumeName, bucketName);
+ DataTestUtil.createKey(bucket, keyName,
"snapshot-payload".getBytes(StandardCharsets.UTF_8));
+
+ store.createSnapshot(volumeName, bucketName, snapshotName);
+
+ OzoneSnapshot createdSnapshot = null;
+ Iterator<OzoneSnapshot> snapshots =
+ store.listSnapshot(volumeName, bucketName, "", null);
+ while (snapshots.hasNext()) {
+ OzoneSnapshot snapshot = snapshots.next();
+ if (snapshotName.equals(snapshot.getName())) {
+ createdSnapshot = snapshot;
+ break;
+ }
+ }
+ assertNotNull(createdSnapshot, "Expected Ozone snapshot to exist before
download.");
+ UUID snapshotId = createdSnapshot.getSnapshotId();
+
+ OzoneManager leader = cluster.getOMLeader();
+ String leaderNodeId = leader.getOMNodeId();
+ Path outputDir = tempDir.resolve("downloaded-metadata-with-snapshots");
+
+ int exitCode = new OzoneRepair().getCmd().execute(withHAConf(new
String[] {
+ "om", "download",
+ "--service-id", OM_SERVICE_ID,
+ "--node-id", leaderNodeId,
+ "--output-dir", outputDir.toString()
+ }));
+
+ assertEquals(0, exitCode, err.getOutput());
+ Path omDbDir = outputDir.resolve(OM_DB_NAME);
+ Path snapshotsDir = outputDir.resolve(OM_SNAPSHOT_DIR);
+ assertTrue(Files.isDirectory(omDbDir));
+ assertTrue(Files.exists(omDbDir.resolve("CURRENT")));
+ assertTrue(Files.isDirectory(snapshotsDir),
+ "Expected db.snapshots in downloaded metadata layout.");
+ Path checkpointState = snapshotsDir.resolve("checkpointState");
+ assertTrue(Files.isDirectory(checkpointState),
+ "Expected db.snapshots/checkpointState in downloaded metadata.");
+
+ String snapshotCheckpointPrefix = OM_DB_NAME + "-" + snapshotId;
+ Path snapshotYaml = checkpointState.resolve(snapshotCheckpointPrefix +
".yaml");
+ Path snapshotCheckpointDir =
checkpointState.resolve(snapshotCheckpointPrefix);
+ assertTrue(Files.exists(snapshotYaml),
+ "Expected snapshot checkpoint YAML for created Ozone snapshot.");
+ assertTrue(Files.isDirectory(snapshotCheckpointDir),
+ "Expected snapshot RocksDB checkpoint dir for created Ozone
snapshot.");
+ } finally {
+ IOUtils.closeQuietly(client);
+ }
+ }
+
+ @Test
+ public void testOverwriteReplacesExistingOutput(@TempDir Path tempDir)
throws Exception {
+ OzoneManager leader = cluster.getOMLeader();
+ String leaderNodeId = leader.getOMNodeId();
+ Path outputDir = tempDir.resolve("downloaded-metadata");
+ Files.createDirectories(outputDir);
+ Files.write(outputDir.resolve("OLD-MARKER"),
"stale".getBytes(StandardCharsets.UTF_8));
+
+ int firstRunExitCode = new OzoneRepair().getCmd().execute(withHAConf(new
String[] {
+ "om", "download",
+ "--service-id", OM_SERVICE_ID,
+ "--node-id", leaderNodeId,
+ "--output-dir", outputDir.toString()
+ }));
+ assertNotEquals(0, firstRunExitCode);
+ assertTrue(Files.exists(outputDir.resolve("OLD-MARKER")), "Marker should
remain without overwrite.");
+ assertThat(err.getOutput()).contains("Output directory already exists");
+
+ out.reset();
+ err.reset();
+ int secondRunExitCode = new OzoneRepair().getCmd().execute(withHAConf(new
String[] {
+ "om", "download",
+ "--service-id", OM_SERVICE_ID,
+ "--node-id", leaderNodeId,
+ "--output-dir", outputDir.toString(),
+ "--overwrite"
+ }));
+ assertEquals(0, secondRunExitCode, err.getOutput());
+ assertFalse(Files.exists(outputDir.resolve("OLD-MARKER")), "Overwrite
should remove stale output.");
+ assertTrue(Files.exists(outputDir.resolve(OM_DB_NAME).resolve("CURRENT")),
+ "Downloaded om.db should exist after overwrite.");
+ }
+
+ private String[] withHAConf(String[] existingArgs) throws IOException {
+ List<String> args = new ArrayList<>();
+ addConf(args, OZONE_OM_SERVICE_IDS_KEY);
+
+ String omNodesKey = ConfUtils.addKeySuffixes(OZONE_OM_NODES_KEY,
OM_SERVICE_ID);
+ addConf(args, omNodesKey);
+
+ Collection<String> omNodes = conf.getTrimmedStringCollection(omNodesKey);
+ for (String omNodeId : omNodes) {
+ addConf(args, ConfUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY,
OM_SERVICE_ID, omNodeId));
+ addOptionalConf(args,
ConfUtils.addKeySuffixes(OZONE_OM_HTTP_ADDRESS_KEY, OM_SERVICE_ID, omNodeId));
+ }
+
+ addOptionalConf(args, OzoneConfigKeys.OZONE_HTTP_POLICY_KEY);
+
+ args.addAll(Arrays.asList(existingArgs));
+ return args.toArray(new String[0]);
+ }
+
+ private void addConf(List<String> args, String key) throws IOException {
+ String value = conf.get(key);
+ if (value == null || value.isEmpty()) {
+ throw new IOException("Missing required config key for CLI test: " +
key);
+ }
+ args.add("-D");
+ args.add(key + "=" + value);
+ }
+
+ private void addOptionalConf(List<String> args, String key) {
+ String value = conf.get(key);
+ if (value == null || value.isEmpty()) {
+ return;
+ }
+ args.add("-D");
+ args.add(key + "=" + value);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
