This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 74ecb3bde9 HDDS-9245. Add tool for upgrading container layout v2 to v3
(#5539)
74ecb3bde9 is described below
commit 74ecb3bde9c33889f89331d66fa6bd436a0ec471
Author: hao guo <[email protected]>
AuthorDate: Wed Jan 10 21:02:15 2024 +0800
HDDS-9245. Add tool for upgrading container layout v2 to v3 (#5539)
---
.../org/apache/hadoop/ozone/common/Storage.java | 2 +-
.../container/common/utils/DatanodeStoreCache.java | 6 +
.../hdds/scm/cli/container/ContainerCommands.java | 3 +-
.../hdds/scm/cli/container/UpgradeSubcommand.java | 205 +++++++++++++++++
.../scm/cli/container/upgrade/UpgradeChecker.java | 111 +++++++++
.../scm/cli/container/upgrade/UpgradeManager.java | 92 +++-----
.../scm/cli/container/upgrade/UpgradeTask.java | 255 +++++++++++++--------
.../scm/cli/container/upgrade/UpgradeUtils.java | 25 ++
.../cli/container/upgrade/TestUpgradeManager.java | 7 +-
.../shell/TestOzoneContainerUpgradeShell.java | 249 ++++++++++++++++++++
10 files changed, 797 insertions(+), 158 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
index ae291c633f..bc83a6cdf1 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
@@ -50,7 +50,7 @@ public abstract class Storage {
private static final Logger LOG = LoggerFactory.getLogger(Storage.class);
public static final String STORAGE_DIR_CURRENT = "current";
- protected static final String STORAGE_FILE_VERSION = "VERSION";
+ public static final String STORAGE_FILE_VERSION = "VERSION";
public static final String CONTAINER_DIR = "containerDir";
private final NodeType nodeType;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DatanodeStoreCache.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DatanodeStoreCache.java
index 7edfcd9969..b0b6f98cbe 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DatanodeStoreCache.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DatanodeStoreCache.java
@@ -61,6 +61,12 @@ public final class DatanodeStoreCache {
getInstance().miniClusterMode = true;
}
+ @VisibleForTesting
+ public static synchronized void setMiniClusterMode(boolean isMiniCluster) {
+ getInstance().miniClusterMode = isMiniCluster;
+ }
+
+
public void addDB(String containerDBPath, RawDB db) {
datanodeStoreMap.putIfAbsent(containerDBPath, db);
LOG.info("Added db {} to cache", containerDBPath);
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
index a3d07728eb..54c69273f0 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
@@ -42,7 +42,8 @@ import picocli.CommandLine.Spec;
InfoSubcommand.class,
CreateSubcommand.class,
CloseSubcommand.class,
- ReportSubcommand.class
+ ReportSubcommand.class,
+ UpgradeSubcommand.class
})
@MetaInfServices(SubcommandWithParent.class)
public class ContainerCommands implements Callable<Void>, SubcommandWithParent
{
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/UpgradeSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/UpgradeSubcommand.java
new file mode 100644
index 0000000000..a94f631b5b
--- /dev/null
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/UpgradeSubcommand.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.container;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.cli.container.upgrade.UpgradeChecker;
+import org.apache.hadoop.hdds.scm.cli.container.upgrade.UpgradeManager;
+import org.apache.hadoop.hdds.scm.cli.container.upgrade.UpgradeUtils;
+import org.apache.hadoop.hdds.server.OzoneAdmins;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
+import org.apache.hadoop.ozone.common.Storage;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+import java.io.File;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Scanner;
+import java.util.concurrent.Callable;
+
+/**
+ * This is the handler that process container upgrade command.
+ */
+@Command(
+ name = "upgrade",
+ description = "Offline upgrade all schema V2 containers to schema V3 " +
+ "for this datanode.",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class UpgradeSubcommand implements Callable<Void> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(UpgradeSubcommand.class);
+
+ @CommandLine.Spec
+ private static CommandLine.Model.CommandSpec spec;
+
+ @CommandLine.Option(names = {"--volume"},
+ required = false,
+ description = "volume path")
+ private String volume;
+
+ @CommandLine.Option(names = {"-y", "--yes"},
+ description = "Continue without interactive user confirmation")
+ private boolean yes;
+
+ private static OzoneConfiguration ozoneConfiguration;
+
+
+ @Override
+ public Void call() throws Exception {
+ OzoneConfiguration configuration = getConfiguration();
+ // Verify admin privilege
+ OzoneAdmins admins = OzoneAdmins.getOzoneAdmins("", configuration);
+ if (!admins.isAdmin(UserGroupInformation.getCurrentUser())) {
+ out().println("It requires ozone administrator privilege. Current user" +
+ " is " + UserGroupInformation.getCurrentUser() + ".");
+ return null;
+ }
+
+ final UpgradeChecker upgradeChecker = new UpgradeChecker();
+ Pair<Boolean, String> pair = upgradeChecker.checkDatanodeRunning();
+ final boolean isRunning = pair.getKey();
+ if (isRunning) {
+ out().println(pair.getValue());
+ return null;
+ }
+
+ DatanodeDetails dnDetail =
+ UpgradeUtils.getDatanodeDetails(configuration);
+
+ Pair<HDDSLayoutFeature, HDDSLayoutFeature> layoutFeature =
+ upgradeChecker.getLayoutFeature(dnDetail, configuration);
+ final HDDSLayoutFeature softwareLayoutFeature = layoutFeature.getLeft();
+ final HDDSLayoutFeature metadataLayoutFeature = layoutFeature.getRight();
+ final int needLayoutVersion =
+ HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion();
+
+ if (metadataLayoutFeature.layoutVersion() < needLayoutVersion ||
+ softwareLayoutFeature.layoutVersion() < needLayoutVersion) {
+ out().println(String.format(
+ "Please upgrade your software version, no less than %s," +
+ " current metadata layout version is %s," +
+ " software layout version is %s",
+ HDDSLayoutFeature.DATANODE_SCHEMA_V3.name(),
+ metadataLayoutFeature.name(), softwareLayoutFeature.name()));
+ return null;
+ }
+
+ if (!Strings.isNullOrEmpty(volume)) {
+ File volumeDir = new File(volume);
+ if (!volumeDir.exists() || !volumeDir.isDirectory()) {
+ out().println(
+ String.format("Volume path %s is not a directory or doesn't exist",
+ volume));
+ return null;
+ }
+ File hddsRootDir = new File(volume + "/" + HddsVolume.HDDS_VOLUME_DIR);
+ File versionFile = new File(volume + "/" + HddsVolume.HDDS_VOLUME_DIR +
+ "/" + Storage.STORAGE_FILE_VERSION);
+ if (!hddsRootDir.exists() || !hddsRootDir.isDirectory() ||
+ !versionFile.exists() || !versionFile.isFile()) {
+ out().println(
+ String.format("Volume path %s is not a valid data volume",
volume));
+ return null;
+ }
+ configuration.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, volume);
+ }
+
+ final HddsProtos.NodeOperationalState opState =
+ dnDetail.getPersistedOpState();
+
+ if (!opState.equals(HddsProtos.NodeOperationalState.IN_MAINTENANCE)) {
+ out().println("This command requires the datanode's " +
+ "NodeOperationalState to be IN_MAINTENANCE, currently is " +
+ opState);
+ return null;
+ }
+
+ List<HddsVolume> allVolume =
+ upgradeChecker.getAllVolume(dnDetail, configuration);
+
+ Iterator<HddsVolume> volumeIterator = allVolume.iterator();
+ while (volumeIterator.hasNext()) {
+ HddsVolume hddsVolume = volumeIterator.next();
+ if (UpgradeChecker.isAlreadyUpgraded(hddsVolume)) {
+ out().println("Volume " + hddsVolume.getVolumeRootDir() +
+ " is already upgraded, skip it.");
+ volumeIterator.remove();
+ }
+ }
+
+ if (allVolume.isEmpty()) {
+ out().println("There is no more volume to upgrade. Exit.");
+ return null;
+ }
+
+ if (!yes) {
+ Scanner scanner = new Scanner(new InputStreamReader(
+ System.in, StandardCharsets.UTF_8));
+ System.out.println(
+ "All volume db stores will be automatically backup," +
+ " should we continue the upgrade ? [yes|no] : ");
+ boolean confirm = scanner.next().trim().equals("yes");
+ scanner.close();
+ if (!confirm) {
+ return null;
+ }
+ }
+
+ // do upgrade
+ final UpgradeManager upgradeManager = new UpgradeManager();
+ upgradeManager.run(configuration, allVolume);
+ return null;
+ }
+
+ @VisibleForTesting
+ public static void setOzoneConfiguration(OzoneConfiguration config) {
+ ozoneConfiguration = config;
+ }
+
+ private OzoneConfiguration getConfiguration() {
+ if (ozoneConfiguration == null) {
+ ozoneConfiguration = new OzoneConfiguration();
+ }
+ return ozoneConfiguration;
+ }
+
+ private static PrintWriter err() {
+ return spec.commandLine().getErr();
+ }
+
+ private static PrintWriter out() {
+ return spec.commandLine().getOut();
+ }
+}
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeChecker.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeChecker.java
new file mode 100644
index 0000000000..44516c0687
--- /dev/null
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeChecker.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.container.upgrade;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
+import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+
+/**
+ * This is the handler that process container upgrade checker.
+ */
+public class UpgradeChecker {
+
+ /*
+ * Verify that the datanode is in the shutdown state or running.
+ */
+ public Pair<Boolean, String> checkDatanodeRunning() {
+ String command =
+ "ps aux | grep org.apache.hadoop.ozone.HddsDatanodeService " +
+ "| grep -v grep";
+ try {
+ Process exec = Runtime.getRuntime().exec(new String[]{"/bin/bash", "-c",
+ command});
+ boolean notTimeout = exec.waitFor(10, TimeUnit.SECONDS);
+ if (!notTimeout) {
+ return Pair.of(true,
+ String.format("Execution of the command '%s' timeout", command));
+ }
+ if (exec.exitValue() == 0) {
+ return Pair.of(true, "HddsDatanodeService is running." +
+ " This upgrade command requires datanode to be off and in" +
+ " the IN_MAINTENANCE mode. Please put the datanode in" +
+ " the desired state first, then try this command later again.");
+ } else if (exec.exitValue() == 1) {
+ return Pair.of(false, "HddsDatanodeService is not running.");
+ } else {
+ return Pair.of(true,
+ String.format("Return code of the command '%s' is %d", command,
+ exec.exitValue()));
+ }
+ } catch (IOException | InterruptedException e) {
+ return Pair.of(true,
+ String.format("Run command '%s' has error '%s'",
+ command, e.getMessage()));
+ }
+ }
+
+ public Pair<HDDSLayoutFeature, HDDSLayoutFeature> getLayoutFeature(
+ DatanodeDetails dnDetail, OzoneConfiguration conf) throws IOException {
+ DatanodeLayoutStorage layoutStorage =
+ new DatanodeLayoutStorage(conf, dnDetail.getUuidString());
+ HDDSLayoutVersionManager layoutVersionManager =
+ new HDDSLayoutVersionManager(layoutStorage.getLayoutVersion());
+
+ final int metadataLayoutVersion =
+ layoutVersionManager.getMetadataLayoutVersion();
+ final HDDSLayoutFeature metadataLayoutFeature =
+ (HDDSLayoutFeature) layoutVersionManager.getFeature(
+ metadataLayoutVersion);
+
+ final int softwareLayoutVersion =
+ layoutVersionManager.getSoftwareLayoutVersion();
+ final HDDSLayoutFeature softwareLayoutFeature =
+ (HDDSLayoutFeature) layoutVersionManager.getFeature(
+ softwareLayoutVersion);
+
+ return Pair.of(softwareLayoutFeature, metadataLayoutFeature);
+ }
+
+ public List<HddsVolume> getAllVolume(DatanodeDetails detail,
+ OzoneConfiguration configuration) throws IOException {
+ final MutableVolumeSet dataVolumeSet = UpgradeUtils
+ .getHddsVolumes(configuration, StorageVolume.VolumeType.DATA_VOLUME,
+ detail.getUuidString());
+ return
StorageVolumeUtil.getHddsVolumesList(dataVolumeSet.getVolumesList());
+ }
+
+ public static boolean isAlreadyUpgraded(HddsVolume hddsVolume) {
+ final File migrateFile =
+ UpgradeUtils.getVolumeUpgradeCompleteFile(hddsVolume);
+ return migrateFile.exists();
+ }
+}
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeManager.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeManager.java
index 9725f26ad3..3e0629e57c 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeManager.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeManager.java
@@ -19,26 +19,23 @@ package org.apache.hadoop.hdds.scm.cli.container.upgrade;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
-import static
org.apache.hadoop.hdds.scm.cli.container.upgrade.UpgradeUtils.getContainerDBPath;
/**
* This class manages v2 to v3 container upgrade.
@@ -49,74 +46,41 @@ public class UpgradeManager {
LoggerFactory.getLogger(UpgradeManager.class);
private final Map<String, DatanodeStoreSchemaThreeImpl>
- volumeStoreMap = new HashMap<>();
-
- public void run(OzoneConfiguration configuration) throws IOException {
- final DatanodeDetails detail =
- UpgradeUtils.getDatanodeDetails(configuration);
+ volumeStoreMap = new ConcurrentHashMap<>();
- final MutableVolumeSet dataVolumeSet = UpgradeUtils
- .getHddsVolumes(configuration, StorageVolume.VolumeType.DATA_VOLUME,
- detail.getUuidString());
-
- HddsVolumeUtil.loadAllHddsVolumeDbStore(dataVolumeSet, null, false, LOG);
- initVolumeStoreMap(dataVolumeSet, configuration);
- upgradeAll(dataVolumeSet, configuration);
- }
-
- public void initVolumeStoreMap(MutableVolumeSet dataVolumeSet,
- OzoneConfiguration configuration) throws IOException {
- for (StorageVolume storageVolume : dataVolumeSet.getVolumesList()) {
- final HddsVolume volume = (HddsVolume) storageVolume;
- final File containerDBPath = getContainerDBPath(volume);
- final DatanodeStoreSchemaThreeImpl datanodeStoreSchemaThree =
- new DatanodeStoreSchemaThreeImpl(configuration,
- containerDBPath.getAbsolutePath(), false);
- volumeStoreMap.put(volume.getStorageDir().getAbsolutePath(),
- datanodeStoreSchemaThree);
- }
- }
-
- public List<Result> upgradeAll(MutableVolumeSet volumeSet,
- OzoneConfiguration configuration) {
+ public List<Result> run(OzoneConfiguration configuration,
+ List<HddsVolume> volumes) throws IOException {
List<Result> results = new ArrayList<>();
- final List<StorageVolume> volumesList = volumeSet.getVolumesList();
- Map<HddsVolume, CompletableFuture<Result>> volumeFutures =
- new HashMap<>();
+ Map<HddsVolume, CompletableFuture<Result>> volumeFutures = new HashMap<>();
long startTime = System.currentTimeMillis();
- LOG.info("Start upgrade {} volumes container LayoutVersion",
- volumesList.size());
-
- for (StorageVolume volume : volumesList) {
+ LOG.info("Start to upgrade {} volume(s)", volumes.size());
+ for (StorageVolume volume : volumes) {
final HddsVolume hddsVolume = (HddsVolume) volume;
final UpgradeTask task =
- new UpgradeTask(configuration, hddsVolume,
- getDBStore(hddsVolume));
-
- final CompletableFuture<Result> future =
- task.getUpgradeFutureByVolume();
+ new UpgradeTask(configuration, hddsVolume, volumeStoreMap);
+ final CompletableFuture<Result> future = task.getUpgradeFuture();
volumeFutures.put(hddsVolume, future);
}
- for (Map.Entry<HddsVolume, CompletableFuture<Result>> entry : volumeFutures
- .entrySet()) {
+ for (Map.Entry<HddsVolume, CompletableFuture<Result>> entry :
+ volumeFutures.entrySet()) {
final HddsVolume hddsVolume = entry.getKey();
final CompletableFuture<Result> volumeFuture = entry.getValue();
try {
final Result result = volumeFuture.get();
results.add(result);
- LOG.info("Finish upgrade containers on volume {}, result {}",
- hddsVolume.getVolumeRootDir(), result);
+ LOG.info("Finish upgrading containers on volume {}, {}",
+ hddsVolume.getVolumeRootDir(), result.toString());
} catch (Exception e) {
- LOG.error("Upgrade containers on volume {} failed",
+ LOG.error("Failed to upgrade containers on volume {}",
hddsVolume.getVolumeRootDir(), e);
}
}
- LOG.info("Upgrade all volume container LayoutVersion costs {}s",
- (System.currentTimeMillis() - startTime) / 1000);
+ LOG.info("It took {}ms to finish all volume upgrade.",
+ (System.currentTimeMillis() - startTime));
return results;
}
@@ -159,6 +123,10 @@ public class UpgradeManager {
return resultMap;
}
+ public boolean isSuccess() {
+ return this.status == Status.SUCCESS;
+ }
+
public void success() {
this.endTimeMs = System.currentTimeMillis();
this.status = Status.SUCCESS;
@@ -173,21 +141,23 @@ public class UpgradeManager {
@Override
public String toString() {
final StringBuilder stringBuilder = new StringBuilder();
- stringBuilder.append("Result{");
- stringBuilder.append("volumeDir=");
+ stringBuilder.append("Result:{");
+ stringBuilder.append("hddsRootDir=");
stringBuilder.append(getHddsVolume().getHddsRootDir());
stringBuilder.append(", resultList=");
AtomicLong total = new AtomicLong(0L);
- resultMap.forEach((k, r) -> {
- stringBuilder.append(r.toString());
- stringBuilder.append("\n");
- total.addAndGet(r.getTotalRow());
- });
+ if (resultMap != null) {
+ resultMap.forEach((k, r) -> {
+ stringBuilder.append(r.toString());
+ stringBuilder.append("\n");
+ total.addAndGet(r.getTotalRow());
+ });
+ }
stringBuilder.append(", totalRow=");
stringBuilder.append(total.get());
stringBuilder.append(", costMs=");
stringBuilder.append(getCost());
- stringBuilder.append("ms, status=");
+ stringBuilder.append(", status=");
stringBuilder.append(status);
if (e != null) {
stringBuilder.append(", Exception=");
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeTask.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeTask.java
index cb21cd978a..5d12b12bc7 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeTask.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeTask.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.cli.container.upgrade;
import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -32,6 +33,8 @@ import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
+import org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache;
+import org.apache.hadoop.ozone.container.common.utils.RawDB;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
@@ -46,11 +49,16 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Date;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_NAME;
+
/**
* This class implements the v2 to v3 container upgrade process.
*/
@@ -61,22 +69,27 @@ public class UpgradeTask {
private final ConfigurationSource config;
private final HddsVolume hddsVolume;
- private final DatanodeStoreSchemaThreeImpl datanodeStoreSchemaThree;
+ private DatanodeStoreSchemaThreeImpl dataStore;
+ private final Map volumeStoreMap;
private static final String BACKUP_CONTAINER_DATA_FILE_SUFFIX = ".backup";
+ public static final String UPGRADE_COMPLETE_FILE_NAME = "upgrade.complete";
+ public static final String UPGRADE_LOCK_FILE_NAME = "upgrade.lock";
+
private static final Set<String> COLUMN_FAMILIES_NAME =
(new DatanodeSchemaTwoDBDefinition("", new OzoneConfiguration()))
.getMap().keySet();
- public UpgradeTask(ConfigurationSource config,
- HddsVolume hddsVolume,
- DatanodeStoreSchemaThreeImpl datanodeStoreSchemaThree) {
+ public UpgradeTask(ConfigurationSource config, HddsVolume hddsVolume,
+ Map storeMap) {
this.config = config;
this.hddsVolume = hddsVolume;
- this.datanodeStoreSchemaThree = datanodeStoreSchemaThree;
+ this.volumeStoreMap = storeMap;
}
- public CompletableFuture<UpgradeManager.Result> getUpgradeFutureByVolume() {
+ public CompletableFuture<UpgradeManager.Result> getUpgradeFuture() {
+ final File lockFile = UpgradeUtils.getVolumeUpgradeLockFile(hddsVolume);
+
return CompletableFuture.supplyAsync(() -> {
final UpgradeManager.Result result =
@@ -88,63 +101,111 @@ public class UpgradeTask {
Preconditions.checkNotNull(hddsVolumeRootDir, "hddsVolumeRootDir" +
"cannot be null");
- //filtering storage directory
- File[] storageDirs = hddsVolumeRootDir.listFiles(File::isDirectory);
-
- if (storageDirs == null) {
+ // check CID directory and current file
+ File clusterIDDir = new File(hddsVolume.getStorageDir(),
+ hddsVolume.getClusterID());
+ if (!clusterIDDir.exists() || !clusterIDDir.isDirectory()) {
+ result.fail(new Exception("Volume " + hddsVolumeRootDir +
+ " is in an inconsistent state. Expected " +
+ "clusterID directory " + clusterIDDir +
+ " is not found or not a directory."));
+ return result;
+ }
+ File currentDir = new File(clusterIDDir, Storage.STORAGE_DIR_CURRENT);
+ if (!currentDir.exists() || !currentDir.isDirectory()) {
result.fail(new Exception(
- "Storage dir not found for the volume " + hddsVolumeRootDir +
- ", skipped upgrade."));
+ "Current dir " + currentDir + " is not found or not a directory,"
+ + " skip upgrade."));
return result;
}
- if (storageDirs.length > 0) {
- File clusterIDDir = new File(hddsVolume.getStorageDir(),
- hddsVolume.getClusterID());
- File idDir = clusterIDDir;
- if (storageDirs.length == 1 && !clusterIDDir.exists()) {
- idDir = storageDirs[0];
- } else {
- if (!clusterIDDir.exists()) {
- result.fail(new Exception("Volume " + hddsVolumeRootDir +
- " is in an inconsistent state. Expected " +
- "clusterID directory " + clusterIDDir + " not found."));
- return result;
- }
- }
-
- LOG.info("Start to upgrade containers on volume {}",
hddsVolumeRootDir);
- File currentDir = new File(idDir, Storage.STORAGE_DIR_CURRENT);
-
- if (!currentDir.exists()) {
- result.fail(new Exception(
- "Storage current dir not found for the volume " +
- currentDir + ", skipped upgrade."));
+ try {
+ // create lock file
+ if (!lockFile.createNewFile()) {
+ result.fail(new Exception("Upgrade lock file already exists " +
+ lockFile.getAbsolutePath() + ", skip upgrade."));
return result;
}
- File[] containerTopDirs = currentDir.listFiles();
- if (containerTopDirs != null) {
- for (File containerTopDir : containerTopDirs) {
- try {
- final List<UpgradeContainerResult> results =
- upgradeSubContainerDir(containerTopDir);
- resultList.addAll(results);
- } catch (IOException e) {
- result.fail(e);
- return result;
- }
- }
+ } catch (IOException e) {
+ result.fail(new Exception("Failed to create upgrade lock file " +
+ lockFile.getAbsolutePath() + ", skip upgrade."));
+ return result;
+ }
+
+ // check complete file again
+ final File completeFile =
+ UpgradeUtils.getVolumeUpgradeCompleteFile(hddsVolume);
+ if (completeFile.exists()) {
+ result.fail(new Exception("Upgrade complete file already exists " +
+ completeFile.getAbsolutePath() + ", skip upgrade."));
+ if (!lockFile.delete()) {
+ LOG.warn("Failed to delete upgrade lock file {}.", lockFile);
}
- } else {
+ return result;
+ }
+
+ // backup DB directory
+ final File volumeDBPath;
+ try {
+ volumeDBPath = getVolumeDBPath(hddsVolume);
+ dbBackup(volumeDBPath);
+ } catch (IOException e) {
+ result.fail(new Exception(e.getMessage() + ", skip upgrade."));
+ return result;
+ }
+
+ // load DB store
+ try {
+ hddsVolume.loadDbStore(false);
+ RawDB db = DatanodeStoreCache.getInstance().getDB(
+ volumeDBPath.getAbsolutePath(), config);
+ dataStore = (DatanodeStoreSchemaThreeImpl) db.getStore();
+ volumeStoreMap.put(
+ hddsVolume.getStorageDir().getAbsolutePath(), dataStore);
+ } catch (IOException e) {
result.fail(new Exception(
- "Storage dir not found for the volume " + hddsVolumeRootDir +
- ", skipped upgrade."));
+ "Failed to load db for volume " + hddsVolume.getVolumeRootDir() +
+ " for " + e.getMessage() + ", skip upgrade."));
return result;
}
+ LOG.info("Start to upgrade containers on volume {}",
+ hddsVolume.getVolumeRootDir());
+ File[] containerTopDirs = currentDir.listFiles();
+ if (containerTopDirs != null) {
+ for (File containerTopDir : containerTopDirs) {
+ try {
+ final List<UpgradeContainerResult> results =
+ upgradeSubContainerDir(containerTopDir);
+ resultList.addAll(results);
+ } catch (IOException e) {
+ result.fail(e);
+ return result;
+ }
+ }
+ }
+
result.setResultList(resultList);
result.success();
return result;
+ }).whenComplete((r, e) -> {
+ final File hddsRootDir = r.getHddsVolume().getHddsRootDir();
+ final File file =
+ UpgradeUtils.getVolumeUpgradeCompleteFile(r.getHddsVolume());
+ // create a flag file
+ if (e == null && r.isSuccess()) {
+ try {
+ UpgradeUtils.createFile(file);
+ } catch (IOException ioe) {
+ LOG.warn("Failed to create upgrade complete file {}.", file, ioe);
+ }
+ }
+ if (lockFile.exists()) {
+ boolean deleted = lockFile.delete();
+ if (!deleted) {
+ LOG.warn("Failed to delete upgrade lock file {}.", file);
+ }
+ }
});
}
@@ -172,58 +233,46 @@ public class UpgradeTask {
private ContainerData parseContainerData(File containerDir) {
try {
- File containerFile = ContainerUtils.getContainerFile(
- containerDir);
- long containerID =
- ContainerUtils.getContainerID(containerDir);
- if (containerFile.exists()) {
- try {
- ContainerData containerData =
- ContainerDataYaml.readContainerFile(
- containerFile);
- if (containerID != containerData.getContainerID()) {
- LOG.error("Invalid ContainerID in file {}. " +
- "Skipping loading of this container.",
- containerFile);
- return null;
- }
- if (containerData.getContainerType().equals(
- ContainerProtos.ContainerType.KeyValueContainer) &&
- containerData instanceof KeyValueContainerData) {
- KeyValueContainerData kvContainerData =
- (KeyValueContainerData)
- containerData;
- containerData.setVolume(hddsVolume);
- KeyValueContainerUtil
- .parseKVContainerData(kvContainerData, config);
-
- return kvContainerData;
- } else {
- LOG.error("Only KeyValueContainer support. ");
- return null;
- }
-
- } catch (IOException ex) {
- LOG.error(
- "Failed to parse ContainerFile for ContainerID: {}",
- containerID, ex);
+ File containerFile = ContainerUtils.getContainerFile(containerDir);
+ long containerID = ContainerUtils.getContainerID(containerDir);
+ if (!containerFile.exists()) {
+ LOG.error("Missing .container file: {}.", containerDir);
+ return null;
+ }
+ try {
+ ContainerData containerData =
+ ContainerDataYaml.readContainerFile(containerFile);
+ if (containerID != containerData.getContainerID()) {
+ LOG.error("ContainerID in file {} mismatch with expected {}.",
+ containerFile, containerID);
+ return null;
+ }
+ if (containerData.getContainerType().equals(
+ ContainerProtos.ContainerType.KeyValueContainer) &&
+ containerData instanceof KeyValueContainerData) {
+ KeyValueContainerData kvContainerData =
+ (KeyValueContainerData) containerData;
+ containerData.setVolume(hddsVolume);
+ KeyValueContainerUtil.parseKVContainerData(kvContainerData, config);
+ return kvContainerData;
+ } else {
+ LOG.error("Container is not KeyValueContainer type: {}.",
+ containerDir);
return null;
}
- } else {
- LOG.error("Missing .container file for ContainerID: {}",
- containerDir.getName());
+ } catch (IOException ex) {
+ LOG.error("Failed to parse ContainerFile: {}.", containerFile, ex);
return null;
}
} catch (Throwable e) {
- LOG.error("Failed to load container from {}",
- containerDir.getAbsolutePath(), e);
+ LOG.error("Failed to load container: {}.", containerDir, e);
return null;
}
}
private void upgradeContainer(ContainerData containerData,
UpgradeContainerResult result) throws IOException {
- final DBStore targetDBStore = datanodeStoreSchemaThree.getStore();
+ final DBStore targetDBStore = dataStore.getStore();
// open container schema v2 rocksdb
final DatanodeStore dbStore = BlockUtils
@@ -306,6 +355,30 @@ public class UpgradeTask {
}
}
+ public File getVolumeDBPath(HddsVolume volume) throws IOException {
+ File clusterIdDir = new File(volume.getStorageDir(),
volume.getClusterID());
+ File storageIdDir = new File(clusterIdDir, volume.getStorageID());
+ File containerDBPath = new File(storageIdDir, CONTAINER_DB_NAME);
+ if (containerDBPath.exists() && containerDBPath.isDirectory()) {
+ return containerDBPath;
+ } else {
+ throw new IOException("DB " + containerDBPath +
+ " doesn't exist or is not a directory");
+ }
+ }
+
+ public void dbBackup(File dbPath) throws IOException {
+ final File backup = new File(dbPath.getParentFile(),
+ new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss").format(new Date()) +
+ "-" + dbPath.getName() + ".backup");
+ if (backup.exists()) {
+ throw new IOException("Backup dir " + backup + "already exists");
+ } else {
+ FileUtils.copyDirectory(dbPath, backup, true);
+ System.out.println("DB " + dbPath + " is backup to " + backup);
+ }
+ }
+
/**
* This class represents upgrade v2 to v3 container result.
*/
@@ -368,7 +441,7 @@ public class UpgradeTask {
@Override
public String toString() {
final StringBuilder stringBuilder = new StringBuilder();
- stringBuilder.append("Result{");
+ stringBuilder.append("Result:{");
stringBuilder.append("containerID=");
stringBuilder.append(originContainerData.getContainerID());
stringBuilder.append(", originContainerSchemaVersion=");
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeUtils.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeUtils.java
index b9fa820d71..9047bd5fc7 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeUtils.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/UpgradeUtils.java
@@ -27,7 +27,12 @@ import
org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.Date;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_NAME;
@@ -59,4 +64,24 @@ public final class UpgradeUtils {
public static File getContainerDBPath(HddsVolume volume) {
return new File(volume.getDbParentDir(), CONTAINER_DB_NAME);
}
+
+ public static File getVolumeUpgradeCompleteFile(HddsVolume volume) {
+ return new File(volume.getHddsRootDir(),
+ UpgradeTask.UPGRADE_COMPLETE_FILE_NAME);
+ }
+
+ public static File getVolumeUpgradeLockFile(HddsVolume volume) {
+ return new File(volume.getHddsRootDir(),
+ UpgradeTask.UPGRADE_LOCK_FILE_NAME);
+ }
+
+ public static boolean createFile(File file) throws IOException {
+ final Date date = new Date();
+ try (Writer writer = new OutputStreamWriter(new FileOutputStream(file),
+ StandardCharsets.UTF_8)) {
+ writer.write(date.toString());
+ }
+ return file.exists();
+ }
+
}
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java
index 9f4c0aa26f..ee02f0e506 100644
---
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/upgrade/TestUpgradeManager.java
@@ -159,9 +159,9 @@ public class TestUpgradeManager {
shutdownAllVolume();
final UpgradeManager upgradeManager = new UpgradeManager();
- upgradeManager.initVolumeStoreMap(volumeSet, CONF);
final List<UpgradeManager.Result> results =
- upgradeManager.upgradeAll(volumeSet, CONF);
+ upgradeManager.run(CONF,
+ StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()));
checkV3MetaData(keyValueContainerBlockDataMap, results, upgradeManager);
}
@@ -269,8 +269,7 @@ public class TestUpgradeManager {
private void checkV3MetaData(Map<KeyValueContainerData,
Map<String, BlockData>> blockDataMap, List<UpgradeManager.Result>
results,
- UpgradeManager upgradeManager)
- throws IOException {
+ UpgradeManager upgradeManager) throws IOException {
Map<Long, UpgradeTask.UpgradeContainerResult> resultMap = new HashMap<>();
for (UpgradeManager.Result result : results) {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java
new file mode 100644
index 0000000000..5f2380e97e
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands;
+import org.apache.hadoop.hdds.scm.cli.container.UpgradeSubcommand;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.hdds.utils.db.CodecTestUtil;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectMetrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.ozone.HddsDatanodeService;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneTestUtils;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
+import org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED;
+
+/**
+ * Test Ozone Container upgrade shell.
+ */
+public class TestOzoneContainerUpgradeShell {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestOzoneContainerUpgradeShell.class);
+ private static String omServiceId;
+ private static String clusterId;
+ private static String scmId;
+ private static MiniOzoneCluster cluster = null;
+ private static OzoneClient client;
+ private static OzoneConfiguration conf = null;
+ private static final String VOLUME_NAME = UUID.randomUUID().toString();
+ private static final String BUCKET_NAME = UUID.randomUUID().toString();
+
+ protected static void startCluster() throws Exception {
+ // Init HA cluster
+ omServiceId = "om-service-test-upgrade-container1";
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ final int numDNs = 3;
+ cluster = MiniOzoneCluster.newBuilder(conf).setClusterId(clusterId)
+ .setScmId(scmId).setOMServiceId(omServiceId).setNumDatanodes(numDNs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ client = cluster.newClient();
+ }
+
+ @BeforeAll
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(OZONE_ADMINISTRATORS, "*");
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
+ TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, SECONDS);
+ ReplicationManager.ReplicationManagerConfiguration replicationConf =
+ conf.getObject(
+ ReplicationManager.ReplicationManagerConfiguration.class);
+ replicationConf.setInterval(Duration.ofSeconds(1));
+ conf.setFromObject(replicationConf);
+ // gen schema v2 container
+ conf.setBoolean(CONTAINER_SCHEMA_V3_ENABLED, false);
+
+ startCluster();
+ }
+
+ public List<OzoneConfiguration> getDatanodeConfigs() {
+ List<OzoneConfiguration> configList = new ArrayList<>();
+ for (HddsDatanodeService datanodeService : cluster.getHddsDatanodes()) {
+ OzoneConfiguration datanodeConf = datanodeService.getConf();
+ configList.add(datanodeConf);
+ }
+ return configList;
+ }
+
+ @Test
+ public void testNormalContainerUpgrade() throws Exception {
+ writeKeyAndCloseContainer();
+ List<OzoneConfiguration> datanodeConfigs = getDatanodeConfigs();
+ HddsDatanodeService hddsDatanodeService =
cluster.getHddsDatanodes().get(0);
+ DatanodeDetails dni = hddsDatanodeService.getDatanodeDetails();
+ OzoneConfiguration datanodeConf = datanodeConfigs.get(0);
+
+ // create volume rocksdb
+ OzoneConfiguration newConf = new OzoneConfiguration(datanodeConf);
+ newConf.setBoolean(CONTAINER_SCHEMA_V3_ENABLED, true);
+ hddsDatanodeService.setConfiguration(newConf);
+ cluster.restartHddsDatanode(dni, true);
+
+ nodeOperationalStateToMaintenance(dni, datanodeConf);
+
+ shutdownCluster();
+
+ // datanode1 test check all pass & upgrade success
+ UpgradeSubcommand.setOzoneConfiguration(datanodeConf);
+ StringWriter stdout = new StringWriter();
+ PrintWriter pstdout = new PrintWriter(stdout);
+ CommandLine commandLine = upgradeCommand(pstdout);
+
+ String[] args = new String[]{"upgrade", "--yes"};
+ int exitCode = commandLine.execute(args);
+ Assertions.assertEquals(0, exitCode);
+
+ // datanode2 NodeOperationalState is IN_SERVICE upgrade fail.
+ OzoneConfiguration datanode2Conf = datanodeConfigs.get(1);
+ UpgradeSubcommand.setOzoneConfiguration(datanode2Conf);
+ StringWriter stdout2 = new StringWriter();
+ PrintWriter pstdout2 = new PrintWriter(stdout2);
+ CommandLine commandLine2 = upgradeCommand(pstdout2);
+
+ String[] args2 = new String[]{"upgrade", "--yes"};
+ int exit2Code = commandLine2.execute(args2);
+
+ Assertions.assertEquals(0, exit2Code);
+ String cmdOut = stdout2.toString();
+ Assertions.assertTrue(cmdOut.contains("IN_MAINTENANCE"));
+ }
+
+ private CommandLine upgradeCommand(PrintWriter pstdout) {
+ return new CommandLine(new ContainerCommands()).setOut(pstdout);
+ }
+
+ private static ContainerInfo writeKeyAndCloseContainer() throws Exception {
+ final String keyName = UUID.randomUUID().toString();
+ writeKey(keyName);
+ return closeContainerForKey(keyName);
+ }
+
+ private static void writeKey(String keyName) throws IOException {
+ try (OzoneClient client = OzoneClientFactory.getRpcClient(conf)) {
+ TestDataUtil.createVolumeAndBucket(client, VOLUME_NAME, BUCKET_NAME);
+ TestDataUtil.createKey(
+
client.getObjectStore().getVolume(VOLUME_NAME).getBucket(BUCKET_NAME),
+ keyName, ReplicationFactor.THREE, ReplicationType.RATIS, "test");
+ }
+ }
+
+ private static ContainerInfo closeContainerForKey(String keyName)
+ throws IOException, TimeoutException, InterruptedException {
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(VOLUME_NAME)
+ .setBucketName(BUCKET_NAME).setKeyName(keyName).build();
+
+ OmKeyLocationInfo omKeyLocationInfo =
+ cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
+ .get(0).getBlocksLatestVersionOnly().get(0);
+
+ ContainerInfo container =
+
cluster.getStorageContainerManager().getContainerManager().getContainer(
+ ContainerID.valueOf(omKeyLocationInfo.getContainerID()));
+
+ OzoneTestUtils.closeContainer(cluster.getStorageContainerManager(),
+ container);
+ return container;
+ }
+
+ public static void shutdownCluster() throws InterruptedException {
+
+ try {
+ IOUtils.closeQuietly(client);
+ if (cluster != null) {
+ List<OzoneConfiguration> dnConfigs =
cluster.getHddsDatanodes().stream()
+ .map(HddsDatanodeService::getConf).collect(Collectors.toList());
+
+ DatanodeStoreCache.setMiniClusterMode(false);
+
+ cluster.stop();
+ ContainerCache.getInstance(conf).shutdownCache();
+
+
+ for (OzoneConfiguration dnConfig : dnConfigs) {
+ ContainerCache.getInstance(dnConfig).shutdownCache();
+ }
+ DefaultMetricsSystem.shutdown();
+ ManagedRocksObjectMetrics.INSTANCE.assertNoLeaks();
+ CodecTestUtil.gc();
+ }
+ } catch (Exception e) {
+ LOG.error("Exception while shutting down the cluster.", e);
+ }
+ }
+
+ private void nodeOperationalStateToMaintenance(DatanodeDetails dnDetails,
+ OzoneConfiguration config) throws IOException {
+ dnDetails.setPersistedOpState(
+ HddsProtos.NodeOperationalState.IN_MAINTENANCE);
+ String idFilePath = HddsServerUtil.getDatanodeIdFilePath(config);
+ Preconditions.checkNotNull(idFilePath);
+ File idFile = new File(idFilePath);
+ ContainerUtils.writeDatanodeDetailsTo(dnDetails, idFile, config);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
