This is an automated email from the ASF dual-hosted git repository.
erose pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7e2cdf0 HDDS-5432. Enable downgrade testing after 1.1.0 release.
(#2484)
7e2cdf0 is described below
commit 7e2cdf009883842c11373d4aae8262081d5d00ed
Author: Ethan Rose <[email protected]>
AuthorDate: Tue Sep 7 09:48:02 2021 -0700
HDDS-5432. Enable downgrade testing after 1.1.0 release. (#2484)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 -
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 -
.../upgrade/AbstractLayoutVersionManager.java | 158 +++--
.../common/src/main/resources/ozone-default.xml | 9 -
.../apache/hadoop/ozone/HddsDatanodeService.java | 5 +-
.../common/interfaces/ContainerDispatcher.java | 7 +-
.../common/statemachine/DatanodeStateMachine.java | 6 +-
.../states/endpoint/VersionEndpointTask.java | 5 +-
.../container/common/utils/HddsVolumeUtil.java | 72 +-
.../ozone/container/common/volume/HddsVolume.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 13 -
.../container/keyvalue/KeyValueContainer.java | 12 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 5 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 7 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 161 ++---
.../ozone/container/ozoneimpl/OzoneContainer.java | 5 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 5 +-
.../upgrade/DatanodeMetadataFeatures.java | 53 --
.../ScmHAFinalizeUpgradeActionDatanode.java | 129 ++++
.../ScmHAFirstUpgradeLayoutChangeAction.java | 45 --
.../upgrade/VersionedDatanodeFeatures.java | 145 ++++
.../hadoop/ozone/container/common/ScmTestMock.java | 9 +
.../ozone/container/common/TestContainerCache.java | 23 +-
.../common/TestKeyValueContainerData.java | 8 +-
.../common/impl/TestContainerDataYaml.java | 6 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 741 +++++++++++++++++++++
.../ozone/container/common/TestEndPoint.java | 4 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 16 -
.../dist/src/main/compose/upgrade/README.md | 35 +-
.../compose/upgrade/delete-and-regenerate-data.sh | 53 --
.../dist/src/main/compose/upgrade/delete-data.sh | 36 -
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 9 +-
.../manual-upgrade/0.5.0-1.1.0/callback.sh | 40 --
.../upgrades/manual-upgrade}/README.md | 7 +-
.../{1.0.0-1.1.0 => 1.1.0-1.2.0}/callback.sh | 17 +-
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 31 +-
.../dist/src/main/compose/versions/0.5.0.sh | 26 -
.../dist/src/main/compose/versions/1.0.0.sh | 26 -
.../dist/src/main/compose/versions/1.1.0.sh | 26 -
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 4 -
.../container/ozoneimpl/TestOzoneContainer.java | 8 +-
.../ozone/dn/TestDatanodeLayoutUpgradeTool.java | 131 ----
.../ozone/om/upgrade/OMLayoutVersionManager.java | 3 +-
.../apache/hadoop/ozone/debug/DatanodeLayout.java | 110 ---
44 files changed, 1310 insertions(+), 910 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index ab8311e..57b80cb 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -454,11 +454,6 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL_DEFAULT
=
"30s";
- public static final String HDDS_DATANODE_UPGRADE_LAYOUT_INLINE =
- "hdds.datanode.upgrade.layout.inline";
- public static final boolean HDDS_DATANODE_UPGRADE_LAYOUT_INLINE_DEFAULT =
- true;
-
public static final String OZONE_SCM_INFO_WAIT_DURATION =
"ozone.scm.info.wait.duration";
public static final long OZONE_SCM_INFO_WAIT_DURATION_DEFAULT =
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 16ef3cd..a373fd1 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -271,8 +271,6 @@ public final class OzoneConsts {
// V2: Metadata, block data, and delete transactions in their own
// column families.
public static final String SCHEMA_V2 = "2";
- // Most recent schema version that all new containers should be created with.
- public static final String SCHEMA_LATEST = SCHEMA_V2;
public static final String[] SCHEMA_VERSIONS =
new String[] {SCHEMA_V1, SCHEMA_V2};
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java
index ef74ec7..2ce6e02 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java
@@ -23,12 +23,14 @@ import static
org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.Status.FINALIZATI
import static
org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.Status.FINALIZATION_REQUIRED;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
-import java.util.stream.Collectors;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.Status;
@@ -45,43 +47,63 @@ public abstract class AbstractLayoutVersionManager<T
extends LayoutFeature>
private static final Logger LOG =
LoggerFactory.getLogger(AbstractLayoutVersionManager.class);
- protected int metadataLayoutVersion; // MLV.
- protected int softwareLayoutVersion; // SLV.
- protected TreeMap<Integer, T> features = new TreeMap<>();
- protected Map<String, T> featureMap = new HashMap<>();
- protected volatile Status currentUpgradeState = FINALIZATION_REQUIRED;
+ private int metadataLayoutVersion; // MLV.
+ private int softwareLayoutVersion; // SLV.
+ @VisibleForTesting
+ protected final TreeMap<Integer, T> features = new TreeMap<>();
+ @VisibleForTesting
+ protected final Map<String, T> featureMap = new HashMap<>();
+ private volatile Status currentUpgradeState = FINALIZATION_REQUIRED;
+ // Allows querying upgrade state while an upgrade is in progress.
+ // Note that MLV may have been incremented during the upgrade
+ // by the time the value is read/used.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
protected void init(int version, T[] lfs) throws IOException {
+ lock.writeLock().lock();
+ try {
+ metadataLayoutVersion = version;
+ initializeFeatures(lfs);
+ softwareLayoutVersion = features.lastKey();
+ if (softwareIsBehindMetaData()) {
+ throw new IOException(
+ String.format("Cannot initialize VersionManager. Metadata " +
+ "layout version (%d) > software layout version (%d)",
+ metadataLayoutVersion, softwareLayoutVersion));
+ } else if (metadataLayoutVersion == softwareLayoutVersion) {
+ currentUpgradeState = ALREADY_FINALIZED;
+ }
- metadataLayoutVersion = version;
- initializeFeatures(lfs);
- softwareLayoutVersion = features.lastKey();
- if (softwareIsBehindMetaData()) {
- throw new IOException(
- String.format("Cannot initialize VersionManager. Metadata " +
- "layout version (%d) > software layout version (%d)",
- metadataLayoutVersion, softwareLayoutVersion));
- } else if (metadataLayoutVersion == softwareLayoutVersion) {
- currentUpgradeState = ALREADY_FINALIZED;
+ LayoutFeature mlvFeature = features.get(metadataLayoutVersion);
+ LayoutFeature slvFeature = features.get(softwareLayoutVersion);
+ LOG.info("Initializing Layout version manager with metadata layout" +
+ " = {} (version = {}), software layout = {} (version = {})",
+ mlvFeature, mlvFeature.layoutVersion(),
+ slvFeature, slvFeature.layoutVersion());
+
+ MBeans.register("LayoutVersionManager",
+ getClass().getSimpleName(), this);
+ } finally {
+ lock.writeLock().unlock();
}
-
- LayoutFeature mlvFeature = features.get(metadataLayoutVersion);
- LayoutFeature slvFeature = features.get(softwareLayoutVersion);
- LOG.info("Initializing Layout version manager with metadata layout" +
- " = {} (version = {}), software layout = {} (version = {})",
- mlvFeature, mlvFeature.layoutVersion(),
- slvFeature, slvFeature.layoutVersion());
-
- MBeans.register("LayoutVersionManager",
- getClass().getSimpleName(), this);
}
public Status getUpgradeState() {
- return currentUpgradeState;
+ lock.readLock().lock();
+ try {
+ return currentUpgradeState;
+ } finally {
+ lock.readLock().unlock();
+ }
}
public void setUpgradeState(Status status) {
- currentUpgradeState = status;
+ lock.writeLock().lock();
+ try {
+ currentUpgradeState = status;
+ } finally {
+ lock.writeLock().unlock();
+ }
}
private void initializeFeatures(T[] lfs) {
@@ -94,35 +116,56 @@ public abstract class AbstractLayoutVersionManager<T
extends LayoutFeature>
}
public void finalized(T layoutFeature) {
- if (layoutFeature.layoutVersion() == metadataLayoutVersion + 1) {
- metadataLayoutVersion = layoutFeature.layoutVersion();
- } else {
- String msgStart = "";
- if (layoutFeature.layoutVersion() < metadataLayoutVersion) {
- msgStart = "Finalize attempt on a layoutFeature which has already "
- + "been finalized.";
+ lock.writeLock().lock();
+ try {
+ if (layoutFeature.layoutVersion() == metadataLayoutVersion + 1) {
+ metadataLayoutVersion = layoutFeature.layoutVersion();
} else {
- msgStart = "Finalize attempt on a layoutFeature that is newer than the"
- + " next feature to be finalized.";
+ String msgStart = "";
+ if (layoutFeature.layoutVersion() < metadataLayoutVersion) {
+ msgStart = "Finalize attempt on a layoutFeature which has already "
+ + "been finalized.";
+ } else {
+ msgStart =
+ "Finalize attempt on a layoutFeature that is newer than the " +
+ "next feature to be finalized.";
+ }
+
+ throw new IllegalArgumentException(
+ msgStart + "Software Layout version: " + softwareLayoutVersion
+ + " Feature Layout version: " + layoutFeature.layoutVersion());
}
-
- throw new IllegalArgumentException(
- msgStart + "Software Layout version: " + softwareLayoutVersion
- + " Feature Layout version: " + layoutFeature.layoutVersion());
+ } finally {
+ lock.writeLock().unlock();
}
}
public void completeFinalization() {
- currentUpgradeState = FINALIZATION_DONE;
+ lock.writeLock().lock();
+ try {
+ currentUpgradeState = FINALIZATION_DONE;
+ } finally {
+ lock.writeLock().unlock();
+ }
}
private boolean softwareIsBehindMetaData() {
- return metadataLayoutVersion > softwareLayoutVersion;
+ lock.readLock().lock();
+ try {
+ return metadataLayoutVersion > softwareLayoutVersion;
+ } finally {
+ lock.readLock().unlock();
+ }
}
@Override
public int getMetadataLayoutVersion() {
- return metadataLayoutVersion;
+ lock.readLock().lock();
+ try {
+ return metadataLayoutVersion;
+ } finally {
+ lock.readLock().unlock();
+ }
}
@Override
@@ -132,12 +175,22 @@ public abstract class AbstractLayoutVersionManager<T
extends LayoutFeature>
@Override
public boolean needsFinalization() {
- return metadataLayoutVersion < softwareLayoutVersion;
+ lock.readLock().lock();
+ try {
+ return metadataLayoutVersion < softwareLayoutVersion;
+ } finally {
+ lock.readLock().unlock();
+ }
}
@Override
public boolean isAllowed(LayoutFeature layoutFeature) {
- return layoutFeature.layoutVersion() <= metadataLayoutVersion;
+ lock.readLock().lock();
+ try {
+ return layoutFeature.layoutVersion() <= metadataLayoutVersion;
+ } finally {
+ lock.readLock().unlock();
+ }
}
@Override
@@ -153,10 +206,13 @@ public abstract class AbstractLayoutVersionManager<T
extends LayoutFeature>
@Override
public Iterable<T> unfinalizedFeatures() {
- return features
- .tailMap(metadataLayoutVersion + 1)
- .values()
- .stream()
- .collect(Collectors.toList());
+ lock.readLock().lock();
+ try {
+ return new ArrayList<>(features
+ .tailMap(metadataLayoutVersion + 1)
+ .values());
+ } finally {
+ lock.readLock().unlock();
+ }
}
}
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 57512a7..db0a7a7 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -145,15 +145,6 @@
</description>
</property>
<property>
- <name>hdds.datanode.upgrade.layout.inline</name>
- <value>true</value>
- <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT</tag>
- <description>Determines whether to upgrade the DN layout on restart
- automatically. If set of false, the tool verifies that the current
- disk format is correct.
- </description>
- </property>
- <property>
<name>hdds.datanode.dir.du.reserved</name>
<value/>
<tag>OZONE, CONTAINER, STORAGE, MANAGEMENT</tag>
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 56003e1..f43abe8 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -309,7 +309,6 @@ public class HddsDatanodeService extends GenericCli
implements ServicePlugin {
* datanode.
*/
private void startRatisForTest() throws IOException {
- String scmId = "scm-01";
String clusterId = "clusterId";
datanodeStateMachine.getContainer().start(clusterId);
MutableVolumeSet volumeSet =
@@ -319,8 +318,8 @@ public class HddsDatanodeService extends GenericCli
implements ServicePlugin {
for (Map.Entry<String, StorageVolume> entry : volumeMap.entrySet()) {
HddsVolume hddsVolume = (HddsVolume) entry.getValue();
- boolean result = HddsVolumeUtil.checkVolume(hddsVolume, scmId,
- clusterId, LOG);
+ boolean result = HddsVolumeUtil.checkVolume(hddsVolume, clusterId,
+ clusterId, conf, LOG);
if (!result) {
volumeSet.failVolume(hddsVolume.getHddsRootDir().getPath());
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
index 77145ca..a2e397d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
@@ -79,8 +79,9 @@ public interface ContainerDispatcher {
Handler getHandler(ContainerProtos.ContainerType containerType);
/**
- * If scmId is not set, this will set scmId, otherwise it is a no-op.
- * @param scmId
+ * If cluster ID is not set, this will set cluster ID, otherwise it is a
+ * no-op.
+ * @param clusterId
*/
- void setClusterId(String scmId);
+ void setClusterId(String clusterId);
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index e748382..d7959b3 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -57,7 +57,7 @@ import
org.apache.hadoop.ozone.container.replication.MeasuredReplicator;
import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
import org.apache.hadoop.ozone.container.upgrade.DataNodeUpgradeFinalizer;
-import org.apache.hadoop.ozone.container.upgrade.DatanodeMetadataFeatures;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
@@ -133,7 +133,7 @@ public class DatanodeStateMachine implements Closeable {
layoutVersionManager = new HDDSLayoutVersionManager(
layoutStorage.getLayoutVersion());
upgradeFinalizer = new DataNodeUpgradeFinalizer(layoutVersionManager);
- DatanodeMetadataFeatures.initialize(layoutVersionManager);
+ VersionedDatanodeFeatures.initialize(layoutVersionManager);
this.dnCRLStore = crlStore;
executorService = Executors.newFixedThreadPool(
@@ -453,8 +453,8 @@ public class DatanodeStateMachine implements Closeable {
public void startDaemon() {
Runnable startStateMachineTask = () -> {
try {
- start();
LOG.info("Ozone container server started.");
+ start();
} catch (Exception ex) {
LOG.error("Unable to start the DatanodeState Machine", ex);
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
index d067000..fa6c937 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
@@ -90,13 +90,12 @@ public class VersionEndpointTask implements
"Reply from SCM: clusterId cannot be null");
// If version file does not exist
- // create version file and also set scmId
-
+ // create version file and also set scm ID or cluster ID.
for (Map.Entry<String, StorageVolume> entry
: volumeMap.entrySet()) {
StorageVolume volume = entry.getValue();
boolean result = HddsVolumeUtil.checkVolume((HddsVolume) volume,
- scmId, clusterId, LOG);
+ scmId, clusterId, configuration, LOG);
if (!result) {
volumeSet.failVolume(volume.getStorageDir().getPath());
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
index 35dab34..83b8615 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
@@ -20,11 +20,12 @@ package org.apache.hadoop.ozone.container.common.utils;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
import org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
@@ -168,18 +169,23 @@ public final class HddsVolumeUtil {
/**
* Check Volume is in consistent state or not.
+ * Prior to SCM HA, volumes used the format {@code <volume>/hdds/<scm-id>}.
+ * Post SCM HA, new volumes will use the format {@code <volume>/hdds/<cluster
+ * -id>}.
+ * Existing volumes using SCM ID would have been reformatted to have {@code
+ * <volume>/hdds/<cluster-id>} as a symlink pointing to {@code <volume
+ * >/hdds/<scm-id>}.
+ *
* @param hddsVolume
- * @param scmId
* @param clusterId
* @param logger
* @return true - if volume is in consistent state, otherwise false.
*/
- public static boolean checkVolume(HddsVolume hddsVolume, String scmId, String
- clusterId, Logger logger) {
+ public static boolean checkVolume(HddsVolume hddsVolume, String scmId,
+ String clusterId, ConfigurationSource conf, Logger logger) {
File hddsRoot = hddsVolume.getHddsRootDir();
String volumeRoot = hddsRoot.getPath();
File clusterDir = new File(hddsRoot, clusterId);
- File scmDir = new File(hddsRoot, scmId);
try {
hddsVolume.format(clusterId);
@@ -191,47 +197,41 @@ public final class HddsVolumeUtil {
File[] hddsFiles = hddsRoot.listFiles();
- if(hddsFiles == null) {
+ if (hddsFiles == null) {
// This is the case for IOException, where listFiles returns null.
// So, we fail the volume.
return false;
} else if (hddsFiles.length == 1) {
// DN started for first time or this is a newly added volume.
- // So we create scm directory.
- if (!clusterDir.mkdir()) {
- logger.error("Unable to create scmDir {}", clusterDir);
+ // The one file is the version file.
+ // So we create cluster ID directory, or SCM ID directory if
+ // pre-finalized for SCM HA.
+ // Either the SCM ID or cluster ID will be used in naming the
+ // volume's subdirectory, depending on the datanode's layout version.
+ String id = VersionedDatanodeFeatures.ScmHA.chooseContainerPathID(conf,
+ scmId, clusterId);
+ File idDir = new File(hddsRoot, id);
+ if (!idDir.mkdir()) {
+ logger.error("Unable to create ID directory {} for datanode.", idDir);
return false;
}
return true;
- } else if(hddsFiles.length == 2) {
- if (scmDir.exists()) {
- String msg = "Volume " + volumeRoot +
- " is in Inconsistent state, and contains the" +
- "SCM Directory:" + scmDir.getAbsolutePath() +
- " which is a older format, please upgrade the volume.";
- logger.error(msg);
- ExitUtil.terminate(-2, msg);
- return false;
- }
- // The files should be Version and SCM directory
- if (clusterDir.exists()) {
- return true;
- } else {
- logger.error("Volume {} is in Inconsistent state, expected cluster " +
- "directory {} does not exist", volumeRoot, clusterDir
- .getAbsolutePath());
+ } else if (hddsFiles.length == 2) {
+ // If we are finalized for SCM HA and there is no cluster ID directory,
+ // the volume may have been unhealthy during finalization and been
+ // skipped. Create cluster ID symlink now.
+ // Else, We are still pre-finalized.
+ // The existing directory should be left for backwards compatibility.
+ return VersionedDatanodeFeatures.ScmHA.
+ upgradeVolumeIfNeeded(hddsVolume, clusterId);
+ } else {
+ if (!clusterDir.exists()) {
+ logger.error("Volume {} is in an inconsistent state. {} files found " +
+ "but cluster ID directory {} does not exist.", volumeRoot,
+ hddsFiles.length, clusterDir);
return false;
}
- } else {
- // The hdds root dir should always have 2 files. One is Version file
- // and other is SCM directory.
- logger.error("The hdds root dir {} should always have 2 files. " +
- "One is Version file and other is Cluster directory. " +
- "Please remove any other extra files from the directory " +
- "so that DataNode startup can proceed.",
- hddsRoot.getAbsolutePath());
- return false;
+ return true;
}
-
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index dd427c7..a1e41ff 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory;
* >>/<<dataDir>>}
* <p>
* Each hdds volume has its own VERSION file. The hdds volume will have one
- * scmUuid directory for each SCM it is a part of (currently only one SCM is
+ * clusterUuid directory for each SCM it is a part of (currently only one SCM
is
* supported).
*
* During DN startup, if the VERSION file exists, we verify that the
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
index 97adc8f..0311f7d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
@@ -39,7 +39,6 @@ import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfigurati
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
@@ -131,18 +130,6 @@ public class MutableVolumeSet implements VolumeSet {
initializeVolumeSet();
}
- public MutableVolumeSet(ConfigurationSource conf) throws IOException {
- this.datanodeUuid = null;
- this.clusterID = null;
- this.conf = conf;
- this.volumeSetRWLock = new ReentrantReadWriteLock();
- this.volumeChecker = new StorageVolumeChecker(conf, new Timer());
- this.volumeType = StorageVolume.VolumeType.DATA_VOLUME;
- this.volumeFactory = new HddsVolumeFactory(conf, null,
- this, null, null);
- initializeVolumeSet();
- }
-
public void setFailedVolumeListener(Runnable runnable) {
failedVolumeListener = runnable;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 37bfff8..4b47e50 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -52,7 +52,7 @@ import
org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import
org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import
org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
-import org.apache.hadoop.ozone.container.upgrade.DatanodeMetadataFeatures;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import com.google.common.base.Preconditions;
@@ -118,13 +118,15 @@ public class KeyValueContainer implements
Container<KeyValueContainerData> {
String hddsVolumeDir = containerVolume.getHddsRootDir().toString();
long containerID = containerData.getContainerID();
+ String idDir = VersionedDatanodeFeatures.ScmHA.chooseContainerPathID(
+ containerVolume, clusterId);
containerMetaDataPath = KeyValueContainerLocationUtil
- .getContainerMetaDataPath(hddsVolumeDir, clusterId, containerID);
+ .getContainerMetaDataPath(hddsVolumeDir, idDir, containerID);
containerData.setMetadataPath(containerMetaDataPath.getPath());
File chunksPath = KeyValueContainerLocationUtil.getChunksLocationPath(
- hddsVolumeDir, clusterId, containerID);
+ hddsVolumeDir, idDir, containerID);
// Check if it is new Container.
ContainerUtils.verifyIsNewContainer(containerMetaDataPath);
@@ -132,10 +134,8 @@ public class KeyValueContainer implements
Container<KeyValueContainerData> {
//Create Metadata path chunks path and metadata db
File dbFile = getContainerDBFile();
- // This method is only called when creating new containers.
- // Therefore, always use the newest schema version.
containerData.setSchemaVersion(
- DatanodeMetadataFeatures.getSchemaVersion());
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion());
KeyValueContainerUtil.createContainerMetaData(containerID,
containerMetaDataPath, chunksPath, dbFile,
containerData.getSchemaVersion(), config);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index e978bf8..1d2c7c7 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -76,6 +76,7 @@ import
org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerFactory;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.util.AutoCloseableLock;
import com.google.common.annotations.VisibleForTesting;
@@ -299,7 +300,9 @@ public class KeyValueHandler extends Handler {
StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()),
container.getContainerData().getMaxSize());
String hddsVolumeDir = containerVolume.getHddsRootDir().toString();
- container.populatePathFields(clusterId, containerVolume, hddsVolumeDir);
+ String idDir = VersionedDatanodeFeatures.ScmHA.chooseContainerPathID(
+ containerVolume, clusterId);
+ container.populatePathFields(idDir, containerVolume, hddsVolumeDir);
} finally {
volumeSet.readUnlock();
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
index 671e61b..f984b20 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
@@ -70,11 +70,8 @@ public final class KeyValueContainerUtil {
* @param containerMetaDataPath Path to the container's metadata directory.
* @param chunksPath Path were chunks for this container should be stored.
* @param dbFile Path to the container's .db file.
- * @param schemaVersion The schema version of the container. Since this
- * method is used when creating new containers, the
- * {@link OzoneConsts#SCHEMA_LATEST} variable can be
- * used to construct the container. If this method has
- * not been updated after a schema version addition
+ * @param schemaVersion The schema version of the container. If this method
+ * has not been updated after a schema version addition
* and does not recognize the latest SchemaVersion, an
* {@link IllegalArgumentException} is thrown.
* @param conf The configuration to use for this container.
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
index d9e0582..44aa92c 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
@@ -19,18 +19,12 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
import java.io.File;
-import java.io.FileFilter;
import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collections;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -44,7 +38,6 @@ import
org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import
org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.Yaml;
/**
* Class used to read .container files from Volume and build container map.
@@ -80,7 +73,6 @@ public class ContainerReader implements Runnable {
private final ConfigurationSource config;
private final File hddsVolumeDir;
private final MutableVolumeSet volumeSet;
- private final boolean isInUpgradeMode;
public ContainerReader(
MutableVolumeSet volSet, HddsVolume volume, ContainerSet cset,
@@ -92,17 +84,8 @@ public class ContainerReader implements Runnable {
this.containerSet = cset;
this.config = conf;
this.volumeSet = volSet;
- this.isInUpgradeMode =
- conf.getBoolean(ScmConfigKeys.HDDS_DATANODE_UPGRADE_LAYOUT_INLINE,
- ScmConfigKeys.HDDS_DATANODE_UPGRADE_LAYOUT_INLINE_DEFAULT);
- LOG.info("Running in upgrade mode:{}", this.isInUpgradeMode);
}
-
- private File getClusterDir() {
- File hddsVolumeRootDir = hddsVolume.getHddsRootDir();
- return new File(hddsVolumeRootDir, hddsVolume.getClusterID());
- }
@Override
public void run() {
try {
@@ -114,35 +97,50 @@ public class ContainerReader implements Runnable {
}
}
- public void readVolume(File hddsVolumeRootDir) throws Exception {
+ public void readVolume(File hddsVolumeRootDir) {
Preconditions.checkNotNull(hddsVolumeRootDir, "hddsVolumeRootDir" +
"cannot be null");
//filtering storage directory
- File[] storageDir = hddsVolumeRootDir.listFiles(new FileFilter() {
- @Override
- public boolean accept(File pathname) {
- return pathname.isDirectory();
- }
- });
+ File[] storageDirs = hddsVolumeRootDir.listFiles(File::isDirectory);
- if (storageDir == null) {
+ if (storageDirs == null) {
LOG.error("IO error for the volume {}, skipped loading",
hddsVolumeRootDir);
volumeSet.failVolume(hddsVolumeRootDir.getPath());
return;
}
- if (storageDir.length > 1) {
- LOG.error("Volume {} is in Inconsistent state", hddsVolumeRootDir);
- volumeSet.failVolume(hddsVolumeRootDir.getPath());
- return;
- }
+ // If there are no storage dirs yet, the volume needs to be formatted
+ // by HddsUtil#checkVolume once we have a cluster ID from SCM. No
+ // operations to perform here in that case.
+ if (storageDirs.length > 0) {
+ File clusterIDDir = new File(hddsVolumeRootDir,
+ hddsVolume.getClusterID());
+ // The subdirectory we should verify containers within.
+ // If this volume was formatted pre SCM HA, this will be the SCM ID.
+ // A cluster ID symlink will exist in this case only if this cluster is
+ // finalized for SCM HA.
+ // If the volume was formatted post SCM HA, this will be the cluster ID.
+ File idDir = clusterIDDir;
+ if (storageDirs.length == 1 && !clusterIDDir.exists()) {
+ // If the one directory is not the cluster ID directory, assume it is
+ // the old SCM ID directory used before SCM HA.
+ idDir = storageDirs[0];
+ } else {
+ // There are 1 or more storage directories. We only care about the
+ // cluster ID directory.
+ if (!clusterIDDir.exists()) {
+ LOG.error("Volume {} is in an inconsistent state. Expected " +
+ "clusterID directory {} not found.", hddsVolumeRootDir,
+ clusterIDDir);
+ volumeSet.failVolume(hddsVolumeRootDir.getPath());
+ return;
+ }
+ }
- LOG.info("Start to verify containers on volume {}", hddsVolumeRootDir);
- for (File storageLoc : storageDir) {
- File location = preProcessStorageLoc(storageLoc);
- File currentDir = new File(location, Storage.STORAGE_DIR_CURRENT);
+ LOG.info("Start to verify containers on volume {}", hddsVolumeRootDir);
+ File currentDir = new File(idDir, Storage.STORAGE_DIR_CURRENT);
File[] containerTopDirs = currentDir.listFiles();
if (containerTopDirs != null) {
for (File containerTopDir : containerTopDirs) {
@@ -156,7 +154,7 @@ public class ContainerReader implements Runnable {
long containerID =
ContainerUtils.getContainerID(containerDir);
if (containerFile.exists()) {
- verifyContainerFile(storageLoc, containerID,
containerFile);
+ verifyContainerFile(containerID, containerFile);
} else {
LOG.error("Missing .container file for ContainerID: {}",
containerDir.getName());
@@ -174,35 +172,7 @@ public class ContainerReader implements Runnable {
LOG.info("Finish verifying containers on volume {}", hddsVolumeRootDir);
}
- public File preProcessStorageLoc(File storageLoc) throws Exception {
- File clusterDir = getClusterDir();
-
- if (!isInUpgradeMode) {
- Preconditions.checkArgument(clusterDir.exists(),
- "Storage Dir: %s doesn't exist", clusterDir);
- Preconditions.checkArgument(storageLoc.equals(clusterDir),
- "configured storage location %s does not contain the clusterId: %s",
- storageLoc, hddsVolume.getClusterID());
- return storageLoc;
- }
-
- if (clusterDir.exists()) {
- return storageLoc;
- }
-
- try {
- LOG.info("Storage dir based on clusterId doesn't exist." +
- "Renaming storage location:{} to {}", storageLoc, clusterDir);
- NativeIO.renameTo(storageLoc, clusterDir);
- return clusterDir;
- } catch (Throwable t) {
- LOG.error("DN Layout upgrade failed. Renaming of storage" +
- "location:{} to {} failed", storageLoc, clusterDir, t);
- throw t;
- }
- }
-
- private void verifyContainerFile(File storageLoc, long containerID,
+ private void verifyContainerFile(long containerID,
File containerFile) {
try {
ContainerData containerData = ContainerDataYaml.readContainerFile(
@@ -212,7 +182,7 @@ public class ContainerReader implements Runnable {
"Skipping loading of this container.", containerFile);
return;
}
- verifyAndFixupContainerData(storageLoc, containerData);
+ verifyAndFixupContainerData(containerData);
} catch (IOException ex) {
LOG.error("Failed to parse ContainerFile for ContainerID: {}",
containerID, ex);
@@ -220,60 +190,23 @@ public class ContainerReader implements Runnable {
}
/**
- * This function upgrades the container layout in following steps.
- * a) Converts the chunk and metadata path to the new clusterID
- * based location.
- * b) Re-computes the new container checksum.
- * b) Persists the new container layout to disk.
- * @param storageLoc
- * @param kvContainerData
- * @return upgraded KeyValueContainer
- * @throws IOException
- */
- public KeyValueContainer upgradeContainerLayout(File storageLoc,
- KeyValueContainerData kvContainerData) throws IOException {
- kvContainerData.setMetadataPath(
- findNormalizedPath(storageLoc,
- kvContainerData.getMetadataPath()));
- kvContainerData.setChunksPath(
- findNormalizedPath(storageLoc,
- kvContainerData.getChunksPath()));
-
- Yaml yaml = ContainerDataYaml.getYamlForContainerType(
- kvContainerData.getContainerType());
- kvContainerData.computeAndSetChecksum(yaml);
-
- KeyValueContainerUtil.parseKVContainerData(kvContainerData, config);
- KeyValueContainer kvContainer = new KeyValueContainer(
- kvContainerData, config);
- kvContainer.update(Collections.emptyMap(), true);
- return kvContainer;
- }
-
- /**
* verify ContainerData loaded from disk and fix-up stale members.
* Specifically blockCommitSequenceId, delete related metadata
* and bytesUsed
* @param containerData
* @throws IOException
*/
- public void verifyAndFixupContainerData(File storageLoc,
- ContainerData containerData) throws IOException {
+ public void verifyAndFixupContainerData(ContainerData containerData)
+ throws IOException {
switch (containerData.getContainerType()) {
case KeyValueContainer:
if (containerData instanceof KeyValueContainerData) {
KeyValueContainerData kvContainerData = (KeyValueContainerData)
containerData;
containerData.setVolume(hddsVolume);
- KeyValueContainer kvContainer = null;
- if (isInUpgradeMode) {
- kvContainer =
- upgradeContainerLayout(storageLoc, kvContainerData);
- } else {
- KeyValueContainerUtil.parseKVContainerData(kvContainerData, config);
- kvContainer = new KeyValueContainer(
- kvContainerData, config);
- }
+ KeyValueContainerUtil.parseKVContainerData(kvContainerData, config);
+ KeyValueContainer kvContainer = new KeyValueContainer(kvContainerData,
+ config);
containerSet.addContainer(kvContainer);
} else {
throw new StorageContainerException("Container File is corrupted. " +
@@ -288,18 +221,4 @@ public class ContainerReader implements Runnable {
ContainerProtos.Result.UNKNOWN_CONTAINER_TYPE);
}
}
-
- public String findNormalizedPath(File storageLoc, String path) {
- Path p = Paths.get(path);
- Path relativePath = storageLoc.toPath().relativize(p);
- Path newPath = getClusterDir().toPath().resolve(relativePath);
-
- if (!isInUpgradeMode) {
- Preconditions.checkArgument(newPath.toFile().exists(),
- "modified path:%s doesn't exist", newPath);
- }
-
- LOG.debug("new Normalized Path is:{}", newPath);
- return newPath.toAbsolutePath().toString();
- }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 81ce1d2..0042913 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -132,7 +132,7 @@ public class OzoneContainer {
containerSet = new ContainerSet();
metadataScanner = null;
- buildContainerSet(volumeSet, containerSet, config);
+ buildContainerSet();
final ContainerMetrics metrics = ContainerMetrics.create(conf);
handlers = Maps.newHashMap();
@@ -208,8 +208,7 @@ public class OzoneContainer {
/**
* Build's container map.
*/
- public static void buildContainerSet(MutableVolumeSet volumeSet,
- ContainerSet containerSet, ConfigurationSource config) {
+ private void buildContainerSet() {
Iterator<StorageVolume> volumeSetIterator = volumeSet.getVolumesList()
.iterator();
ArrayList<Thread> volumeThreads = new ArrayList<>();
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java
index 58c5c4a..9ff4b0a 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java
@@ -65,11 +65,12 @@ public class DataNodeUpgradeFinalizer extends
while (containerIt.hasNext()) {
Container ctr = containerIt.next();
ContainerProtos.ContainerDataProto.State state = ctr.getContainerState();
+ long id = ctr.getContainerData().getContainerID();
switch (state) {
case OPEN:
case CLOSING:
- LOG.warn("FinalizeUpgrade : Waiting for container to close, current "
- + "state is: {}", state);
+ LOG.warn("FinalizeUpgrade : Waiting for container {} to close, current
"
+ + "state is: {}", id, state);
return false;
default:
continue;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeMetadataFeatures.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeMetadataFeatures.java
deleted file mode 100644
index dc71a6e..0000000
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeMetadataFeatures.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.upgrade;
-
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.io.IOException;
-
-/**
- * Utility class to retrieve the version of a feature that corresponds to the
- * metadata layout version specified by the provided
- * {@link HDDSLayoutVersionManager}.
- */
-public final class DatanodeMetadataFeatures {
- private static HDDSLayoutVersionManager versionManager;
-
- private DatanodeMetadataFeatures() { }
-
- public static synchronized void initialize(
- HDDSLayoutVersionManager manager) {
- versionManager = manager;
- }
-
- public static synchronized String getSchemaVersion() throws IOException {
- if (versionManager == null) {
- // version manager can be null for testing. Use the latest version in
- // this case.
- return OzoneConsts.SCHEMA_V2;
- } else if (versionManager.getMetadataLayoutVersion() <
- HDDSLayoutFeature.DATANODE_SCHEMA_V2.layoutVersion()) {
- return OzoneConsts.SCHEMA_V1;
- } else {
- return OzoneConsts.SCHEMA_V2;
- }
- }
-}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
new file mode 100644
index 0000000..b4c130c
--- /dev/null
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ *
+ */
+
+package org.apache.hadoop.ozone.container.upgrade;
+
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.SCM_HA;
+import static
org.apache.hadoop.ozone.upgrade.LayoutFeature.UpgradeActionType.ON_FINALIZE;
+import static
org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Action to run upgrade flow for SCM HA exactly once.
+ */
+@UpgradeActionHdds(feature = SCM_HA, component = DATANODE,
+ type = ON_FINALIZE)
+public class ScmHAFinalizeUpgradeActionDatanode
+ implements HDDSUpgradeAction<DatanodeStateMachine> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ScmHAFinalizeUpgradeActionDatanode.class);
+
+ @Override
+ public void execute(DatanodeStateMachine dsm) throws Exception {
+ LOG.info("Upgrading Datanode volume layout for SCM HA support.");
+ MutableVolumeSet volumeSet = dsm.getContainer().getVolumeSet();
+
+ for (StorageVolume volume: volumeSet.getVolumesList()) {
+ volumeSet.writeLock();
+ try {
+ if (volume instanceof HddsVolume) {
+ HddsVolume hddsVolume = (HddsVolume) volume;
+ if (!upgradeVolume(hddsVolume, hddsVolume.getClusterID())) {
+ volumeSet.failVolume(volume.getStorageDir().getAbsolutePath());
+ }
+ }
+ } finally {
+ volumeSet.writeUnlock();
+ }
+ }
+ }
+
+ /**
+ * Upgrade the specified volume to be compatible with SCM HA layout feature.
+ * @return true if the volume upgrade succeeded, false otherwise.
+ */
+ public static boolean upgradeVolume(HddsVolume volume, String clusterID) {
+ Preconditions.checkNotNull(clusterID, "Cannot upgrade volume with null " +
+ "cluster ID");
+ File hddsVolumeDir = volume.getStorageDir();
+ File clusterIDDir = new File(hddsVolumeDir, clusterID);
+ File[] storageDirs = volume.getStorageDir().listFiles(File::isDirectory);
+ boolean success = true;
+
+ if (storageDirs == null) {
+ LOG.error("IO error for the volume {}. " +
+ "Unable to process it for finalizing layout for SCM HA" +
+ "support. Formatting will be retried on datanode restart.",
+ volume.getStorageDir());
+ success = false;
+ } else if (storageDirs.length == 0) {
+ LOG.info("Skipping finalize for SCM HA for unformatted volume {}, no " +
+ "action required.", hddsVolumeDir);
+ } else if (storageDirs.length == 1) {
+ if (!clusterIDDir.exists()) {
+ // If the one directory is not the cluster ID directory, assume it is
+ // the old SCM ID directory.
+ File scmIDDir = storageDirs[0];
+ Path relativeScmIDDir =
+ hddsVolumeDir.toPath().relativize(scmIDDir.toPath());
+ LOG.info("Creating symlink {} -> {} as part of SCM HA " +
+ "finalization for datanode.", clusterIDDir.getAbsolutePath(),
+ relativeScmIDDir);
+ try {
+ Files.createSymbolicLink(clusterIDDir.toPath(), relativeScmIDDir);
+ } catch (IOException ex) {
+ LOG.error("IO error for the volume {}. " +
+ "Unable to process it for finalizing layout for SCM HA" +
+ "support. Formatting will be retried on datanode restart.",
+ volume.getStorageDir(), ex);
+ success = false;
+ }
+ } else {
+ LOG.info("Volume already contains cluster ID directory {}. No " +
+ "action required for SCM HA finalization.", clusterIDDir);
+ }
+ } else {
+ // More than one subdirectory. As long as the cluster ID directory
+ // exists we are ok.
+ if (!clusterIDDir.exists()) {
+ LOG.error("Volume {} is in an inconsistent state. Expected directory" +
+ "{} not found.", hddsVolumeDir, clusterIDDir);
+ success = false;
+ } else {
+ LOG.info("Volume already contains cluster ID directory {}. No " +
+ "action required for SCM HA finalization.", clusterIDDir);
+ }
+ }
+
+ return success;
+ }
+}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFirstUpgradeLayoutChangeAction.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFirstUpgradeLayoutChangeAction.java
deleted file mode 100644
index b871942..0000000
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFirstUpgradeLayoutChangeAction.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.container.upgrade;
-
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.SCM_HA;
-import static
org.apache.hadoop.ozone.upgrade.LayoutFeature.UpgradeActionType.ON_FIRST_UPGRADE_START;
-import static
org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
-
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
-import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Action to run upgrade flow for SCM HA exactly once.
- */
-@UpgradeActionHdds(feature = SCM_HA, component = DATANODE,
- type = ON_FIRST_UPGRADE_START)
-public class ScmHAFirstUpgradeLayoutChangeAction
- implements HDDSUpgradeAction<DatanodeStateMachine> {
- private static final Logger LOG =
- LoggerFactory.getLogger(ScmHAFirstUpgradeLayoutChangeAction.class);
-
- @Override
- public void execute(DatanodeStateMachine arg) throws Exception {
- LOG.info("Upgrade Datanode container layout for SCM HA support.");
- }
-}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java
new file mode 100644
index 0000000..ec84946
--- /dev/null
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.upgrade;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Utility class to retrieve the version of a feature that corresponds to the
+ * metadata layout version specified by the provided
+ * {@link HDDSLayoutVersionManager}.
+ */
+public final class VersionedDatanodeFeatures {
+ private static HDDSLayoutVersionManager versionManager;
+
+ private VersionedDatanodeFeatures() { }
+
+ public static void initialize(
+ HDDSLayoutVersionManager manager) {
+ versionManager = manager;
+ }
+
+ public static boolean isFinalized(
+ HDDSLayoutFeature layoutFeature) {
+ // version manager can be null for testing. Use the latest version in
+ // this case.
+ return versionManager == null ||
+ versionManager.isAllowed(layoutFeature);
+ }
+
+ /**
+ * Utilities for container Schema V2 layout feature.
+ * Containers created prior to the feature's finalization will use schema
+ * v1, and schema v2 will be used for all containers created after
+ * finalization.
+ */
+ public static class SchemaV2 {
+ public static String chooseSchemaVersion() {
+ if (isFinalized(HDDSLayoutFeature.DATANODE_SCHEMA_V2)) {
+ return OzoneConsts.SCHEMA_V2;
+ } else {
+ return OzoneConsts.SCHEMA_V1;
+ }
+ }
+ }
+
+ /**
+ * Utilities for SCM HA layout feature.
+ * Prior to SCM HA finalization, datanode volumes used the format
+ * {@literal <volume>/hdds/<scm-id>}. After SCM HA, the expected format is
+ * {@literal <volume>/hdds/<cluster-id>}.
+ *
+ * In pre-finalize for SCM HA, datanodes
+ * will still use the SCM ID for container file paths. The exception is if
+ * the cluster is already using cluster ID paths (since SCM HA was merged
+ * before the upgrade framework). In this case, cluster ID paths should
+ * continue to be used.
+ *
+ * On finalization of SCM HA, datanodes
+ * will create a symlink from the SCM ID directory to the cluster ID
+ * directory and use cluster ID in container file paths. If a cluster ID
+ * directory is already present, no changes are made.
+ */
+ public static class ScmHA {
+ /**
+ * Choose whether to use cluster ID or SCM ID based on the format of the
+ * volume and SCM HA finalization status.
+ */
+ public static String chooseContainerPathID(StorageVolume volume,
+ String clusterID) throws IOException {
+ File clusterIDDir = new File(volume.getStorageDir(), clusterID);
+
+ // SCM ID may be null for testing, but these non-upgrade tests will use
+ // the latest version with cluster ID anyways.
+ if (isFinalized(HDDSLayoutFeature.SCM_HA) || clusterIDDir.exists()) {
+ return clusterID;
+ } else {
+ File[] subdirs = volume.getStorageDir().listFiles(File::isDirectory);
+ if (subdirs == null) {
+ throw new IOException("Failed to read volume " +
+ volume.getStorageDir());
+ } else if (subdirs.length != 1) {
+ throw new IOException("Invalid volume directory " +
+ volume.getStorageDir() +
+ " has more than one directory before SCM HA finalization.");
+ }
+ return subdirs[0].getName();
+ }
+ }
+
+ /**
+ * Choose whether to use SCM ID or cluster ID based on SCM HA
+ * finalization status and SCM HA configuration.
+ */
+ public static String chooseContainerPathID(ConfigurationSource conf,
+ String scmID, String clusterID) {
+ boolean scmHAEnabled =
+ conf.getBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY,
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT);
+ if (isFinalized(HDDSLayoutFeature.SCM_HA) || scmHAEnabled){
+ return clusterID;
+ } else {
+ return scmID;
+ }
+ }
+
+ public static boolean upgradeVolumeIfNeeded(HddsVolume volume,
+ String clusterID) {
+ File clusterIDDir = new File(volume.getStorageDir(), clusterID);
+ boolean needsUpgrade = isFinalized(HDDSLayoutFeature.SCM_HA) &&
+ !clusterIDDir.exists();
+
+ boolean success = true;
+
+ if (needsUpgrade) {
+ success = ScmHAFinalizeUpgradeActionDatanode.upgradeVolume(volume,
+ clusterID);
+ }
+ return success;
+ }
+ }
+}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
index 55d5114..157dee6 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
@@ -69,6 +69,11 @@ public class ScmTestMock implements
StorageContainerDatanodeProtocol {
scmId = UUID.randomUUID().toString();
}
+ public ScmTestMock(String clusterId, String scmId) {
+ this.clusterId = clusterId;
+ this.scmId = scmId;
+ }
+
// Map of datanode to containers
private Map<DatanodeDetails,
Map<String, ContainerReplicaProto>> nodeContainers =
@@ -352,4 +357,8 @@ public class ScmTestMock implements
StorageContainerDatanodeProtocol {
public void setClusterId(String clusterId) {
this.clusterId = clusterId;
}
+
+ public void setScmId(String scmId) {
+ this.scmId = scmId;
+ }
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java
index e7f6388..562775d 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java
@@ -21,12 +21,12 @@ package org.apache.hadoop.ozone.container.common;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
import org.apache.hadoop.ozone.container.common.utils.ContainerCacheMetrics;
import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
import org.apache.hadoop.ozone.container.metadata.DatanodeStore;
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -90,11 +90,13 @@ public class TestContainerCache {
long numCacheMisses = metrics.getNumCacheMisses();
// Get 2 references out of the same db and verify the objects are same.
ReferenceCountedDB db1 = cache.getDB(1, "RocksDB",
- containerDir1.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
+ containerDir1.getPath(),
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(), conf);
Assert.assertEquals(1, db1.getReferenceCount());
Assert.assertEquals(numDbGetCount + 1, metrics.getNumDbGetOps());
ReferenceCountedDB db2 = cache.getDB(1, "RocksDB",
- containerDir1.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
+ containerDir1.getPath(),
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(), conf);
Assert.assertEquals(2, db2.getReferenceCount());
Assert.assertEquals(numCacheMisses + 1, metrics.getNumCacheMisses());
Assert.assertEquals(2, db1.getReferenceCount());
@@ -104,7 +106,8 @@ public class TestContainerCache {
// add one more references to ContainerCache.
ReferenceCountedDB db3 = cache.getDB(2, "RocksDB",
- containerDir2.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
+ containerDir2.getPath(),
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(), conf);
Assert.assertEquals(1, db3.getReferenceCount());
// and close the reference
@@ -114,7 +117,8 @@ public class TestContainerCache {
// add one more reference to ContainerCache and verify that it will not
// evict the least recent entry as it has reference.
ReferenceCountedDB db4 = cache.getDB(3, "RocksDB",
- containerDir3.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
+ containerDir3.getPath(),
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(), conf);
Assert.assertEquals(1, db4.getReferenceCount());
Assert.assertEquals(2, cache.size());
@@ -130,7 +134,8 @@ public class TestContainerCache {
// The reference count for container1 is 0 but it is not evicted.
ReferenceCountedDB db5 = cache.getDB(1, "RocksDB",
- containerDir1.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
+ containerDir1.getPath(),
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(), conf);
Assert.assertEquals(1, db5.getReferenceCount());
Assert.assertEquals(db1, db5);
db5.close();
@@ -159,7 +164,8 @@ public class TestContainerCache {
Runnable task = () -> {
try {
ReferenceCountedDB db1 = cache.getDB(1, "RocksDB",
- containerDir.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
+ containerDir.getPath(),
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(), conf);
Assert.assertNotNull(db1);
} catch (IOException e) {
Assert.fail("Should get the DB instance");
@@ -177,7 +183,8 @@ public class TestContainerCache {
}
ReferenceCountedDB db = cache.getDB(1, "RocksDB",
- containerDir.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
+ containerDir.getPath(),
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(), conf);
db.close();
db.close();
db.close();
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
index 7171a81..3814fdd 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.ozone.container.common;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
import org.apache.hadoop.ozone.container.keyvalue.ChunkLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -93,7 +93,8 @@ public class TestKeyValueContainerData {
kvData.incrWriteCount();
kvData.incrKeyCount();
kvData.incrPendingDeletionBlocks(1);
- kvData.setSchemaVersion(OzoneConsts.SCHEMA_LATEST);
+ kvData.setSchemaVersion(
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion());
assertEquals(state, kvData.getState());
assertEquals(containerDBType, kvData.getContainerDBType());
@@ -108,7 +109,8 @@ public class TestKeyValueContainerData {
assertEquals(1, kvData.getNumPendingDeletionBlocks());
assertEquals(pipelineId.toString(), kvData.getOriginPipelineId());
assertEquals(datanodeId.toString(), kvData.getOriginNodeId());
- assertEquals(OzoneConsts.SCHEMA_LATEST, kvData.getSchemaVersion());
+ assertEquals(VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(),
+ kvData.getSchemaVersion());
}
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
index ca88474..4dc38e9 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.keyvalue.ChunkLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -89,7 +90,8 @@ public class TestContainerDataYaml {
keyValueContainerData.setMetadataPath(testRoot);
keyValueContainerData.setChunksPath(testRoot);
keyValueContainerData.updateDataScanTime(SCAN_TIME);
- keyValueContainerData.setSchemaVersion(OzoneConsts.SCHEMA_LATEST);
+ keyValueContainerData.setSchemaVersion(
+ VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion());
File containerFile = new File(testRoot, containerPath);
@@ -133,7 +135,7 @@ public class TestContainerDataYaml {
kvData.lastDataScanTime().get().toEpochMilli());
assertEquals(SCAN_TIME.toEpochMilli(),
kvData.getDataScanTimestamp().longValue());
- assertEquals(OzoneConsts.SCHEMA_LATEST,
+ assertEquals(VersionedDatanodeFeatures.SchemaV2.chooseSchemaVersion(),
kvData.getSchemaVersion());
// Update ContainerData.
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java
new file mode 100644
index 0000000..cb5257d
--- /dev/null
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.upgrade;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
+import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
+import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.ozone.container.common.ScmTestMock;
+import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import
org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
+import
org.apache.hadoop.ozone.container.common.states.endpoint.VersionEndpointTask;
+import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
+import
org.apache.hadoop.ozone.container.replication.ContainerReplicationSource;
+import
org.apache.hadoop.ozone.container.replication.DownloadAndImportReplicator;
+import
org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
+import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
+import org.apache.ozone.test.LambdaTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * Tests upgrading a single datanode from pre-SCM HA volume format that used
+ * SCM ID to the post-SCM HA volume format using cluster ID. If SCM HA was
+ * already being used before the upgrade, there should be no changes.
+ */
+@RunWith(Parameterized.class)
+public class TestDatanodeUpgradeToScmHA {
+ @Rule
+ public TemporaryFolder tempFolder;
+
+ private DatanodeStateMachine dsm;
+ private final OzoneConfiguration conf;
+ private static final String CLUSTER_ID = "clusterID";
+ private final boolean scmHAAlreadyEnabled;
+
+ private RPC.Server scmRpcServer;
+ private InetSocketAddress address;
+ private ScmTestMock scmServerImpl;
+
+ private Random random;
+
+ @Parameterized.Parameters(name = "{index}: scmHAAlreadyEnabled={0}")
+ public static Collection<Object[]> getSchemaFiles() {
+ Collection<Object[]> parameters = new ArrayList<>();
+ parameters.add(new Boolean[]{false});
+ parameters.add(new Boolean[]{true});
+ return parameters;
+ }
+
+ public TestDatanodeUpgradeToScmHA(boolean scmHAAlreadyEnabled) {
+ this.scmHAAlreadyEnabled = scmHAAlreadyEnabled;
+ conf = new OzoneConfiguration();
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY,
scmHAAlreadyEnabled);
+ }
+
+ @Before
+ public void setup() throws Exception {
+ tempFolder = new TemporaryFolder();
+ tempFolder.create();
+ random = new Random();
+
+ address = SCMTestUtils.getReuseableAddress();
+ conf.setSocketAddr(ScmConfigKeys.OZONE_SCM_NAMES, address);
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (scmRpcServer != null) {
+ scmRpcServer.stop();
+ }
+
+ if (dsm != null) {
+ dsm.close();
+ }
+ }
+
+ @Test
+ public void testReadsDuringFinalization() throws Exception {
+ // start DN and SCM
+ startScmServer();
+ addVolume();
+ startPreFinalizedDatanode();
+ final Pipeline pipeline = getPipeline();
+
+ // Add data to read.
+ final long containerID = addContainer(pipeline);
+ ContainerProtos.WriteChunkRequestProto writeChunk = putBlock(containerID,
+ pipeline);
+ closeContainer(containerID, pipeline);
+
+ // Create thread to keep reading during finalization.
+ ExecutorService executor = Executors.newFixedThreadPool(1);
+ Future<Void> readFuture = executor.submit(() -> {
+ // Layout version check should be thread safe.
+ while(!dsm.getLayoutVersionManager()
+ .isAllowed(HDDSLayoutFeature.SCM_HA)) {
+ readChunk(writeChunk, pipeline);
+ }
+ // Make sure we can read after finalizing too.
+ readChunk(writeChunk, pipeline);
+ return null;
+ });
+
+ dsm.finalizeUpgrade();
+ // If there was a failure reading during the upgrade, the exception will
+ // be thrown here.
+ readFuture.get();
+ }
+
+ @Test
+ public void testImportContainer() throws Exception {
+ // start DN and SCM
+ startScmServer();
+ addVolume();
+ startPreFinalizedDatanode();
+ final Pipeline pipeline = getPipeline();
+
+ // Pre-export a container to continuously import and delete.
+ final long exportContainerID = addContainer(pipeline);
+ ContainerProtos.WriteChunkRequestProto exportWriteChunk =
+ putBlock(exportContainerID, pipeline);
+ closeContainer(exportContainerID, pipeline);
+ File exportedContainerFile = exportContainer(exportContainerID);
+ deleteContainer(exportContainerID, pipeline);
+
+ // Export another container to import while pre-finalized and read
+ // finalized.
+ final long exportContainerID2 = addContainer(pipeline);
+ ContainerProtos.WriteChunkRequestProto exportWriteChunk2 =
+ putBlock(exportContainerID2, pipeline);
+ closeContainer(exportContainerID2, pipeline);
+ File exportedContainerFile2 = exportContainer(exportContainerID2);
+ deleteContainer(exportContainerID2, pipeline);
+
+ // Make sure we can import and read a container pre-finalized.
+ importContainer(exportContainerID2, exportedContainerFile2);
+ readChunk(exportWriteChunk2, pipeline);
+
+ // Now SCM and enough other DNs finalize to enable SCM HA. This DN is
+ // restarted with SCM HA config and gets a different SCM ID.
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+ changeScmID();
+ restartDatanode(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
+ // Make sure the existing container can be read.
+ readChunk(exportWriteChunk2, pipeline);
+
+ // Create thread to keep importing containers during the upgrade.
+ // Since the datanode's MLV is behind SCM's, container creation is not
+ // allowed. We will keep importing and deleting the same container since
+ // we cannot create new ones to import here.
+ ExecutorService executor = Executors.newFixedThreadPool(1);
+ Future<Void> importFuture = executor.submit(() -> {
+ // Layout version check should be thread safe.
+ while(!dsm.getLayoutVersionManager()
+ .isAllowed(HDDSLayoutFeature.SCM_HA)) {
+ importContainer(exportContainerID, exportedContainerFile);
+ readChunk(exportWriteChunk, pipeline);
+ deleteContainer(exportContainerID, pipeline);
+ }
+ // Make sure we can import after finalizing too.
+ importContainer(exportContainerID, exportedContainerFile);
+ readChunk(exportWriteChunk, pipeline);
+ return null;
+ });
+
+ dsm.finalizeUpgrade();
+ // If there was a failure importing during the upgrade, the exception will
+ // be thrown here.
+ importFuture.get();
+
+ // Make sure we can read the container that was imported while
+ // pre-finalized after finalizing.
+ readChunk(exportWriteChunk2, pipeline);
+ }
+
+ @Test
+ public void testFailedVolumeDuringFinalization() throws Exception {
+ /// SETUP ///
+
+ String originalScmID = startScmServer();
+ File volume = addVolume();
+ startPreFinalizedDatanode();
+ final Pipeline pipeline = getPipeline();
+
+ /// PRE-FINALIZED: Write and Read from formatted volume ///
+
+ Assert.assertEquals(1,
+ dsm.getContainer().getVolumeSet().getVolumesList().size());
+ Assert.assertEquals(0,
+ dsm.getContainer().getVolumeSet().getFailedVolumesList().size());
+
+ // Add container with data, make sure it can be read and written.
+ final long containerID = addContainer(pipeline);
+ ContainerProtos.WriteChunkRequestProto writeChunk = putBlock(containerID,
+ pipeline);
+ readChunk(writeChunk, pipeline);
+
+ checkPreFinalizedVolumePathID(volume, originalScmID, CLUSTER_ID);
+ checkContainerPathID(containerID, originalScmID, CLUSTER_ID);
+
+ // FINALIZE: With failed volume ///
+
+ failVolume(volume);
+ // Since volume is failed, container should be marked unhealthy.
+ // Finalization should proceed anyways.
+ closeContainer(containerID, pipeline,
+ ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR);
+ State containerState = dsm.getContainer().getContainerSet()
+ .getContainer(containerID).getContainerState();
+ Assert.assertEquals(State.UNHEALTHY, containerState);
+ dsm.finalizeUpgrade();
+ LambdaTestUtils.await(2000, 500,
+ () -> dsm.getLayoutVersionManager()
+ .isAllowed(HDDSLayoutFeature.SCM_HA));
+
+ /// FINALIZED: Volume marked failed but gets restored on disk ///
+
+ // Check that volume is marked failed during finalization.
+ Assert.assertEquals(0,
+ dsm.getContainer().getVolumeSet().getVolumesList().size());
+ Assert.assertEquals(1,
+ dsm.getContainer().getVolumeSet().getFailedVolumesList().size());
+
+ // Since the volume was out during the upgrade, it should maintain its
+ // original format.
+ checkPreFinalizedVolumePathID(volume, originalScmID, CLUSTER_ID);
+ checkContainerPathID(containerID, originalScmID, CLUSTER_ID);
+
+ // Now that we are done finalizing, restore the volume.
+ restoreVolume(volume);
+ // After restoring the failed volume, its containers are readable again.
+ // However, since it is marked as failed no containers can be created or
+ // imported to it.
+ // This should log a warning about reading from an unhealthy container
+ // but otherwise proceed successfully.
+ readChunk(writeChunk, pipeline);
+
+ /// FINALIZED: Restart datanode to upgrade the failed volume ///
+
+ restartDatanode(HDDSLayoutFeature.SCM_HA.layoutVersion());
+
+ Assert.assertEquals(1,
+ dsm.getContainer().getVolumeSet().getVolumesList().size());
+ Assert.assertEquals(0,
+ dsm.getContainer().getVolumeSet().getFailedVolumesList().size());
+
+ checkFinalizedVolumePathID(volume, originalScmID, CLUSTER_ID);
+ checkContainerPathID(containerID, originalScmID, CLUSTER_ID);
+
+ // Read container from before upgrade. The upgrade required it to be
closed.
+ readChunk(writeChunk, pipeline);
+ // Write and read container after upgrade.
+ long newContainerID = addContainer(pipeline);
+ ContainerProtos.WriteChunkRequestProto newWriteChunk =
+ putBlock(newContainerID, pipeline);
+ readChunk(newWriteChunk, pipeline);
+ // The new container should use cluster ID in its path.
+ // The volume it is placed on is up to the implementation.
+ checkContainerPathID(newContainerID, CLUSTER_ID);
+ }
+
+ @Test
+ public void testFormattingNewVolumes() throws Exception {
+ /// SETUP ///
+
+ String originalScmID = startScmServer();
+ File preFinVolume1 = addVolume();
+ startPreFinalizedDatanode();
+ final Pipeline pipeline = getPipeline();
+
+ /// PRE-FINALIZED: Write and Read from formatted volume ///
+
+ Assert.assertEquals(1,
+ dsm.getContainer().getVolumeSet().getVolumesList().size());
+ Assert.assertEquals(0,
+ dsm.getContainer().getVolumeSet().getFailedVolumesList().size());
+
+ // Add container with data, make sure it can be read and written.
+ final long containerID = addContainer(pipeline);
+ ContainerProtos.WriteChunkRequestProto writeChunk = putBlock(containerID,
+ pipeline);
+ readChunk(writeChunk, pipeline);
+
+ checkPreFinalizedVolumePathID(preFinVolume1, originalScmID, CLUSTER_ID);
+ checkContainerPathID(containerID, originalScmID, CLUSTER_ID);
+
+ /// PRE-FINALIZED: Restart with SCM HA enabled and new SCM ID ///
+
+ // Now SCM and enough other DNs finalize to enable SCM HA. This DN is
+ // restarted with SCM HA config and gets a different SCM ID.
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+ changeScmID();
+ // A new volume is added that must be formatted.
+ File preFinVolume2 = addVolume();
+ restartDatanode(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
+
+ Assert.assertEquals(2,
+ dsm.getContainer().getVolumeSet().getVolumesList().size());
+ Assert.assertEquals(0,
+ dsm.getContainer().getVolumeSet().getFailedVolumesList().size());
+
+ // Because DN mlv would be behind SCM mlv, only reads are allowed.
+ readChunk(writeChunk, pipeline);
+
+ // On restart, there should have been no changes to the paths already used.
+ checkPreFinalizedVolumePathID(preFinVolume1, originalScmID, CLUSTER_ID);
+ checkContainerPathID(containerID, originalScmID, CLUSTER_ID);
+ // No new containers can be created on this volume since SCM MLV is ahead
+ // of DN MLV at this point.
+ // cluster ID should always be used for the new volume since SCM HA is now
+ // enabled.
+ checkVolumePathID(preFinVolume2, CLUSTER_ID);
+
+ /// FINALIZE ///
+
+ closeContainer(containerID, pipeline);
+ dsm.finalizeUpgrade();
+ LambdaTestUtils.await(2000, 500,
+ () -> dsm.getLayoutVersionManager()
+ .isAllowed(HDDSLayoutFeature.SCM_HA));
+
+ /// FINALIZED: Add a new volume and check its formatting ///
+
+ // Add a new volume that should be formatted with cluster ID only, since
+ // DN has finalized.
+ File finVolume = addVolume();
+ // Yet another SCM ID is received this time, but it should not matter.
+ changeScmID();
+ restartDatanode(HDDSLayoutFeature.SCM_HA.layoutVersion());
+ Assert.assertEquals(3,
+ dsm.getContainer().getVolumeSet().getVolumesList().size());
+ Assert.assertEquals(0,
+ dsm.getContainer().getVolumeSet().getFailedVolumesList().size());
+
+ checkFinalizedVolumePathID(preFinVolume1, originalScmID, CLUSTER_ID);
+ checkVolumePathID(preFinVolume2, CLUSTER_ID);
+ checkContainerPathID(containerID, originalScmID, CLUSTER_ID);
+ // New volume should have been formatted with cluster ID only, since the
+ // datanode is finalized.
+ checkVolumePathID(finVolume, CLUSTER_ID);
+
+ /// FINALIZED: Read old data and write + read new data ///
+
+ // Read container from before upgrade. The upgrade required it to be
closed.
+ readChunk(writeChunk, pipeline);
+ // Write and read container after upgrade.
+ long newContainerID = addContainer(pipeline);
+ ContainerProtos.WriteChunkRequestProto newWriteChunk =
+ putBlock(newContainerID, pipeline);
+ readChunk(newWriteChunk, pipeline);
+ // The new container should use cluster ID in its path.
+ // The volume it is placed on is up to the implementation.
+ checkContainerPathID(newContainerID, CLUSTER_ID);
+ }
+
+ /// CHECKS FOR TESTING ///
+
+ public void checkContainerPathID(long containerID, String scmID,
+ String clusterID) {
+ if (scmHAAlreadyEnabled) {
+ checkContainerPathID(containerID, clusterID);
+ } else {
+ checkContainerPathID(containerID, scmID);
+ }
+ }
+
+ public void checkContainerPathID(long containerID, String expectedID) {
+ KeyValueContainerData data =
+ (KeyValueContainerData) dsm.getContainer().getContainerSet()
+ .getContainer(containerID).getContainerData();
+ Assert.assertTrue(data.getChunksPath().contains(expectedID));
+ Assert.assertTrue(data.getMetadataPath().contains(expectedID));
+ }
+
+ public void checkFinalizedVolumePathID(File volume, String scmID,
+ String clusterID) throws Exception {
+
+ if (scmHAAlreadyEnabled) {
+ checkVolumePathID(volume, clusterID);
+ } else {
+ List<File> subdirs = getHddsSubdirs(volume);
+ File hddsRoot = getHddsRoot(volume);
+
+ // Volume should have SCM ID and cluster ID directory, where cluster ID
+ // is a symlink to SCM ID.
+ Assert.assertEquals(2, subdirs.size());
+
+ File scmIDDir = new File(hddsRoot, scmID);
+ Assert.assertTrue(subdirs.contains(scmIDDir));
+
+ File clusterIDDir = new File(hddsRoot, CLUSTER_ID);
+ Assert.assertTrue(subdirs.contains(clusterIDDir));
+ Assert.assertTrue(Files.isSymbolicLink(clusterIDDir.toPath()));
+ Path symlinkTarget = Files.readSymbolicLink(clusterIDDir.toPath());
+ Assert.assertEquals(scmID, symlinkTarget.toString());
+ }
+ }
+
+ public void checkPreFinalizedVolumePathID(File volume, String scmID,
+ String clusterID) {
+
+ if (scmHAAlreadyEnabled) {
+ checkVolumePathID(volume, clusterID);
+ } else {
+ checkVolumePathID(volume, scmID);
+ }
+
+ }
+
+ public void checkVolumePathID(File volume, String expectedID) {
+ List<File> subdirs;
+ File hddsRoot;
+ if (dnThinksVolumeFailed(volume)) {
+ // If the volume is failed, read from the failed location it was
+ // moved to.
+ subdirs = getHddsSubdirs(getFailedVolume(volume));
+ hddsRoot = getHddsRoot(getFailedVolume(volume));
+ } else {
+ subdirs = getHddsSubdirs(volume);
+ hddsRoot = getHddsRoot(volume);
+ }
+
+ // Volume should only have the specified ID directory.
+ Assert.assertEquals(1, subdirs.size());
+ File idDir = new File(hddsRoot, expectedID);
+ Assert.assertTrue(subdirs.contains(idDir));
+ }
+
+ public List<File> getHddsSubdirs(File volume) {
+ File[] subdirsArray = getHddsRoot(volume).listFiles(File::isDirectory);
+ Assert.assertNotNull(subdirsArray);
+ return Arrays.asList(subdirsArray);
+ }
+
+ public File getHddsRoot(File volume) {
+ return new File(HddsVolumeUtil.getHddsRoot(volume.getAbsolutePath()));
+ }
+
+ /// CLUSTER OPERATIONS ///
+
+ /**
+ * Starts the datanode with the first layout version, and calls the version
+ * endpoint task to get cluster ID and SCM ID.
+ *
+ * The daemon for the datanode state machine is not started in this test.
+ * This greatly speeds up execution time.
+ * It means we do not have heartbeat functionality or pre-finalize
+ * upgrade actions, but neither of those things are needed for these tests.
+ */
+ public void startPreFinalizedDatanode() throws Exception {
+ // Set layout version.
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
+ tempFolder.getRoot().getAbsolutePath());
+ DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(conf,
+ UUID.randomUUID().toString(),
+ HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
+ layoutStorage.initialize();
+
+ // Build and start the datanode.
+ DatanodeDetails dd = ContainerTestUtils.createDatanodeDetails();
+ DatanodeStateMachine newDsm = new DatanodeStateMachine(dd,
+ conf, null, null,
+ null);
+ int actualMlv =
newDsm.getLayoutVersionManager().getMetadataLayoutVersion();
+ Assert.assertEquals(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(),
+ actualMlv);
+ dsm = newDsm;
+
+ callVersionEndpointTask();
+ }
+
+ public void restartDatanode(int expectedMlv)
+ throws Exception {
+ // Stop existing datanode.
+ DatanodeDetails dd = dsm.getDatanodeDetails();
+ dsm.close();
+
+ // Start new datanode with the same configuration.
+ dsm = new DatanodeStateMachine(dd,
+ conf, null, null,
+ null);
+ int mlv = dsm.getLayoutVersionManager().getMetadataLayoutVersion();
+ Assert.assertEquals(expectedMlv, mlv);
+
+ callVersionEndpointTask();
+ }
+
+ /**
+ * Get the cluster ID and SCM ID from SCM to the datanode.
+ */
+ public void callVersionEndpointTask() throws Exception {
+ try(EndpointStateMachine esm = ContainerTestUtils.createEndpoint(conf,
+ address, 1000)) {
+ VersionEndpointTask vet = new VersionEndpointTask(esm, conf,
+ dsm.getContainer());
+ esm.setState(EndpointStateMachine.EndPointStates.GETVERSION);
+ vet.call();
+ }
+ }
+
+ public String startScmServer() throws Exception {
+ String scmID = UUID.randomUUID().toString();
+ scmServerImpl = new ScmTestMock(CLUSTER_ID, scmID);
+ scmRpcServer = SCMTestUtils.startScmRpcServer(conf,
+ scmServerImpl, address, 10);
+ return scmID;
+ }
+
+ /**
+ * Updates the SCM ID on the SCM server. Datanode will not be aware of this
+ * until {@link this#callVersionEndpointTask} is called.
+ * @return the new scm ID.
+ */
+ public String changeScmID() {
+ String scmID = UUID.randomUUID().toString();
+ scmServerImpl.setScmId(scmID);
+ return scmID;
+ }
+
+ /// CONTAINER OPERATIONS ///
+
+ public void readChunk(ContainerProtos.WriteChunkRequestProto writeChunk,
+ Pipeline pipeline) throws Exception {
+ ContainerProtos.ContainerCommandRequestProto readChunkRequest =
+ ContainerTestHelper.getReadChunkRequest(pipeline, writeChunk);
+
+ dispatchRequest(readChunkRequest);
+ }
+
+ public ContainerProtos.WriteChunkRequestProto putBlock(long containerID,
+ Pipeline pipeline) throws Exception {
+ ContainerProtos.ContainerCommandRequestProto writeChunkRequest =
+ getWriteChunk(containerID, pipeline);
+ dispatchRequest(writeChunkRequest);
+
+ ContainerProtos.ContainerCommandRequestProto putBlockRequest =
+ ContainerTestHelper.getPutBlockRequest(pipeline,
+ writeChunkRequest.getWriteChunk());
+ dispatchRequest(putBlockRequest);
+
+ return writeChunkRequest.getWriteChunk();
+ }
+
+ public ContainerProtos.ContainerCommandRequestProto getWriteChunk(
+ long containerID, Pipeline pipeline) throws Exception {
+ return ContainerTestHelper.getWriteChunkRequest(pipeline,
+ ContainerTestHelper.getTestBlockID(containerID), 100, null);
+ }
+
+ public Pipeline getPipeline() {
+ return MockPipeline.createPipeline(
+ Collections.singletonList(dsm.getDatanodeDetails()));
+ }
+
+ public long addContainer(Pipeline pipeline)
+ throws Exception {
+ long containerID = random.nextInt(Integer.MAX_VALUE);
+ ContainerProtos.ContainerCommandRequestProto createContainerRequest =
+ ContainerTestHelper.getCreateContainerRequest(containerID, pipeline);
+ dispatchRequest(createContainerRequest);
+
+ return containerID;
+ }
+
+ public void deleteContainer(long containerID, Pipeline pipeline)
+ throws Exception {
+ ContainerProtos.ContainerCommandRequestProto deleteContainerRequest =
+ ContainerTestHelper.getDeleteContainer(pipeline, containerID, true);
+ dispatchRequest(deleteContainerRequest);
+ }
+
+ public void closeContainer(long containerID, Pipeline pipeline)
+ throws Exception {
+ closeContainer(containerID, pipeline, ContainerProtos.Result.SUCCESS);
+ }
+
+ public void closeContainer(long containerID, Pipeline pipeline,
+ ContainerProtos.Result expectedResult) throws Exception {
+ ContainerProtos.ContainerCommandRequestProto closeContainerRequest =
+ ContainerTestHelper.getCloseContainer(pipeline, containerID);
+ dispatchRequest(closeContainerRequest, expectedResult);
+ }
+
+ /**
+ * Exports the specified container to a temporary file and returns the file.
+ */
+ public File exportContainer(long containerId) throws Exception {
+ final ContainerReplicationSource replicationSource =
+ new OnDemandContainerReplicationSource(
+ dsm.getContainer().getController());
+
+ replicationSource.prepare(containerId);
+
+ File destination = tempFolder.newFile();
+ try (FileOutputStream fos = new FileOutputStream(destination)) {
+ replicationSource.copyData(containerId, fos);
+ }
+ return destination;
+ }
+
+ /**
+ * Imports the container found in {@code source} to the datanode with the ID
+ * {@code containerID}.
+ */
+ public void importContainer(long containerID, File source) throws Exception {
+ DownloadAndImportReplicator replicator =
+ new DownloadAndImportReplicator(dsm.getContainer().getContainerSet(),
+ dsm.getContainer().getController(),
+ new SimpleContainerDownloader(conf, null),
+ new TarContainerPacker());
+
+ File tempFile = tempFolder.newFile();
+ Files.copy(source.toPath(), tempFile.toPath(),
+ StandardCopyOption.REPLACE_EXISTING);
+ replicator.importContainer(containerID, tempFile.toPath());
+ }
+
+ public void dispatchRequest(
+ ContainerProtos.ContainerCommandRequestProto request) {
+ dispatchRequest(request, ContainerProtos.Result.SUCCESS);
+ }
+
+ public void dispatchRequest(
+ ContainerProtos.ContainerCommandRequestProto request,
+ ContainerProtos.Result expectedResult) {
+ ContainerProtos.ContainerCommandResponseProto response =
+ dsm.getContainer().getDispatcher().dispatch(request, null);
+ Assert.assertEquals(expectedResult, response.getResult());
+ }
+
+ /// VOLUME OPERATIONS ///
+
+ /**
+ * Append a datanode volume to the existing volumes in the configuration.
+ * @return The root directory for the new volume.
+ */
+ public File addVolume() throws Exception {
+ File vol = tempFolder.newFolder(UUID.randomUUID().toString());
+ String[] existingVolumes =
+ conf.getStrings(ScmConfigKeys.HDDS_DATANODE_DIR_KEY);
+ List<String> allVolumes = new ArrayList<>();
+ if (existingVolumes != null) {
+ allVolumes.addAll(Arrays.asList(existingVolumes));
+ }
+
+ allVolumes.add(vol.getAbsolutePath());
+ conf.setStrings(ScmConfigKeys.HDDS_DATANODE_DIR_KEY,
+ allVolumes.toArray(new String[0]));
+
+ return vol;
+ }
+
+ /**
+ * Renames the specified volume directory so it will appear as failed to
+ * the datanode.
+ */
+ public void failVolume(File volume) {
+ File failedVolume = getFailedVolume(volume);
+ Assert.assertTrue(volume.renameTo(failedVolume));
+ }
+
+ /**
+ * Convert the specified volume from its failed name back to its original
+ * name. The File passed should be the original volume path, not the one it
+ * was renamed to to fail it.
+ */
+ public void restoreVolume(File volume) {
+ File failedVolume = getFailedVolume(volume);
+ Assert.assertTrue(failedVolume.renameTo(volume));
+ }
+
+ /**
+ * @return The file name that will be used to rename a volume to fail it.
+ */
+ public File getFailedVolume(File volume) {
+ return new File(volume.getParent(), volume.getName() + "-failed");
+ }
+
+ /**
+ * Checks whether the datanode thinks the volume has failed.
+ * This could be outdated information if the volume was restored already
+ * and the datanode has not been restarted since then.
+ */
+ public boolean dnThinksVolumeFailed(File volume) {
+ return dsm.getContainer().getVolumeSet().getFailedVolumesList().stream()
+ .anyMatch(v ->
+ getHddsRoot(v.getStorageDir()).equals(getHddsRoot(volume)));
+ }
+}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index 2cf2835..dc11ece 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -190,8 +190,8 @@ public class TestEndPoint {
// Now rpcEndpoint should remember the version it got from SCM
Assert.assertNotNull(rpcEndPoint.getVersion());
- // Now change server scmId, so datanode scmId will be
- // different from SCM server response scmId
+ // Now change server cluster ID, so datanode cluster ID will be
+ // different from SCM server response cluster ID.
String newClusterId = UUID.randomUUID().toString();
scmServerImpl.setClusterId(newClusterId);
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh
b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 89a743d..9c3d6c4 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -407,19 +407,3 @@ prepare_for_runner_image() {
export OZONE_IMAGE="apache/ozone-runner:${v}"
}
-## @description Activate the version-specific behavior for a given release
-## @param the release for which definitions should be loaded
-load_version_specifics() {
- local v="$1"
-
- # shellcheck source=/dev/null
- source "${_testlib_dir}/versions/${v}.sh"
-
- ozone_version_load
-}
-
-## @description Deactivate the previously version-specific behavior,
-## reverting to the current version's definitions
-unload_version_specifics() {
- ozone_version_unload
-}
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/README.md
b/hadoop-ozone/dist/src/main/compose/upgrade/README.md
index e63bf71..27d6afa 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/README.md
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/README.md
@@ -23,34 +23,17 @@ an older release of Ozone and a later release (which may be
the local build).
1. Backwards Incompatibility
- These tests will not catch backwards incompatible changes against
commits in between releases.
- Example:
- 1. After 1.0.0, a change *c1* is made that is backwards compatible
with *1.0.0*.
+ 1. After 1.0.0, a change *c1* is made that is backwards compatible
with 1.0.0.
2. After *c1*, a new change *c2* is made that is also backwards
compatible with 1.0.0 but backwards *incompatible* with *c1*.
- This test suite will not raise an error for *c2*, because it
only tests against the last release
(1.0.0), and not the last commit (*c1*).
-2. Downgrade Support
- - Downgrades will not be supported until upgrading from 1.1.0 to 1.2.0
-
- - Until 1.1.0 is released, downgrades cannot be tested, so they are
commented out of the current non-rolling upgrade tests.
-
## Directory Layout
### upgrades
-Each type of upgrade has a subdirectory under the *upgrades* directory. Each
upgrade's steps are controlled by a *test.sh* script in its
*upgrades/\<upgrade-type>* directory. Callbacks to execute throughout the
upgrade are called by this script and should be placed in a file called
*callback.sh* in the *upgrades/\<upgrade-type>/\<upgrade-from>-\<upgrade-to>*
directory. After the test is run, results and docker volume data for the
upgrade for these versions will also be placed in this dir [...]
-
-#### manual-upgrade
-
-- Any necessary conversion of on disk structures from the old version to the
new version must be done explicitly.
-
-- This is primarily for testing upgrades from versions before the non-rolling
upgrade framework was introduced.
-
-- Supported Callbacks:
- 1. `setup_with_old_version`: Run before ozone is started in the old
version.
- 3. `with_old_version`: Run while ozone is running in the old version.
- 3. `setup_with_new_version`: Run after ozone is stopped in the old
version, but before it is restarted in the new version.
- 4. `with_new_version`: Run while ozone is running in the new version.
+Each type of upgrade has a subdirectory under the *upgrades* directory. Each
upgrade's steps are controlled by a *driver.sh* script in its
*upgrades/\<upgrade-type>* directory. Callbacks to execute throughout the
upgrade are called by this script and should be placed in a file called
*callback.sh* in the *upgrades/\<upgrade-type>/\<upgrade-from>-\<upgrade-to>*
directory. After the test is run, results and docker volume data for the
upgrade for these versions will also be placed in this d [...]
#### non-rolling-upgrade
@@ -68,6 +51,20 @@ Each type of upgrade has a subdirectory under the *upgrades*
directory. Each upg
- The non-rolling upgrade framework can still be used, the only difference
is that OMs cannot be prepared before moving from the old version to the new
version.
- Set the variable `OZONE_PREPARE_OMS` to `false` in `callback.sh` setup
function to disable OM preparation as part of the upgrade.
+#### manual-upgrade
+
+- This is a legacy option that was used before the upgrade framework was
introduced in 1.2.0. This option is left as an example in case it needs to be
used for some reason in the future.
+
+- Any necessary conversion of on disk structures from the old version to the
new version must be done explicitly.
+
+- This is primarily for testing upgrades from versions before the non-rolling
upgrade framework was introduced.
+
+- Supported Callbacks:
+ 1. `setup_with_old_version`: Run before ozone is started in the old
version.
+ 3. `with_old_version`: Run while ozone is running in the old version.
+ 3. `setup_with_new_version`: Run after ozone is stopped in the old
version, but before it is restarted in the new version.
+ 4. `with_new_version`: Run while ozone is running in the new version.
+
### compose
Docker compose cluster definitions to be used in upgrade testing are defined
in the *compose* directory. A compose cluster can be selected by sourcing the
*load.sh* script in the compose cluster's directory on the setup callback for
the upgrade test.
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/delete-and-regenerate-data.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/delete-and-regenerate-data.sh
deleted file mode 100755
index a9991c4..0000000
--- a/hadoop-ozone/dist/src/main/compose/upgrade/delete-and-regenerate-data.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script can be run only if the cluster is already started
-# one, and initialized (but not data is written, yet).
-
-set -e
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-cd "${COMPOSE_DIR}"
-
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/../testlib.sh"
-
-#read OZONE_VOLUME from here
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR"/.env
-
-rm -rf "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-
-
-#During the first start, all the required VERSION and metadata files will be
created
-start_docker_env
-
-#data generation requires offline cluster
-docker-compose stop
-
-#generate metadadata (-n1 means: only one container is generated)
-docker-compose run scm ozone freon cgscm -u hadoop -n 1
-docker-compose run om ozone freon cgom -u hadoop -n 1
-
-#generate real data (and metadata) on datanodes.
-docker-compose run dn1 ozone freon cgdn -u hadoop -n 1 --datanodes=3 --index=1
-docker-compose run dn2 ozone freon cgdn -u hadoop -n 1 --datanodes=3 --index=2
-docker-compose run dn3 ozone freon cgdn -u hadoop -n 1 --datanodes=3 --index=3
-
-#start docker env with the generated data
-start_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/delete-data.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/delete-data.sh
deleted file mode 100755
index bc4dbad..0000000
--- a/hadoop-ozone/dist/src/main/compose/upgrade/delete-data.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script can be run only if the cluster is already started
-# one, and initialized (but not data is written, yet).
-
-set -e
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-cd "${COMPOSE_DIR}"
-
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/../testlib.sh"
-
-#read OZONE_VOLUME from here
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR"/.env
-
-rm -rf "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-
-
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index a323310..687f9bd 100755
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -16,7 +16,7 @@
# limitations under the License.
# Version that will be run using the local build.
-: "${OZONE_CURRENT_VERSION:=1.1.0}"
+: "${OZONE_CURRENT_VERSION:=1.2.0}"
export OZONE_CURRENT_VERSION
TEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
@@ -32,12 +32,7 @@ run_test_scripts ${tests} || RESULT=$?
RESULT_DIR="$ALL_RESULT_DIR" create_results_dir
# Upgrade tests to be run.
-# Run all upgrades even if one fails.
-# Any failure will save a failing return code to $RESULT.
-set +e
-run_test manual-upgrade 0.5.0 1.1.0
-run_test non-rolling-upgrade 1.0.0 1.1.0
-set -e
+run_test non-rolling-upgrade 1.1.0 1.2.0
generate_report "upgrade" "$ALL_RESULT_DIR"
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/0.5.0-1.1.0/callback.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/0.5.0-1.1.0/callback.sh
deleted file mode 100755
index d2f43de..0000000
---
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/0.5.0-1.1.0/callback.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source "$TEST_DIR"/testlib.sh
-
-setup_old_version() {
- load_version_specifics "$OZONE_UPGRADE_FROM"
-}
-
-with_old_version() {
- generate old
- validate old
-}
-
-setup_new_version() {
- unload_version_specifics "$OZONE_UPGRADE_FROM"
- # Reformat SCM DB from 0.5.0 to format for versions after 1.0.0.
- "$TEST_DIR"/../../libexec/upgrade/1.0.0.sh
- load_version_specifics "$OZONE_UPGRADE_TO"
-}
-
-with_new_version() {
- validate old
- generate new
- validate new
-}
diff --git a/hadoop-ozone/dist/src/main/compose/versions/README.md
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/README.md
similarity index 59%
rename from hadoop-ozone/dist/src/main/compose/versions/README.md
rename to
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/README.md
index c974adc..20c1c3d 100644
--- a/hadoop-ozone/dist/src/main/compose/versions/README.md
+++
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/README.md
@@ -12,9 +12,8 @@
limitations under the License. See accompanying LICENSE file.
-->
-The scripts in this directory define version-specific behavior required for
[`testlib.sh`](../testlib.sh). For example the `ozone admin` command was
renamed from `ozone scmcli` in 1.0.0.
+# Manual Upgrade Testing
-Interface:
+- Since the release of Ozone's non-rolling upgrade framework in 1.2.0, no
manual reformatting steps need to be run on nodes between upgrades.
- * `ozone_version_load`: define version-specific variables for the test library
- * `ozone_version_unload`: unset version-specific variables; this reverts test
library behavior to the "current" one.
+- The driver for testing this type of upgrade is left here as an example in
case it needs to be used for some reason in the future.
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.0.0-1.1.0/callback.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.1.0-1.2.0/callback.sh
similarity index 86%
rename from
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.0.0-1.1.0/callback.sh
rename to
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.1.0-1.2.0/callback.sh
index dae4953..b533e6c 100755
---
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.0.0-1.1.0/callback.sh
+++
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.1.0-1.2.0/callback.sh
@@ -37,7 +37,6 @@ _check_om_mlvs() {
setup() {
# OM preparation is not implemented until 1.2.0.
export OZONE_OM_PREPARE='false'
- load_version_specifics "$OZONE_UPGRADE_FROM"
}
with_old_version() {
@@ -55,14 +54,13 @@ with_new_version_pre_finalized() {
validate new1
}
-# TODO: Run when 1.1.0 is released.
-# with_old_version_downgraded() {
-# validate old1
-# validate new1
+with_old_version_downgraded() {
+ validate old1
+ validate new1
-# generate old2
-# validate old2
-# }
+ generate old2
+ validate old2
+}
with_new_version_finalized() {
_check_hdds_mlvs 2
@@ -71,8 +69,7 @@ with_new_version_finalized() {
validate old1
validate new1
- # TODO: Run when 1.1.0 is released.
- #validate old2
+ validate old2
generate new2
validate new2
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh
index 1fcb904..e59bec5 100755
---
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh
+++
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh
@@ -69,29 +69,28 @@ prepare_for_image "$OZONE_UPGRADE_TO"
export OM_HA_ARGS='--upgrade'
echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO PRE-FINALIZED ---"
-OUTPUT_NAME="$OZONE_UPGRADE_TO"
+OUTPUT_NAME="$OZONE_UPGRADE_TO"-pre-finalized
OZONE_KEEP_RESULTS=true start_docker_env
callback with_new_version_pre_finalized
-# TODO: Start testing downgrades when 1.1.0 is released.
-# prepare_oms
-# stop_docker_env
-# prepare_for_image "$OZONE_UPGRADE_FROM"
-# set_downgrade_om_args
+prepare_oms
+stop_docker_env
+prepare_for_image "$OZONE_UPGRADE_FROM"
+set_downgrade_om_args
-# echo "--- RUNNING WITH OLD VERSION $OZONE_UPGRADE_FROM AFTER DOWNGRADE ---"
-# OUTPUT_NAME="$OZONE_UPGRADE_FROM"-downgraded
-# OZONE_KEEP_RESULTS=true start_docker_env
-# callback with_old_version_downgraded
+echo "--- RUNNING WITH OLD VERSION $OZONE_UPGRADE_FROM AFTER DOWNGRADE ---"
+OUTPUT_NAME="$OZONE_UPGRADE_FROM"-downgraded
+OZONE_KEEP_RESULTS=true start_docker_env
+callback with_old_version_downgraded
-# prepare_oms
-# stop_docker_env
-# prepare_for_image "$OZONE_UPGRADE_TO"
-# export OM_HA_ARGS='--upgrade'
+prepare_oms
+stop_docker_env
+prepare_for_image "$OZONE_UPGRADE_TO"
+export OM_HA_ARGS='--upgrade'
echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO FINALIZED ---"
-# OUTPUT_NAME="$OZONE_UPGRADE_TO"-finalized
-# OZONE_KEEP_RESULTS=true start_docker_env
+OUTPUT_NAME="$OZONE_UPGRADE_TO"-finalized
+OZONE_KEEP_RESULTS=true start_docker_env
# Sends commands to finalize OM and SCM.
execute_robot_test scm upgrade/finalize.robot
diff --git a/hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh
b/hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh
deleted file mode 100644
index 3c76415..0000000
--- a/hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ozone_version_load() {
- export OZONE_ADMIN_COMMAND=scmcli
- export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status'
-}
-
-ozone_version_unload() {
- unset OZONE_ADMIN_COMMAND
- unset OZONE_SAFEMODE_STATUS_COMMAND
-}
diff --git a/hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh
b/hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh
deleted file mode 100644
index 9565b95..0000000
--- a/hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ozone_version_load() {
- export OZONE_ADMIN_COMMAND=admin
- export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose'
-}
-
-ozone_version_unload() {
- unset OZONE_ADMIN_COMMAND
- unset OZONE_SAFEMODE_STATUS_COMMAND
-}
diff --git a/hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh
b/hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh
deleted file mode 100644
index 9565b95..0000000
--- a/hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ozone_version_load() {
- export OZONE_ADMIN_COMMAND=admin
- export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose'
-}
-
-ozone_version_unload() {
- unset OZONE_ADMIN_COMMAND
- unset OZONE_SAFEMODE_STATUS_COMMAND
-}
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
index 7d7ee1d..b15c6e5 100755
--- a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
@@ -81,12 +81,8 @@ create_results_dir
COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${current_version}
test_cross_compatibility
for cluster_version in 1.0.0; do
- load_version_specifics ${cluster_version}
-
export OZONE_VERSION=${cluster_version}
COMPOSE_FILE=old-cluster.yaml:clients.yaml test_cross_compatibility
-
- unload_version_specifics
done
generate_report
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 57ebd0d..4f0c437 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -85,7 +85,7 @@ public class TestOzoneContainer {
Mockito.when(dsm.getDatanodeDetails()).thenReturn(datanodeDetails);
Mockito.when(context.getParent()).thenReturn(dsm);
container = new OzoneContainer(datanodeDetails, conf, context, null);
- //Set scmId and manually start ozone container.
+ //Set clusterId and manually start ozone container.
container.start(UUID.randomUUID().toString());
XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
@@ -128,10 +128,10 @@ public class TestOzoneContainer {
container = new OzoneContainer(datanodeDetails, conf,
context, null);
- String scmId = UUID.randomUUID().toString();
- container.start(scmId);
+ String clusterId = UUID.randomUUID().toString();
+ container.start(clusterId);
try {
- container.start(scmId);
+ container.start(clusterId);
} catch (Exception e) {
Assert.fail();
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeLayoutUpgradeTool.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeLayoutUpgradeTool.java
deleted file mode 100644
index 07bc3fc..0000000
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeLayoutUpgradeTool.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- */
-package org.apache.hadoop.ozone.dn;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
-import org.apache.hadoop.ozone.HddsDatanodeService;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.client.ObjectStore;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.debug.DatanodeLayout;
-import org.junit.Before;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.time.Instant;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.UUID;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE;
-import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
-
-/**
- * Test Datanode Layout Upgrade Tool.
- */
-public class TestDatanodeLayoutUpgradeTool {
- private MiniOzoneCluster cluster = null;
-
- @Before
- public void setup() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3).setTotalPipelineNumLimit(2).build();
- cluster.waitForClusterToBeReady();
- }
-
- @After
- public void destroy() throws Exception {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- private void writeData() throws Exception {
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- Instant testStartTime = Instant.now();
-
- ObjectStore store = cluster.getClient().getObjectStore();
-
- String value = "sample value";
- store.createVolume(volumeName);
- OzoneVolume volume = store.getVolume(volumeName);
- volume.createBucket(bucketName);
- OzoneBucket bucket = volume.getBucket(bucketName);
-
- for (int i = 0; i < 10; i++) {
- String keyName = UUID.randomUUID().toString();
-
- OzoneOutputStream out = bucket.createKey(keyName,
- value.getBytes(UTF_8).length, RATIS,
- THREE, new HashMap<>());
- out.write(value.getBytes(UTF_8));
- out.close();
- OzoneKey key = bucket.getKey(keyName);
- Assert.assertEquals(keyName, key.getName());
- OzoneInputStream is = bucket.readKey(keyName);
- byte[] fileContent = new byte[value.getBytes(UTF_8).length];
- is.read(fileContent);
- Assert.assertEquals(value, new String(fileContent, UTF_8));
- Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
- Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
- }
-
- // wait for the container report to propagate to SCM
- Thread.sleep(5000);
- }
-
- @Test
- public void testDatanodeLayoutVerify() throws Exception {
- writeData();
- cluster.stop();
-
- List<HddsDatanodeService> dns = cluster.getHddsDatanodes();
- OzoneConfiguration c1 = dns.get(0).getConf();
- Collection<String> paths = HddsServerUtil.getDatanodeStorageDirs(c1);
-
- for (String p : paths) {
- // Verify that tool is able to verify the storage path
- List<HddsVolume> volumes = DatanodeLayout.runUpgrade(c1, p, true);
- Assert.assertEquals(0, volumes.size());
-
- HddsVolume.Builder volumeBuilder = new HddsVolume.Builder(p)
- .conf(c1);
- HddsVolume vol = volumeBuilder.build();
-
- // Rename the path and verify that the tool fails
- File clusterDir = new File(vol.getHddsRootDir(), vol.getClusterID());
- File renamePath = new File(vol.getHddsRootDir(),
- UUID.randomUUID().toString());
- Assert.assertTrue(clusterDir.renameTo(renamePath));
-
- List<HddsVolume> failedVols = DatanodeLayout.runUpgrade(c1, p, true);
- Assert.assertEquals(1, failedVols.size());
- }
- }
-}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
index 449e340..94c7dc0 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
@@ -73,7 +73,6 @@ public final class OMLayoutVersionManager
/**
* Initialize the OM Layout Features and current Layout Version.
- * @param storage to read the current layout version.
* @throws OMException on error.
*/
private void init(int layoutVersion) throws OMException {
@@ -83,7 +82,7 @@ public final class OMLayoutVersionManager
throw new OMException(
String.format("Cannot initialize VersionManager. Metadata " +
"layout version (%d) > software layout version (%d)",
- metadataLayoutVersion, softwareLayoutVersion),
+ getMetadataLayoutVersion(), getSoftwareLayoutVersion()),
e,
NOT_SUPPORTED_OPERATION);
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java
deleted file mode 100644
index c35e00e..0000000
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.debug;
-
-import org.apache.hadoop.hdds.cli.GenericCli;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.cli.SubcommandWithParent;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-
-import org.kohsuke.MetaInfServices;
-import picocli.CommandLine.Spec;
-import picocli.CommandLine.Model.CommandSpec;
-
-import picocli.CommandLine;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
-
-/**
- * Tool to upgrade Datanode layout.
- */
[email protected](
- name = "dnlayout",
- description = "Shell of updating datanode layout format",
- versionProvider = HddsVersionProvider.class,
- mixinStandardHelpOptions = true)
-@MetaInfServices(SubcommandWithParent.class)
-public class DatanodeLayout extends GenericCli
- implements Callable<Void>, SubcommandWithParent{
-
- @CommandLine.Option(names = {"--path"},
- description = "File Path")
- private String storagePath;
-
- @CommandLine.Option(names = {"--verify"},
- hidden = true,
- description = "Verify that the datanode layout is correct")
- private boolean verify;
-
- @Spec
- private CommandSpec spec;
-
- @Override
- public Void call() throws Exception {
- OzoneConfiguration conf = createOzoneConfiguration();
-
- runUpgrade(conf, storagePath, verify);
- return null;
- }
-
- public static void main(String[] args) {
- new DatanodeLayout().run(args);
- }
-
- @Override
- public Class<?> getParentType() {
- return OzoneDebug.class;
- }
-
- public static List<HddsVolume> runUpgrade(OzoneConfiguration conf,
- String storagePath, boolean verify) throws Exception {
- if (storagePath != null) {
- conf.unset(HDDS_DATANODE_DIR_KEY);
- conf.set(HDDS_DATANODE_DIR_KEY, storagePath);
- }
-
- if (verify) {
- conf.setBoolean(
- ScmConfigKeys.HDDS_DATANODE_UPGRADE_LAYOUT_INLINE, false);
- }
-
- MutableVolumeSet volumeSet = new MutableVolumeSet(conf);
- ContainerSet containerSet = new ContainerSet();
- OzoneContainer.buildContainerSet(volumeSet, containerSet, conf);
- volumeSet.shutdown();
-
- List<HddsVolume> failedVolumes = StorageVolumeUtil.getHddsVolumesList(
- volumeSet.getFailedVolumesList());
-
- if (verify) {
- for (HddsVolume vol : failedVolumes) {
- System.out.println("Failed Volume:" + vol.getHddsRootDir());
- }
- }
- return failedVolumes;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
