Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 bef53683b -> c10bf788e
HDFS-9730. Storage ID update does not happen when there is a layout change.
Contributed by Tsz Wo Nicholas Sze.
(cherry picked from commit 496f33de0ce80dc455cfd51f19612da6f9b914f9)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c10bf788
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c10bf788
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c10bf788
Branch: refs/heads/branch-2.7
Commit: c10bf788ebf7c0c639fce463cd4e9595e6104fe1
Parents: bef5368
Author: Kihwal Lee <[email protected]>
Authored: Thu Feb 4 13:32:11 2016 -0600
Committer: Kihwal Lee <[email protected]>
Committed: Thu Feb 4 13:32:11 2016 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/datanode/DataStorage.java | 47 +++++++++-----------
2 files changed, 23 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c10bf788/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1ed6416..17561f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -86,6 +86,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9740. Use a reasonable limit in DFSTestUtil.waitForMetric()
(Chang Li via vinayakumarb)
+ HDFS-9730. Storage ID update does not happen when there is a layout change
+ (Tsz Wo Nicholas Sze via kihwal)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c10bf788/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index ad91ad3..19cb1eb 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -533,11 +533,7 @@ public class DataStorage extends Storage {
this.cTime = 0;
setDatanodeUuid(datanodeUuid);
- if (sd.getStorageUuid() == null) {
- // Assign a new Storage UUID.
- sd.setStorageUuid(DatanodeStorage.generateUuid());
- }
-
+ createStorageID(sd, false);
writeProperties(sd);
}
@@ -709,7 +705,13 @@ public class DataStorage extends Storage {
// do upgrade
if (this.layoutVersion > HdfsConstants.DATANODE_LAYOUT_VERSION) {
- doUpgrade(sd, nsInfo, conf); // upgrade
+ if (federationSupported) {
+ // If the existing on-disk layout version supports federation,
+ // simply update the properties.
+ upgradeProperties(sd);
+ } else {
+ doUpgradePreFederation(sd, nsInfo, conf);
+ }
return true; // doUgrade already has written properties
}
@@ -723,7 +725,8 @@ public class DataStorage extends Storage {
}
/**
- * Upgrade -- Move current storage into a backup directory,
+ * Upgrade from a pre-federation layout.
+ * Move current storage into a backup directory,
* and hardlink all its blocks into the new current directory.
*
* Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ =>
0.22/0.23
@@ -742,25 +745,9 @@ public class DataStorage extends Storage {
* There should be only ONE namenode in the cluster for first
* time upgrade to 0.22
* @param sd storage directory
- * @throws IOException on error
*/
- void doUpgrade(final StorageDirectory sd, final NamespaceInfo nsInfo,
- final Configuration conf) throws IOException {
- // If the existing on-disk layout version supportes federation, simply
- // update its layout version.
- if (DataNodeLayoutVersion.supports(
- LayoutVersion.Feature.FEDERATION, layoutVersion)) {
- // The VERSION file is already read in. Override the layoutVersion
- // field and overwrite the file. The upgrade work is handled by
- // {@link BlockPoolSliceStorage#doUpgrade}
- LOG.info("Updating layout version from " + layoutVersion + " to "
- + HdfsConstants.DATANODE_LAYOUT_VERSION + " for storage "
- + sd.getRoot());
- layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
- writeProperties(sd);
- return;
- }
-
+ void doUpgradePreFederation(final StorageDirectory sd,
+ final NamespaceInfo nsInfo, final Configuration conf) throws IOException
{
final int oldLV = getLayoutVersion();
LOG.info("Upgrading storage directory " + sd.getRoot()
+ ".\n old LV = " + oldLV
@@ -804,15 +791,21 @@ public class DataStorage extends Storage {
linkAllBlocks(tmpDir, bbwDir, toDir, oldLV, conf);
// 4. Write version file under <SD>/current
- layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
clusterID = nsInfo.getClusterID();
- writeProperties(sd);
+ upgradeProperties(sd);
// 5. Rename <SD>/previous.tmp to <SD>/previous
rename(tmpDir, prevDir);
LOG.info("Upgrade of " + sd.getRoot()+ " is complete");
+ }
+ void upgradeProperties(StorageDirectory sd) throws IOException {
createStorageID(sd, layoutVersion);
+ LOG.info("Updating layout version from " + layoutVersion
+ + " to " + HdfsConstants.DATANODE_LAYOUT_VERSION
+ + " for storage " + sd.getRoot());
+ layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
+ writeProperties(sd);
}
/**
