Author: rangadi
Date: Thu Jul 17 11:24:24 2008
New Revision: 677667
URL: http://svn.apache.org/viewvc?rev=677667&view=rev
Log:
HADOOP-3677. Simplify generation stamp upgrade by making is a
local upgrade on datandodes. Deleted distributed upgrade. (rangadi)
Removed:
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/GenerationStampUpgrade.java
Modified:
hadoop/core/branches/branch-0.18/CHANGES.txt
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/DataStorage.java
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/FSDataset.java
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/Storage.java
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/UpgradeObjectCollection.java
Modified: hadoop/core/branches/branch-0.18/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/CHANGES.txt?rev=677667&r1=677666&r2=677667&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.18/CHANGES.txt Thu Jul 17 11:24:24 2008
@@ -180,6 +180,10 @@
in hadoop user guide. (shv)
IMPROVEMENTS
+
+ HADOOP-3677. Simplify generation stamp upgrade by making is a
+ local upgrade on datandodes. Deleted distributed upgrade.
+ (rangadi)
HADOOP-2928. Remove deprecated FileSystem.getContentLength().
(Lohit Vjayarenu via rangadi)
Modified:
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/DataStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/DataStorage.java?rev=677667&r1=677666&r2=677667&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/DataStorage.java
(original)
+++
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/DataStorage.java
Thu Jul 17 11:24:24 2008
@@ -28,6 +28,8 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.hadoop.dfs.FSConstants.StartupOption;
import org.apache.hadoop.dfs.FSConstants.NodeType;
@@ -270,7 +272,7 @@
// rename current to tmp
rename(curDir, tmpDir);
// hardlink blocks
- linkBlocks(tmpDir, curDir);
+ linkBlocks(tmpDir, curDir, this.getLayoutVersion());
// write version file
this.layoutVersion = FSConstants.LAYOUT_VERSION;
assert this.namespaceID == nsInfo.getNamespaceID() :
@@ -352,12 +354,19 @@
}
}
- static void linkBlocks(File from, File to) throws IOException {
+ static void linkBlocks(File from, File to, int oldLV) throws IOException {
if (!from.isDirectory()) {
if (from.getName().startsWith(COPY_FILE_PREFIX)) {
IOUtils.copyBytes(new FileInputStream(from),
new FileOutputStream(to), 16*1024, true);
} else {
+
+ //check if we are upgrading from pre-generation stamp version.
+ if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) {
+ // Link to the new file name.
+ to = new File(convertMetatadataFileName(to.getAbsolutePath()));
+ }
+
HardLink.createHardLink(from, to);
}
return;
@@ -374,7 +383,8 @@
});
for(int i = 0; i < blockNames.length; i++)
- linkBlocks(new File(from, blockNames[i]), new File(to, blockNames[i]));
+ linkBlocks(new File(from, blockNames[i]),
+ new File(to, blockNames[i]), oldLV);
}
protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
@@ -401,4 +411,22 @@
um.setUpgradeState(false, getLayoutVersion());
um.initializeUpgrade(nsInfo);
}
+
+ private static final Pattern PRE_GENSTAMP_META_FILE_PATTERN =
+ Pattern.compile("(.*blk_[-]*\\d+)\\.meta$");
+ /**
+ * This is invoked on target file names when upgrading from pre generation
+ * stamp version (version -13) to correct the metatadata file name.
+ * @param oldFileName
+ * @return the new metadata file name with the default generation stamp.
+ */
+ private static String convertMetatadataFileName(String oldFileName) {
+ Matcher matcher = PRE_GENSTAMP_META_FILE_PATTERN.matcher(oldFileName);
+ if (matcher.matches()) {
+ //return the current metadata file name
+ return FSDataset.getMetaFileName(matcher.group(1),
+ Block.GRANDFATHER_GENERATION_STAMP);
+ }
+ return oldFileName;
+ }
}
Modified:
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/FSDataset.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/FSDataset.java?rev=677667&r1=677666&r2=677667&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/FSDataset.java
(original)
+++
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/FSDataset.java
Thu Jul 17 11:24:24 2008
@@ -559,9 +559,13 @@
}
}
+ static String getMetaFileName(String blockFileName, long genStamp) {
+ return blockFileName + "_" + genStamp + METADATA_EXTENSION;
+ }
+
static File getMetaFile(File f , Block b) {
- return new File( f.getAbsolutePath() +
- "_" + b.getGenerationStamp() + METADATA_EXTENSION );
+ return new File(getMetaFileName(f.getAbsolutePath(),
+ b.getGenerationStamp()));
}
protected File getMetaFile(Block b) throws IOException {
return getMetaFile(getBlockFile(b), b);
Modified:
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/Storage.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/Storage.java?rev=677667&r1=677666&r2=677667&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/Storage.java
(original)
+++
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/Storage.java
Thu Jul 17 11:24:24 2008
@@ -101,6 +101,10 @@
// this corresponds to Hadoop-0.14.
protected static final int LAST_UPGRADABLE_LAYOUT_VERSION = -7;
protected static final String LAST_UPGRADABLE_HADOOP_VERSION = "Hadoop-0.14";
+
+ /* this should be removed when LAST_UPGRADABLE_LV goes beyond -13.
+ * any upgrade code that uses this constant should also be removed. */
+ public static final int PRE_GENERATIONSTAMP_LAYOUT_VERSION = -13;
private static final String STORAGE_FILE_LOCK = "in_use.lock";
protected static final String STORAGE_FILE_VERSION = "VERSION";
Modified:
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/UpgradeObjectCollection.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/UpgradeObjectCollection.java?rev=677667&r1=677666&r2=677667&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/UpgradeObjectCollection.java
(original)
+++
hadoop/core/branches/branch-0.18/src/hdfs/org/apache/hadoop/dfs/UpgradeObjectCollection.java
Thu Jul 17 11:24:24 2008
@@ -33,8 +33,6 @@
initialize();
// Registered distributed upgrade objects here
// registerUpgrade(new UpgradeObject());
- registerUpgrade(new GenerationStampUpgradeNamenode());
- registerUpgrade(new GenerationStampUpgradeDatanode());
}
static class UOSignature implements Comparable<UOSignature> {
