Author: rangadi
Date: Thu Jul 17 11:22:34 2008
New Revision: 677665
URL: http://svn.apache.org/viewvc?rev=677665&view=rev
Log:
HADOOP-3677. Simplify generation stamp upgrade by making is a
local upgrade on datandodes. Deleted distributed upgrade. (rangadi)
Removed:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/GenerationStampStatsUpgradeCommand.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/GenerationStampUpgradeDatanode.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/GenerationStampUpgradeNamenode.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Jul 17 11:22:34 2008
@@ -307,6 +307,10 @@
in hadoop user guide. (shv)
IMPROVEMENTS
+
+ HADOOP-3677. Simplify generation stamp upgrade by making is a
+ local upgrade on datandodes. Deleted distributed upgrade.
+ (rangadi)
HADOOP-2928. Remove deprecated FileSystem.getContentLength().
(Lohit Vjayarenu via rangadi)
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
Thu Jul 17 11:22:34 2008
@@ -69,6 +69,10 @@
// this corresponds to Hadoop-0.14.
public static final int LAST_UPGRADABLE_LAYOUT_VERSION = -7;
protected static final String LAST_UPGRADABLE_HADOOP_VERSION = "Hadoop-0.14";
+
+ /* this should be removed when LAST_UPGRADABLE_LV goes beyond -13.
+ * any upgrade code that uses this constant should also be removed. */
+ public static final int PRE_GENERATIONSTAMP_LAYOUT_VERSION = -13;
private static final String STORAGE_FILE_LOCK = "in_use.lock";
protected static final String STORAGE_FILE_VERSION = "VERSION";
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java
Thu Jul 17 11:22:34 2008
@@ -22,8 +22,6 @@
import java.util.TreeSet;
import org.apache.hadoop.hdfs.protocol.FSConstants;
-import org.apache.hadoop.hdfs.server.datanode.GenerationStampUpgradeDatanode;
-import org.apache.hadoop.hdfs.server.namenode.GenerationStampUpgradeNamenode;
import org.apache.hadoop.util.StringUtils;
/**
@@ -36,8 +34,6 @@
initialize();
// Registered distributed upgrade objects here
// registerUpgrade(new UpgradeObject());
- registerUpgrade(new GenerationStampUpgradeNamenode());
- registerUpgrade(new GenerationStampUpgradeDatanode());
}
static class UOSignature implements Comparable<UOSignature> {
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
Thu Jul 17 11:22:34 2008
@@ -28,7 +28,10 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.FSConstants.NodeType;
import org.apache.hadoop.hdfs.protocol.FSConstants.StartupOption;
@@ -275,7 +278,7 @@
// rename current to tmp
rename(curDir, tmpDir);
// hardlink blocks
- linkBlocks(tmpDir, curDir);
+ linkBlocks(tmpDir, curDir, this.getLayoutVersion());
// write version file
this.layoutVersion = FSConstants.LAYOUT_VERSION;
assert this.namespaceID == nsInfo.getNamespaceID() :
@@ -357,12 +360,19 @@
}
}
- static void linkBlocks(File from, File to) throws IOException {
+ static void linkBlocks(File from, File to, int oldLV) throws IOException {
if (!from.isDirectory()) {
if (from.getName().startsWith(COPY_FILE_PREFIX)) {
IOUtils.copyBytes(new FileInputStream(from),
new FileOutputStream(to), 16*1024, true);
} else {
+
+ //check if we are upgrading from pre-generation stamp version.
+ if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) {
+ // Link to the new file name.
+ to = new File(convertMetatadataFileName(to.getAbsolutePath()));
+ }
+
HardLink.createHardLink(from, to);
}
return;
@@ -379,7 +389,8 @@
});
for(int i = 0; i < blockNames.length; i++)
- linkBlocks(new File(from, blockNames[i]), new File(to, blockNames[i]));
+ linkBlocks(new File(from, blockNames[i]),
+ new File(to, blockNames[i]), oldLV);
}
protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
@@ -406,4 +417,22 @@
um.setUpgradeState(false, getLayoutVersion());
um.initializeUpgrade(nsInfo);
}
+
+ private static final Pattern PRE_GENSTAMP_META_FILE_PATTERN =
+ Pattern.compile("(.*blk_[-]*\\d+)\\.meta$");
+ /**
+ * This is invoked on target file names when upgrading from pre generation
+ * stamp version (version -13) to correct the metatadata file name.
+ * @param oldFileName
+ * @return the new metadata file name with the default generation stamp.
+ */
+ private static String convertMetatadataFileName(String oldFileName) {
+ Matcher matcher = PRE_GENSTAMP_META_FILE_PATTERN.matcher(oldFileName);
+ if (matcher.matches()) {
+ //return the current metadata file name
+ return FSDataset.getMetaFileName(matcher.group(1),
+ Block.GRANDFATHER_GENERATION_STAMP);
+ }
+ return oldFileName;
+ }
}
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
Thu Jul 17 11:22:34 2008
@@ -564,9 +564,13 @@
}
}
+ static String getMetaFileName(String blockFileName, long genStamp) {
+ return blockFileName + "_" + genStamp + METADATA_EXTENSION;
+ }
+
static File getMetaFile(File f , Block b) {
- return new File( f.getAbsolutePath() +
- "_" + b.getGenerationStamp() + METADATA_EXTENSION );
+ return new File(getMetaFileName(f.getAbsolutePath(),
+ b.getGenerationStamp()));
}
protected File getMetaFile(Block b) throws IOException {
return getMetaFile(getBlockFile(b), b);
