Author: eli
Date: Fri Oct 29 16:36:46 2010
New Revision: 1028825
URL: http://svn.apache.org/viewvc?rev=1028825&view=rev
Log:
HDFS-259. Remove intentionally corrupt 0.13 directory layout creation.
Contributed by Todd Lipcon.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1028825&r1=1028824&r2=1028825&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Oct 29 16:36:46 2010
@@ -154,6 +154,9 @@ Trunk (unreleased changes)
HADOOP-7007. Update the hudson-test-patch ant target to work with the
latest test-patch.sh script (gkesavan)
+ HDFS-259. Remove intentionally corrupt 0.13 directory layout creation.
+ (Todd Lipcon via eli)
+
OPTIMIZATIONS
HDFS-1140. Speedup INode.getPathComponents. (Dmytro Molkov via shv)
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1028825&r1=1028824&r2=1028825&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
Fri Oct 29 16:36:46 2010
@@ -64,7 +64,7 @@ public abstract class Storage extends St
// Constants
- // last layout version that did not suppot upgrades
+ // last layout version that did not support upgrades
protected static final int LAST_PRE_UPGRADE_LAYOUT_VERSION = -3;
// this corresponds to Hadoop-0.14.
@@ -80,7 +80,7 @@ public abstract class Storage extends St
private static final String STORAGE_FILE_LOCK = "in_use.lock";
protected static final String STORAGE_FILE_VERSION = "VERSION";
- public static final String STORAGE_DIR_CURRENT = "current";
+ public static final String STORAGE_DIR_CURRENT = "current";
private static final String STORAGE_DIR_PREVIOUS = "previous";
private static final String STORAGE_TMP_REMOVED = "removed.tmp";
private static final String STORAGE_TMP_PREVIOUS = "previous.tmp";
@@ -256,7 +256,6 @@ public abstract class Storage extends St
* @throws IOException
*/
public void write() throws IOException {
- corruptPreUpgradeStorage(root);
write(getVersionFile());
}
@@ -472,8 +471,7 @@ public abstract class Storage extends St
if (startOpt == HdfsConstants.StartupOption.FORMAT)
return StorageState.NOT_FORMATTED;
if (startOpt != HdfsConstants.StartupOption.IMPORT) {
- //make sure no conversion is required
- checkConversionNeeded(this);
+ checkOldLayoutStorage(this);
}
// check whether current directory is valid
@@ -686,16 +684,22 @@ public abstract class Storage extends St
protected void addStorageDir(StorageDirectory sd) {
storageDirs.add(sd);
}
-
- public abstract boolean isConversionNeeded(StorageDirectory sd) throws
IOException;
- /*
- * Coversion is no longer supported. So this should throw exception if
- * conversion is needed.
- */
- private void checkConversionNeeded(StorageDirectory sd) throws IOException {
- if (isConversionNeeded(sd)) {
- //throw an exception
+ /**
+ * Return true if the layout of the given storage directory is from a version
+ * of Hadoop prior to the introduction of the "current" and "previous"
+ * directories which allow upgrade and rollback.
+ */
+ public abstract boolean isPreUpgradableLayout(StorageDirectory sd)
+ throws IOException;
+
+ /**
+ * Check if the given storage directory comes from a version of Hadoop
+ * prior to when the directory layout changed (ie 0.13). If this is
+ * the case, this method throws an IOException.
+ */
+ private void checkOldLayoutStorage(StorageDirectory sd) throws IOException {
+ if (isPreUpgradableLayout(sd)) {
checkVersionUpgradable(0);
}
}
@@ -851,20 +855,4 @@ public abstract class Storage extends St
+ "-" + Integer.toString(storage.getLayoutVersion())
+ "-" + Long.toString(storage.getCTime());
}
-
- // Pre-upgrade version compatibility
- protected abstract void corruptPreUpgradeStorage(File rootDir) throws
IOException;
-
- protected void writeCorruptedData(RandomAccessFile file) throws IOException {
- final String messageForPreUpgradeVersion =
- "\nThis file is INTENTIONALLY CORRUPTED so that versions\n"
- + "of Hadoop prior to 0.13 (which are incompatible\n"
- + "with this directory layout) will fail to start.\n";
-
- file.seek(0);
- file.writeInt(FSConstants.LAYOUT_VERSION);
- org.apache.hadoop.hdfs.DeprecatedUTF8.writeString(file, "");
- file.writeBytes(messageForPreUpgradeVersion);
- file.getFD().sync();
- }
}
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1028825&r1=1028824&r2=1028825&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
Fri Oct 29 16:36:46 2010
@@ -190,7 +190,8 @@ public class DataStorage extends Storage
storageID = ssid;
}
- public boolean isConversionNeeded(StorageDirectory sd) throws IOException {
+ @Override
+ public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException
{
File oldF = new File(sd.getRoot(), "storage");
if (!oldF.exists())
return false;
@@ -462,22 +463,6 @@ public class DataStorage extends Storage
new File(to, blockNames[i]), oldLV);
}
- protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
- File oldF = new File(rootDir, "storage");
- if (oldF.exists())
- return;
- // recreate old storage file to let pre-upgrade versions fail
- if (!oldF.createNewFile())
- throw new IOException("Cannot create file " + oldF);
- RandomAccessFile oldFile = new RandomAccessFile(oldF, "rws");
- // write new version into old storage file
- try {
- writeCorruptedData(oldFile);
- } finally {
- oldFile.close();
- }
- }
-
private void verifyDistributedUpgradeProgress(
NamespaceInfo nsInfo
) throws IOException {
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java?rev=1028825&r1=1028824&r2=1028825&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
Fri Oct 29 16:36:46 2010
@@ -60,7 +60,7 @@ public class BackupStorage extends FSIma
}
@Override
- public boolean isConversionNeeded(StorageDirectory sd) {
+ public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException
{
return false;
}
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1028825&r1=1028824&r2=1028825&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Fri Oct 29 16:36:46 2010
@@ -862,12 +862,9 @@ public class FSImage extends Storage {
return editLog;
}
- public boolean isConversionNeeded(StorageDirectory sd) throws IOException {
+ public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException
{
File oldImageDir = new File(sd.getRoot(), "image");
if (!oldImageDir.exists()) {
- if(sd.getVersionFile().exists())
- throw new InconsistentFSStateException(sd.getRoot(),
- oldImageDir + " does not exist.");
return false;
}
// check the layout version inside the image file
@@ -883,7 +880,7 @@ public class FSImage extends Storage {
}
return true;
}
-
+
//
// Atomic move sequence, to recover from interrupted checkpoint
//
@@ -2099,25 +2096,6 @@ public class FSImage extends Storage {
}
}
- protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
- File oldImageDir = new File(rootDir, "image");
- if (!oldImageDir.exists())
- if (!oldImageDir.mkdir())
- throw new IOException("Cannot create directory " + oldImageDir);
- File oldImage = new File(oldImageDir, "fsimage");
- if (!oldImage.exists())
- // recreate old image file to let pre-upgrade versions fail
- if (!oldImage.createNewFile())
- throw new IOException("Cannot create file " + oldImage);
- RandomAccessFile oldFile = new RandomAccessFile(oldImage, "rws");
- // write new version into old image file
- try {
- writeCorruptedData(oldFile);
- } finally {
- oldFile.close();
- }
- }
-
private boolean getDistributedUpgradeState() {
FSNamesystem ns = getFSNamesystem();
return ns == null ? false : ns.getDistributedUpgradeState();
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1028825&r1=1028824&r2=1028825&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Fri Oct 29 16:36:46 2010
@@ -585,12 +585,6 @@ public class SecondaryNameNode implement
super(conf);
}
- @Override
- public
- boolean isConversionNeeded(StorageDirectory sd) {
- return false;
- }
-
/**
* Analyze checkpoint directories.
* Create directories if they do not exist.
Modified:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1028825&r1=1028824&r2=1028825&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
(original)
+++
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
Fri Oct 29 16:36:46 2010
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.util.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -48,6 +49,8 @@ public class TestDFSUpgradeFromImage ext
private static final Log LOG = LogFactory.getLog(
"org.apache.hadoop.hdfs.TestDFSUpgradeFromImage");
+ private static File TEST_ROOT_DIR =
+ new File(MiniDFSCluster.getBaseDirectory());
public int numDataNodes = 4;
@@ -203,4 +206,47 @@ public class TestDFSUpgradeFromImage ext
if (cluster != null) { cluster.shutdown(); }
}
}
+
+ /**
+ * Test that sets up a fake image from Hadoop 0.3.0 and tries to start a
+ * NN, verifying that the correct error message is thrown.
+ */
+ public void testFailOnPreUpgradeImage() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+
+ File namenodeStorage = new File(TEST_ROOT_DIR, "nnimage-0.3.0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
namenodeStorage.toString());
+
+ // Set up a fake NN storage that looks like an ancient Hadoop dir circa
0.3.0
+ FileUtil.fullyDelete(namenodeStorage);
+ assertTrue("Make " + namenodeStorage, namenodeStorage.mkdirs());
+ File imageDir = new File(namenodeStorage, "image");
+ assertTrue("Make " + imageDir, imageDir.mkdirs());
+
+ // Hex dump of a formatted image from Hadoop 0.3.0
+ File imageFile = new File(imageDir, "fsimage");
+ byte[] imageBytes = StringUtils.hexStringToByte(
+ "fffffffee17c0d2700000000");
+ FileOutputStream fos = new FileOutputStream(imageFile);
+ try {
+ fos.write(imageBytes);
+ } finally {
+ fos.close();
+ }
+
+ // Now try to start an NN from it
+
+ try {
+ new MiniDFSCluster.Builder(conf).numDataNodes(0)
+ .format(false)
+ .manageDataDfsDirs(false)
+ .manageNameDfsDirs(false)
+ .startupOption(StartupOption.REGULAR)
+ .build();
+ fail("Was able to start NN from 0.3.0 image");
+ } catch (IOException ioe) {
+ LOG.info("Got expected exception", ioe);
+ assertTrue(ioe.toString().contains("Old layout version is 'too old'"));
+ }
+ }
}
Modified:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=1028825&r1=1028824&r2=1028825&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
(original)
+++
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
Fri Oct 29 16:36:46 2010
@@ -279,23 +279,11 @@ public class UpgradeUtilities {
localFS.copyToLocalFile(new Path(namenodeStorage.toString(),
"current"),
new Path(newDir.toString()),
false);
- Path newImgDir = new Path(newDir.getParent(), "image");
- if (!localFS.exists(newImgDir))
- localFS.copyToLocalFile(
- new Path(namenodeStorage.toString(), "image"),
- newImgDir,
- false);
break;
case DATA_NODE:
localFS.copyToLocalFile(new Path(datanodeStorage.toString(),
"current"),
new Path(newDir.toString()),
false);
- Path newStorageFile = new Path(newDir.getParent(), "storage");
- if (!localFS.exists(newStorageFile))
- localFS.copyToLocalFile(
- new Path(datanodeStorage.toString(), "storage"),
- newStorageFile,
- false);
break;
}
retVal[i] = newDir;
