Author: szetszwo
Date: Wed Mar 14 18:36:30 2012
New Revision: 1300680
URL: http://svn.apache.org/viewvc?rev=1300680&view=rev
Log:
HDFS-3075. Backport HADOOP-4885: Try to restore failed name-node storage
directories at checkpoint time. Contributed by Brandon Li
Added:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1300680&r1=1300679&r2=1300680&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Wed Mar 14 18:36:30 2012
@@ -163,6 +163,9 @@ Release 1.1.0 - unreleased
MAPREDUCE-4001. Improve MAPREDUCE-3789's fix logic by looking at job's
slot demands instead. (harsh)
+ HDFS-3075. Backport HADOOP-4885: Try to restore failed name-node storage
+ directories at checkpoint time. (Brandon Li via szetszwo)
+
Release 1.0.2 - unreleased
NEW FEATURES
Modified:
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1300680&r1=1300679&r2=1300680&view=diff
==============================================================================
---
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Wed Mar 14 18:36:30 2012
@@ -69,6 +69,9 @@ class FSDirectory implements FSConstants
ns.createFsOwnerPermissions(new FsPermission((short)0755)),
Integer.MAX_VALUE, -1);
this.fsImage = fsImage;
+ fsImage.setRestoreRemovedDirs(conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT));
namesystem = ns;
int configuredLimit = conf.getInt(
DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
Modified:
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1300680&r1=1300679&r2=1300680&view=diff
==============================================================================
---
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Wed Mar 14 18:36:30 2012
@@ -1238,18 +1238,26 @@ public class FSEditLog {
if (existsNew()) {
Iterator<StorageDirectory> it =
fsimage.dirIterator(NameNodeDirType.EDITS);
+ StringBuilder b = new StringBuilder();
while (it.hasNext()) {
File editsNew = getEditNewFile(it.next());
+ b.append("\n ").append(editsNew);
if (!editsNew.exists()) {
throw new IOException(
"Inconsistent existence of edits.new " + editsNew);
}
}
- return; // nothing to do, edits.new exists!
+ FSNamesystem.LOG.warn("Cannot roll edit log," +
+ " edits.new files already exists in all healthy directories:" + b);
+ return;
}
close(); // close existing edit log
+ // After edit streams are closed, healthy edits files should be identical,
+ // and same to fsimage files
+ fsimage.restoreStorageDirs();
+
//
// Open edits.new
//
Modified:
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1300680&r1=1300679&r2=1300680&view=diff
==============================================================================
---
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Wed Mar 14 18:36:30 2012
@@ -28,39 +28,41 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.Random;
-import java.util.Map;
-import java.util.HashMap;
-import java.lang.Math;
-import java.nio.ByteBuffer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
-import org.apache.hadoop.io.UTF8;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.namenode.BlocksMap.BlockInfo;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLog.EditLogFileInputStream;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.common.UpgradeManager;
+import org.apache.hadoop.hdfs.server.namenode.BlocksMap.BlockInfo;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLog.EditLogFileInputStream;
import org.apache.hadoop.hdfs.util.AtomicFileOutputStream;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MultipleIOException;
+import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.io.Writable;
/**
* FSImage handles checkpointing and logging of the namespace edits.
@@ -138,6 +140,9 @@ public class FSImage extends Storage {
static private final FsPermission FILE_PERM = new FsPermission((short)0);
static private final byte[] PATH_SEPARATOR =
DFSUtil.string2Bytes(Path.SEPARATOR);
+ /** Flag to restore removed storage directories at checkpointing */
+ private boolean restoreRemovedDirs =
DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT;
+
/**
*/
FSImage() {
@@ -1215,7 +1220,99 @@ public class FSImage extends Storage {
newID = r.nextInt(0x7FFFFFFF); // use 31 bits only
return newID;
}
+
+ void setRestoreRemovedDirs(boolean allow) {
+ this.restoreRemovedDirs = allow;
+ }
+
+ /** restore a metadata file */
+ private static void restoreFile(File src, File dstdir, String dstfile)
+ throws IOException {
+ File dst = new File(dstdir, dstfile);
+ IOUtils.copyBytes(new FileInputStream(src), new FileOutputStream(dst),
+ DFSConfigKeys.DFS_STREAM_BUFFER_SIZE_DEFAULT, true);
+ }
+ /**
+ * Refresh storage dirs by copying files from good storage dir
+ */
+ void restoreStorageDirs() throws IOException {
+ if (!restoreRemovedDirs || getRemovedStorageDirs().isEmpty()) {
+ return;
+ }
+
+ Iterator<StorageDirectory> it = dirIterator(NameNodeDirType.EDITS);
+ if (!it.hasNext()) {
+ throw new IOException("No healthy edits directory");
+ }
+ StorageDirectory goodSd = it.next();
+ File goodEdits = getEditFile(goodSd);
+
+ it = dirIterator(NameNodeDirType.IMAGE);
+ if (!it.hasNext()) {
+ throw new IOException("No healthy fsimage directory");
+ }
+ goodSd = it.next();
+ File goodImage = getImageFile(goodSd, NameNodeFile.IMAGE);
+ File goodFstime = getImageFile(goodSd, NameNodeFile.TIME);
+ File goodVersion = goodSd.getVersionFile();
+ //for Hadoop version < 0.13 to fail to start
+ File goodImage013 = new File(goodSd.getRoot(), "image/fsimage");
+
+ List<IOException> exceptions = new ArrayList<IOException>();
+ for (Iterator<StorageDirectory> i = removedStorageDirs.iterator();
+ i.hasNext();) {
+ StorageDirectory sd = i.next();
+ FSNamesystem.LOG.info("Try to recover removed directory " + sd.getRoot()
+ + " by reformatting");
+ try {
+ // don't create dir if it doesn't exist, since it may should be mounted
+ if (!sd.getRoot().exists()) {
+ throw new IOException("Directory " + sd.getRoot() + "doesn't
exist");
+ }
+ if (!FileUtil.fullyDeleteContents(sd.getRoot())) {
+ throw new IOException("Can't fully delete content of " +
sd.getRoot());
+ }
+ sd.clearDirectory(); // create empty "current" dir
+ restoreFile(goodVersion, sd.getCurrentDir(),
Storage.STORAGE_FILE_VERSION);
+ restoreFile(goodFstime, sd.getCurrentDir(),
NameNodeFile.TIME.getName());
+
+ // Create image directory
+ File imageDir = new File(sd.getRoot(), "image");
+ if (!imageDir.mkdir()) {
+ throw new IOException("Can't make directory 'image'.");
+ }
+ restoreFile(goodImage013, imageDir, NameNodeFile.IMAGE.getName());
+
+ if (sd.getStorageDirType().equals(NameNodeDirType.EDITS)) {
+ restoreFile(goodEdits, sd.getCurrentDir(),
NameNodeFile.EDITS.getName());
+ } else if (sd.getStorageDirType().equals(NameNodeDirType.IMAGE)) {
+ restoreFile(goodImage, sd.getCurrentDir(),
NameNodeFile.IMAGE.getName());
+ } else if (sd.getStorageDirType().equals(
+ NameNodeDirType.IMAGE_AND_EDITS)) {
+ restoreFile(goodEdits, sd.getCurrentDir(),
NameNodeFile.EDITS.getName());
+ restoreFile(goodImage, sd.getCurrentDir(),
NameNodeFile.IMAGE.getName());
+ } else {
+ throw new IOException("Invalid NameNodeDirType: "
+ + sd.getStorageDirType());
+ }
+
+ //remove from removedStorageDirs and add back to healthy.
+ i.remove();
+ addStorageDir(new StorageDirectory(sd.getRoot(),
sd.getStorageDirType()));
+ } catch (IOException e) {
+ FSNamesystem.LOG.warn("Failed to recover removed directory "
+ + sd.getRoot() + " with " + e);
+ exceptions.add(e);
+ }
+ }
+
+ if (!exceptions.isEmpty()) {
+ throw MultipleIOException.createIOException(exceptions);
+ }
+ }
+
+
/** Create new dfs name directory. Caution: this destroys all files
* in this filesystem. */
void format(StorageDirectory sd) throws IOException {
Added:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java?rev=1300680&view=auto
==============================================================================
---
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
(added)
+++
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Wed Mar 14 18:36:30 2012
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.util.Iterator;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeDirType;
+import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
+
+/**
+ * Startup and checkpoint tests
+ *
+ */
+public class TestStorageRestore extends TestCase {
+ public static final String NAME_NODE_HOST = "localhost:";
+ public static final String NAME_NODE_HTTP_HOST = "0.0.0.0:";
+ private static final Log LOG = LogFactory.getLog(TestStorageRestore.class
+ .getName());
+ private Configuration config;
+ private File hdfsDir = null;
+ static final long seed = 0xAAAAEEFL;
+ static final int blockSize = 4096;
+ static final int fileSize = 8192;
+ private File path1, path2, path3;
+ private MiniDFSCluster cluster;
+
+ private void writeFile(FileSystem fileSys, Path name, int repl)
+ throws IOException {
+ FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
+ .getInt("io.file.buffer.size", 4096), (short) repl, (long) blockSize);
+ byte[] buffer = new byte[fileSize];
+ Random rand = new Random(seed);
+ rand.nextBytes(buffer);
+ stm.write(buffer);
+ stm.close();
+ }
+
+ protected void setUp() throws Exception {
+ config = new Configuration();
+ String baseDir = System.getProperty("test.build.data", "/tmp");
+
+ hdfsDir = new File(baseDir, "dfs");
+ if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
+ throw new IOException("Could not delete hdfs directory '" + hdfsDir +
"'");
+ }
+
+ hdfsDir.mkdir();
+ path1 = new File(hdfsDir, "name1");
+ path2 = new File(hdfsDir, "name2");
+ path3 = new File(hdfsDir, "name3");
+
+ path1.mkdir();
+ path2.mkdir();
+ path3.mkdir();
+ if (!path2.exists() || !path3.exists() || !path1.exists()) {
+ throw new IOException("Couldn't create dfs.name dirs");
+ }
+
+ String dfs_name_dir = new String(path1.getPath() + "," + path2.getPath());
+ System.out.println("configuring hdfsdir is " + hdfsDir.getAbsolutePath()
+ + "; dfs_name_dir = " + dfs_name_dir + ";dfs_name_edits_dir(only)="
+ + path3.getPath());
+
+ config.set("dfs.name.dir", dfs_name_dir);
+ config.set("dfs.name.edits.dir", dfs_name_dir + "," + path3.getPath());
+
+ config.set("fs.checkpoint.dir", new File(hdfsDir, "secondary").getPath());
+
+ FileSystem.setDefaultUri(config, "hdfs://" + NAME_NODE_HOST + "0");
+
+ config.set("dfs.secondary.http.address", "0.0.0.0:0");
+
+ // set the restore feature on
+ config.setBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, true);
+ }
+
+ /**
+ * clean up
+ */
+ public void tearDown() throws Exception {
+ if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
+ throw new IOException("Could not delete hdfs directory in tearDown '"
+ + hdfsDir + "'");
+ }
+ }
+
+ /**
+ * invalidate storage by removing sub-directory "current" in name2 and name3
+ */
+ public void invalidateStorage(FSImage fi) throws IOException {
+ for (Iterator<StorageDirectory> it = fi.dirIterator(); it.hasNext();) {
+ StorageDirectory sd = it.next();
+
+ if (sd.getRoot().equals(path2) || sd.getRoot().equals(path3)) {
+ fi.getEditLog().removeEditsForStorageDir(sd);
+ fi.updateRemovedDirs(sd, null);
+ it.remove();
+ }
+ }
+ }
+
+ /**
+ * test
+ */
+ public void printStorages(FSImage fs) {
+ LOG.info("current storages and corresoponding sizes:");
+ for (Iterator<StorageDirectory> it = fs.dirIterator(); it.hasNext();) {
+ StorageDirectory sd = it.next();
+
+ if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) {
+ File imf = FSImage.getImageFile(sd, NameNodeFile.IMAGE);
+ LOG.info(" image file " + imf.getAbsolutePath() + "; len = "
+ + imf.length());
+ }
+ if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
+ File edf = FSImage.getImageFile(sd, NameNodeFile.EDITS);
+ LOG.info(" edits file " + edf.getAbsolutePath() + "; len = "
+ + edf.length());
+ }
+ }
+ }
+
+ /**
+ * This function returns a md5 hash of a file.
+ *
+ * @param file input file
+ * @return The md5 string
+ */
+ public String getFileMD5(File file) throws Exception {
+ String res = new String();
+ MessageDigest mD = MessageDigest.getInstance("MD5");
+ DataInputStream dis = new DataInputStream(new FileInputStream(file));
+
+ try {
+ while(true) {
+ mD.update(dis.readByte());
+ }
+ } catch (EOFException eof) {}
+
+ BigInteger bigInt = new BigInteger(1, mD.digest());
+ res = bigInt.toString(16);
+ dis.close();
+
+ return res;
+ }
+
+
+ /**
+ * check if files exist/not exist
+ */
+ public void checkFiles(boolean valid) {
+ // look at the valid storage
+ File fsImg1 = new File(path1, Storage.STORAGE_DIR_CURRENT + "/"
+ + NameNodeFile.IMAGE.getName());
+ File fsImg2 = new File(path2, Storage.STORAGE_DIR_CURRENT + "/"
+ + NameNodeFile.IMAGE.getName());
+ File fsImg3 = new File(path3, Storage.STORAGE_DIR_CURRENT + "/"
+ + NameNodeFile.IMAGE.getName());
+
+ File fsEdits1 = new File(path1, Storage.STORAGE_DIR_CURRENT + "/"
+ + NameNodeFile.EDITS.getName());
+ File fsEdits2 = new File(path2, Storage.STORAGE_DIR_CURRENT + "/"
+ + NameNodeFile.EDITS.getName());
+ File fsEdits3 = new File(path3, Storage.STORAGE_DIR_CURRENT + "/"
+ + NameNodeFile.EDITS.getName());
+
+ this.printStorages(cluster.getNameNode().getFSImage());
+
+ String md5_1 = null,md5_2 = null,md5_3 = null;
+ try {
+ md5_1 = getFileMD5(fsEdits1);
+ md5_2 = getFileMD5(fsEdits2);
+ md5_3 = getFileMD5(fsEdits3);
+ } catch (Exception e) {
+ System.err.println("md 5 calculation failed:" + e.getLocalizedMessage());
+ }
+
+ LOG.info("++++ image files = " + fsImg1.getAbsolutePath() + ","
+ + fsImg2.getAbsolutePath() + "," + fsImg3.getAbsolutePath());
+ LOG.info("++++ edits files = " + fsEdits1.getAbsolutePath() + ","
+ + fsEdits2.getAbsolutePath() + "," + fsEdits3.getAbsolutePath());
+ LOG.info("checkFiles compares lengths: img1=" + fsImg1.length() + ",img2="
+ + fsImg2.length() + ",img3=" + fsImg3.length());
+ LOG.info("checkFiles compares lengths: edits1=" + fsEdits1.length()
+ + ",edits2=" + fsEdits2.length() + ",edits3=" + fsEdits3.length());
+
+ if (valid) {
+ // should be the same
+ assertTrue(fsImg1.length() == fsImg2.length());
+ assertTrue(0 == fsImg3.length()); // shouldn't be created
+ assertTrue(fsEdits1.length() == fsEdits2.length());
+ assertTrue(md5_1.equals(md5_2));
+ assertTrue(md5_1.equals(md5_3));
+ } else {
+ // should be different
+ assertFalse(md5_1.equals(md5_2));
+ assertFalse(md5_1.equals(md5_2));
+ }
+ }
+
+ /**
+ * test
+ * 1. create DFS cluster with 3 storage directories - 2 EDITS_IMAGE, 1 EDITS
+ * 2. create a cluster and write a file
+ * 3. corrupt/disable one storage (or two) by removing
+ * 4. run doCheckpoint - it will fail on removed dirs (which will invalidate
the storages)
+ * 5. write another file
+ * 6. check that edits and fsimage differ
+ * 7. run doCheckpoint
+ * 8. verify that all the image and edits files are the same.
+ */
+ public void testStorageRestore() throws Exception {
+ int numDatanodes = 2;
+ cluster = new MiniDFSCluster(0, config, numDatanodes, true, false, true,
+ null, null, null, null);
+ cluster.waitActive();
+
+ SecondaryNameNode secondary = new SecondaryNameNode(config);
+ System.out.println("****testStorageRestore: Cluster and SNN started");
+ printStorages(cluster.getNameNode().getFSImage());
+
+ FileSystem fs = cluster.getFileSystem();
+ Path path = new Path("/", "test");
+ writeFile(fs, path, 2);
+
+ System.out
+ .println("****testStorageRestore: file test written, invalidating
storage...");
+
+ invalidateStorage(cluster.getNameNode().getFSImage());
+ printStorages(cluster.getNameNode().getFSImage());
+ System.out
+ .println("****testStorageRestore: storage invalidated + doCheckpoint");
+
+ path = new Path("/", "test1");
+ writeFile(fs, path, 2);
+ System.out.println("****testStorageRestore: file test1 written");
+
+ checkFiles(false); // SHOULD BE FALSE
+
+ System.out.println("****testStorageRestore: checkfiles(false) run");
+
+ secondary.doCheckpoint(); // /should enable storage..
+
+ checkFiles(true);
+
+ //use the recovered dir to restart nn
+ if (!FileUtil.fullyDelete(path1)) {
+ throw new Exception("Can't fully delete " + path1);
+ }
+ if (!FileUtil.fullyDelete(path3)) {
+ throw new Exception("Can't fully delete " + path3);
+ }
+ cluster.restartDataNodes();
+ cluster.waitActive();
+ File tmpfile = new File("/test");
+ assert(tmpfile.exists());
+ tmpfile = new File("/test1");
+ assert(tmpfile.exists());
+
+ System.out
+ .println("****testStorageRestore: second Checkpoint done and
checkFiles(true) run");
+ secondary.shutdown();
+ cluster.shutdown();
+ }
+}
