Author: cos
Date: Tue May 25 23:56:10 2010
New Revision: 948260
URL: http://svn.apache.org/viewvc?rev=948260&view=rev
Log:
HDFS-1132. Refactor TestFileStatus. Contributed by Eli Collins.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileStatus.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=948260&r1=948259&r2=948260&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue May 25 23:56:10 2010
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
IMPROVEMENTS
+ HDFS-1132. Refactor TestFileStatus (Eli Collins via cos)
+
HDFS-1163. normalize property names for JT/NN kerberos principal
names in configuration (from HADOOP 6633) (boryas)
Modified:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileStatus.java?rev=948260&r1=948259&r2=948260&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileStatus.java
(original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileStatus.java
Tue May 25 23:56:10 2010
@@ -22,8 +22,6 @@ import java.io.IOException;
import java.util.Iterator;
import java.util.Random;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -31,16 +29,22 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.log4j.Level;
+import static org.junit.Assert.*;
+import org.junit.Test;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+
/**
* This class tests the FileStatus API.
*/
-public class TestFileStatus extends TestCase {
+public class TestFileStatus {
{
((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
((Log4JLogger)FileSystem.LOG).getLogger().setLevel(Level.ALL);
@@ -50,232 +54,235 @@ public class TestFileStatus extends Test
static final int blockSize = 8192;
static final int fileSize = 16384;
- private void writeFile(FileSystem fileSys, Path name, int repl,
- int fileSize, int blockSize)
- throws IOException {
- // create and write a file that contains three blocks of data
+ private static Configuration conf;
+ private static MiniDFSCluster cluster;
+ private static FileSystem fs;
+ private static FileContext fc;
+ private static HftpFileSystem hftpfs;
+ private static DFSClient dfsClient;
+ private static Path file1;
+
+ @BeforeClass
+ public static void testSetUp() throws Exception {
+ conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 2);
+ cluster = new MiniDFSCluster(conf, 1, true, null);
+ fs = cluster.getFileSystem();
+ fc = FileContext.getFileContext(cluster.getURI(), conf);
+ hftpfs = cluster.getHftpFileSystem();
+ dfsClient = new DFSClient(NameNode.getAddress(conf), conf);
+ file1 = new Path("filestatus.dat");
+ writeFile(fs, file1, 1, fileSize, blockSize);
+ }
+
+ @AfterClass
+ public static void testTearDown() throws Exception {
+ fs.close();
+ cluster.shutdown();
+ }
+
+ private static void writeFile(FileSystem fileSys, Path name, int repl,
+ int fileSize, int blockSize) throws IOException {
+ // Create and write a file that contains three blocks of data
FSDataOutputStream stm = fileSys.create(name, true,
-
fileSys.getConf().getInt("io.file.buffer.size", 4096),
- (short)repl, (long)blockSize);
+ FSConstants.BUFFER_SIZE, (short)repl, (long)blockSize);
byte[] buffer = new byte[fileSize];
Random rand = new Random(seed);
rand.nextBytes(buffer);
stm.write(buffer);
stm.close();
}
-
+
private void checkFile(FileSystem fileSys, Path name, int repl)
- throws IOException {
+ throws IOException {
DFSTestUtil.waitReplication(fileSys, name, (short) repl);
}
+
+ /** Test calling getFileInfo directly on the client */
+ @Test
+ public void testGetFileInfo() throws IOException {
+ // Check that / exists
+ Path path = new Path("/");
+ assertTrue("/ should be a directory",
+ fs.getFileStatus(path).isDir());
+
+ // Make sure getFileInfo returns null for files which do not exist
+ HdfsFileStatus fileInfo = dfsClient.getFileInfo("/noSuchFile");
+ assertEquals("Non-existant file should result in null", null, fileInfo);
-
- /**
- * Tests various options of DFSShell.
- */
- public void testFileStatus() throws IOException {
- Configuration conf = new HdfsConfiguration();
- conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 2);
- MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
- FileSystem fs = cluster.getFileSystem();
- final HftpFileSystem hftpfs = cluster.getHftpFileSystem();
- final DFSClient dfsClient = new DFSClient(NameNode.getAddress(conf), conf);
- FileContext fc = FileContext.getFileContext(cluster.getURI(), conf);
+ // Test getFileInfo throws the right exception given a non-absolute path.
try {
+ dfsClient.getFileInfo("non-absolute");
+ fail("getFileInfo for a non-absolute path did not throw IOException");
+ } catch (RemoteException re) {
+ assertTrue("Wrong exception for invalid file name",
+ re.toString().contains("Invalid file name"));
+ }
+ }
+
- //
- // check that / exists
- //
- Path path = new Path("/");
- System.out.println("Path : \"" + path.toString() + "\"");
- assertTrue("/ should be a directory",
- fs.getFileStatus(path).isDir() == true);
-
- // make sure getFileInfo returns null for files which do not exist
- HdfsFileStatus fileInfo = dfsClient.getFileInfo("/noSuchFile");
- assertTrue(fileInfo == null);
-
- // make sure getFileInfo throws the appropriate exception for
non-relative
- // filenames
- try {
- dfsClient.getFileInfo("non-relative");
- fail("getFileInfo for a non-relative path did not thro IOException");
- } catch (RemoteException re) {
- assertTrue(re.toString().contains("Invalid file name"));
- }
-
- // create a file in home directory
- //
- Path file1 = new Path("filestatus.dat");
- writeFile(fs, file1, 1, fileSize, blockSize);
- System.out.println("Created file filestatus.dat with one "
- + " replicas.");
- checkFile(fs, file1, 1);
- System.out.println("Path : \"" + file1 + "\"");
-
- // test getFileStatus on a file
- FileStatus status = fs.getFileStatus(file1);
- assertTrue(file1 + " should be a file",
- status.isDir() == false);
- assertTrue(status.getBlockSize() == blockSize);
- assertTrue(status.getReplication() == 1);
- assertTrue(status.getLen() == fileSize);
- assertEquals(file1.makeQualified(fs.getUri(),
- fs.getWorkingDirectory()).toString(),
- status.getPath().toString());
-
- // test listStatus on a file
- FileStatus[] stats = fs.listStatus(file1);
- assertEquals(1, stats.length);
- status = stats[0];
- assertTrue(file1 + " should be a file",
- status.isDir() == false);
- assertTrue(status.getBlockSize() == blockSize);
- assertTrue(status.getReplication() == 1);
- assertTrue(status.getLen() == fileSize);
- assertEquals(file1.makeQualified(fs.getUri(),
- fs.getWorkingDirectory()).toString(),
- status.getPath().toString());
-
- Iterator<FileStatus> itor = fc.listStatus(file1);
- status = itor.next();
- assertEquals(stats[0], status);
- assertTrue(file1 + " should be a file",
- status.isDir() == false);
-
- // test file status on a directory
- Path dir = new Path("/test/mkdirs");
-
- // test listStatus on a non-existent file/directory
- try {
- stats = fs.listStatus(dir);
- fail("listStatus of non-existent path should fail");
- } catch (FileNotFoundException fe) {
- assertTrue(fe.getMessage().equals("File " + dir + " does not exist."));
- }
-
- try {
- itor = fc.listStatus(dir);
- fail("listStatus of non-existent path should fail");
- } catch (FileNotFoundException fe) {
- assertTrue(fe.getMessage().equals("File " + dir + " does not exist."));
- }
- try {
- status = fs.getFileStatus(dir);
- fail("getFileStatus of non-existent path should fail");
- } catch (FileNotFoundException fe) {
- assertTrue(fe.getMessage().startsWith("File does not exist"));
- }
-
- // create the directory
- assertTrue(fs.mkdirs(dir));
- assertTrue(fs.exists(dir));
- System.out.println("Dir : \"" + dir + "\"");
-
- // test getFileStatus on an empty directory
- status = fs.getFileStatus(dir);
- assertTrue(dir + " should be a directory", status.isDir());
- assertTrue(dir + " should be zero size ", status.getLen() == 0);
- assertEquals(dir.makeQualified(fs.getUri(),
- fs.getWorkingDirectory()).toString(),
- status.getPath().toString());
-
- // test listStatus on an empty directory
- stats = fs.listStatus(dir);
- assertEquals(dir + " should be empty", 0, stats.length);
- assertEquals(dir + " should be zero size ",
- 0, fs.getContentSummary(dir).getLength());
- assertEquals(dir + " should be zero size using hftp",
- 0, hftpfs.getContentSummary(dir).getLength());
-
- itor = fc.listStatus(dir);
- assertFalse(dir + " should be empty", itor.hasNext());
-
- // create another file that is smaller than a block.
- //
- Path file2 = new Path(dir, "filestatus2.dat");
- writeFile(fs, file2, 1, blockSize/4, blockSize);
- System.out.println("Created file filestatus2.dat with one "
- + " replicas.");
- checkFile(fs, file2, 1);
- System.out.println("Path : \"" + file2 + "\"");
-
- // verify file attributes
- status = fs.getFileStatus(file2);
- assertTrue(status.getBlockSize() == blockSize);
- assertTrue(status.getReplication() == 1);
- file2 = fs.makeQualified(file2);
- assertEquals(file2.toString(), status.getPath().toString());
-
- // create another file in the same directory
- Path file3 = new Path(dir, "filestatus3.dat");
- writeFile(fs, file3, 1, blockSize/4, blockSize);
- System.out.println("Created file filestatus3.dat with one "
- + " replicas.");
- checkFile(fs, file3, 1);
- file3 = fs.makeQualified(file3);
-
- // verify that the size of the directory increased by the size
- // of the two files
- final int expected = blockSize/2;
- assertEquals(dir + " size should be " + expected,
- expected, fs.getContentSummary(dir).getLength());
- assertEquals(dir + " size should be " + expected + " using hftp",
- expected, hftpfs.getContentSummary(dir).getLength());
-
- // test listStatus on a non-empty directory
- stats = fs.listStatus(dir);
- assertEquals(dir + " should have two entries", 2, stats.length);
- assertEquals(file2.toString(), stats[0].getPath().toString());
- assertEquals(file3.toString(), stats[1].getPath().toString());
-
- itor = fc.listStatus(dir);
- assertEquals(file2.toString(), itor.next().getPath().toString());
- assertEquals(file3.toString(), itor.next().getPath().toString());
- assertFalse(itor.hasNext());
-
- // test iterative listing
- // now dir has 2 entries, create one more
- Path dir3 = fs.makeQualified(new Path(dir, "dir3"));
- fs.mkdirs(dir3);
- dir3 = fs.makeQualified(dir3);
- stats = fs.listStatus(dir);
- assertEquals(dir + " should have three entries", 3, stats.length);
- assertEquals(dir3.toString(), stats[0].getPath().toString());
- assertEquals(file2.toString(), stats[1].getPath().toString());
- assertEquals(file3.toString(), stats[2].getPath().toString());
-
- itor = fc.listStatus(dir);
- assertEquals(dir3.toString(), itor.next().getPath().toString());
- assertEquals(file2.toString(), itor.next().getPath().toString());
- assertEquals(file3.toString(), itor.next().getPath().toString());
- assertFalse(itor.hasNext());
-
- // now dir has 3 entries, create two more
- Path dir4 = fs.makeQualified(new Path(dir, "dir4"));
- fs.mkdirs(dir4);
- dir4 = fs.makeQualified(dir4);
- Path dir5 = fs.makeQualified(new Path(dir, "dir5"));
- fs.mkdirs(dir5);
- dir5 = fs.makeQualified(dir5);
- stats = fs.listStatus(dir);
- assertEquals(dir + " should have five entries", 5, stats.length);
- assertEquals(dir3.toString(), stats[0].getPath().toString());
- assertEquals(dir4.toString(), stats[1].getPath().toString());
- assertEquals(dir5.toString(), stats[2].getPath().toString());
- assertEquals(file2.toString(), stats[3].getPath().toString());
- assertEquals(file3.toString(), stats[4].getPath().toString());
-
- itor = fc.listStatus(dir);
- assertEquals(dir3.toString(), itor.next().getPath().toString());
- assertEquals(dir4.toString(), itor.next().getPath().toString());
- assertEquals(dir5.toString(), itor.next().getPath().toString());
- assertEquals(file2.toString(), itor.next().getPath().toString());
- assertEquals(file3.toString(), itor.next().getPath().toString());
- assertFalse(itor.hasNext());
- } finally {
- fs.close();
- cluster.shutdown();
+ /** Test the FileStatus obtained calling getFileStatus on a file */
+ @Test
+ public void testGetFileStatusOnFile() throws IOException {
+ checkFile(fs, file1, 1);
+ // test getFileStatus on a file
+ FileStatus status = fs.getFileStatus(file1);
+ assertFalse(file1 + " should be a file", status.isDir());
+ assertEquals(blockSize, status.getBlockSize());
+ assertEquals(1, status.getReplication());
+ assertEquals(fileSize, status.getLen());
+ assertEquals(file1.makeQualified(fs.getUri(),
+ fs.getWorkingDirectory()).toString(),
+ status.getPath().toString());
+ }
+
+ /** Test the FileStatus obtained calling listStatus on a file */
+ @Test
+ public void testListStatusOnFile() throws IOException {
+ FileStatus[] stats = fs.listStatus(file1);
+ assertEquals(1, stats.length);
+ FileStatus status = stats[0];
+ assertFalse(file1 + " should be a file", status.isDir());
+ assertEquals(blockSize, status.getBlockSize());
+ assertEquals(1, status.getReplication());
+ assertEquals(fileSize, status.getLen());
+ assertEquals(file1.makeQualified(fs.getUri(),
+ fs.getWorkingDirectory()).toString(),
+ status.getPath().toString());
+
+ Iterator<FileStatus> itor = fc.listStatus(file1);
+ status = itor.next();
+ assertEquals(stats[0], status);
+ assertFalse(file1 + " should be a file", status.isDir());
+ }
+
+ /** Test getting a FileStatus object using a non-existant path */
+ @Test
+ public void testGetFileStatusOnNonExistantFileDir() throws IOException {
+ Path dir = new Path("/test/mkdirs");
+ try {
+ fs.listStatus(dir);
+ fail("listStatus of non-existent path should fail");
+ } catch (FileNotFoundException fe) {
+ assertEquals("File " + dir + " does not exist.",fe.getMessage());
}
+
+ try {
+ fc.listStatus(dir);
+ fail("listStatus of non-existent path should fail");
+ } catch (FileNotFoundException fe) {
+ assertEquals("File " + dir + " does not exist.", fe.getMessage());
+ }
+ try {
+ fs.getFileStatus(dir);
+ fail("getFileStatus of non-existent path should fail");
+ } catch (FileNotFoundException fe) {
+ assertTrue("Exception doesn't indicate non-existant path",
+ fe.getMessage().startsWith("File does not exist"));
+ }
+ }
+
+ /** Test FileStatus objects obtained from a directory */
+ @Test
+ public void testGetFileStatusOnDir() throws IOException {
+ // Create the directory
+ Path dir = new Path("/test/mkdirs");
+ assertTrue("mkdir failed", fs.mkdirs(dir));
+ assertTrue("mkdir failed", fs.exists(dir));
+
+ // test getFileStatus on an empty directory
+ FileStatus status = fs.getFileStatus(dir);
+ assertTrue(dir + " should be a directory", status.isDir());
+ assertTrue(dir + " should be zero size ", status.getLen() == 0);
+ assertEquals(dir.makeQualified(fs.getUri(),
+ fs.getWorkingDirectory()).toString(),
+ status.getPath().toString());
+
+ // test listStatus on an empty directory
+ FileStatus[] stats = fs.listStatus(dir);
+ assertEquals(dir + " should be empty", 0, stats.length);
+ assertEquals(dir + " should be zero size ",
+ 0, fs.getContentSummary(dir).getLength());
+ assertEquals(dir + " should be zero size using hftp",
+ 0, hftpfs.getContentSummary(dir).getLength());
+
+ Iterator<FileStatus> itor = fc.listStatus(dir);
+ assertFalse(dir + " should be empty", itor.hasNext());
+
+ // create another file that is smaller than a block.
+ Path file2 = new Path(dir, "filestatus2.dat");
+ writeFile(fs, file2, 1, blockSize/4, blockSize);
+ checkFile(fs, file2, 1);
+
+ // verify file attributes
+ status = fs.getFileStatus(file2);
+ assertEquals(blockSize, status.getBlockSize());
+ assertEquals(1, status.getReplication());
+ file2 = fs.makeQualified(file2);
+ assertEquals(file2.toString(), status.getPath().toString());
+
+ // Create another file in the same directory
+ Path file3 = new Path(dir, "filestatus3.dat");
+ writeFile(fs, file3, 1, blockSize/4, blockSize);
+ checkFile(fs, file3, 1);
+ file3 = fs.makeQualified(file3);
+
+ // Verify that the size of the directory increased by the size
+ // of the two files
+ final int expected = blockSize/2;
+ assertEquals(dir + " size should be " + expected,
+ expected, fs.getContentSummary(dir).getLength());
+ assertEquals(dir + " size should be " + expected + " using hftp",
+ expected, hftpfs.getContentSummary(dir).getLength());
+
+ // Test listStatus on a non-empty directory
+ stats = fs.listStatus(dir);
+ assertEquals(dir + " should have two entries", 2, stats.length);
+ assertEquals(file2.toString(), stats[0].getPath().toString());
+ assertEquals(file3.toString(), stats[1].getPath().toString());
+
+ itor = fc.listStatus(dir);
+ assertEquals(file2.toString(), itor.next().getPath().toString());
+ assertEquals(file3.toString(), itor.next().getPath().toString());
+ assertFalse("Unexpected addtional file", itor.hasNext());
+
+ // Test iterative listing. Now dir has 2 entries, create one more.
+ Path dir3 = fs.makeQualified(new Path(dir, "dir3"));
+ fs.mkdirs(dir3);
+ dir3 = fs.makeQualified(dir3);
+ stats = fs.listStatus(dir);
+ assertEquals(dir + " should have three entries", 3, stats.length);
+ assertEquals(dir3.toString(), stats[0].getPath().toString());
+ assertEquals(file2.toString(), stats[1].getPath().toString());
+ assertEquals(file3.toString(), stats[2].getPath().toString());
+
+ itor = fc.listStatus(dir);
+ assertEquals(dir3.toString(), itor.next().getPath().toString());
+ assertEquals(file2.toString(), itor.next().getPath().toString());
+ assertEquals(file3.toString(), itor.next().getPath().toString());
+ assertFalse("Unexpected addtional file", itor.hasNext());
+
+ // Now dir has 3 entries, create two more
+ Path dir4 = fs.makeQualified(new Path(dir, "dir4"));
+ fs.mkdirs(dir4);
+ dir4 = fs.makeQualified(dir4);
+ Path dir5 = fs.makeQualified(new Path(dir, "dir5"));
+ fs.mkdirs(dir5);
+ dir5 = fs.makeQualified(dir5);
+ stats = fs.listStatus(dir);
+ assertEquals(dir + " should have five entries", 5, stats.length);
+ assertEquals(dir3.toString(), stats[0].getPath().toString());
+ assertEquals(dir4.toString(), stats[1].getPath().toString());
+ assertEquals(dir5.toString(), stats[2].getPath().toString());
+ assertEquals(file2.toString(), stats[3].getPath().toString());
+ assertEquals(file3.toString(), stats[4].getPath().toString());
+
+ itor = fc.listStatus(dir);
+ assertEquals(dir3.toString(), itor.next().getPath().toString());
+ assertEquals(dir4.toString(), itor.next().getPath().toString());
+ assertEquals(dir5.toString(), itor.next().getPath().toString());
+ assertEquals(file2.toString(), itor.next().getPath().toString());
+ assertEquals(file3.toString(), itor.next().getPath().toString());
+ assertFalse(itor.hasNext());
}
}
