Repository: hadoop Updated Branches: refs/heads/branch-2 14bacd2b9 -> a868cf404
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a868cf40/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 88f0c95..6068978 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -66,10 +66,6 @@ import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; -import org.junit.rules.Timeout; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Rule; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; @@ -99,37 +95,6 @@ public class TestDFSShell { private static final byte[] RAW_A1_VALUE = new byte[]{0x32, 0x32, 0x32}; private static final byte[] TRUSTED_A1_VALUE = new byte[]{0x31, 0x31, 0x31}; private static final byte[] USER_A1_VALUE = new byte[]{0x31, 0x32, 0x33}; - private static final int BLOCK_SIZE = 1024; - - private static MiniDFSCluster miniCluster; - private static DistributedFileSystem dfs; - - @BeforeClass - public static void setup() throws IOException { - final Configuration conf = new Configuration(); - conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true); - conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); - // set up the shared miniCluster directory so individual tests can launch - // new clusters without conflict - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, - GenericTestUtils.getTestDir("TestDFSShell").getAbsolutePath()); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); - - miniCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - miniCluster.waitActive(); - dfs = miniCluster.getFileSystem(); - } - - @AfterClass - public static void tearDown() { - if (miniCluster != null) { - miniCluster.shutdown(true, true); - } - } - - @Rule - public Timeout globalTimeout= new Timeout(30 * 1000); // 30s static Path writeFile(FileSystem fs, Path f) throws IOException { DataOutputStream out = fs.create(f); @@ -181,74 +146,102 @@ public class TestDFSShell { @Test (timeout = 30000) public void testZeroSizeFile() throws IOException { - //create a zero size file - final File f1 = new File(TEST_ROOT_DIR, "f1"); - assertTrue(!f1.exists()); - assertTrue(f1.createNewFile()); - assertTrue(f1.exists()); - assertTrue(f1.isFile()); - assertEquals(0L, f1.length()); - - //copy to remote - final Path root = mkdir(dfs, new Path("/test/zeroSizeFile")); - final Path remotef = new Path(root, "dst"); - show("copy local " + f1 + " to remote " + remotef); - dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), remotef); - - //getBlockSize() should not throw exception - show("Block size = " + dfs.getFileStatus(remotef).getBlockSize()); - - //copy back - final File f2 = new File(TEST_ROOT_DIR, "f2"); - assertTrue(!f2.exists()); - dfs.copyToLocalFile(remotef, new Path(f2.getPath())); - assertTrue(f2.exists()); - assertTrue(f2.isFile()); - assertEquals(0L, f2.length()); - - f1.delete(); - f2.delete(); + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + FileSystem fs = cluster.getFileSystem(); + assertTrue("Not a HDFS: "+fs.getUri(), + fs instanceof DistributedFileSystem); + final DistributedFileSystem dfs = (DistributedFileSystem)fs; + + try { + //create a zero size file + final File f1 = new File(TEST_ROOT_DIR, "f1"); + assertTrue(!f1.exists()); + assertTrue(f1.createNewFile()); + assertTrue(f1.exists()); + assertTrue(f1.isFile()); + assertEquals(0L, f1.length()); + + //copy to remote + final Path root = mkdir(dfs, new Path("/test/zeroSizeFile")); + final Path remotef = new Path(root, "dst"); + show("copy local " + f1 + " to remote " + remotef); + dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), remotef); + + //getBlockSize() should not throw exception + show("Block size = " + dfs.getFileStatus(remotef).getBlockSize()); + + //copy back + final File f2 = new File(TEST_ROOT_DIR, "f2"); + assertTrue(!f2.exists()); + dfs.copyToLocalFile(remotef, new Path(f2.getPath())); + assertTrue(f2.exists()); + assertTrue(f2.isFile()); + assertEquals(0L, f2.length()); + + f1.delete(); + f2.delete(); + } finally { + try {dfs.close();} catch (Exception e) {} + cluster.shutdown(); + } } @Test (timeout = 30000) public void testRecursiveRm() throws IOException { - final Path parent = new Path("/testRecursiveRm", "parent"); - final Path child = new Path(parent, "child"); - dfs.mkdirs(child); - dfs.mkdirs(new Path(new Path("parent"), "child")); - try { - dfs.delete(parent, false); - fail("Should have failed because dir is not empty"); - } catch(IOException e) { - //should have thrown an exception + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + FileSystem fs = cluster.getFileSystem(); + assertTrue("Not a HDFS: " + fs.getUri(), + fs instanceof DistributedFileSystem); + try { + fs.mkdirs(new Path(new Path("parent"), "child")); + try { + fs.delete(new Path("parent"), false); + assert(false); // should never reach here. + } catch(IOException e) { + //should have thrown an exception + } + try { + fs.delete(new Path("parent"), true); + } catch(IOException e) { + assert(false); + } + } finally { + try { fs.close();}catch(IOException e){}; + cluster.shutdown(); } - dfs.delete(parent, true); - assertFalse(dfs.exists(parent)); } @Test (timeout = 30000) public void testDu() throws IOException { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + DistributedFileSystem fs = cluster.getFileSystem(); PrintStream psBackup = System.out; ByteArrayOutputStream out = new ByteArrayOutputStream(); PrintStream psOut = new PrintStream(out); System.setOut(psOut); - FsShell shell = new FsShell(dfs.getConf()); + FsShell shell = new FsShell(); + shell.setConf(conf); + try { - Path myPath = new Path("/testDu/dir"); - assertTrue(dfs.mkdirs(myPath)); - assertTrue(dfs.exists(myPath)); - Path myFile = new Path(myPath, "file"); - writeFile(dfs, myFile); - assertTrue(dfs.exists(myFile)); - Path myFile2 = new Path(myPath, "file2"); - writeFile(dfs, myFile2); - assertTrue(dfs.exists(myFile2)); - Long myFileLength = dfs.getFileStatus(myFile).getLen(); - Long myFile2Length = dfs.getFileStatus(myFile2).getLen(); - + cluster.waitActive(); + Path myPath = new Path("/test/dir"); + assertTrue(fs.mkdirs(myPath)); + assertTrue(fs.exists(myPath)); + Path myFile = new Path("/test/dir/file"); + writeFile(fs, myFile); + assertTrue(fs.exists(myFile)); + Path myFile2 = new Path("/test/dir/file2"); + writeFile(fs, myFile2); + assertTrue(fs.exists(myFile2)); + Long myFileLength = fs.getFileStatus(myFile).getLen(); + Long myFile2Length = fs.getFileStatus(myFile2).getLen(); + String[] args = new String[2]; args[0] = "-du"; - args[1] = myPath.toString(); + args[1] = "/test/dir"; int val = -1; try { val = shell.run(args); @@ -266,10 +259,10 @@ public class TestDFSShell { // Check that -du -s reports the state of the snapshot String snapshotName = "ss1"; Path snapshotPath = new Path(myPath, ".snapshot/" + snapshotName); - dfs.allowSnapshot(myPath); - assertThat(dfs.createSnapshot(myPath, snapshotName), is(snapshotPath)); - assertThat(dfs.delete(myFile, false), is(true)); - assertThat(dfs.exists(myFile), is(false)); + fs.allowSnapshot(myPath); + assertThat(fs.createSnapshot(myPath, snapshotName), is(snapshotPath)); + assertThat(fs.delete(myFile, false), is(true)); + assertThat(fs.exists(myFile), is(false)); args = new String[3]; args[0] = "-du"; @@ -289,6 +282,7 @@ public class TestDFSShell { assertThat(returnString, containsString(combinedLength.toString())); } finally { System.setOut(psBackup); + cluster.shutdown(); } } @@ -296,14 +290,20 @@ public class TestDFSShell { @Test (timeout = 180000) public void testDuSnapshots() throws IOException { final int replication = 2; + final Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(replication).build(); + final DistributedFileSystem dfs = cluster.getFileSystem(); final PrintStream psBackup = System.out; final ByteArrayOutputStream out = new ByteArrayOutputStream(); final PrintStream psOut = new PrintStream(out); - final FsShell shell = new FsShell(dfs.getConf()); + final FsShell shell = new FsShell(); + shell.setConf(conf); try { System.setOut(psOut); - final Path parent = new Path("/testDuSnapshots"); + cluster.waitActive(); + final Path parent = new Path("/test"); final Path dir = new Path(parent, "dir"); mkdir(dfs, dir); final Path file = new Path(dir, "file"); @@ -406,19 +406,27 @@ public class TestDFSShell { out.reset(); } finally { System.setOut(psBackup); + cluster.shutdown(); } } @Test (timeout = 180000) public void testCountSnapshots() throws IOException { + final int replication = 2; + final Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(replication).build(); + final DistributedFileSystem dfs = cluster.getFileSystem(); final PrintStream psBackup = System.out; final ByteArrayOutputStream out = new ByteArrayOutputStream(); final PrintStream psOut = new PrintStream(out); System.setOut(psOut); - final FsShell shell = new FsShell(dfs.getConf()); + final FsShell shell = new FsShell(); + shell.setConf(conf); try { - final Path parent = new Path("/testCountSnapshots"); + cluster.waitActive(); + final Path parent = new Path("/test"); final Path dir = new Path(parent, "dir"); mkdir(dfs, dir); final Path file = new Path(dir, "file"); @@ -490,100 +498,117 @@ public class TestDFSShell { out.reset(); } finally { System.setOut(psBackup); + cluster.shutdown(); } } @Test (timeout = 30000) public void testPut() throws IOException { - // remove left over crc files: - new File(TEST_ROOT_DIR, ".f1.crc").delete(); - new File(TEST_ROOT_DIR, ".f2.crc").delete(); - final File f1 = createLocalFile(new File(TEST_ROOT_DIR, "f1")); - final File f2 = createLocalFile(new File(TEST_ROOT_DIR, "f2")); - - final Path root = mkdir(dfs, new Path("/testPut")); - final Path dst = new Path(root, "dst"); - - show("begin"); + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + FileSystem fs = cluster.getFileSystem(); + assertTrue("Not a HDFS: "+fs.getUri(), + fs instanceof DistributedFileSystem); + final DistributedFileSystem dfs = (DistributedFileSystem)fs; - final Thread copy2ndFileThread = new Thread() { - @Override - public void run() { - try { - show("copy local " + f2 + " to remote " + dst); - dfs.copyFromLocalFile(false, false, new Path(f2.getPath()), dst); - } catch (IOException ioe) { - show("good " + StringUtils.stringifyException(ioe)); - return; + try { + // remove left over crc files: + new File(TEST_ROOT_DIR, ".f1.crc").delete(); + new File(TEST_ROOT_DIR, ".f2.crc").delete(); + final File f1 = createLocalFile(new File(TEST_ROOT_DIR, "f1")); + final File f2 = createLocalFile(new File(TEST_ROOT_DIR, "f2")); + + final Path root = mkdir(dfs, new Path("/test/put")); + final Path dst = new Path(root, "dst"); + + show("begin"); + + final Thread copy2ndFileThread = new Thread() { + @Override + public void run() { + try { + show("copy local " + f2 + " to remote " + dst); + dfs.copyFromLocalFile(false, false, new Path(f2.getPath()), dst); + } catch (IOException ioe) { + show("good " + StringUtils.stringifyException(ioe)); + return; + } + //should not be here, must got IOException + assertTrue(false); } - //should not be here, must got IOException - assertTrue(false); - } - }; - - //use SecurityManager to pause the copying of f1 and begin copying f2 - SecurityManager sm = System.getSecurityManager(); - System.out.println("SecurityManager = " + sm); - System.setSecurityManager(new SecurityManager() { - private boolean firstTime = true; - - @Override - public void checkPermission(Permission perm) { - if (firstTime) { - Thread t = Thread.currentThread(); - if (!t.toString().contains("DataNode")) { - String s = "" + Arrays.asList(t.getStackTrace()); - if (s.contains("FileUtil.copyContent")) { - //pause at FileUtil.copyContent - - firstTime = false; - copy2ndFileThread.start(); - try {Thread.sleep(5000);} catch (InterruptedException e) {} + }; + + //use SecurityManager to pause the copying of f1 and begin copying f2 + SecurityManager sm = System.getSecurityManager(); + System.out.println("SecurityManager = " + sm); + System.setSecurityManager(new SecurityManager() { + private boolean firstTime = true; + + @Override + public void checkPermission(Permission perm) { + if (firstTime) { + Thread t = Thread.currentThread(); + if (!t.toString().contains("DataNode")) { + String s = "" + Arrays.asList(t.getStackTrace()); + if (s.contains("FileUtil.copyContent")) { + //pause at FileUtil.copyContent + + firstTime = false; + copy2ndFileThread.start(); + try {Thread.sleep(5000);} catch (InterruptedException e) {} + } } } } - } - }); - show("copy local " + f1 + " to remote " + dst); - dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), dst); - show("done"); - - try {copy2ndFileThread.join();} catch (InterruptedException e) { } - System.setSecurityManager(sm); - - // copy multiple files to destination directory - final Path destmultiple = mkdir(dfs, new Path(root, "putmultiple")); - Path[] srcs = new Path[2]; - srcs[0] = new Path(f1.getPath()); - srcs[1] = new Path(f2.getPath()); - dfs.copyFromLocalFile(false, false, srcs, destmultiple); - srcs[0] = new Path(destmultiple,"f1"); - srcs[1] = new Path(destmultiple,"f2"); - assertTrue(dfs.exists(srcs[0])); - assertTrue(dfs.exists(srcs[1])); - - // move multiple files to destination directory - final Path destmultiple2 = mkdir(dfs, new Path(root, "movemultiple")); - srcs[0] = new Path(f1.getPath()); - srcs[1] = new Path(f2.getPath()); - dfs.moveFromLocalFile(srcs, destmultiple2); - assertFalse(f1.exists()); - assertFalse(f2.exists()); - srcs[0] = new Path(destmultiple2, "f1"); - srcs[1] = new Path(destmultiple2, "f2"); - assertTrue(dfs.exists(srcs[0])); - assertTrue(dfs.exists(srcs[1])); - - f1.delete(); - f2.delete(); + }); + show("copy local " + f1 + " to remote " + dst); + dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), dst); + show("done"); + + try {copy2ndFileThread.join();} catch (InterruptedException e) { } + System.setSecurityManager(sm); + + // copy multiple files to destination directory + final Path destmultiple = mkdir(dfs, new Path("/test/putmultiple")); + Path[] srcs = new Path[2]; + srcs[0] = new Path(f1.getPath()); + srcs[1] = new Path(f2.getPath()); + dfs.copyFromLocalFile(false, false, srcs, destmultiple); + srcs[0] = new Path(destmultiple,"f1"); + srcs[1] = new Path(destmultiple,"f2"); + assertTrue(dfs.exists(srcs[0])); + assertTrue(dfs.exists(srcs[1])); + + // move multiple files to destination directory + final Path destmultiple2 = mkdir(dfs, new Path("/test/movemultiple")); + srcs[0] = new Path(f1.getPath()); + srcs[1] = new Path(f2.getPath()); + dfs.moveFromLocalFile(srcs, destmultiple2); + assertFalse(f1.exists()); + assertFalse(f2.exists()); + srcs[0] = new Path(destmultiple2, "f1"); + srcs[1] = new Path(destmultiple2, "f2"); + assertTrue(dfs.exists(srcs[0])); + assertTrue(dfs.exists(srcs[1])); + + f1.delete(); + f2.delete(); + } finally { + try {dfs.close();} catch (Exception e) {} + cluster.shutdown(); + } } /** check command error outputs and exit statuses. */ @Test (timeout = 30000) public void testErrOutPut() throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; PrintStream bak = null; try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + FileSystem srcFs = cluster.getFileSystem(); Path root = new Path("/nonexistentfile"); bak = System.err; ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -600,7 +625,8 @@ public class TestDFSShell { out.reset(); argv[0] = "-rm"; argv[1] = root.toString(); - FsShell shell = new FsShell(dfs.getConf()); + FsShell shell = new FsShell(); + shell.setConf(conf); ret = ToolRunner.run(shell, argv); assertEquals(" -rm returned 1 ", 1, ret); returned = out.toString(); @@ -641,7 +667,7 @@ public class TestDFSShell { ret = ToolRunner.run(shell, argv); assertEquals(" -lsr should fail ", 1, ret); out.reset(); - dfs.mkdirs(new Path("/testdir")); + srcFs.mkdirs(new Path("/testdir")); argv[0] = "-ls"; argv[1] = "/testdir"; ret = ToolRunner.run(shell, argv); @@ -662,7 +688,7 @@ public class TestDFSShell { assertTrue(" -mkdir returned File exists", (returned.lastIndexOf("File exists") != -1)); Path testFile = new Path("/testfile"); - OutputStream outtmp = dfs.create(testFile); + OutputStream outtmp = srcFs.create(testFile); outtmp.write(testFile.toString().getBytes()); outtmp.close(); out.reset(); @@ -677,7 +703,7 @@ public class TestDFSShell { argv = new String[3]; argv[0] = "-mv"; argv[1] = "/testfile"; - argv[2] = "/no-such-dir/file"; + argv[2] = "file"; ret = ToolRunner.run(shell, argv); assertEquals("mv failed to rename", 1, ret); out.reset(); @@ -700,7 +726,7 @@ public class TestDFSShell { out.reset(); argv = new String[1]; argv[0] = "-du"; - dfs.mkdirs(dfs.getHomeDirectory()); + srcFs.mkdirs(srcFs.getHomeDirectory()); ret = ToolRunner.run(shell, argv); returned = out.toString(); assertEquals(" no error ", 0, ret); @@ -719,6 +745,9 @@ public class TestDFSShell { if (bak != null) { System.setErr(bak); } + if (cluster != null) { + cluster.shutdown(); + } } } @@ -760,7 +789,7 @@ public class TestDFSShell { Configuration dstConf = new HdfsConfiguration(); MiniDFSCluster srcCluster = null; MiniDFSCluster dstCluster = null; - File bak = new File(PathUtils.getTestDir(getClass()), "testURIPaths"); + File bak = new File(PathUtils.getTestDir(getClass()), "dfs_tmp_uri"); bak.mkdirs(); try{ srcCluster = new MiniDFSCluster.Builder(srcConf).numDataNodes(2).build(); @@ -817,7 +846,7 @@ public class TestDFSShell { Path path = new Path(file); Path parent = new Path("/tmp"); Path root = new Path("/"); - writeFile(dstFs, path); + TestDFSShell.writeFile(dstFs, path); runCmd(shell, "-chgrp", "-R", "herbivores", dstFs.getUri().toString() +"/*"); confirmOwner(null, "herbivores", dstFs, parent, path); runCmd(shell, "-chown", "-R", ":reptiles", dstFs.getUri().toString() + "/"); @@ -850,25 +879,35 @@ public class TestDFSShell { */ @Test (timeout = 30000) public void testTail() throws Exception { - final int fileLen = 5 * BLOCK_SIZE; - // create a text file with multiple KB bytes (and multiple blocks) - final Path testFile = new Path("testTail", "file1"); - final String text = RandomStringUtils.randomAscii(fileLen); - try (OutputStream pout = dfs.create(testFile)) { - pout.write(text.getBytes()); + final int blockSize = 1024; + final int fileLen = 5 * blockSize; + final Configuration conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + + try (MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + + // create a text file with multiple KB bytes (and multiple blocks) + final Path testFile = new Path("testTail", "file1"); + final String text = RandomStringUtils.randomAscii(fileLen); + try (OutputStream pout = dfs.create(testFile)) { + pout.write(text.getBytes()); + } + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + final String[] argv = new String[]{"-tail", testFile.toString()}; + final int ret = ToolRunner.run(new FsShell(conf), argv); + + assertEquals(Arrays.toString(argv) + " returned " + ret, 0, ret); + assertEquals("-tail returned " + out.size() + " bytes data, expected 1KB", + 1024, out.size()); + // tailed out last 1KB of the file content + assertArrayEquals("Tail output doesn't match input", + text.substring(fileLen - 1024).getBytes(), out.toByteArray()); + out.reset(); } - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); - final String[] argv = new String[]{"-tail", testFile.toString()}; - final int ret = ToolRunner.run(new FsShell(dfs.getConf()), argv); - - assertEquals(Arrays.toString(argv) + " returned " + ret, 0, ret); - assertEquals("-tail returned " + out.size() + " bytes data, expected 1KB", - 1024, out.size()); - // tailed out last 1KB of the file content - assertArrayEquals("Tail output doesn't match input", - text.substring(fileLen - 1024).getBytes(), out.toByteArray()); - out.reset(); } /** @@ -876,56 +915,75 @@ public class TestDFSShell { */ @Test(timeout = 30000) public void testTailWithFresh() throws Exception { - final Path testFile = new Path("testTailWithFresh", "file1"); - dfs.create(testFile); + final int blockSize = 1024; + final Configuration conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); - final Thread tailer = new Thread() { - @Override - public void run() { - final String[] argv = new String[]{"-tail", "-f", - testFile.toString()}; - try { - ToolRunner.run(new FsShell(dfs.getConf()), argv); - } catch (Exception e) { - LOG.error("Client that tails the test file fails", e); - } finally { - out.reset(); + try (MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final Path testFile = new Path("testTailWithFresh", "file1"); + dfs.create(testFile); + + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + final Thread tailer = new Thread() { + @Override + public void run() { + final String[] argv = new String[]{"-tail", "-f", + testFile.toString()}; + try { + ToolRunner.run(new FsShell(conf), argv); + } catch (Exception e) { + LOG.error("Client that tails the test file fails", e); + } finally { + out.reset(); + } } + }; + tailer.start(); + // wait till the tailer is sleeping + GenericTestUtils.waitFor(new Supplier<Boolean>() { + @Override + public Boolean get() { + return tailer.getState() == Thread.State.TIMED_WAITING; + } + }, 100, 10000); + + final String text = RandomStringUtils.randomAscii(blockSize / 2); + try (OutputStream pout = dfs.create(testFile)) { + pout.write(text.getBytes()); } - }; - tailer.start(); - // wait till the tailer is sleeping - GenericTestUtils.waitFor(new Supplier<Boolean>() { - @Override - public Boolean get() { - return tailer.getState() == Thread.State.TIMED_WAITING; - } - }, 100, 10000); - - final String text = RandomStringUtils.randomAscii(BLOCK_SIZE / 2); - try (OutputStream pout = dfs.create(testFile)) { - pout.write(text.getBytes()); + // The tailer should eventually show the file contents + GenericTestUtils.waitFor(new Supplier<Boolean>() { + @Override + public Boolean get() { + return Arrays.equals(text.getBytes(), out.toByteArray()); + } + }, 100, 10000); } - // The tailer should eventually show the file contents - GenericTestUtils.waitFor(new Supplier<Boolean>() { - @Override - public Boolean get() { - return Arrays.equals(text.getBytes(), out.toByteArray()); - } - }, 100, 10000); } @Test (timeout = 30000) public void testText() throws Exception { - final Configuration conf = dfs.getConf(); - textTest(new Path("/texttest").makeQualified(dfs.getUri(), - dfs.getWorkingDirectory()), conf); - - final FileSystem lfs = FileSystem.getLocal(conf); - textTest(new Path(TEST_ROOT_DIR, "texttest").makeQualified(lfs.getUri(), - lfs.getWorkingDirectory()), conf); + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + final FileSystem dfs = cluster.getFileSystem(); + textTest(new Path("/texttest").makeQualified(dfs.getUri(), + dfs.getWorkingDirectory()), conf); + + conf.set("fs.defaultFS", dfs.getUri().toString()); + final FileSystem lfs = FileSystem.getLocal(conf); + textTest(new Path(TEST_ROOT_DIR, "texttest").makeQualified(lfs.getUri(), + lfs.getWorkingDirectory()), conf); + } finally { + if (null != cluster) { + cluster.shutdown(); + } + } } private void textTest(Path root, Configuration conf) throws Exception { @@ -1024,60 +1082,75 @@ public class TestDFSShell { @Test (timeout = 30000) public void testCopyToLocal() throws IOException { - FsShell shell = new FsShell(dfs.getConf()); - - String root = createTree(dfs, "copyToLocal"); - - // Verify copying the tree - { - try { - assertEquals(0, - runCmd(shell, "-copyToLocal", root + "*", TEST_ROOT_DIR)); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + FileSystem fs = cluster.getFileSystem(); + assertTrue("Not a HDFS: "+fs.getUri(), + fs instanceof DistributedFileSystem); + DistributedFileSystem dfs = (DistributedFileSystem)fs; + FsShell shell = new FsShell(); + shell.setConf(conf); - File localroot = new File(TEST_ROOT_DIR, "copyToLocal"); - File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2"); + try { + String root = createTree(dfs, "copyToLocal"); - File f1 = new File(localroot, "f1"); - assertTrue("Copying failed.", f1.isFile()); + // Verify copying the tree + { + try { + assertEquals(0, + runCmd(shell, "-copyToLocal", root + "*", TEST_ROOT_DIR)); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } - File f2 = new File(localroot, "f2"); - assertTrue("Copying failed.", f2.isFile()); + File localroot = new File(TEST_ROOT_DIR, "copyToLocal"); + File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2"); + + File f1 = new File(localroot, "f1"); + assertTrue("Copying failed.", f1.isFile()); - File sub = new File(localroot, "sub"); - assertTrue("Copying failed.", sub.isDirectory()); + File f2 = new File(localroot, "f2"); + assertTrue("Copying failed.", f2.isFile()); - File f3 = new File(sub, "f3"); - assertTrue("Copying failed.", f3.isFile()); + File sub = new File(localroot, "sub"); + assertTrue("Copying failed.", sub.isDirectory()); - File f4 = new File(sub, "f4"); - assertTrue("Copying failed.", f4.isFile()); + File f3 = new File(sub, "f3"); + assertTrue("Copying failed.", f3.isFile()); - File f5 = new File(localroot2, "f1"); - assertTrue("Copying failed.", f5.isFile()); + File f4 = new File(sub, "f4"); + assertTrue("Copying failed.", f4.isFile()); + + File f5 = new File(localroot2, "f1"); + assertTrue("Copying failed.", f5.isFile()); - f1.delete(); - f2.delete(); - f3.delete(); - f4.delete(); - f5.delete(); - sub.delete(); - } - // Verify copying non existing sources do not create zero byte - // destination files - { - String[] args = {"-copyToLocal", "nosuchfile", TEST_ROOT_DIR}; + f1.delete(); + f2.delete(); + f3.delete(); + f4.delete(); + f5.delete(); + sub.delete(); + } + // Verify copying non existing sources do not create zero byte + // destination files + { + String[] args = {"-copyToLocal", "nosuchfile", TEST_ROOT_DIR}; + try { + assertEquals(1, shell.run(args)); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + File f6 = new File(TEST_ROOT_DIR, "nosuchfile"); + assertTrue(!f6.exists()); + } + } finally { try { - assertEquals(1, shell.run(args)); + dfs.close(); } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); } - File f6 = new File(TEST_ROOT_DIR, "nosuchfile"); - assertTrue(!f6.exists()); + cluster.shutdown(); } } @@ -1107,42 +1180,63 @@ public class TestDFSShell { @Test (timeout = 30000) public void testCount() throws Exception { - FsShell shell = new FsShell(dfs.getConf()); - - String root = createTree(dfs, "count"); - - // Verify the counts - runCount(root, 2, 4, shell); - runCount(root + "2", 2, 1, shell); - runCount(root + "2/f1", 0, 1, shell); - runCount(root + "2/sub", 1, 0, shell); - - final FileSystem localfs = FileSystem.getLocal(dfs.getConf()); - Path localpath = new Path(TEST_ROOT_DIR, "testcount"); - localpath = localpath.makeQualified(localfs.getUri(), - localfs.getWorkingDirectory()); - localfs.mkdirs(localpath); + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + DistributedFileSystem dfs = cluster.getFileSystem(); + FsShell shell = new FsShell(); + shell.setConf(conf); - final String localstr = localpath.toString(); - System.out.println("localstr=" + localstr); - runCount(localstr, 1, 0, shell); - assertEquals(0, runCmd(shell, "-count", root, localstr)); + try { + String root = createTree(dfs, "count"); + + // Verify the counts + runCount(root, 2, 4, shell); + runCount(root + "2", 2, 1, shell); + runCount(root + "2/f1", 0, 1, shell); + runCount(root + "2/sub", 1, 0, shell); + + final FileSystem localfs = FileSystem.getLocal(conf); + Path localpath = new Path(TEST_ROOT_DIR, "testcount"); + localpath = localpath.makeQualified(localfs.getUri(), + localfs.getWorkingDirectory()); + localfs.mkdirs(localpath); + + final String localstr = localpath.toString(); + System.out.println("localstr=" + localstr); + runCount(localstr, 1, 0, shell); + assertEquals(0, runCmd(shell, "-count", root, localstr)); + } finally { + try { + dfs.close(); + } catch (Exception e) { + } + cluster.shutdown(); + } } @Test(timeout = 30000) public void testTotalSizeOfAllFiles() throws Exception { - final Path root = new Path("/testTotalSizeOfAllFiles"); - dfs.mkdirs(root); - // create file under root - FSDataOutputStream File1 = dfs.create(new Path(root, "File1")); - File1.write("hi".getBytes()); - File1.close(); - // create file under sub-folder - FSDataOutputStream File2 = dfs.create(new Path(root, "Folder1/File2")); - File2.write("hi".getBytes()); - File2.close(); - // getUsed() should return total length of all the files in Filesystem - assertEquals(4, dfs.getUsed(root)); + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + FileSystem fs = cluster.getFileSystem(); + // create file under root + FSDataOutputStream File1 = fs.create(new Path("/File1")); + File1.write("hi".getBytes()); + File1.close(); + // create file under sub-folder + FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2")); + File2.write("hi".getBytes()); + File2.close(); + // getUsed() should return total length of all the files in Filesystem + assertEquals(4, fs.getUsed()); + } finally { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } } private static void runCount(String path, long dirs, long files, FsShell shell @@ -1208,7 +1302,7 @@ public class TestDFSShell { //create an empty file Path file = new Path(chmodDir, "file"); - writeFile(fs, file); + TestDFSShell.writeFile(fs, file); //test octal mode confirmPermissionChange("644", "rw-r--r--", fs, shell, file); @@ -1253,6 +1347,7 @@ public class TestDFSShell { } finally { try { + fs.close(); shell.close(); } catch (IOException ignored) {} } @@ -1295,14 +1390,16 @@ public class TestDFSShell { conf.set(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, "true"); //test chmod on DFS - fs = dfs; - conf = dfs.getConf(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + fs = cluster.getFileSystem(); testChmod(conf, fs, "/tmp/chmodTest"); // test chown and chgrp on DFS: FsShell shell = new FsShell(); shell.setConf(conf); + fs = cluster.getFileSystem(); + /* For dfs, I am the super user and I can change owner of any file to * anything. "-R" option is already tested by chmod test above. */ @@ -1311,7 +1408,7 @@ public class TestDFSShell { Path path = new Path(file); Path parent = new Path("/tmp"); Path root = new Path("/"); - writeFile(fs, path); + TestDFSShell.writeFile(fs, path); runCmd(shell, "-chgrp", "-R", "herbivores", "/*", "unknownFile*"); confirmOwner(null, "herbivores", fs, parent, path); @@ -1341,423 +1438,439 @@ public class TestDFSShell { runCmd(shell, "-chgrp", "hadoop-c...@apache.org/100", file); confirmOwner(null, "hadoop-c...@apache.org/100", fs, path); + + cluster.shutdown(); } - /** * Tests various options of DFSShell. */ @Test (timeout = 120000) public void testDFSShell() throws Exception { + final Configuration conf = new HdfsConfiguration(); /* This tests some properties of ChecksumFileSystem as well. * Make sure that we create ChecksumDFS */ - FsShell shell = new FsShell(dfs.getConf()); - - // First create a new directory with mkdirs - Path myPath = new Path("/test/mkdirs"); - assertTrue(dfs.mkdirs(myPath)); - assertTrue(dfs.exists(myPath)); - assertTrue(dfs.mkdirs(myPath)); - - // Second, create a file in that directory. - Path myFile = new Path("/test/mkdirs/myFile"); - writeFile(dfs, myFile); - assertTrue(dfs.exists(myFile)); - Path myFile2 = new Path("/test/mkdirs/myFile2"); - writeFile(dfs, myFile2); - assertTrue(dfs.exists(myFile2)); - - // Verify that rm with a pattern - { - String[] args = new String[2]; - args[0] = "-rm"; - args[1] = "/test/mkdirs/myFile*"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertTrue(val == 0); - assertFalse(dfs.exists(myFile)); - assertFalse(dfs.exists(myFile2)); - - //re-create the files for other tests - writeFile(dfs, myFile); - assertTrue(dfs.exists(myFile)); - writeFile(dfs, myFile2); - assertTrue(dfs.exists(myFile2)); - } + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + FileSystem fs = cluster.getFileSystem(); + assertTrue("Not a HDFS: "+fs.getUri(), + fs instanceof DistributedFileSystem); + DistributedFileSystem fileSys = (DistributedFileSystem)fs; + FsShell shell = new FsShell(); + shell.setConf(conf); - // Verify that we can read the file - { - String[] args = new String[3]; - args[0] = "-cat"; - args[1] = "/test/mkdirs/myFile"; - args[2] = "/test/mkdirs/myFile2"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run: " + - StringUtils.stringifyException(e)); - } - assertTrue(val == 0); - } - dfs.delete(myFile2, true); + try { + // First create a new directory with mkdirs + Path myPath = new Path("/test/mkdirs"); + assertTrue(fileSys.mkdirs(myPath)); + assertTrue(fileSys.exists(myPath)); + assertTrue(fileSys.mkdirs(myPath)); + + // Second, create a file in that directory. + Path myFile = new Path("/test/mkdirs/myFile"); + writeFile(fileSys, myFile); + assertTrue(fileSys.exists(myFile)); + Path myFile2 = new Path("/test/mkdirs/myFile2"); + writeFile(fileSys, myFile2); + assertTrue(fileSys.exists(myFile2)); + + // Verify that rm with a pattern + { + String[] args = new String[2]; + args[0] = "-rm"; + args[1] = "/test/mkdirs/myFile*"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertTrue(val == 0); + assertFalse(fileSys.exists(myFile)); + assertFalse(fileSys.exists(myFile2)); - // Verify that we get an error while trying to read an nonexistent file - { - String[] args = new String[2]; - args[0] = "-cat"; - args[1] = "/test/mkdirs/myFile1"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + //re-create the files for other tests + writeFile(fileSys, myFile); + assertTrue(fileSys.exists(myFile)); + writeFile(fileSys, myFile2); + assertTrue(fileSys.exists(myFile2)); } - assertTrue(val != 0); - } - // Verify that we get an error while trying to delete an nonexistent file - { - String[] args = new String[2]; - args[0] = "-rm"; - args[1] = "/test/mkdirs/myFile1"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify that we can read the file + { + String[] args = new String[3]; + args[0] = "-cat"; + args[1] = "/test/mkdirs/myFile"; + args[2] = "/test/mkdirs/myFile2"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run: " + + StringUtils.stringifyException(e)); + } + assertTrue(val == 0); } - assertTrue(val != 0); - } + fileSys.delete(myFile2, true); - // Verify that we succeed in removing the file we created - { - String[] args = new String[2]; - args[0] = "-rm"; - args[1] = "/test/mkdirs/myFile"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify that we get an error while trying to read an nonexistent file + { + String[] args = new String[2]; + args[0] = "-cat"; + args[1] = "/test/mkdirs/myFile1"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertTrue(val != 0); } - assertTrue(val == 0); - } - // Verify touch/test - { - String[] args; - int val; - - args = new String[3]; - args[0] = "-test"; - args[1] = "-e"; - args[2] = "/test/mkdirs/noFileHere"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify that we get an error while trying to delete an nonexistent file + { + String[] args = new String[2]; + args[0] = "-rm"; + args[1] = "/test/mkdirs/myFile1"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertTrue(val != 0); } - assertEquals(1, val); - args[1] = "-z"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify that we succeed in removing the file we created + { + String[] args = new String[2]; + args[0] = "-rm"; + args[1] = "/test/mkdirs/myFile"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertTrue(val == 0); } - assertEquals(1, val); - args = new String[2]; - args[0] = "-touchz"; - args[1] = "/test/mkdirs/isFileHere"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + // Verify touch/test + { + String[] args; + int val; + + args = new String[3]; + args[0] = "-test"; + args[1] = "-e"; + args[2] = "/test/mkdirs/noFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); - args = new String[2]; - args[0] = "-touchz"; - args[1] = "/test/mkdirs/thisDirNotExists/isFileHere"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); + args[1] = "-z"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); + args = new String[2]; + args[0] = "-touchz"; + args[1] = "/test/mkdirs/isFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); - args = new String[3]; - args[0] = "-test"; - args[1] = "-e"; - args[2] = "/test/mkdirs/isFileHere"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + args = new String[2]; + args[0] = "-touchz"; + args[1] = "/test/mkdirs/thisDirNotExists/isFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); - args[1] = "-d"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); - args[1] = "-z"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); - } + args = new String[3]; + args[0] = "-test"; + args[1] = "-e"; + args[2] = "/test/mkdirs/isFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); - // Verify that cp from a directory to a subdirectory fails - { - String[] args = new String[2]; - args[0] = "-mkdir"; - args[1] = "/test/dir1"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + args[1] = "-d"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); - // this should fail - String[] args1 = new String[3]; - args1[0] = "-cp"; - args1[1] = "/test/dir1"; - args1[2] = "/test/dir1/dir2"; - val = 0; - try { - val = shell.run(args1); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + args[1] = "-z"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); } - assertEquals(1, val); - // this should succeed - args1[0] = "-cp"; - args1[1] = "/test/dir1"; - args1[2] = "/test/dir1foo"; - val = -1; - try { - val = shell.run(args1); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + // Verify that cp from a directory to a subdirectory fails + { + String[] args = new String[2]; + args[0] = "-mkdir"; + args[1] = "/test/dir1"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + + // this should fail + String[] args1 = new String[3]; + args1[0] = "-cp"; + args1[1] = "/test/dir1"; + args1[2] = "/test/dir1/dir2"; + val = 0; + try { + val = shell.run(args1); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); - // this should fail - args1[0] = "-cp"; - args1[1] = "/"; - args1[2] = "/test"; - val = 0; - try { - val = shell.run(args1); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); - } + // this should succeed + args1[0] = "-cp"; + args1[1] = "/test/dir1"; + args1[2] = "/test/dir1foo"; + val = -1; + try { + val = shell.run(args1); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); - // Verify -test -f negative case (missing file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-f"; - args[2] = "/test/mkdirs/noFileHere"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // this should fail + args1[0] = "-cp"; + args1[1] = "/"; + args1[2] = "/test"; + val = 0; + try { + val = shell.run(args1); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); } - assertEquals(1, val); - } - // Verify -test -f negative case (directory rather than file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-f"; - args[2] = "/test/mkdirs"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify -test -f negative case (missing file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-f"; + args[2] = "/test/mkdirs/noFileHere"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); } - assertEquals(1, val); - } - // Verify -test -f positive case - { - writeFile(dfs, myFile); - assertTrue(dfs.exists(myFile)); - - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-f"; - args[2] = myFile.toString(); - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify -test -f negative case (directory rather than file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-f"; + args[2] = "/test/mkdirs"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); } - assertEquals(0, val); - } - // Verify -test -s negative case (missing file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-s"; - args[2] = "/test/mkdirs/noFileHere"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify -test -f positive case + { + writeFile(fileSys, myFile); + assertTrue(fileSys.exists(myFile)); + + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-f"; + args[2] = myFile.toString(); + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); } - assertEquals(1, val); - } - // Verify -test -s negative case (zero length file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-s"; - args[2] = "/test/mkdirs/isFileHere"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify -test -s negative case (missing file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-s"; + args[2] = "/test/mkdirs/noFileHere"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); } - assertEquals(1, val); - } - // Verify -test -s positive case (nonzero length file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-s"; - args[2] = myFile.toString(); - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify -test -s negative case (zero length file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-s"; + args[2] = "/test/mkdirs/isFileHere"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); } - assertEquals(0, val); - } - // Verify -test -w/-r - { - Path permDir = new Path("/test/permDir"); - Path permFile = new Path("/test/permDir/permFile"); - mkdir(dfs, permDir); - writeFile(dfs, permFile); - - // Verify -test -w positive case (dir exists and can write) - final String[] wargs = new String[3]; - wargs[0] = "-test"; - wargs[1] = "-w"; - wargs[2] = permDir.toString(); - int val = -1; - try { - val = shell.run(wargs); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); + // Verify -test -s positive case (nonzero length file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-s"; + args[2] = myFile.toString(); + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); } - assertEquals(0, val); - // Verify -test -r positive case (file exists and can read) - final String[] rargs = new String[3]; - rargs[0] = "-test"; - rargs[1] = "-r"; - rargs[2] = permFile.toString(); - try { - val = shell.run(rargs); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + // Verify -test -w/-r + { + Path permDir = new Path("/test/permDir"); + Path permFile = new Path("/test/permDir/permFile"); + mkdir(fs, permDir); + writeFile(fs, permFile); + + // Verify -test -w positive case (dir exists and can write) + final String[] wargs = new String[3]; + wargs[0] = "-test"; + wargs[1] = "-w"; + wargs[2] = permDir.toString(); + int val = -1; + try { + val = shell.run(wargs); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + + // Verify -test -r positive case (file exists and can read) + final String[] rargs = new String[3]; + rargs[0] = "-test"; + rargs[1] = "-r"; + rargs[2] = permFile.toString(); + try { + val = shell.run(rargs); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + + // Verify -test -r negative case (file exists but cannot read) + runCmd(shell, "-chmod", "600", permFile.toString()); + + UserGroupInformation smokeUser = + UserGroupInformation.createUserForTesting("smokeUser", + new String[] {"hadoop"}); + smokeUser.doAs(new PrivilegedExceptionAction<String>() { + @Override + public String run() throws Exception { + FsShell shell = new FsShell(conf); + int exitCode = shell.run(rargs); + assertEquals(1, exitCode); + return null; + } + }); - // Verify -test -r negative case (file exists but cannot read) - runCmd(shell, "-chmod", "600", permFile.toString()); + // Verify -test -w negative case (dir exists but cannot write) + runCmd(shell, "-chown", "-R", "not_allowed", permDir.toString()); + runCmd(shell, "-chmod", "-R", "700", permDir.toString()); - UserGroupInformation smokeUser = - UserGroupInformation.createUserForTesting("smokeUser", - new String[] {"hadoop"}); - smokeUser.doAs(new PrivilegedExceptionAction<String>() { + smokeUser.doAs(new PrivilegedExceptionAction<String>() { @Override public String run() throws Exception { - FsShell shell = new FsShell(dfs.getConf()); - int exitCode = shell.run(rargs); + FsShell shell = new FsShell(conf); + int exitCode = shell.run(wargs); assertEquals(1, exitCode); return null; } }); - // Verify -test -w negative case (dir exists but cannot write) - runCmd(shell, "-chown", "-R", "not_allowed", permDir.toString()); - runCmd(shell, "-chmod", "-R", "700", permDir.toString()); - - smokeUser.doAs(new PrivilegedExceptionAction<String>() { - @Override - public String run() throws Exception { - FsShell shell = new FsShell(dfs.getConf()); - int exitCode = shell.run(wargs); - assertEquals(1, exitCode); - return null; - } - }); - - // cleanup - dfs.delete(permDir, true); + // cleanup + fs.delete(permDir, true); + } + } finally { + try { + fileSys.close(); + } catch (Exception e) { + } + cluster.shutdown(); } } @@ -1797,17 +1910,21 @@ public class TestDFSShell { public void testRemoteException() throws Exception { UserGroupInformation tmpUGI = UserGroupInformation.createUserForTesting("tmpname", new String[] {"mygroup"}); + MiniDFSCluster dfs = null; PrintStream bak = null; try { + final Configuration conf = new HdfsConfiguration(); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + FileSystem fs = dfs.getFileSystem(); Path p = new Path("/foo"); - dfs.mkdirs(p); - dfs.setPermission(p, new FsPermission((short)0700)); + fs.mkdirs(p); + fs.setPermission(p, new FsPermission((short)0700)); bak = System.err; tmpUGI.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws Exception { - FsShell fshell = new FsShell(dfs.getConf()); + FsShell fshell = new FsShell(conf); ByteArrayOutputStream out = new ByteArrayOutputStream(); PrintStream tmp = new PrintStream(out); System.setErr(tmp); @@ -1827,6 +1944,9 @@ public class TestDFSShell { if (bak != null) { System.setErr(bak); } + if (dfs != null) { + dfs.shutdown(); + } } } @@ -1884,13 +2004,13 @@ public class TestDFSShell { // find block files to modify later List<MaterializedReplica> replicas = getMaterializedReplicas(cluster); - // Shut down miniCluster and then corrupt the block files by overwriting a - // portion with junk data. We must shut down the miniCluster so that threads + // Shut down cluster and then corrupt the block files by overwriting a + // portion with junk data. We must shut down the cluster so that threads // in the data node do not hold locks on the block files while we try to // write into them. Particularly on Windows, the data node's use of the // FileChannel.transferTo method can cause block files to be memory mapped // in read-only mode during the transfer to a client, and this causes a - // locking conflict. The call to shutdown the miniCluster blocks until all + // locking conflict. The call to shutdown the cluster blocks until all // DataXceiver threads exit, preventing this problem. dfs.close(); cluster.shutdown(); @@ -1898,7 +2018,7 @@ public class TestDFSShell { show("replicas=" + replicas); corrupt(replicas, localfcontent); - // Start the miniCluster again, but do not reformat, so prior files remain. + // Start the cluster again, but do not reformat, so prior files remain. cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(false) .build(); dfs = cluster.getFileSystem(); @@ -1927,45 +2047,55 @@ public class TestDFSShell { */ @Test (timeout = 30000) public void testStat() throws Exception { - final SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - fmt.setTimeZone(TimeZone.getTimeZone("UTC")); - final Path testDir1 = new Path("testStat", "dir1"); - dfs.mkdirs(testDir1); - final Path testFile2 = new Path(testDir1, "file2"); - DFSTestUtil.createFile(dfs, testFile2, 2 * BLOCK_SIZE, (short) 3, 0); - final FileStatus status1 = dfs.getFileStatus(testDir1); - final String mtime1 = fmt.format(new Date(status1.getModificationTime())); - final FileStatus status2 = dfs.getFileStatus(testFile2); - final String mtime2 = fmt.format(new Date(status2.getModificationTime())); + final int blockSize = 1024; + final Configuration conf = new HdfsConfiguration(); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + + try (MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + + final SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + fmt.setTimeZone(TimeZone.getTimeZone("UTC")); + final Path testDir1 = new Path("testStat", "dir1"); + dfs.mkdirs(testDir1); + final Path testFile2 = new Path(testDir1, "file2"); + DFSTestUtil.createFile(dfs, testFile2, 2 * blockSize, (short) 3, 0); + final FileStatus status1 = dfs.getFileStatus(testDir1); + final String mtime1 = fmt.format(new Date(status1.getModificationTime())); + final FileStatus status2 = dfs.getFileStatus(testFile2); + final String mtime2 = fmt.format(new Date(status2.getModificationTime())); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); - doFsStat(dfs.getConf(), null); + doFsStat(conf, null); - out.reset(); - doFsStat(dfs.getConf(), null, testDir1); - assertEquals("Unexpected -stat output: " + out, - out.toString(), String.format("%s%n", mtime1)); + out.reset(); + doFsStat(conf, null, testDir1); + assertEquals("Unexpected -stat output: " + out, + out.toString(), String.format("%s%n", mtime1)); - out.reset(); - doFsStat(dfs.getConf(), null, testDir1, testFile2); - assertEquals("Unexpected -stat output: " + out, - out.toString(), String.format("%s%n%s%n", mtime1, mtime2)); + out.reset(); + doFsStat(conf, null, testDir1, testFile2); + assertEquals("Unexpected -stat output: " + out, + out.toString(), String.format("%s%n%s%n", mtime1, mtime2)); - doFsStat(dfs.getConf(), "%F %u:%g %b %y %n"); + doFsStat(conf, "%F %u:%g %b %y %n"); - out.reset(); - doFsStat(dfs.getConf(), "%F %u:%g %b %y %n", testDir1); - assertTrue(out.toString(), out.toString().contains(mtime1)); - assertTrue(out.toString(), out.toString().contains("directory")); - assertTrue(out.toString(), out.toString().contains(status1.getGroup())); + out.reset(); + doFsStat(conf, "%F %u:%g %b %y %n", testDir1); + assertTrue(out.toString(), out.toString().contains(mtime1)); + assertTrue(out.toString(), out.toString().contains("directory")); + assertTrue(out.toString(), out.toString().contains(status1.getGroup())); - out.reset(); - doFsStat(dfs.getConf(), "%F %u:%g %b %y %n", testDir1, testFile2); - assertTrue(out.toString(), out.toString().contains(mtime1)); - assertTrue(out.toString(), out.toString().contains("regular file")); - assertTrue(out.toString(), out.toString().contains(mtime2)); + out.reset(); + doFsStat(conf, "%F %u:%g %b %y %n", testDir1, testFile2); + assertTrue(out.toString(), out.toString().contains(mtime1)); + assertTrue(out.toString(), out.toString().contains("regular file")); + assertTrue(out.toString(), out.toString().contains(mtime2)); + } } private static void doFsStat(Configuration conf, String format, Path... files) @@ -1992,26 +2122,33 @@ public class TestDFSShell { @Test (timeout = 30000) public void testLsr() throws Exception { - final Configuration conf = dfs.getConf(); - final String root = createTree(dfs, "lsr"); - dfs.mkdirs(new Path(root, "zzz")); - - runLsr(new FsShell(conf), root, 0); - - final Path sub = new Path(root, "sub"); - dfs.setPermission(sub, new FsPermission((short)0)); + final Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + DistributedFileSystem dfs = cluster.getFileSystem(); - final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - final String tmpusername = ugi.getShortUserName() + "1"; - UserGroupInformation tmpUGI = UserGroupInformation.createUserForTesting( - tmpusername, new String[] {tmpusername}); - String results = tmpUGI.doAs(new PrivilegedExceptionAction<String>() { - @Override - public String run() throws Exception { - return runLsr(new FsShell(conf), root, 1); - } - }); - assertTrue(results.contains("zzz")); + try { + final String root = createTree(dfs, "lsr"); + dfs.mkdirs(new Path(root, "zzz")); + + runLsr(new FsShell(conf), root, 0); + + final Path sub = new Path(root, "sub"); + dfs.setPermission(sub, new FsPermission((short)0)); + + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + final String tmpusername = ugi.getShortUserName() + "1"; + UserGroupInformation tmpUGI = UserGroupInformation.createUserForTesting( + tmpusername, new String[] {tmpusername}); + String results = tmpUGI.doAs(new PrivilegedExceptionAction<String>() { + @Override + public String run() throws Exception { + return runLsr(new FsShell(conf), root, 1); + } + }); + assertTrue(results.contains("zzz")); + } finally { + cluster.shutdown(); + } } private static String runLsr(final FsShell shell, String root, int returnvalue @@ -2055,33 +2192,40 @@ public class TestDFSShell { // ACLs) @Test (timeout = 120000) public void testCopyCommandsWithPreserveOption() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .format(true).build(); FsShell shell = null; + FileSystem fs = null; final String testdir = "/tmp/TestDFSShell-testCopyCommandsWithPreserveOption-" + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); try { - dfs.mkdirs(hdfsTestDir); + fs = cluster.getFileSystem(); + fs.mkdirs(hdfsTestDir); Path src = new Path(hdfsTestDir, "srcfile"); - dfs.create(src).close(); + fs.create(src).close(); - dfs.setAcl(src, Lists.newArrayList( + fs.setAcl(src, Lists.newArrayList( aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, USER, "foo", ALL), aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(ACCESS, GROUP, "bar", READ_EXECUTE), aclEntry(ACCESS, OTHER, EXECUTE))); - FileStatus status = dfs.getFileStatus(src); + FileStatus status = fs.getFileStatus(src); final long mtime = status.getModificationTime(); final long atime = status.getAccessTime(); final String owner = status.getOwner(); final String group = status.getGroup(); final FsPermission perm = status.getPermission(); - - dfs.setXAttr(src, USER_A1, USER_A1_VALUE); - dfs.setXAttr(src, TRUSTED_A1, TRUSTED_A1_VALUE); - shell = new FsShell(dfs.getConf()); + fs.setXAttr(src, USER_A1, USER_A1_VALUE); + fs.setXAttr(src, TRUSTED_A1, TRUSTED_A1_VALUE); + + shell = new FsShell(conf); // -p Path target1 = new Path(hdfsTestDir, "targetfile1"); @@ -2089,16 +2233,16 @@ public class TestDFSShell { target1.toUri().toString() }; int ret = ToolRunner.run(shell, argv); assertEquals("cp -p is not working", SUCCESS, ret); - FileStatus targetStatus = dfs.getFileStatus(target1); + FileStatus targetStatus = fs.getFileStatus(target1); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); FsPermission targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - Map<String, byte[]> xattrs = dfs.getXAttrs(target1); + Map<String, byte[]> xattrs = fs.getXAttrs(target1); assertTrue(xattrs.isEmpty()); - List<AclEntry> acls = dfs.getAclStatus(target1).getEntries(); + List<AclEntry> acls = fs.getAclStatus(target1).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2108,16 +2252,16 @@ public class TestDFSShell { target2.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptop is not working", SUCCESS, ret); - targetStatus = dfs.getFileStatus(target2); + targetStatus = fs.getFileStatus(target2); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = dfs.getXAttrs(target2); + xattrs = fs.getXAttrs(target2); assertTrue(xattrs.isEmpty()); - acls = dfs.getAclStatus(target2).getEntries(); + acls = fs.getAclStatus(target2).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2127,18 +2271,18 @@ public class TestDFSShell { target3.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptopx is not working", SUCCESS, ret); - targetStatus = dfs.getFileStatus(target3); + targetStatus = fs.getFileStatus(target3); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = dfs.getXAttrs(target3); + xattrs = fs.getXAttrs(target3); assertEquals(xattrs.size(), 2); assertArrayEquals(USER_A1_VALUE, xattrs.get(USER_A1)); assertArrayEquals(TRUSTED_A1_VALUE, xattrs.get(TRUSTED_A1)); - acls = dfs.getAclStatus(target3).getEntries(); + acls = fs.getAclStatus(target3).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2148,19 +2292,19 @@ public class TestDFSShell { target4.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptopa is not working", SUCCESS, ret); - targetStatus = dfs.getFileStatus(target4); + targetStatus = fs.getFileStatus(target4); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = dfs.getXAttrs(target4); + xattrs = fs.getXAttrs(target4); assertTrue(xattrs.isEmpty()); - acls = dfs.getAclStatus(target4).getEntries(); + acls = fs.getAclStatus(target4).getEntries(); assertFalse(acls.isEmpty()); assertTrue(targetPerm.getAclBit()); - assertEquals(dfs.getAclStatus(src), dfs.getAclStatus(target4)); + assertEquals(fs.getAclStatus(src), fs.getAclStatus(target4)); // -ptoa (verify -pa option will preserve permissions also) Path target5 = new Path(hdfsTestDir, "targetfile5"); @@ -2168,47 +2312,59 @@ public class TestDFSShell { target5.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptoa is not working", SUCCESS, ret); - targetStatus = dfs.getFileStatus(target5); + targetStatus = fs.getFileStatus(target5); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = dfs.getXAttrs(target5); + xattrs = fs.getXAttrs(target5); assertTrue(xattrs.isEmpty()); - acls = dfs.getAclStatus(target5).getEntries(); + acls = fs.getAclStatus(target5).getEntries(); assertFalse(acls.isEmpty()); assertTrue(targetPerm.getAclBit()); - assertEquals(dfs.getAclStatus(src), dfs.getAclStatus(target5)); + assertEquals(fs.getAclStatus(src), fs.getAclStatus(target5)); } finally { if (null != shell) { shell.close(); } + + if (null != fs) { + fs.delete(hdfsTestDir, true); + fs.close(); + } + cluster.shutdown(); } } @Test (timeout = 120000) public void testCopyCommandsWithRawXAttrs() throws Exception { + final Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf). + numDataNodes(1).format(true).build(); FsShell shell = null; + FileSystem fs = null; final String testdir = "/tmp/TestDFSShell-testCopyCommandsWithRawXAttrs-" + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); final Path rawHdfsTestDir = new Path("/.reserved/raw" + testdir); try { - dfs.mkdirs(hdfsTestDir); + fs = cluster.getFileSystem(); + fs.mkdirs(hdfsTestDir); final Path src = new Path(hdfsTestDir, "srcfile"); final String rawSrcBase = "/.reserved/raw" + testdir; final Path rawSrc = new Path(rawSrcBase, "srcfile"); - dfs.create(src).close(); + fs.create(src).close(); final Path srcDir = new Path(hdfsTestDir, "srcdir"); final Path rawSrcDir = new Path("/.reserved/raw" + testdir, "srcdir"); - dfs.mkdirs(srcDir); + fs.mkdirs(srcDir); final Path srcDirFile = new Path(srcDir, "srcfile"); final Path rawSrcDirFile = new Path("/.reserved/raw" + srcDirFile); - dfs.create(srcDirFile).close(); + fs.create(srcDirFile).close(); final Path[] paths = { rawSrc, rawSrcDir, rawSrcDirFile }; final String[] xattrNames = { USER_A1, RAW_A1 }; @@ -2216,35 +2372,35 @@ public class TestDFSShell { for (int i = 0; i < paths.length; i++) { for (int j = 0; j < xattrNames.length; j++) { - dfs.setXAttr(paths[i], xattrNames[j], xattrVals[j]); + fs.setXAttr(paths[i], xattrNames[j], xattrVals[j]); } } - shell = new FsShell(dfs.getConf()); + shell = new FsShell(conf); /* Check that a file as the source path works ok. */ - doTestCopyCommandsWithRawXAttrs(shell, dfs, src, hdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, dfs, rawSrc, hdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, dfs, src, rawHdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, dfs, rawSrc, rawHdfsTestDir, true); + doTestCopyCommandsWithRawXAttrs(shell, fs, src, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrc, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, src, rawHdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrc, rawHdfsTestDir, true); /* Use a relative /.reserved/raw path. */ - final Path savedWd = dfs.getWorkingDirectory(); + final Path savedWd = fs.getWorkingDirectory(); try { - dfs.setWorkingDirectory(new Path(rawSrcBase)); + fs.setWorkingDirectory(new Path(rawSrcBase)); final Path relRawSrc = new Path("../srcfile"); final Path relRawHdfsTestDir = new Path(".."); - doTestCopyCommandsWithRawXAttrs(shell, dfs, relRawSrc, - relRawHdfsTestDir, true); + doTestCopyCommandsWithRawXAttrs(shell, fs, relRawSrc, relRawHdfsTestDir, + true); } finally { - dfs.setWorkingDirectory(savedWd); + fs.setWorkingDirectory(savedWd); } /* Check that a directory as the source path works ok. */ - doTestCopyCommandsWithRawXAttrs(shell, dfs, srcDir, hdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, dfs, rawSrcDir, hdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, dfs, srcDir, rawHdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, dfs, rawSrcDir, rawHdfsTestDir, + doTestCopyCommandsWithRawXAttrs(shell, fs, srcDir, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrcDir, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, srcDir, rawHdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrcDir, rawHdfsTestDir, true); /* Use relative in an absolute path. */ @@ -2252,13 +2408,18 @@ public class TestDFSShell { testdir + "/srcdir"; final String relRawDstDir = "./.reserved/../.reserved/raw/../raw" + testdir; - doTestCopyCommandsWithRawXAttrs(shell, dfs, new Path(relRawSrcDir), + doTestCopyCommandsWithRawXAttrs(shell, fs, new Path(relRawSrcDir), new Path(relRawDstDir), true); } finally { if (null != shell) { shell.close(); } - dfs.delete(hdfsTestDir, true); + + if (null != fs) { + fs.delete(hdfsTestDir, true); + fs.close(); + } + cluster.shutdown(); } } @@ -2335,24 +2496,31 @@ public class TestDFSShell { @Test (timeout = 120000) public void testCopyCommandsToDirectoryWithPreserveOption() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .format(true).build(); FsShell shell = null; + FileSystem fs = null; final String testdir = "/tmp/TestDFSShell-testCopyCommandsToDirectoryWithPreserveOption-" + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); try { - dfs.mkdirs(hdfsTestDir); + fs = cluster.getFileSystem(); + fs.mkdirs(hdfsTestDir); Path srcDir = new Path(hdfsTestDir, "srcDir"); - dfs.mkdirs(srcDir); + fs.mkdirs(srcDir); - dfs.setAcl(srcDir, Lists.newArrayList( + fs.setAcl(srcDir, Lists.newArrayList( aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, USER, "foo", ALL), aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(DEFAULT, GROUP, "bar", READ_EXECUTE), aclEntry(ACCESS, OTHER, EXECUTE))); // set sticky bit - dfs.setPermission(srcDir, + fs.setPermission(srcDir, new FsPermission(ALL, READ_EXECUTE, EXECUTE, true)); // Create a file in srcDir to check if modification time of @@ -2360,19 +2528,19 @@ public class TestDFSShell { // If cp -p command is to preserve modification time and then copy child // (srcFile), modification time will not be preserved. Path srcFile = new Path(srcDir, "srcFile"); - dfs.create(srcFile).close(); + fs.create(srcFile).close(); - FileStatus status = dfs.getFileStatus(srcDir); + FileStatus status = fs.getFileStatus(srcDir); final long mtime = status.getModificationTime(); final long atime = status.getAccessTime(); final String owner = status.getOwner(); final String group = status.getGroup(); final FsPermission perm = status.getPermission(); - dfs.setXAttr(srcDir, USER_A1, USER_A1_VALUE); - dfs.setXAttr(srcDir, TRUSTED_A1, TRUSTED_A1_VALUE); + fs.setXAttr(srcDir, USER_A1, USER_A1_VALUE); + fs.setXAttr(srcDir, TRUSTED_A1, TRUSTED_A1_VALUE); - shell = new FsShell(dfs.getConf()); + shell = new FsShell(conf); // -p Path targetDir1 = new Path(hdfsTestDir, "targetDir1"); @@ -2380,16 +2548,16 @@ public class TestDFSShell { targetDir1.toUri().toString() }; int ret = ToolRunner.run(shell, argv); assertEquals("cp -p is not working", SUCCESS, ret); - FileStatus targetStatus = dfs.getFileStatus(targetDir1); + FileStatus targetStatus = fs.getFileStatus(targetDir1); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); FsPermission targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - Map<String, byte[]> xattrs = dfs.getXAttrs(targetDir1); + Map<String, byte[]> xattrs = fs.getXAttrs(targetDir1); assertTrue(xattrs.isEmpty()); - List<AclEntry> acls = dfs.getAclStatus(targetDir1).getEntries(); + List<AclEntry> acls = fs.getAclStatus(targetDir1).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2399,16 +2567,16 @@ public class TestDFSShell { targetDir2.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptop is not working", SUCCESS, ret); - targetStatus = dfs.getFileStatus(targetDir2); + targetStatus = fs.getFileStatus(targetDir2); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = dfs.getXAttrs(targetDir2); + xattrs = fs.getXAttrs(targetDir2); assertTrue(xattrs.isEmpty()); - acls = dfs.getAclStatus(targetDir2).getEntries(); + acls = fs.getAclStatus(targetDir2).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2418,18 +2586,18 @@ public class TestDFSShell { targetDir3.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptopx is not working", SUCCESS, ret); - targetStatus = dfs.getFileStatus(targetDir3); + targetStatus = fs.getFileStatus(targetDir3); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); <TRUNCATED> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org