Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm?rev=1562670&r1=1562669&r2=1562670&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm Thu Jan 30 01:55:14 2014 @@ -18,8 +18,6 @@ WebHDFS REST API - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * {Document Conventions} @@ -54,7 +52,7 @@ WebHDFS REST API * {{{Status of a File/Directory}<<<GETFILESTATUS>>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.getFileStatus) - * {{<<<LISTSTATUS>>>}} + * {{{List a Directory}<<<LISTSTATUS>>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.listStatus) * {{{Get Content Summary of a Directory}<<<GETCONTENTSUMMARY>>>}} @@ -109,7 +107,7 @@ WebHDFS REST API * {{{Append to a File}<<<APPEND>>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.append) - * {{{Concatenate Files}<<<CONCAT>>>}} + * {{{Concat File(s)}<<<CONCAT>>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat) * HTTP DELETE @@ -871,7 +869,7 @@ Content-Length: 0 * {Error Responses} When an operation fails, the server may throw an exception. - The JSON schema of error responses is defined in {{<<<RemoteException>>> JSON schema}}. + The JSON schema of error responses is defined in {{RemoteException JSON Schema}}. The table below shows the mapping from exceptions to HTTP response codes. ** {HTTP Response Codes} @@ -1119,7 +1117,7 @@ Transfer-Encoding: chunked See also: {{{FileStatus Properties}<<<FileStatus>>> Properties}}, {{{Status of a File/Directory}<<<GETFILESTATUS>>>}}, - {{{../../api/org/apache/hadoop/fs/FileStatus}FileStatus}} + {{{../../api/org/apache/hadoop/fs/FileStatus.html}FileStatus}} *** {FileStatus Properties} @@ -1232,7 +1230,7 @@ var fileStatusProperties = See also: {{{FileStatus Properties}<<<FileStatus>>> Properties}}, {{{List a Directory}<<<LISTSTATUS>>>}}, - {{{../../api/org/apache/hadoop/fs/FileStatus}FileStatus}} + {{{../../api/org/apache/hadoop/fs/FileStatus.html}FileStatus}} ** {Long JSON Schema} @@ -1275,7 +1273,7 @@ var fileStatusProperties = See also: {{{Get Home Directory}<<<GETHOMEDIRECTORY>>>}}, - {{{../../api/org/apache/hadoop/fs/Path}Path}} + {{{../../api/org/apache/hadoop/fs/Path.html}Path}} ** {RemoteException JSON Schema}
Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1561770-1562668 Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1562670&r1=1562669&r2=1562670&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Thu Jan 30 01:55:14 2014 @@ -22,6 +22,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Joiner; import com.google.common.collect.Lists; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -891,21 +892,7 @@ public class DFSTestUtil { /** Copy one file's contents into the other **/ public static void copyFile(File src, File dest) throws IOException { - InputStream in = null; - OutputStream out = null; - - try { - in = new FileInputStream(src); - out = new FileOutputStream(dest); - - byte [] b = new byte[1024]; - while( in.read(b) > 0 ) { - out.write(b); - } - } finally { - if(in != null) in.close(); - if(out != null) out.close(); - } + FileUtils.copyFile(src, dest); } public static class Builder { Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1562670&r1=1562669&r2=1562670&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Thu Jan 30 01:55:14 2014 @@ -118,6 +118,20 @@ public class TestDFSUtil { assertEquals(0, bs.length); } + /** + * Test constructing LocatedBlock with null cachedLocs + */ + @Test + public void testLocatedBlockConstructorWithNullCachedLocs() { + DatanodeInfo d = DFSTestUtil.getLocalDatanodeInfo(); + DatanodeInfo[] ds = new DatanodeInfo[1]; + ds[0] = d; + + ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1); + LocatedBlock l1 = new LocatedBlock(b1, ds, null, null, 0, false, null); + final DatanodeInfo[] cachedLocs = l1.getCachedLocations(); + assertTrue(cachedLocs.length == 0); + } private Configuration setupAddress(String key) { HdfsConfiguration conf = new HdfsConfiguration(); Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java?rev=1562670&r1=1562669&r2=1562670&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java Thu Jan 30 01:55:14 2014 @@ -69,6 +69,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CachePoolStats; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -528,77 +529,111 @@ public class TestCacheDirectives { @Test(timeout=60000) public void testCacheManagerRestart() throws Exception { - // Create and validate a pool - final String pool = "poolparty"; - String groupName = "partygroup"; - FsPermission mode = new FsPermission((short)0777); - long limit = 747; - dfs.addCachePool(new CachePoolInfo(pool) - .setGroupName(groupName) - .setMode(mode) - .setLimit(limit)); - RemoteIterator<CachePoolEntry> pit = dfs.listCachePools(); - assertTrue("No cache pools found", pit.hasNext()); - CachePoolInfo info = pit.next().getInfo(); - assertEquals(pool, info.getPoolName()); - assertEquals(groupName, info.getGroupName()); - assertEquals(mode, info.getMode()); - assertEquals(limit, (long)info.getLimit()); - assertFalse("Unexpected # of cache pools found", pit.hasNext()); - - // Create some cache entries - int numEntries = 10; - String entryPrefix = "/party-"; - long prevId = -1; - final Date expiry = new Date(); - for (int i=0; i<numEntries; i++) { - prevId = dfs.addCacheDirective( - new CacheDirectiveInfo.Builder(). - setPath(new Path(entryPrefix + i)).setPool(pool). - setExpiration( - CacheDirectiveInfo.Expiration.newAbsolute(expiry.getTime())). - build()); - } - RemoteIterator<CacheDirectiveEntry> dit - = dfs.listCacheDirectives(null); - for (int i=0; i<numEntries; i++) { - assertTrue("Unexpected # of cache entries: " + i, dit.hasNext()); - CacheDirectiveInfo cd = dit.next().getInfo(); - assertEquals(i+1, cd.getId().longValue()); - assertEquals(entryPrefix + i, cd.getPath().toUri().getPath()); - assertEquals(pool, cd.getPool()); - } - assertFalse("Unexpected # of cache directives found", dit.hasNext()); - - // Restart namenode - cluster.restartNameNode(); + SecondaryNameNode secondary = null; + try { + // Start a secondary namenode + conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, + "0.0.0.0:0"); + secondary = new SecondaryNameNode(conf); - // Check that state came back up - pit = dfs.listCachePools(); - assertTrue("No cache pools found", pit.hasNext()); - info = pit.next().getInfo(); - assertEquals(pool, info.getPoolName()); - assertEquals(pool, info.getPoolName()); - assertEquals(groupName, info.getGroupName()); - assertEquals(mode, info.getMode()); - assertEquals(limit, (long)info.getLimit()); - assertFalse("Unexpected # of cache pools found", pit.hasNext()); + // Create and validate a pool + final String pool = "poolparty"; + String groupName = "partygroup"; + FsPermission mode = new FsPermission((short)0777); + long limit = 747; + dfs.addCachePool(new CachePoolInfo(pool) + .setGroupName(groupName) + .setMode(mode) + .setLimit(limit)); + RemoteIterator<CachePoolEntry> pit = dfs.listCachePools(); + assertTrue("No cache pools found", pit.hasNext()); + CachePoolInfo info = pit.next().getInfo(); + assertEquals(pool, info.getPoolName()); + assertEquals(groupName, info.getGroupName()); + assertEquals(mode, info.getMode()); + assertEquals(limit, (long)info.getLimit()); + assertFalse("Unexpected # of cache pools found", pit.hasNext()); + + // Create some cache entries + int numEntries = 10; + String entryPrefix = "/party-"; + long prevId = -1; + final Date expiry = new Date(); + for (int i=0; i<numEntries; i++) { + prevId = dfs.addCacheDirective( + new CacheDirectiveInfo.Builder(). + setPath(new Path(entryPrefix + i)).setPool(pool). + setExpiration( + CacheDirectiveInfo.Expiration.newAbsolute(expiry.getTime())). + build()); + } + RemoteIterator<CacheDirectiveEntry> dit + = dfs.listCacheDirectives(null); + for (int i=0; i<numEntries; i++) { + assertTrue("Unexpected # of cache entries: " + i, dit.hasNext()); + CacheDirectiveInfo cd = dit.next().getInfo(); + assertEquals(i+1, cd.getId().longValue()); + assertEquals(entryPrefix + i, cd.getPath().toUri().getPath()); + assertEquals(pool, cd.getPool()); + } + assertFalse("Unexpected # of cache directives found", dit.hasNext()); + + // Checkpoint once to set some cache pools and directives on 2NN side + secondary.doCheckpoint(); + + // Add some more CacheManager state + final String imagePool = "imagePool"; + dfs.addCachePool(new CachePoolInfo(imagePool)); + prevId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder() + .setPath(new Path("/image")).setPool(imagePool).build()); + + // Save a new image to force a fresh fsimage download + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + dfs.saveNamespace(); + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + // Checkpoint again forcing a reload of FSN state + boolean fetchImage = secondary.doCheckpoint(); + assertTrue("Secondary should have fetched a new fsimage from NameNode", + fetchImage); + + // Remove temp pool and directive + dfs.removeCachePool(imagePool); + + // Restart namenode + cluster.restartNameNode(); + + // Check that state came back up + pit = dfs.listCachePools(); + assertTrue("No cache pools found", pit.hasNext()); + info = pit.next().getInfo(); + assertEquals(pool, info.getPoolName()); + assertEquals(pool, info.getPoolName()); + assertEquals(groupName, info.getGroupName()); + assertEquals(mode, info.getMode()); + assertEquals(limit, (long)info.getLimit()); + assertFalse("Unexpected # of cache pools found", pit.hasNext()); + + dit = dfs.listCacheDirectives(null); + for (int i=0; i<numEntries; i++) { + assertTrue("Unexpected # of cache entries: " + i, dit.hasNext()); + CacheDirectiveInfo cd = dit.next().getInfo(); + assertEquals(i+1, cd.getId().longValue()); + assertEquals(entryPrefix + i, cd.getPath().toUri().getPath()); + assertEquals(pool, cd.getPool()); + assertEquals(expiry.getTime(), cd.getExpiration().getMillis()); + } + assertFalse("Unexpected # of cache directives found", dit.hasNext()); - dit = dfs.listCacheDirectives(null); - for (int i=0; i<numEntries; i++) { - assertTrue("Unexpected # of cache entries: " + i, dit.hasNext()); - CacheDirectiveInfo cd = dit.next().getInfo(); - assertEquals(i+1, cd.getId().longValue()); - assertEquals(entryPrefix + i, cd.getPath().toUri().getPath()); - assertEquals(pool, cd.getPool()); - assertEquals(expiry.getTime(), cd.getExpiration().getMillis()); + long nextId = dfs.addCacheDirective( + new CacheDirectiveInfo.Builder(). + setPath(new Path("/foobar")).setPool(pool).build()); + assertEquals(prevId + 1, nextId); + } finally { + if (secondary != null) { + secondary.shutdown(); + } } - assertFalse("Unexpected # of cache directives found", dit.hasNext()); - - long nextId = dfs.addCacheDirective( - new CacheDirectiveInfo.Builder(). - setPath(new Path("/foobar")).setPool(pool).build()); - assertEquals(prevId + 1, nextId); } /** Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1562670&r1=1562669&r2=1562670&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Thu Jan 30 01:55:14 2014 @@ -1634,7 +1634,7 @@ public class TestCheckpoint { * Test that the secondary namenode correctly deletes temporary edits * on startup. */ - @Test(timeout = 30000) + @Test(timeout = 60000) public void testDeleteTemporaryEditsOnStartup() throws IOException { Configuration conf = new HdfsConfiguration(); SecondaryNameNode secondary = null; Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java?rev=1562670&r1=1562669&r2=1562670&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java Thu Jan 30 01:55:14 2014 @@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.http.HttpServer; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.authorize.AccessControlList; @@ -66,7 +66,7 @@ public class TestGetImageServlet { AccessControlList acls = Mockito.mock(AccessControlList.class); Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false); ServletContext context = Mockito.mock(ServletContext.class); - Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls); + Mockito.when(context.getAttribute(HttpServer2.ADMINS_ACL)).thenReturn(acls); // Make sure that NN2 is considered a valid fsimage/edits requestor. assertTrue(GetImageServlet.isValidRequestor(context, Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java?rev=1562670&r1=1562669&r2=1562670&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java Thu Jan 30 01:55:14 2014 @@ -37,7 +37,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.http.HttpServer; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.http.HttpServerFunctionalTest; import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.StringUtils; @@ -119,7 +119,7 @@ public class TestTransferFsImage { */ @Test(timeout = 5000) public void testImageTransferTimeout() throws Exception { - HttpServer testServer = HttpServerFunctionalTest.createServer("hdfs"); + HttpServer2 testServer = HttpServerFunctionalTest.createServer("hdfs"); try { testServer.addServlet("GetImage", "/getimage", TestGetImageServlet.class); testServer.start(); Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java?rev=1562670&r1=1562669&r2=1562670&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java Thu Jan 30 01:55:14 2014 @@ -58,7 +58,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.http.HttpServer; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.ipc.ProtobufRpcEngine.Server; import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; import org.apache.hadoop.security.UserGroupInformation; @@ -89,7 +89,7 @@ public class SnapshotTestHelper { setLevel2OFF(LogFactory.getLog(MetricsSystemImpl.class)); setLevel2OFF(DataBlockScanner.LOG); - setLevel2OFF(HttpServer.LOG); + setLevel2OFF(HttpServer2.LOG); setLevel2OFF(DataNode.LOG); setLevel2OFF(BlockPoolSliceStorage.LOG); setLevel2OFF(LeaseManager.LOG);
