Author: szetszwo
Date: Wed Jun 27 23:29:18 2012
New Revision: 1354774
URL: http://svn.apache.org/viewvc?rev=1354774&view=rev
Log:
svn merge -c 1354773 from branch-1 for HDFS-3551.
Added:
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/server/namenode/web/
- copied from r1354773,
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/web/
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/server/namenode/web/resources/
- copied from r1354773,
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/web/resources/
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java
- copied unchanged from r1354773,
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java
Modified:
hadoop/common/branches/branch-1.1/ (props changed)
hadoop/common/branches/branch-1.1/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
Propchange: hadoop/common/branches/branch-1.1/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-1:r1354773
Modified: hadoop/common/branches/branch-1.1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/CHANGES.txt?rev=1354774&r1=1354773&r2=1354774&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.1/CHANGES.txt Wed Jun 27 23:29:18 2012
@@ -259,6 +259,9 @@ Release 1.1.0 - unreleased
HDFS-3522. If a namenode is in safemode, it should throw SafeModeException
when getBlockLocations has zero locations. (Brandon Li via szetszwo)
+ HDFS-3551. WebHDFS CREATE should use client location for HTTP redirection.
+ (szetszwo)
+
Release 1.0.3 - 2012.05.07
NEW FEATURES
Propchange: hadoop/common/branches/branch-1.1/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-1/CHANGES.txt:r1354773
Modified:
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1354774&r1=1354773&r2=1354774&view=diff
==============================================================================
---
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Wed Jun 27 23:29:18 2012
@@ -3100,6 +3100,20 @@ public class FSNamesystem implements FSC
return true;
}
+ /** Choose a datanode near to the given address. */
+ public DatanodeInfo chooseDatanode(String address, long blocksize) {
+ final DatanodeDescriptor clientNode = host2DataNodeMap.getDatanodeByHost(
+ address);
+ if (clientNode != null) {
+ final DatanodeDescriptor[] datanodes = replicator.chooseTarget(
+ 1, clientNode, null, blocksize);
+ if (datanodes.length > 0) {
+ return datanodes[0];
+ }
+ }
+ return null;
+ }
+
/**
* Parse the data-nodes the block belongs to and choose one,
* which will be the replication source.
@@ -5929,4 +5943,9 @@ public class FSNamesystem implements FSC
}
}
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ": " + host2DataNodeMap;
+ }
}
Modified:
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java?rev=1354774&r1=1354773&r2=1354774&view=diff
==============================================================================
---
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java
(original)
+++
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java
Wed Jun 27 23:29:18 2012
@@ -185,4 +185,14 @@ class Host2NodesMap {
hostmapLock.readLock().unlock();
}
}
+
+ @Override
+ public String toString() {
+ final StringBuilder b = new StringBuilder(getClass().getSimpleName())
+ .append("[");
+ for(Map.Entry<String, DatanodeDescriptor[]> e : map.entrySet()) {
+ b.append("\n " + e.getKey() + " => " + Arrays.asList(e.getValue()));
+ }
+ return b.append("\n]").toString();
+ }
}
Modified:
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1354774&r1=1354773&r2=1354774&view=diff
==============================================================================
---
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
(original)
+++
hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
Wed Jun 27 23:29:18 2012
@@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.JspHelper;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.web.JsonUtil;
@@ -107,6 +108,11 @@ public class NamenodeWebHdfsMethods {
return REMOTE_ADDRESS.get();
}
+ /** Set the remote client address. */
+ static void setRemoteAddress(String remoteAddress) {
+ REMOTE_ADDRESS.set(remoteAddress);
+ }
+
private @Context ServletContext context;
private @Context HttpServletRequest request;
private @Context HttpServletResponse response;
@@ -126,12 +132,21 @@ public class NamenodeWebHdfsMethods {
response.setContentType(null);
}
- private static DatanodeInfo chooseDatanode(final NameNode namenode,
- final String path, final HttpOpParam.Op op, final long openOffset
- ) throws IOException {
- if (op == GetOpParam.Op.OPEN
+ static DatanodeInfo chooseDatanode(final NameNode namenode,
+ final String path, final HttpOpParam.Op op, final long openOffset,
+ final long blocksize) throws IOException {
+ final FSNamesystem ns = namenode.getNamesystem();
+
+ if (op == PutOpParam.Op.CREATE) {
+ //choose a datanode near to client
+ final DatanodeInfo dn = ns.chooseDatanode(getRemoteAddress(), blocksize);
+ if (dn != null) {
+ return dn;
+ }
+ } else if (op == GetOpParam.Op.OPEN
|| op == GetOpParam.Op.GETFILECHECKSUM
|| op == PostOpParam.Op.APPEND) {
+ //choose a datanode containing a replica
final HdfsFileStatus status = namenode.getFileInfo(path);
if (status == null) {
throw new FileNotFoundException("File " + path + " not found.");
@@ -155,7 +170,7 @@ public class NamenodeWebHdfsMethods {
}
}
- return namenode.getNamesystem().getRandomDatanode();
+ return ns.getRandomDatanode();
}
private Token<? extends TokenIdentifier> generateDelegationToken(
@@ -173,8 +188,10 @@ public class NamenodeWebHdfsMethods {
final UserGroupInformation ugi, final DelegationParam delegation,
final UserParam username, final DoAsParam doAsUser,
final String path, final HttpOpParam.Op op, final long openOffset,
+ final long blocksize,
final Param<?, ?>... parameters) throws URISyntaxException, IOException {
- final DatanodeInfo dn = chooseDatanode(namenode, path, op, openOffset);
+ final DatanodeInfo dn = chooseDatanode(namenode, path, op, openOffset,
+ blocksize);
final String delegationQuery;
if (!UserGroupInformation.isSecurityEnabled()) {
@@ -302,7 +319,7 @@ public class NamenodeWebHdfsMethods {
case CREATE:
{
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser,
- fullpath, op.getValue(), -1L,
+ fullpath, op.getValue(), -1L, blockSize.getValue(conf),
permission, overwrite, bufferSize, replication, blockSize);
return
Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
@@ -425,7 +442,7 @@ public class NamenodeWebHdfsMethods {
case APPEND:
{
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser,
- fullpath, op.getValue(), -1L, bufferSize);
+ fullpath, op.getValue(), -1L, -1L, bufferSize);
return
Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
default:
@@ -507,7 +524,7 @@ public class NamenodeWebHdfsMethods {
case OPEN:
{
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser,
- fullpath, op.getValue(), offset.getValue(), offset, length,
bufferSize);
+ fullpath, op.getValue(), offset.getValue(), -1L, offset, length,
bufferSize);
return
Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
case GET_BLOCK_LOCATIONS:
@@ -543,7 +560,7 @@ public class NamenodeWebHdfsMethods {
case GETFILECHECKSUM:
{
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser,
- fullpath, op.getValue(), -1L);
+ fullpath, op.getValue(), -1L, -1L);
return
Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
case GETDELEGATIONTOKEN:
Modified:
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1354774&r1=1354773&r2=1354774&view=diff
==============================================================================
---
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
(original)
+++
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
Wed Jun 27 23:29:18 2012
@@ -17,9 +17,20 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
+
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
public abstract class NameNodeAdapter {
+ /**
+ * Get block locations within the specified range.
+ */
+ public static LocatedBlocks getBlockLocations(NameNode namenode,
+ String src, long offset, long length) throws IOException {
+ return namenode.getNamesystem().getBlockLocations(
+ src, offset, length, false, true, true);
+ }
+
public static boolean checkFileProgress(FSNamesystem fsn, String path,
boolean checkall) throws IOException {
INodeFile f = fsn.dir.getFileINode(path);
return fsn.checkFileProgress(f, checkall);
Modified:
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java?rev=1354774&r1=1354773&r2=1354774&view=diff
==============================================================================
---
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
(original)
+++
hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
Wed Jun 27 23:29:18 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.security.UserGroupInformation;
@@ -39,6 +40,12 @@ import org.junit.Assert;
public class WebHdfsTestUtil {
public static final Log LOG = LogFactory.getLog(WebHdfsTestUtil.class);
+ public static Configuration createConf() {
+ final Configuration conf = new Configuration();
+ conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
+ return conf;
+ }
+
public static WebHdfsFileSystem getWebHdfsFileSystem(final Configuration conf
) throws IOException, URISyntaxException {
final String uri = WebHdfsFileSystem.SCHEME + "://"
