Author: wang
Date: Mon May 19 19:29:30 2014
New Revision: 1596000
URL: http://svn.apache.org/r1596000
Log:
Merge trunk r1595999 to branch.
Added:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
- copied unchanged from r1595999,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
- copied unchanged from r1595999,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/resources/
- copied from r1595999,
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/test/resources/
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/resources/log4j.properties
- copied unchanged from r1595999,
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/test/resources/log4j.properties
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/CHANGES.txt
Mon May 19 19:29:30 2014
@@ -328,6 +328,8 @@ Trunk (Unreleased)
HADOOP-10583. bin/hadoop key throws NPE with no args and assorted other
fixups. (clamb via tucu)
+ HADOOP-10586. KeyShell doesn't allow setting Options via CLI. (clamb via
tucu)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -388,6 +390,11 @@ Release 2.5.0 - UNRELEASED
HADOOP-10572. Example NFS mount command must pass noacl as it isn't
supported by the server yet. (Harsh J via brandonli)
+ HADOOP-10609. .gitignore should ignore .orig and .rej files. (kasha)
+
+ HADOOP-10614. CBZip2InputStream is not threadsafe (Xiangrui Meng via
+ Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
Propchange:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1594887-1595999
Propchange:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1588992-1595999
Propchange:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1594887-1595999
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
Mon May 19 19:29:30 2014
@@ -89,6 +89,8 @@ public class KeyShell extends Configured
* @throws IOException
*/
private int init(String[] args) throws IOException {
+ final Options options = KeyProvider.options(getConf());
+
for (int i = 0; i < args.length; i++) { // parse command line
boolean moreTokens = (i < args.length - 1);
if (args[i].equals("create")) {
@@ -97,7 +99,7 @@ public class KeyShell extends Configured
keyName = args[++i];
}
- command = new CreateCommand(keyName);
+ command = new CreateCommand(keyName, options);
if ("--help".equals(keyName)) {
printKeyShellUsage();
return -1;
@@ -127,9 +129,11 @@ public class KeyShell extends Configured
} else if ("list".equals(args[i])) {
command = new ListCommand();
} else if ("--size".equals(args[i]) && moreTokens) {
- getConf().set(KeyProvider.DEFAULT_BITLENGTH_NAME, args[++i]);
+ options.setBitLength(Integer.parseInt(args[++i]));
} else if ("--cipher".equals(args[i]) && moreTokens) {
- getConf().set(KeyProvider.DEFAULT_CIPHER_NAME, args[++i]);
+ options.setCipher(args[++i]);
+ } else if ("--description".equals(args[i]) && moreTokens) {
+ options.setDescription(args[++i]);
} else if ("--provider".equals(args[i]) && moreTokens) {
userSuppliedProvider = true;
getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
@@ -399,6 +403,7 @@ public class KeyShell extends Configured
private class CreateCommand extends Command {
public static final String USAGE =
"create <keyname> [--cipher <cipher>] [--size <size>]\n" +
+ " [--description <description>]\n" +
" [--provider <provider>] [--help]";
public static final String DESC =
"The create subcommand creates a new key for the name specified\n" +
@@ -408,10 +413,12 @@ public class KeyShell extends Configured
"The default keysize is 256. You may specify the requested key\n" +
"length using the --size argument.\n";
- String keyName = null;
+ final String keyName;
+ final Options options;
- public CreateCommand(String keyName) {
+ public CreateCommand(String keyName, Options options) {
this.keyName = keyName;
+ this.options = options;
}
public boolean validate() {
@@ -434,7 +441,6 @@ public class KeyShell extends Configured
public void execute() throws IOException, NoSuchAlgorithmException {
warnIfTransientProvider();
try {
- Options options = KeyProvider.options(getConf());
provider.createKey(keyName, options);
out.println(keyName + " has been successfully created.");
provider.flush();
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
Mon May 19 19:29:30 2014
@@ -100,6 +100,21 @@ public class FileStatus implements Writa
}
/**
+ * Copy constructor.
+ *
+ * @param other FileStatus to copy
+ */
+ public FileStatus(FileStatus other) throws IOException {
+ // It's important to call the getters here instead of directly accessing
the
+ // members. Subclasses like ViewFsFileStatus can override the getters.
+ this(other.getLen(), other.isDirectory(), other.getReplication(),
+ other.getBlockSize(), other.getModificationTime(), other.getAccessTime(),
+ other.getPermission(), other.getOwner(), other.getGroup(),
+ (other.isSymlink() ? other.getSymlink() : null),
+ other.getPath());
+ }
+
+ /**
* Get the length of this file, in bytes.
* @return the length of this file, in bytes.
*/
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
Mon May 19 19:29:30 2014
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.shell;
import java.io.IOException;
-import java.util.Iterator;
+import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@@ -31,8 +31,10 @@ import org.apache.hadoop.fs.permission.A
import org.apache.hadoop.fs.permission.AclEntryScope;
import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.ScopedAclEntries;
/**
* Acl related operations
@@ -84,67 +86,34 @@ class AclCommands extends FsCommand {
(perm.getOtherAction().implies(FsAction.EXECUTE) ? "t" : "T"));
}
- if (perm.getAclBit()) {
- AclStatus aclStatus = item.fs.getAclStatus(item.path);
- List<AclEntry> entries = aclStatus.getEntries();
- printExtendedAcl(perm, entries);
- } else {
- printMinimalAcl(perm);
- }
-
+ List<AclEntry> entries = perm.getAclBit() ?
+ item.fs.getAclStatus(item.path).getEntries() :
+ Collections.<AclEntry>emptyList();
+ ScopedAclEntries scopedEntries = new ScopedAclEntries(
+ AclUtil.getAclFromPermAndEntries(perm, entries));
+ printAclEntriesForSingleScope(scopedEntries.getAccessEntries());
+ printAclEntriesForSingleScope(scopedEntries.getDefaultEntries());
out.println();
}
/**
- * Prints an extended ACL, including all extended ACL entries and also the
- * base entries implied by the permission bits.
+ * Prints all the ACL entries in a single scope.
*
- * @param perm FsPermission of file
* @param entries List<AclEntry> containing ACL entries of file
*/
- private void printExtendedAcl(FsPermission perm, List<AclEntry> entries) {
- // Print owner entry implied by owner permission bits.
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.USER)
- .setPermission(perm.getUserAction())
- .build());
-
- // Print all extended access ACL entries.
- boolean hasAccessAcl = false;
- Iterator<AclEntry> entryIter = entries.iterator();
- AclEntry curEntry = null;
- while (entryIter.hasNext()) {
- curEntry = entryIter.next();
- if (curEntry.getScope() == AclEntryScope.DEFAULT) {
- break;
- }
- hasAccessAcl = true;
- printExtendedAclEntry(curEntry, perm.getGroupAction());
+ private void printAclEntriesForSingleScope(List<AclEntry> entries) {
+ if (entries.isEmpty()) {
+ return;
}
-
- // Print mask entry implied by group permission bits, or print group
entry
- // if there is no access ACL (only default ACL).
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(hasAccessAcl ? AclEntryType.MASK : AclEntryType.GROUP)
- .setPermission(perm.getGroupAction())
- .build());
-
- // Print other entry implied by other bits.
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.OTHER)
- .setPermission(perm.getOtherAction())
- .build());
-
- // Print default ACL entries.
- if (curEntry != null && curEntry.getScope() == AclEntryScope.DEFAULT) {
- out.println(curEntry);
- // ACL sort order guarantees default mask is the second-to-last entry.
+ if (AclUtil.isMinimalAcl(entries)) {
+ for (AclEntry entry: entries) {
+ out.println(entry);
+ }
+ } else {
+ // ACL sort order guarantees mask is the second-to-last entry.
FsAction maskPerm = entries.get(entries.size() - 2).getPermission();
- while (entryIter.hasNext()) {
- printExtendedAclEntry(entryIter.next(), maskPerm);
+ for (AclEntry entry: entries) {
+ printExtendedAclEntry(entry, maskPerm);
}
}
}
@@ -172,30 +141,6 @@ class AclCommands extends FsCommand {
out.println(entry);
}
}
-
- /**
- * Prints a minimal ACL, consisting of exactly 3 ACL entries implied by the
- * permission bits.
- *
- * @param perm FsPermission of file
- */
- private void printMinimalAcl(FsPermission perm) {
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.USER)
- .setPermission(perm.getUserAction())
- .build());
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.GROUP)
- .setPermission(perm.getGroupAction())
- .build());
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.OTHER)
- .setPermission(perm.getOtherAction())
- .build());
- }
}
/**
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
Mon May 19 19:29:30 2014
@@ -129,7 +129,7 @@ public class CBZip2InputStream extends I
private int computedBlockCRC, computedCombinedCRC;
private boolean skipResult = false;// used by skipToNextMarker
- private static boolean skipDecompression = false;
+ private boolean skipDecompression = false;
// Variables used by setup* methods exclusively
@@ -281,12 +281,18 @@ public class CBZip2InputStream extends I
*/
public CBZip2InputStream(final InputStream in, READ_MODE readMode)
throws IOException {
+ this(in, readMode, false);
+ }
+
+ private CBZip2InputStream(final InputStream in, READ_MODE readMode, boolean
skipDecompression)
+ throws IOException {
super();
int blockSize = 0X39;// i.e 9
this.blockSize100k = blockSize - '0';
this.in = new BufferedInputStream(in, 1024 * 9);// >1 MB buffer
this.readMode = readMode;
+ this.skipDecompression = skipDecompression;
if (readMode == READ_MODE.CONTINUOUS) {
currentState = STATE.START_BLOCK_STATE;
lazyInitialization = (in.available() == 0)?true:false;
@@ -316,11 +322,7 @@ public class CBZip2InputStream extends I
*
*/
public static long numberOfBytesTillNextMarker(final InputStream in) throws
IOException{
- CBZip2InputStream.skipDecompression = true;
- CBZip2InputStream anObject = null;
-
- anObject = new CBZip2InputStream(in, READ_MODE.BYBLOCK);
-
+ CBZip2InputStream anObject = new CBZip2InputStream(in, READ_MODE.BYBLOCK,
true);
return anObject.getProcessedByteCount();
}
@@ -397,7 +399,7 @@ public class CBZip2InputStream extends I
if(skipDecompression){
changeStateToProcessABlock();
- CBZip2InputStream.skipDecompression = false;
+ skipDecompression = false;
}
final int hi = offs + len;
Propchange:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1588992-1595999
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
Mon May 19 19:29:30 2014
@@ -111,6 +111,30 @@ public class TestKeyShell {
assertFalse(outContent.toString(), outContent.toString().contains("key1"));
}
+ /* HADOOP-10586 KeyShell didn't allow -description. */
+ @Test
+ public void testKeySuccessfulCreationWithDescription() throws Exception {
+ outContent.reset();
+ String[] args1 = {"create", "key1", "--provider",
+ "jceks://file" + tmpDir + "/keystore.jceks",
+ "--description", "someDescription"};
+ int rc = 0;
+ KeyShell ks = new KeyShell();
+ ks.setConf(new Configuration());
+ rc = ks.run(args1);
+ assertEquals(0, rc);
+ assertTrue(outContent.toString().contains("key1 has been successfully " +
+ "created."));
+
+ outContent.reset();
+ String[] args2a = {"list", "--metadata", "--provider",
+ "jceks://file" + tmpDir + "/keystore.jceks"};
+ rc = ks.run(args2a);
+ assertEquals(0, rc);
+ assertTrue(outContent.toString().contains("description"));
+ assertTrue(outContent.toString().contains("someDescription"));
+ }
+
@Test
public void testInvalidKeySize() throws Exception {
String[] args1 = {"create", "key1", "--size", "56", "--provider",
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
Mon May 19 19:29:30 2014
@@ -113,7 +113,23 @@ public class IdUserGroup {
"The new entry is to be ignored for the following reason.",
DUPLICATE_NAME_ID_DEBUG_INFO));
}
-
+
+ /**
+ * uid and gid are defined as uint32 in linux. Some systems create
+ * (intended or unintended) <nfsnobody, 4294967294> kind of <name,Id>
+ * mapping, where 4294967294 is 2**32-2 as unsigned int32. As an example,
+ * https://bugzilla.redhat.com/show_bug.cgi?id=511876.
+ * Because user or group id are treated as Integer (signed integer or int32)
+ * here, the number 4294967294 is out of range. The solution is to convert
+ * uint32 to int32, so to map the out-of-range ID to the negative side of
+ * Integer, e.g. 4294967294 maps to -2 and 4294967295 maps to -1.
+ */
+ private static Integer parseId(final String idStr) {
+ Long longVal = Long.parseLong(idStr);
+ int intVal = longVal.intValue();
+ return Integer.valueOf(intVal);
+ }
+
/**
* Get the whole list of users and groups and save them in the maps.
* @throws IOException
@@ -134,8 +150,8 @@ public class IdUserGroup {
}
LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" +
nameId[1]);
// HDFS can't differentiate duplicate names with simple authentication
- final Integer key = Integer.valueOf(nameId[1]);
- final String value = nameId[0];
+ final Integer key = parseId(nameId[1]);
+ final String value = nameId[0];
if (map.containsKey(key)) {
final String prevValue = map.get(key);
if (value.equals(prevValue)) {
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
Mon May 19 19:29:30 2014
@@ -19,11 +19,14 @@ package org.apache.hadoop.oncrpc;
import java.io.IOException;
import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.oncrpc.RpcAcceptedReply.AcceptState;
import org.apache.hadoop.oncrpc.security.Verifier;
+import org.apache.hadoop.oncrpc.security.VerifierNone;
import org.apache.hadoop.portmap.PortmapMapping;
import org.apache.hadoop.portmap.PortmapRequest;
import org.jboss.netty.buffer.ChannelBuffer;
@@ -37,7 +40,7 @@ import org.jboss.netty.channel.SimpleCha
* and implement {@link #handleInternal} to handle the requests received.
*/
public abstract class RpcProgram extends SimpleChannelUpstreamHandler {
- private static final Log LOG = LogFactory.getLog(RpcProgram.class);
+ static final Log LOG = LogFactory.getLog(RpcProgram.class);
public static final int RPCB_PORT = 111;
private final String program;
private final String host;
@@ -45,6 +48,7 @@ public abstract class RpcProgram extends
private final int progNumber;
private final int lowProgVersion;
private final int highProgVersion;
+ private final boolean allowInsecurePorts;
/**
* If not null, this will be used as the socket to use to connect to the
@@ -61,10 +65,14 @@ public abstract class RpcProgram extends
* @param progNumber program number as defined in RFC 1050
* @param lowProgVersion lowest version of the specification supported
* @param highProgVersion highest version of the specification supported
+ * @param DatagramSocket registrationSocket if not null, use this socket to
+ * register with portmap daemon
+ * @param allowInsecurePorts true to allow client connections from
+ * unprivileged ports, false otherwise
*/
protected RpcProgram(String program, String host, int port, int progNumber,
int lowProgVersion, int highProgVersion,
- DatagramSocket registrationSocket) {
+ DatagramSocket registrationSocket, boolean allowInsecurePorts) {
this.program = program;
this.host = host;
this.port = port;
@@ -72,6 +80,9 @@ public abstract class RpcProgram extends
this.lowProgVersion = lowProgVersion;
this.highProgVersion = highProgVersion;
this.registrationSocket = registrationSocket;
+ this.allowInsecurePorts = allowInsecurePorts;
+ LOG.info("Will " + (allowInsecurePorts ? "" : "not ") + "accept client "
+ + "connections from unprivileged ports");
}
/**
@@ -133,43 +144,82 @@ public abstract class RpcProgram extends
throws Exception {
RpcInfo info = (RpcInfo) e.getMessage();
RpcCall call = (RpcCall) info.header();
+
+ SocketAddress remoteAddress = info.remoteAddress();
+ if (!allowInsecurePorts) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Will not allow connections from unprivileged ports. " +
+ "Checking for valid client port...");
+ }
+ if (remoteAddress instanceof InetSocketAddress) {
+ InetSocketAddress inetRemoteAddress = (InetSocketAddress)
remoteAddress;
+ if (inetRemoteAddress.getPort() > 1023) {
+ LOG.warn("Connection attempted from '" + inetRemoteAddress + "' "
+ + "which is an unprivileged port. Rejecting connection.");
+ sendRejectedReply(call, remoteAddress, ctx);
+ return;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Accepting connection from '" + remoteAddress + "'");
+ }
+ }
+ } else {
+ LOG.warn("Could not determine remote port of socket address '" +
+ remoteAddress + "'. Rejecting connection.");
+ sendRejectedReply(call, remoteAddress, ctx);
+ return;
+ }
+ }
+
if (LOG.isTraceEnabled()) {
LOG.trace(program + " procedure #" + call.getProcedure());
}
if (this.progNumber != call.getProgram()) {
LOG.warn("Invalid RPC call program " + call.getProgram());
- RpcAcceptedReply reply = RpcAcceptedReply.getInstance(call.getXid(),
- AcceptState.PROG_UNAVAIL, Verifier.VERIFIER_NONE);
-
- XDR out = new XDR();
- reply.write(out);
- ChannelBuffer b = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap()
- .buffer());
- RpcResponse rsp = new RpcResponse(b, info.remoteAddress());
- RpcUtil.sendRpcResponse(ctx, rsp);
+ sendAcceptedReply(call, remoteAddress, AcceptState.PROG_UNAVAIL, ctx);
return;
}
int ver = call.getVersion();
if (ver < lowProgVersion || ver > highProgVersion) {
LOG.warn("Invalid RPC call version " + ver);
- RpcAcceptedReply reply = RpcAcceptedReply.getInstance(call.getXid(),
- AcceptState.PROG_MISMATCH, Verifier.VERIFIER_NONE);
-
- XDR out = new XDR();
- reply.write(out);
- out.writeInt(lowProgVersion);
- out.writeInt(highProgVersion);
- ChannelBuffer b = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap()
- .buffer());
- RpcResponse rsp = new RpcResponse(b, info.remoteAddress());
- RpcUtil.sendRpcResponse(ctx, rsp);
+ sendAcceptedReply(call, remoteAddress, AcceptState.PROG_MISMATCH, ctx);
return;
}
handleInternal(ctx, info);
}
+
+ private void sendAcceptedReply(RpcCall call, SocketAddress remoteAddress,
+ AcceptState acceptState, ChannelHandlerContext ctx) {
+ RpcAcceptedReply reply = RpcAcceptedReply.getInstance(call.getXid(),
+ acceptState, Verifier.VERIFIER_NONE);
+
+ XDR out = new XDR();
+ reply.write(out);
+ if (acceptState == AcceptState.PROG_MISMATCH) {
+ out.writeInt(lowProgVersion);
+ out.writeInt(highProgVersion);
+ }
+ ChannelBuffer b = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap()
+ .buffer());
+ RpcResponse rsp = new RpcResponse(b, remoteAddress);
+ RpcUtil.sendRpcResponse(ctx, rsp);
+ }
+
+ private static void sendRejectedReply(RpcCall call,
+ SocketAddress remoteAddress, ChannelHandlerContext ctx) {
+ XDR out = new XDR();
+ RpcDeniedReply reply = new RpcDeniedReply(call.getXid(),
+ RpcReply.ReplyState.MSG_DENIED,
+ RpcDeniedReply.RejectState.AUTH_ERROR, new VerifierNone());
+ reply.write(out);
+ ChannelBuffer buf = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap()
+ .buffer());
+ RpcResponse rsp = new RpcResponse(buf, remoteAddress);
+ RpcUtil.sendRpcResponse(ctx, rsp);
+ }
protected abstract void handleInternal(ChannelHandlerContext ctx, RpcInfo
info);
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java
Mon May 19 19:29:30 2014
@@ -67,6 +67,51 @@ public class TestIdUserGroup {
}
@Test
+ public void testIdOutOfIntegerRange() throws IOException {
+ String GET_ALL_USERS_CMD = "echo \""
+ + "nfsnobody:x:4294967294:4294967294:Anonymous NFS
User:/var/lib/nfs:/sbin/nologin\n"
+ + "nfsnobody1:x:4294967295:4294967295:Anonymous NFS
User:/var/lib/nfs1:/sbin/nologin\n"
+ + "maxint:x:2147483647:2147483647:Grid Distributed File
System:/home/maxint:/bin/bash\n"
+ + "minint:x:2147483648:2147483648:Grid Distributed File
System:/home/minint:/bin/bash\n"
+ + "archivebackup:*:1031:4294967294:Archive
Backup:/home/users/archivebackup:/bin/sh\n"
+ + "hdfs:x:11501:10787:Grid Distributed File
System:/home/hdfs:/bin/bash\n"
+ + "daemon:x:2:2:daemon:/sbin:/sbin/nologin\""
+ + " | cut -d: -f1,3";
+ String GET_ALL_GROUPS_CMD = "echo \""
+ + "hdfs:*:11501:hrt_hdfs\n"
+ + "rpcuser:*:29:\n"
+ + "nfsnobody:*:4294967294:\n"
+ + "nfsnobody1:*:4294967295:\n"
+ + "maxint:*:2147483647:\n"
+ + "minint:*:2147483648:\n"
+ + "mapred3:x:498\""
+ + " | cut -d: -f1,3";
+ // Maps for id to name map
+ BiMap<Integer, String> uMap = HashBiMap.create();
+ BiMap<Integer, String> gMap = HashBiMap.create();
+
+ IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":");
+ assertTrue(uMap.size() == 7);
+ assertEquals("nfsnobody", uMap.get(-2));
+ assertEquals("nfsnobody1", uMap.get(-1));
+ assertEquals("maxint", uMap.get(2147483647));
+ assertEquals("minint", uMap.get(-2147483648));
+ assertEquals("archivebackup", uMap.get(1031));
+ assertEquals("hdfs",uMap.get(11501));
+ assertEquals("daemon", uMap.get(2));
+
+ IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":");
+ assertTrue(gMap.size() == 7);
+ assertEquals("hdfs",gMap.get(11501));
+ assertEquals("rpcuser", gMap.get(29));
+ assertEquals("nfsnobody", gMap.get(-2));
+ assertEquals("nfsnobody1", gMap.get(-1));
+ assertEquals("maxint", gMap.get(2147483647));
+ assertEquals("minint", gMap.get(-2147483648));
+ assertEquals("mapred3", gMap.get(498));
+ }
+
+ @Test
public void testUserUpdateSetting() throws IOException {
IdUserGroup iug = new IdUserGroup();
assertEquals(iug.getTimeout(), IdUserGroup.TIMEOUT_DEFAULT);
Modified:
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
Mon May 19 19:29:30 2014
@@ -28,6 +28,8 @@ import java.util.Random;
import org.apache.hadoop.oncrpc.RpcUtil.RpcFrameDecoder;
import org.apache.hadoop.oncrpc.security.CredentialsNone;
import org.apache.hadoop.oncrpc.security.VerifierNone;
+import org.apache.log4j.Level;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.jboss.netty.buffer.ByteBufferBackedChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
@@ -38,10 +40,16 @@ import org.junit.Test;
import org.mockito.Mockito;
public class TestFrameDecoder {
+
+ static {
+ ((Log4JLogger) RpcProgram.LOG).getLogger().setLevel(Level.ALL);
+ }
private static int resultSize;
static void testRequest(XDR request, int serverPort) {
+ // Reset resultSize so as to avoid interference from other tests in this
class.
+ resultSize = 0;
SimpleTcpClient tcpClient = new SimpleTcpClient("localhost", serverPort,
request,
true);
tcpClient.run();
@@ -50,9 +58,10 @@ public class TestFrameDecoder {
static class TestRpcProgram extends RpcProgram {
protected TestRpcProgram(String program, String host, int port,
- int progNumber, int lowProgVersion, int highProgVersion) {
+ int progNumber, int lowProgVersion, int highProgVersion,
+ boolean allowInsecurePorts) {
super(program, host, port, progNumber, lowProgVersion, highProgVersion,
- null);
+ null, allowInsecurePorts);
}
@Override
@@ -149,7 +158,41 @@ public class TestFrameDecoder {
@Test
public void testFrames() {
+ int serverPort = startRpcServer(true);
+ XDR xdrOut = createGetportMount();
+ int headerSize = xdrOut.size();
+ int bufsize = 2 * 1024 * 1024;
+ byte[] buffer = new byte[bufsize];
+ xdrOut.writeFixedOpaque(buffer);
+ int requestSize = xdrOut.size() - headerSize;
+
+ // Send the request to the server
+ testRequest(xdrOut, serverPort);
+
+ // Verify the server got the request with right size
+ assertEquals(requestSize, resultSize);
+ }
+
+ @Test
+ public void testUnprivilegedPort() {
+ // Don't allow connections from unprivileged ports. Given that this test is
+ // presumably not being run by root, this will be the case.
+ int serverPort = startRpcServer(false);
+
+ XDR xdrOut = createGetportMount();
+ int bufsize = 2 * 1024 * 1024;
+ byte[] buffer = new byte[bufsize];
+ xdrOut.writeFixedOpaque(buffer);
+
+ // Send the request to the server
+ testRequest(xdrOut, serverPort);
+
+ // Verify the server rejected the request.
+ assertEquals(0, resultSize);
+ }
+
+ private static int startRpcServer(boolean allowInsecurePorts) {
Random rand = new Random();
int serverPort = 30000 + rand.nextInt(10000);
int retries = 10; // A few retries in case initial choice is in use.
@@ -157,7 +200,7 @@ public class TestFrameDecoder {
while (true) {
try {
RpcProgram program = new
TestFrameDecoder.TestRpcProgram("TestRpcProgram",
- "localhost", serverPort, 100000, 1, 2);
+ "localhost", serverPort, 100000, 1, 2, allowInsecurePorts);
SimpleTcpServer tcpServer = new SimpleTcpServer(serverPort, program,
1);
tcpServer.run();
break; // Successfully bound a port, break out.
@@ -169,19 +212,7 @@ public class TestFrameDecoder {
}
}
}
-
- XDR xdrOut = createGetportMount();
- int headerSize = xdrOut.size();
- int bufsize = 2 * 1024 * 1024;
- byte[] buffer = new byte[bufsize];
- xdrOut.writeFixedOpaque(buffer);
- int requestSize = xdrOut.size() - headerSize;
-
- // Send the request to the server
- testRequest(xdrOut, serverPort);
-
- // Verify the server got the request with right size
- assertEquals(requestSize, resultSize);
+ return serverPort;
}
static void createPortmapXDRheader(XDR xdr_out, int procedure) {
