Author: wang
Date: Fri May 16 18:50:52 2014
New Revision: 1595303
URL: http://svn.apache.org/r1595303
Log:
Merge trunk r1595301 to branch
Added:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
- copied unchanged from r1595301,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
- copied unchanged from r1595301,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
Fri May 16 18:50:52 2014
@@ -328,6 +328,8 @@ Trunk (Unreleased)
HADOOP-10583. bin/hadoop key throws NPE with no args and assorted other
fixups. (clamb via tucu)
+ HADOOP-10586. KeyShell doesn't allow setting Options via CLI. (clamb via
tucu)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -476,6 +478,9 @@ Release 2.5.0 - UNRELEASED
HADOOP-10585. Retry polices ignore interrupted exceptions (Daryn Sharp via
jeagles)
+ HADOOP-10401. ShellBasedUnixGroupsMapping#getGroups does not always return
+ primary group first (Akira AJISAKA via Colin Patrick McCabe)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1594376-1595301
Propchange:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1594376-1595301
Propchange:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1594376-1595301
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
Fri May 16 18:50:52 2014
@@ -89,6 +89,8 @@ public class KeyShell extends Configured
* @throws IOException
*/
private int init(String[] args) throws IOException {
+ final Options options = KeyProvider.options(getConf());
+
for (int i = 0; i < args.length; i++) { // parse command line
boolean moreTokens = (i < args.length - 1);
if (args[i].equals("create")) {
@@ -97,7 +99,7 @@ public class KeyShell extends Configured
keyName = args[++i];
}
- command = new CreateCommand(keyName);
+ command = new CreateCommand(keyName, options);
if ("--help".equals(keyName)) {
printKeyShellUsage();
return -1;
@@ -127,9 +129,11 @@ public class KeyShell extends Configured
} else if ("list".equals(args[i])) {
command = new ListCommand();
} else if ("--size".equals(args[i]) && moreTokens) {
- getConf().set(KeyProvider.DEFAULT_BITLENGTH_NAME, args[++i]);
+ options.setBitLength(Integer.parseInt(args[++i]));
} else if ("--cipher".equals(args[i]) && moreTokens) {
- getConf().set(KeyProvider.DEFAULT_CIPHER_NAME, args[++i]);
+ options.setCipher(args[++i]);
+ } else if ("--description".equals(args[i]) && moreTokens) {
+ options.setDescription(args[++i]);
} else if ("--provider".equals(args[i]) && moreTokens) {
userSuppliedProvider = true;
getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
@@ -399,6 +403,7 @@ public class KeyShell extends Configured
private class CreateCommand extends Command {
public static final String USAGE =
"create <keyname> [--cipher <cipher>] [--size <size>]\n" +
+ " [--description <description>]\n" +
" [--provider <provider>] [--help]";
public static final String DESC =
"The create subcommand creates a new key for the name specified\n" +
@@ -408,10 +413,12 @@ public class KeyShell extends Configured
"The default keysize is 256. You may specify the requested key\n" +
"length using the --size argument.\n";
- String keyName = null;
+ final String keyName;
+ final Options options;
- public CreateCommand(String keyName) {
+ public CreateCommand(String keyName, Options options) {
this.keyName = keyName;
+ this.options = options;
}
public boolean validate() {
@@ -434,7 +441,6 @@ public class KeyShell extends Configured
public void execute() throws IOException, NoSuchAlgorithmException {
warnIfTransientProvider();
try {
- Options options = KeyProvider.options(getConf());
provider.createKey(keyName, options);
out.println(keyName + " has been successfully created.");
provider.flush();
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
Fri May 16 18:50:52 2014
@@ -100,6 +100,21 @@ public class FileStatus implements Writa
}
/**
+ * Copy constructor.
+ *
+ * @param other FileStatus to copy
+ */
+ public FileStatus(FileStatus other) throws IOException {
+ // It's important to call the getters here instead of directly accessing
the
+ // members. Subclasses like ViewFsFileStatus can override the getters.
+ this(other.getLen(), other.isDirectory(), other.getReplication(),
+ other.getBlockSize(), other.getModificationTime(), other.getAccessTime(),
+ other.getPermission(), other.getOwner(), other.getGroup(),
+ (other.isSymlink() ? other.getSymlink() : null),
+ other.getPath());
+ }
+
+ /**
* Get the length of this file, in bytes.
* @return the length of this file, in bytes.
*/
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
Fri May 16 18:50:52 2014
@@ -158,6 +158,17 @@ public class FsPermission implements Wri
return (short)s;
}
+ /**
+ * Encodes the object to a short. Unlike {@link #toShort()}, this method may
+ * return values outside the fixed range 00000 - 01777 if extended features
+ * are encoded into this permission, such as the ACL bit.
+ *
+ * @return short extended short representation of this permission
+ */
+ public short toExtendedShort() {
+ return toShort();
+ }
+
@Override
public boolean equals(Object obj) {
if (obj instanceof FsPermission) {
@@ -273,6 +284,16 @@ public class FsPermission implements Wri
return stickyBit;
}
+ /**
+ * Returns true if there is also an ACL (access control list).
+ *
+ * @return boolean true if there is also an ACL (access control list).
+ */
+ public boolean getAclBit() {
+ // File system subclasses that support the ACL bit would override this.
+ return false;
+ }
+
/** Set the user file creation mask (umask) */
public static void setUMask(Configuration conf, FsPermission umask) {
conf.set(UMASK_LABEL, String.format("%1$03o", umask.toShort()));
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
Fri May 16 18:50:52 2014
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.shell;
import java.io.IOException;
-import java.util.Iterator;
+import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@@ -31,8 +31,10 @@ import org.apache.hadoop.fs.permission.A
import org.apache.hadoop.fs.permission.AclEntryScope;
import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.ScopedAclEntries;
/**
* Acl related operations
@@ -75,84 +77,43 @@ class AclCommands extends FsCommand {
@Override
protected void processPath(PathData item) throws IOException {
- AclStatus aclStatus = item.fs.getAclStatus(item.path);
out.println("# file: " + item);
- out.println("# owner: " + aclStatus.getOwner());
- out.println("# group: " + aclStatus.getGroup());
- List<AclEntry> entries = aclStatus.getEntries();
- if (aclStatus.isStickyBit()) {
- String stickyFlag = "T";
- for (AclEntry aclEntry : entries) {
- if (aclEntry.getType() == AclEntryType.OTHER
- && aclEntry.getScope() == AclEntryScope.ACCESS
- && aclEntry.getPermission().implies(FsAction.EXECUTE)) {
- stickyFlag = "t";
- break;
- }
- }
- out.println("# flags: --" + stickyFlag);
- }
-
+ out.println("# owner: " + item.stat.getOwner());
+ out.println("# group: " + item.stat.getGroup());
FsPermission perm = item.stat.getPermission();
- if (entries.isEmpty()) {
- printMinimalAcl(perm);
- } else {
- printExtendedAcl(perm, entries);
+ if (perm.getStickyBit()) {
+ out.println("# flags: --" +
+ (perm.getOtherAction().implies(FsAction.EXECUTE) ? "t" : "T"));
}
+ List<AclEntry> entries = perm.getAclBit() ?
+ item.fs.getAclStatus(item.path).getEntries() :
+ Collections.<AclEntry>emptyList();
+ ScopedAclEntries scopedEntries = new ScopedAclEntries(
+ AclUtil.getAclFromPermAndEntries(perm, entries));
+ printAclEntriesForSingleScope(scopedEntries.getAccessEntries());
+ printAclEntriesForSingleScope(scopedEntries.getDefaultEntries());
out.println();
}
/**
- * Prints an extended ACL, including all extended ACL entries and also the
- * base entries implied by the permission bits.
+ * Prints all the ACL entries in a single scope.
*
- * @param perm FsPermission of file
* @param entries List<AclEntry> containing ACL entries of file
*/
- private void printExtendedAcl(FsPermission perm, List<AclEntry> entries) {
- // Print owner entry implied by owner permission bits.
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.USER)
- .setPermission(perm.getUserAction())
- .build());
-
- // Print all extended access ACL entries.
- boolean hasAccessAcl = false;
- Iterator<AclEntry> entryIter = entries.iterator();
- AclEntry curEntry = null;
- while (entryIter.hasNext()) {
- curEntry = entryIter.next();
- if (curEntry.getScope() == AclEntryScope.DEFAULT) {
- break;
- }
- hasAccessAcl = true;
- printExtendedAclEntry(curEntry, perm.getGroupAction());
+ private void printAclEntriesForSingleScope(List<AclEntry> entries) {
+ if (entries.isEmpty()) {
+ return;
}
-
- // Print mask entry implied by group permission bits, or print group
entry
- // if there is no access ACL (only default ACL).
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(hasAccessAcl ? AclEntryType.MASK : AclEntryType.GROUP)
- .setPermission(perm.getGroupAction())
- .build());
-
- // Print other entry implied by other bits.
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.OTHER)
- .setPermission(perm.getOtherAction())
- .build());
-
- // Print default ACL entries.
- if (curEntry != null && curEntry.getScope() == AclEntryScope.DEFAULT) {
- out.println(curEntry);
- // ACL sort order guarantees default mask is the second-to-last entry.
+ if (AclUtil.isMinimalAcl(entries)) {
+ for (AclEntry entry: entries) {
+ out.println(entry);
+ }
+ } else {
+ // ACL sort order guarantees mask is the second-to-last entry.
FsAction maskPerm = entries.get(entries.size() - 2).getPermission();
- while (entryIter.hasNext()) {
- printExtendedAclEntry(entryIter.next(), maskPerm);
+ for (AclEntry entry: entries) {
+ printExtendedAclEntry(entry, maskPerm);
}
}
}
@@ -180,30 +141,6 @@ class AclCommands extends FsCommand {
out.println(entry);
}
}
-
- /**
- * Prints a minimal ACL, consisting of exactly 3 ACL entries implied by the
- * permission bits.
- *
- * @param perm FsPermission of file
- */
- private void printMinimalAcl(FsPermission perm) {
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.USER)
- .setPermission(perm.getUserAction())
- .build());
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.GROUP)
- .setPermission(perm.getGroupAction())
- .build());
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.OTHER)
- .setPermission(perm.getOtherAction())
- .build());
- }
}
/**
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
Fri May 16 18:50:52 2014
@@ -31,8 +31,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.RpcNoSuchMethodException;
import com.google.common.collect.Sets;
@@ -116,7 +114,7 @@ class Ls extends FsCommand {
FileStatus stat = item.stat;
String line = String.format(lineFormat,
(stat.isDirectory() ? "d" : "-"),
- stat.getPermission() + (hasAcl(item) ? "+" : " "),
+ stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
(stat.isFile() ? stat.getReplication() : "-"),
stat.getOwner(),
stat.getGroup(),
@@ -153,49 +151,6 @@ class Ls extends FsCommand {
lineFormat = fmt.toString();
}
- /**
- * Calls getAclStatus to determine if the given item has an ACL. For
- * compatibility, this method traps errors caused by the RPC method missing
- * from the server side. This would happen if the client was connected to an
- * old NameNode that didn't have the ACL APIs. This method also traps the
- * case of the client-side FileSystem not implementing the ACL APIs.
- * FileSystem instances that do not support ACLs are remembered. This
- * prevents the client from sending multiple failing RPC calls during a
- * recursive ls.
- *
- * @param item PathData item to check
- * @return boolean true if item has an ACL
- * @throws IOException if there is a failure
- */
- private boolean hasAcl(PathData item) throws IOException {
- FileSystem fs = item.fs;
- if (aclNotSupportedFsSet.contains(fs.getUri())) {
- // This FileSystem failed to run the ACL API in an earlier iteration.
- return false;
- }
- try {
- return !fs.getAclStatus(item.path).getEntries().isEmpty();
- } catch (RemoteException e) {
- // If this is a RpcNoSuchMethodException, then the client is connected to
- // an older NameNode that doesn't support ACLs. Keep going.
- IOException e2 = e.unwrapRemoteException(RpcNoSuchMethodException.class);
- if (!(e2 instanceof RpcNoSuchMethodException)) {
- throw e;
- }
- } catch (IOException e) {
- // The NameNode supports ACLs, but they are not enabled. Keep going.
- String message = e.getMessage();
- if (message != null && !message.contains("ACLs has been disabled")) {
- throw e;
- }
- } catch (UnsupportedOperationException e) {
- // The underlying FileSystem doesn't implement ACLs. Keep going.
- }
- // Remember that this FileSystem cannot support ACLs.
- aclNotSupportedFsSet.add(fs.getUri());
- return false;
- }
-
private int maxLength(int n, Object value) {
return Math.max(n, (value != null) ? String.valueOf(value).length() : 0);
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
Fri May 16 18:50:52 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.ha;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Map;
import org.apache.commons.cli.Options;
@@ -33,6 +34,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
import org.apache.hadoop.util.Tool;
@@ -66,7 +68,7 @@ public abstract class HAAdmin extends Co
protected final static Map<String, UsageInfo> USAGE =
ImmutableMap.<String, UsageInfo>builder()
.put("-transitionToActive",
- new UsageInfo("<serviceId>", "Transitions the service into Active
state"))
+ new UsageInfo(" <serviceId> [--"+FORCEACTIVE+"]", "Transitions the
service into Active state"))
.put("-transitionToStandby",
new UsageInfo("<serviceId>", "Transitions the service into Standby
state"))
.put("-failover",
@@ -100,6 +102,10 @@ public abstract class HAAdmin extends Co
}
protected abstract HAServiceTarget resolveTarget(String string);
+
+ protected Collection<String> getTargetIds(String targetNodeToActivate) {
+ return Arrays.asList(new String[]{targetNodeToActivate});
+ }
protected String getUsageString() {
return "Usage: HAAdmin";
@@ -133,6 +139,11 @@ public abstract class HAAdmin extends Co
printUsage(errOut, "-transitionToActive");
return -1;
}
+ /* returns true if other target node is active or some exception occurred
+ and forceActive was not set */
+ if(isOtherTargetNodeActive(argv[0], cmd.hasOption(FORCEACTIVE))) {
+ return -1;
+ }
HAServiceTarget target = resolveTarget(argv[0]);
if (!checkManualStateManagementOK(target)) {
return -1;
@@ -142,7 +153,48 @@ public abstract class HAAdmin extends Co
HAServiceProtocolHelper.transitionToActive(proto, createReqInfo());
return 0;
}
-
+
+ /**
+ * Checks whether other target node is active or not
+ * @param targetNodeToActivate
+ * @return true if other target node is active or some other exception
+ * occurred and forceActive was set otherwise false
+ * @throws IOException
+ */
+ private boolean isOtherTargetNodeActive(String targetNodeToActivate, boolean
forceActive)
+ throws IOException {
+ Collection<String> targetIds = getTargetIds(targetNodeToActivate);
+ if(targetIds == null) {
+ errOut.println("transitionToActive: No target node in the "
+ + "current configuration");
+ printUsage(errOut, "-transitionToActive");
+ return true;
+ }
+ targetIds.remove(targetNodeToActivate);
+ for(String targetId : targetIds) {
+ HAServiceTarget target = resolveTarget(targetId);
+ if (!checkManualStateManagementOK(target)) {
+ return true;
+ }
+ try {
+ HAServiceProtocol proto = target.getProxy(getConf(), 5000);
+ if(proto.getServiceStatus().getState() == HAServiceState.ACTIVE) {
+ errOut.println("transitionToActive: Node " + targetId +" is already
active");
+ printUsage(errOut, "-transitionToActive");
+ return true;
+ }
+ } catch (Exception e) {
+ //If forceActive switch is false then return true
+ if(!forceActive) {
+ errOut.println("Unexpected error occurred " + e.getMessage());
+ printUsage(errOut, "-transitionToActive");
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
private int transitionToStandby(final CommandLine cmd)
throws IOException, ServiceFailedException {
String[] argv = cmd.getArgs();
@@ -364,6 +416,9 @@ public abstract class HAAdmin extends Co
if ("-failover".equals(cmd)) {
addFailoverCliOpts(opts);
}
+ if("-transitionToActive".equals(cmd)) {
+ addTransitionToActiveCliOpts(opts);
+ }
// Mutative commands take FORCEMANUAL option
if ("-transitionToActive".equals(cmd) ||
"-transitionToStandby".equals(cmd) ||
@@ -433,6 +488,14 @@ public abstract class HAAdmin extends Co
// that change state.
}
+ /**
+ * Add CLI options which are specific to the transitionToActive command and
+ * no others.
+ */
+ private void addTransitionToActiveCliOpts(Options transitionToActiveCliOpts)
{
+ transitionToActiveCliOpts.addOption(FORCEACTIVE, false, "force active");
+ }
+
private CommandLine parseOpts(String cmdName, Options opts, String[] argv) {
try {
// Strip off the first arg, since that's just the command name
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
Fri May 16 18:50:52 2014
@@ -74,7 +74,8 @@ public class ShellBasedUnixGroupsMapping
* Get the current user's group list from Unix by running the command
'groups'
* NOTE. For non-existing user it will return EMPTY list
* @param user user name
- * @return the groups list that the <code>user</code> belongs to
+ * @return the groups list that the <code>user</code> belongs to. The primary
+ * group is returned first.
* @throws IOException if encounter any error when running the command
*/
private static List<String> getUnixGroups(final String user) throws
IOException {
@@ -84,6 +85,7 @@ public class ShellBasedUnixGroupsMapping
} catch (ExitCodeException e) {
// if we didn't get the group - just return empty list;
LOG.warn("got exception trying to get groups for user " + user, e);
+ return new LinkedList<String>();
}
StringTokenizer tokenizer =
@@ -92,6 +94,17 @@ public class ShellBasedUnixGroupsMapping
while (tokenizer.hasMoreTokens()) {
groups.add(tokenizer.nextToken());
}
+
+ // remove duplicated primary group
+ if (!Shell.WINDOWS) {
+ for (int i = 1; i < groups.size(); i++) {
+ if (groups.get(i).equals(groups.get(0))) {
+ groups.remove(i);
+ break;
+ }
+ }
+ }
+
return groups;
}
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Fri May 16 18:50:52 2014
@@ -132,11 +132,17 @@ abstract public class Shell {
: new String[]{"bash", "-c", "groups"};
}
- /** a Unix command to get a given user's groups list */
+ /**
+ * a Unix command to get a given user's groups list.
+ * If the OS is not WINDOWS, the command will get the user's primary group
+ * first and finally get the groups list which includes the primary group.
+ * i.e. the user's primary group will be included twice.
+ */
public static String[] getGroupsForUserCommand(final String user) {
//'groups username' command return is non-consistent across different
unixes
return (WINDOWS)? new String[] { WINUTILS, "groups", "-F", "\"" + user +
"\""}
- : new String [] {"bash", "-c", "id -Gn " + user};
+ : new String [] {"bash", "-c", "id -gn " + user
+ + "&& id -Gn " + user};
}
/** a Unix command to get a given netgroup's user list */
Propchange:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1594376-1595301
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
Fri May 16 18:50:52 2014
@@ -111,6 +111,30 @@ public class TestKeyShell {
assertFalse(outContent.toString(), outContent.toString().contains("key1"));
}
+ /* HADOOP-10586 KeyShell didn't allow -description. */
+ @Test
+ public void testKeySuccessfulCreationWithDescription() throws Exception {
+ outContent.reset();
+ String[] args1 = {"create", "key1", "--provider",
+ "jceks://file" + tmpDir + "/keystore.jceks",
+ "--description", "someDescription"};
+ int rc = 0;
+ KeyShell ks = new KeyShell();
+ ks.setConf(new Configuration());
+ rc = ks.run(args1);
+ assertEquals(0, rc);
+ assertTrue(outContent.toString().contains("key1 has been successfully " +
+ "created."));
+
+ outContent.reset();
+ String[] args2a = {"list", "--metadata", "--provider",
+ "jceks://file" + tmpDir + "/keystore.jceks"};
+ rc = ks.run(args2a);
+ assertEquals(0, rc);
+ assertTrue(outContent.toString().contains("description"));
+ assertTrue(outContent.toString().contains("someDescription"));
+ }
+
@Test
public void testInvalidKeySize() throws Exception {
String[] args1 = {"create", "key1", "--size", "56", "--provider",
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
Fri May 16 18:50:52 2014
@@ -113,7 +113,23 @@ public class IdUserGroup {
"The new entry is to be ignored for the following reason.",
DUPLICATE_NAME_ID_DEBUG_INFO));
}
-
+
+ /**
+ * uid and gid are defined as uint32 in linux. Some systems create
+ * (intended or unintended) <nfsnobody, 4294967294> kind of <name,Id>
+ * mapping, where 4294967294 is 2**32-2 as unsigned int32. As an example,
+ * https://bugzilla.redhat.com/show_bug.cgi?id=511876.
+ * Because user or group id are treated as Integer (signed integer or int32)
+ * here, the number 4294967294 is out of range. The solution is to convert
+ * uint32 to int32, so to map the out-of-range ID to the negative side of
+ * Integer, e.g. 4294967294 maps to -2 and 4294967295 maps to -1.
+ */
+ private static Integer parseId(final String idStr) {
+ Long longVal = Long.parseLong(idStr);
+ int intVal = longVal.intValue();
+ return Integer.valueOf(intVal);
+ }
+
/**
* Get the whole list of users and groups and save them in the maps.
* @throws IOException
@@ -134,8 +150,8 @@ public class IdUserGroup {
}
LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" +
nameId[1]);
// HDFS can't differentiate duplicate names with simple authentication
- final Integer key = Integer.valueOf(nameId[1]);
- final String value = nameId[0];
+ final Integer key = parseId(nameId[1]);
+ final String value = nameId[0];
if (map.containsKey(key)) {
final String prevValue = map.get(key);
if (value.equals(prevValue)) {
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java?rev=1595303&r1=1595302&r2=1595303&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java
Fri May 16 18:50:52 2014
@@ -67,6 +67,51 @@ public class TestIdUserGroup {
}
@Test
+ public void testIdOutOfIntegerRange() throws IOException {
+ String GET_ALL_USERS_CMD = "echo \""
+ + "nfsnobody:x:4294967294:4294967294:Anonymous NFS
User:/var/lib/nfs:/sbin/nologin\n"
+ + "nfsnobody1:x:4294967295:4294967295:Anonymous NFS
User:/var/lib/nfs1:/sbin/nologin\n"
+ + "maxint:x:2147483647:2147483647:Grid Distributed File
System:/home/maxint:/bin/bash\n"
+ + "minint:x:2147483648:2147483648:Grid Distributed File
System:/home/minint:/bin/bash\n"
+ + "archivebackup:*:1031:4294967294:Archive
Backup:/home/users/archivebackup:/bin/sh\n"
+ + "hdfs:x:11501:10787:Grid Distributed File
System:/home/hdfs:/bin/bash\n"
+ + "daemon:x:2:2:daemon:/sbin:/sbin/nologin\""
+ + " | cut -d: -f1,3";
+ String GET_ALL_GROUPS_CMD = "echo \""
+ + "hdfs:*:11501:hrt_hdfs\n"
+ + "rpcuser:*:29:\n"
+ + "nfsnobody:*:4294967294:\n"
+ + "nfsnobody1:*:4294967295:\n"
+ + "maxint:*:2147483647:\n"
+ + "minint:*:2147483648:\n"
+ + "mapred3:x:498\""
+ + " | cut -d: -f1,3";
+ // Maps for id to name map
+ BiMap<Integer, String> uMap = HashBiMap.create();
+ BiMap<Integer, String> gMap = HashBiMap.create();
+
+ IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":");
+ assertTrue(uMap.size() == 7);
+ assertEquals("nfsnobody", uMap.get(-2));
+ assertEquals("nfsnobody1", uMap.get(-1));
+ assertEquals("maxint", uMap.get(2147483647));
+ assertEquals("minint", uMap.get(-2147483648));
+ assertEquals("archivebackup", uMap.get(1031));
+ assertEquals("hdfs",uMap.get(11501));
+ assertEquals("daemon", uMap.get(2));
+
+ IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":");
+ assertTrue(gMap.size() == 7);
+ assertEquals("hdfs",gMap.get(11501));
+ assertEquals("rpcuser", gMap.get(29));
+ assertEquals("nfsnobody", gMap.get(-2));
+ assertEquals("nfsnobody1", gMap.get(-1));
+ assertEquals("maxint", gMap.get(2147483647));
+ assertEquals("minint", gMap.get(-2147483648));
+ assertEquals("mapred3", gMap.get(498));
+ }
+
+ @Test
public void testUserUpdateSetting() throws IOException {
IdUserGroup iug = new IdUserGroup();
assertEquals(iug.getTimeout(), IdUserGroup.TIMEOUT_DEFAULT);
