Author: sradia
Date: Fri Apr 6 22:17:08 2012
New Revision: 1310617
URL: http://svn.apache.org/viewvc?rev=1310617&view=rev
Log:
HADOOP-8234 - Enable user group mappings on Windows (Bikas Saha via Sanjay)
Modified:
hadoop/common/branches/branch-1-win/CHANGES.txt
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/Credentials.java
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.txt?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.txt Fri Apr 6 22:17:08 2012
@@ -9,6 +9,8 @@ branch-hadoop-1-win - unreleased
HADOOP-8223 - Initial patch for branch-1-win (David Lao via Sanjay)
+ HADOOP-8234 - Enable user group mappings on Windows (Bikas Saha via Sanjay)
+
Release 1.1.0 - unreleased
NEW FEATURES
Modified:
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/Credentials.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/Credentials.java?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/Credentials.java
(original)
+++
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/Credentials.java
Fri Apr 6 22:17:08 2012
@@ -126,8 +126,6 @@ public class Credentials implements Writ
) throws IOException {
FSDataInputStream in = null;
Credentials credentials = new Credentials();
- if (Shell.DISABLEWINDOWS_TEMPORARILY)
- return credentials;
try {
in = filename.getFileSystem(conf).open(filename);
credentials.readTokenStorageStream(in);
Modified:
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
(original)
+++
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
Fri Apr 6 22:17:08 2012
@@ -20,11 +20,7 @@ package org.apache.hadoop.security;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.HashSet;
import java.util.StringTokenizer;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,7 +29,7 @@ import org.apache.hadoop.util.Shell.Exit
/**
* A simple shell-based implementation of {@link GroupMappingServiceProvider}
- * that exec's the <code>groups</code> shell command to fetch the group
+ * that exec's a shell command to fetch the group
* memberships of a given user.
*/
public class ShellBasedUnixGroupsMapping implements
GroupMappingServiceProvider {
@@ -42,7 +38,7 @@ public class ShellBasedUnixGroupsMapping
@Override
public List<String> getGroups(String user) throws IOException {
- return getUnixGroups(user);
+ return getUserGroups(user);
}
@Override
@@ -62,24 +58,53 @@ public class ShellBasedUnixGroupsMapping
* @return the groups list that the <code>user</code> belongs to
* @throws IOException if encounter any error when running the command
*/
- private static List<String> getUnixGroups(final String user) throws
IOException {
- String result = "";
- if (Shell.DISABLEWINDOWS_TEMPORARILY)
- result = "hadoopusers";
+ private static List<String> getUserGroups(final String user) throws
IOException {
+ List<String> groups = new LinkedList<String>();
+ if (Shell.WINDOWS) {
+ String result = Shell.execCommand(Shell.getGroupsForUserCommand(user));
+ String[] lines = result.split("\\r\\n");
+ String line = lines[0];
+ if (!line.startsWith("User name")) {
+ throw new IOException(
+ "Command result did not start with \"User name\"");
+ }
+ String[] splits = line.substring(9).split("\\s");
+ if (splits.length == 0 || !splits[splits.length-1].equals(user)) {
+ throw new IOException("Bad user name returned");
+ }
+ for (int i=1; i<lines.length; ++i) {
+ line = lines[i];
+ // not handling global group memberships now
+ // it might be better to handle them via a specific domain controller
+ // plugin
+ if (line.startsWith("Local Group Memberships")) {
+ splits = line.substring(23).split("\\s");
+ for (String group : splits) {
+ if (group.length() > 0) {
+ if (group.charAt(0) == '*') {
+ group = group.substring(1);
+ }
+ groups.add(group);
+ }
+ }
+ }
+ }
+ }
else {
+ String result = "";
try {
result = Shell.execCommand(Shell.getGroupsForUserCommand(user));
} catch (ExitCodeException e) {
// if we didn't get the group - just return empty list;
LOG.warn("got exception trying to get groups for user " + user, e);
}
+ StringTokenizer tokenizer = new StringTokenizer(result);
+
+ while (tokenizer.hasMoreTokens()) {
+ groups.add(tokenizer.nextToken());
+ }
}
-
- StringTokenizer tokenizer = new StringTokenizer(result);
- List<String> groups = new LinkedList<String>();
- while (tokenizer.hasMoreTokens()) {
- groups.add(tokenizer.nextToken());
- }
+
return groups;
}
}
Modified:
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
Fri Apr 6 22:17:08 2012
@@ -431,7 +431,7 @@ public class UserGroupInformation {
static UserGroupInformation getCurrentUser() throws IOException {
AccessControlContext context = AccessController.getContext();
Subject subject = Subject.getSubject(context);
- return subject == null ? getLoginUser() : Shell.WINDOWS? getLoginUser() :
new UserGroupInformation(subject);
+ return subject == null ? getLoginUser() : new
UserGroupInformation(subject);
}
/**
Modified:
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java
Fri Apr 6 22:17:08 2012
@@ -51,7 +51,7 @@ abstract public class Shell {
/** a Unix command to get a given user's groups list */
public static String[] getGroupsForUserCommand(final String user) {
//'groups username' command return is non-consistent across different
unixes
- return (WINDOWS)? new String[] {"cmd", "/c", "id -Gn " + user}:
+ return (WINDOWS)? new String[] {"cmd", "/c", "net user " + user}:
new String [] {"bash", "-c", "id -Gn " + user};
}
/** a Unix command to get a given netgroup's user list */
Modified:
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java
(original)
+++
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java
Fri Apr 6 22:17:08 2012
@@ -89,14 +89,19 @@ class Child {
// file name is passed thru env
String jobTokenFile =
System.getenv().get(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
+ if(Shell.WINDOWS) {
+ if(jobTokenFile.charAt(0)=='"')
+ jobTokenFile = jobTokenFile.substring(1);
+ if(jobTokenFile.charAt(jobTokenFile.length()-1) == '"')
+ jobTokenFile = jobTokenFile.substring(0, jobTokenFile.length()-1);
+ }
Credentials credentials =
TokenCache.loadTokens(jobTokenFile, defaultConf);
LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() +
"; from file=" + jobTokenFile);
Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
- if (!Shell.WINDOWS)
- SecurityUtil.setTokenService(jt, address);
+ SecurityUtil.setTokenService(jt, address);
UserGroupInformation current = UserGroupInformation.getCurrentUser();
current.addToken(jt);
@@ -210,9 +215,8 @@ class Child {
job.setBoolean("fs.file.impl.disable.cache", false);
// set the jobTokenFile into task
- if (!Shell.WINDOWS)
- task.setJobTokenSecret(JobTokenSecretManager.
- createSecretKey(jt.getPassword()));
+ task.setJobTokenSecret(JobTokenSecretManager.
+ createSecretKey(jt.getPassword()));
// setup the child's mapred-local-dir. The child is now sandboxed and
// can only see files down and under attemtdir only.
Modified:
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Apr 6 22:17:08 2012
@@ -489,8 +489,7 @@ public class JobInProgress {
//At this point, this constructor is called in the context of an RPC, and
//hence the "current user" is actually referring to the kerberos
//authenticated user (if security is ON).
- if (!Shell.DISABLEWINDOWS_TEMPORARILY)
- FileSystem.closeAllForUGI(UserGroupInformation.getCurrentUser());
+ FileSystem.closeAllForUGI(UserGroupInformation.getCurrentUser());
}
}
Modified:
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
(original)
+++
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Fri Apr 6 22:17:08 2012
@@ -1485,11 +1485,7 @@ class ReduceTask extends Task {
URL url = mapOutputLoc.getOutputLocation();
URLConnection connection = url.openConnection();
- InputStream input;
- if (Shell.WINDOWS)
- input = getInputStream(connection, shuffleConnectionTimeout,
shuffleReadTimeout);
- else
- input = setupSecureConnection(mapOutputLoc, connection);
+ InputStream input = setupSecureConnection(mapOutputLoc, connection);
// Validate header from map output
TaskAttemptID mapId = null;
@@ -1666,10 +1662,7 @@ class ReduceTask extends Task {
// Reconnect
try {
connection = mapOutputLoc.getOutputLocation().openConnection();
- if (Shell.WINDOWS)
- input = getInputStream(connection, shuffleConnectionTimeout,
shuffleReadTimeout);
- else
- input = setupSecureConnection(mapOutputLoc, connection);
+ input = setupSecureConnection(mapOutputLoc, connection);
} catch (IOException ioe) {
LOG.info("Failed reopen connection to fetch map-output from " +
mapOutputLoc.getHost());
Modified:
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1310617&r1=1310616&r2=1310617&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Fri Apr 6 22:17:08 2012
@@ -3266,8 +3266,6 @@ public class TaskTracker implements MRCo
private void authorizeJVM(org.apache.hadoop.mapreduce.JobID jobId)
throws IOException {
- if (Shell.DISABLEWINDOWS_TEMPORARILY)
- return;
String currentJobId =
UserGroupInformation.getCurrentUser().getUserName();
if (!currentJobId.equals(jobId.toString())) {
@@ -3286,8 +3284,7 @@ public class TaskTracker implements MRCo
*/
public synchronized JvmTask getTask(JvmContext context)
throws IOException {
- if (!Shell.WINDOWS)
- authorizeJVM(context.jvmId.getJobId());
+ authorizeJVM(context.jvmId.getJobId());
JVMId jvmId = context.jvmId;
LOG.debug("JVM with ID : " + jvmId + " asked for a task");
// save pid of task JVM sent by child
@@ -3815,8 +3812,7 @@ public class TaskTracker implements MRCo
String exceptionMsgRegex =
(String) context.getAttribute("exceptionMsgRegex");
- if (!Shell.WINDOWS)
- verifyRequest(request, response, tracker, jobId);
+ verifyRequest(request, response, tracker, jobId);
long startTime = 0;
try {
