Author: omalley
Date: Tue Mar 8 05:59:22 2011
New Revision: 1079240
URL: http://svn.apache.org/viewvc?rev=1079240&view=rev
Log:
commit fe88ef0e0c14fb683463de1799c22d03a1f5050e
Author: Ravi Gummadi <[email protected]>
Date: Tue Jan 11 20:24:51 2011 +0530
: Fix a bug in RoundRobinUserResolver sothat each user from
trace is mapped to a single proxy user. Also makes it nonmandatory for
users-list-file to contain groups. Patch is available at
(gravi)
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java?rev=1079240&r1=1079239&r2=1079240&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
Tue Mar 8 05:59:22 2011
@@ -38,12 +38,18 @@ public class RoundRobinUserResolver impl
private int uidx = 0;
private List<UserGroupInformation> users = Collections.emptyList();
- private final HashMap<UserGroupInformation,UserGroupInformation> usercache =
- new HashMap<UserGroupInformation,UserGroupInformation>();
+
+ /**
+ * Mapping between user names of original cluster and UGIs of proxy users of
+ * simulated cluster
+ */
+ private final HashMap<String,UserGroupInformation> usercache =
+ new HashMap<String,UserGroupInformation>();
/**
- * Userlist assumes one UGI per line, each UGI matching
- * <username>,<group>[,group]*
+ * Userlist assumes one user per line.
+ * Each line in users-list-file is of the form <username>[,group]*
+ * <br> Group names are ignored(they are not read at all).
*/
private List<UserGroupInformation> parseUserList(URI userUri,
Configuration conf)
@@ -53,47 +59,53 @@ public class RoundRobinUserResolver impl
}
final Path userloc = new Path(userUri.toString());
- final Text rawUgi = new Text();
+ final Text rawUgi = new Text();// is of the form username[,group]*
final FileSystem fs = userloc.getFileSystem(conf);
- final ArrayList<UserGroupInformation> ret = new ArrayList();
+ final ArrayList<UserGroupInformation> ugiList =
+ new ArrayList<UserGroupInformation>();
LineReader in = null;
try {
- final ArrayList<String> groups = new ArrayList();
in = new LineReader(fs.open(userloc));
while (in.readLine(rawUgi) > 0) {
+ // e is end position of user name in this line
int e = rawUgi.find(",");
- if (e <= 0) {
+ if (rawUgi.getLength() == 0 || e == 0) {
throw new IOException("Missing username: " + rawUgi);
}
+ if (e == -1) {
+ e = rawUgi.getLength();
+ }
final String username = Text.decode(rawUgi.getBytes(), 0, e);
- int s = e;
- while ((e = rawUgi.find(",", ++s)) != -1) {
- groups.add(Text.decode(rawUgi.getBytes(), s, e - s));
- s = e;
+ UserGroupInformation ugi = null;
+ try {
+ ugi = UserGroupInformation.createProxyUser(
+ username, UserGroupInformation.getLoginUser());
+ } catch (IOException ioe) {
+ LOG.error("Error while creating a proxy user " ,ioe);
}
- groups.add(Text.decode(rawUgi.getBytes(), s, rawUgi.getLength() - s));
- if (groups.size() == 0) {
- throw new IOException("Missing groups: " + rawUgi);
+ if (ugi != null) {
+ ugiList.add(ugi);
}
- ret.add(UserGroupInformation.createRemoteUser(username));
+ // No need to read groups, even if they exist. Go to next line
}
} finally {
if (in != null) {
in.close();
}
}
- return ret;
+ return ugiList;
}
@Override
public synchronized boolean setTargetUsers(URI userloc, Configuration conf)
throws IOException {
+ uidx = 0;
users = parseUserList(userloc, conf);
if (users.size() == 0) {
throw new IOException(buildEmptyUsersErrorMsg(userloc));
}
- usercache.keySet().retainAll(users);
+ usercache.clear();
return true;
}
@@ -105,20 +117,13 @@ public class RoundRobinUserResolver impl
@Override
public synchronized UserGroupInformation getTargetUgi(
UserGroupInformation ugi) {
- UserGroupInformation ret = usercache.get(ugi);
- if (null == ret) {
- ret = users.get(uidx++ % users.size());
- usercache.put(ugi, ret);
- }
- UserGroupInformation val = null;
- try {
- val =
- UserGroupInformation.createProxyUser(ret.getUserName(),
- UserGroupInformation.getLoginUser());
- } catch (IOException e) {
- LOG.error("Error while creating the proxy user " ,e);
+ // UGI of proxy user
+ UserGroupInformation targetUGI = usercache.get(ugi.getUserName());
+ if (null == targetUGI) {
+ targetUGI = users.get(uidx++ % users.size());
+ usercache.put(ugi.getUserName(), targetUGI);
}
- return val;
+ return targetUGI;
}
/**
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java?rev=1079240&r1=1079239&r2=1079240&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
Tue Mar 8 05:59:22 2011
@@ -115,27 +115,47 @@ public class TestUserResolve {
RoundRobinUserResolver.buildEmptyUsersErrorMsg(userRsrc);
validateBadUsersFile(rslv, userRsrc, expectedErrorMsg);
- // Create user resource file with valid content
+ // Create user resource file with valid content(older users list file)
writeUserList(usersFilePath,
- "user0,groupA,groupB,groupC\nuser1,groupA,groupC\n"
- + "user2,groupB\nuser3,groupA,groupB,groupC\n");
+ "user0,groupA,groupB,groupC\nuser1,groupA,groupC");
+ validateValidUsersFile(rslv, userRsrc);
+
+ // Create user resource file with valid content
+ writeUserList(usersFilePath, "user0,groupA,groupB\nuser1,");
+ validateValidUsersFile(rslv, userRsrc);
- // Validate RoundRobinUserResolver for the case of
- // user resource file with valid content.
- assertTrue(rslv.setTargetUsers(new URI(usersFilePath.toString()), conf));
+ // Create user resource file with valid content
+ writeUserList(usersFilePath, "user0\nuser1");
+ validateValidUsersFile(rslv, userRsrc);
+ }
+
+ // Validate RoundRobinUserResolver for the case of
+ // user resource file with valid content.
+ private void validateValidUsersFile(UserResolver rslv, URI userRsrc)
+ throws IOException {
+ assertTrue(rslv.setTargetUsers(userRsrc, conf));
UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("hfre0");
assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
assertEquals("user1",
rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre1"))
.getUserName());
- assertEquals("user2",
+ assertEquals("user0",
rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre2"))
.getUserName());
assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
- assertEquals("user3",
+ assertEquals("user1",
rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre3"))
.getUserName());
- assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
+ // Verify if same user comes again, its mapped user name should be
+ // correct even though UGI is constructed again.
+ assertEquals("user0", rslv.getTargetUgi(
+ UserGroupInformation.createRemoteUser("hfre0")).getUserName());
+ assertEquals("user0",
+ rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre5"))
+ .getUserName());
+ assertEquals("user0",
+ rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre0"))
+ .getUserName());
}
@Test
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml?rev=1079240&r1=1079239&r2=1079240&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml
Tue Mar 8 05:59:22 2011
@@ -480,28 +480,27 @@ hadoop jar <gridmix-jar> org.apach
</table>
<p>If the parameter <code>gridmix.user.resolve.class</code> is set to
<code>org.apache.hadoop.mapred.gridmix.RoundRobinUserResolver</code>,
- we need to define a users-list file with a list of test users and groups.
+ we need to define a users-list file with a list of test users.
This is specified using the <code>-users</code> option to GridMix.</p>
<note>
Specifying a users-list file using the <code>-users</code> option is
mandatory when using the round-robin user-resolver. Other user-resolvers
ignore this option.
</note>
- <p>A users-list file has one user-group-information (UGI) per line, each
- UGI of the format:</p>
+ <p>A users-list file has one user per line, each line of the format:</p>
<source>
- <username>,<group>[,group]*
+ <username>
</source>
<p>For example:</p>
<source>
- user1,group1
- user2,group2,group3
- user3,group3,group4
+ user1
+ user2
+ user3
</source>
<p>In the above example we have defined three users <code>user1</code>,
- <code>user2</code> and <code>user3</code> with their respective groups.
+ <code>user2</code> and <code>user3</code>.
Now we would associate each unique user in the trace to the above users
- defined in round-robin fashion. For example, if traces users are
+ defined in round-robin fashion. For example, if trace's users are
<code>tuser1</code>, <code>tuser2</code>, <code>tuser3</code>,
<code>tuser4</code> and <code>tuser5</code>, then the mappings would
be:</p>
