HBASE-10831 IntegrationTestIngestWithACL is not setting up LoadTestTool correctly (Vandana Ayyalasomayajula)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d89342e4 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d89342e4 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d89342e4 Branch: refs/heads/0.98 Commit: d89342e4166c3f969d79e6be57d7efd0c37cd5b2 Parents: c93ceed Author: Andrew Purtell <[email protected]> Authored: Thu May 22 18:56:20 2014 -0700 Committer: Andrew Purtell <[email protected]> Committed: Thu May 22 18:56:31 2014 -0700 ---------------------------------------------------------------------- .../hbase/IntegrationTestIngestWithACL.java | 32 ++++++++- .../apache/hadoop/hbase/util/LoadTestTool.java | 76 +++++++++++++++++--- .../hbase/util/MultiThreadedReaderWithACL.java | 7 +- .../hbase/util/MultiThreadedUpdaterWithACL.java | 19 +++-- 4 files changed, 116 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/d89342e4/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java index 6b369f0..d635b98 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.List; import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.security.access.AccessController; @@ -45,8 +46,10 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest { private static final int SPECIAL_PERM_CELL_INSERTION_FACTOR = 100; public static final String OPT_SUPERUSER = "superuser"; public static final String OPT_USERS = "userlist"; + public static final String OPT_AUTHN = "authinfo"; private String superUser = "owner"; private String userNames = "user1,user2,user3,user4"; + private String authnFileName; @Override public void setUpCluster() throws Exception { util = getTestingUtil(null); @@ -66,6 +69,10 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest { tmp.add(HYPHEN + LoadTestTool.OPT_GENERATOR); StringBuilder sb = new StringBuilder(LoadTestDataGeneratorWithACL.class.getName()); sb.append(COLON); + if (LoadTestTool.isSecure(getConf())) { + sb.append(authnFileName); + sb.append(COLON); + } sb.append(superUser); sb.append(COLON); sb.append(userNames); @@ -80,7 +87,15 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest { super.addOptWithArg(OPT_SUPERUSER, "Super user name used to add the ACL permissions"); super.addOptWithArg(OPT_USERS, - "List of users to be added with the ACLs. Should be comma seperated."); + "List of users to be added with the ACLs. Should be comma seperated."); + super + .addOptWithArg( + OPT_AUTHN, + "The name of the properties file that contains kerberos key tab file and principal definitions. " + + "The principal key in the file should be of the form hbase.<username>.kerberos.principal." + + " The keytab key in the file should be of the form hbase.<username>.keytab.file. Example: " + + "hbase.user1.kerberos.principal=user1/[email protected], " + + "hbase.user1.keytab.file=<filelocation>."); } @Override @@ -92,6 +107,21 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest { if (cmd.hasOption(OPT_USERS)) { userNames = cmd.getOptionValue(OPT_USERS); } + if (LoadTestTool.isSecure(getConf())) { + boolean authFileNotFound = false; + if (cmd.hasOption(OPT_AUTHN)) { + authnFileName = cmd.getOptionValue(OPT_AUTHN); + if (StringUtils.isEmpty(authnFileName)) { + authFileNotFound = true; + } + } else { + authFileNotFound = true; + } + if (authFileNotFound) { + super.printUsage(); + System.exit(EXIT_FAILURE); + } + } } public static void main(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/hbase/blob/d89342e4/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java index ead7a19..a43af55 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java @@ -19,10 +19,12 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; import java.io.InterruptedIOException; import java.lang.reflect.Constructor; +import java.net.InetAddress; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Properties; import java.util.Random; import java.util.concurrent.atomic.AtomicReference; @@ -31,6 +33,7 @@ import javax.crypto.spec.SecretKeySpec; import org.apache.commons.cli.CommandLine; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -51,6 +54,8 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessControlClient; import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator; import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; /** @@ -185,6 +190,8 @@ public class LoadTestTool extends AbstractHBaseTool { private String superUser; private String userNames = "user1, user2, user3, user4"; + //This file is used to read authentication information in secure clusters. + private String authnFileName; // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad, // console tool itself should only be used from console. @@ -473,14 +480,30 @@ public class LoadTestTool extends AbstractHBaseTool { if (cmd.hasOption(OPT_GENERATOR)) { String[] clazzAndArgs = cmd.getOptionValue(OPT_GENERATOR).split(COLON); dataGen = getLoadGeneratorInstance(clazzAndArgs[0]); - String args[]; + String[] args; if (dataGen instanceof LoadTestDataGeneratorWithACL) { LOG.info("ACL is on"); - superUser = clazzAndArgs[1]; - userNames = clazzAndArgs[2]; - args = Arrays.copyOfRange(clazzAndArgs, 1, - clazzAndArgs.length); - userOwner = User.createUserForTesting(conf, superUser, new String[0]); + if (isSecure(conf)) { + LOG.info("Security is on."); + authnFileName = clazzAndArgs[1]; + superUser = clazzAndArgs[2]; + userNames = clazzAndArgs[3]; + args = Arrays.copyOfRange(clazzAndArgs, 2, clazzAndArgs.length); + Properties authConfig = new Properties(); + authConfig.load(this.getClass().getClassLoader().getResourceAsStream(authnFileName)); + try { + addAuthInfoToConf(authConfig, conf, superUser, userNames); + } catch (IOException exp) { + LOG.error(exp); + return EXIT_FAILURE; + } + userOwner = User.create(loginAndReturnUGI(conf, superUser)); + } else { + superUser = clazzAndArgs[1]; + userNames = clazzAndArgs[2]; + args = Arrays.copyOfRange(clazzAndArgs, 1, clazzAndArgs.length); + userOwner = User.createUserForTesting(conf, superUser, new String[0]); + } } else { args = clazzAndArgs.length == 1 ? new String[0] : Arrays.copyOfRange(clazzAndArgs, 1, clazzAndArgs.length); @@ -492,9 +515,7 @@ public class LoadTestTool extends AbstractHBaseTool { minColsPerKey, maxColsPerKey, COLUMN_FAMILY); } - if(userOwner != null) { - conf.set("hadoop.security.authorization", "false"); - conf.set("hadoop.security.authentication", "simple"); + if (userOwner != null) { LOG.info("Granting permission for the user " + userOwner.getShortName()); HTable table = new HTable(conf, tableName); AccessControlProtos.Permission.Action[] actions = { @@ -738,4 +759,41 @@ public class LoadTestTool extends AbstractHBaseTool { } } } + + private void addAuthInfoToConf(Properties authConfig, Configuration conf, String owner, + String userList) throws IOException { + List<String> users = Arrays.asList(userList.split(",")); + users.add(owner); + for (String user : users) { + String keyTabFileConfKey = "hbase." + user + ".keytab.file"; + String principalConfKey = "hbase." + user + ".kerberos.principal"; + if (!authConfig.containsKey(keyTabFileConfKey) || !authConfig.containsKey(principalConfKey)) { + throw new IOException("Authentication configs missing for user : " + user); + } + } + for (String key : authConfig.stringPropertyNames()) { + conf.set(key, authConfig.getProperty(key)); + } + LOG.debug("Added authentication properties to config successfully."); + } + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username) + throws IOException { + String hostname = InetAddress.getLocalHost().getHostName(); + String keyTabFileConfKey = "hbase." + username + ".keytab.file"; + String keyTabFileLocation = conf.get(keyTabFileConfKey); + String principalConfKey = "hbase." + username + ".kerberos.principal"; + String principal = SecurityUtil.getServerPrincipal(conf.get(principalConfKey), hostname); + if (keyTabFileLocation == null || principal == null) { + LOG.warn("Principal or key tab file null for : " + principalConfKey + ", " + + keyTabFileConfKey); + } + UserGroupInformation ugi = + UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation); + return ugi; + } + + public static boolean isSecure(Configuration conf) { + return ("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"))); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/d89342e4/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java index 0067003..9f57453 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java @@ -119,8 +119,13 @@ public class MultiThreadedReaderWithACL extends MultiThreadedReader { if (userNames != null && userNames.length > 0) { int mod = ((int) keyToRead % userNames.length); User user; + UserGroupInformation realUserUgi; if(!users.containsKey(userNames[mod])) { - UserGroupInformation realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); + if(LoadTestTool.isSecure(conf)) { + realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]); + } else { + realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); + } user = User.create(realUserUgi); users.put(userNames[mod], user); } else { http://git-wip-us.apache.org/repos/asf/hbase/blob/d89342e4/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java index b4e24e5..8ea6d04 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java @@ -133,14 +133,19 @@ public class MultiThreadedUpdaterWithACL extends MultiThreadedUpdater { if (userNames != null && userNames.length > 0) { int mod = ((int) rowKeyBase % userNames.length); User user; - if(!users.containsKey(userNames[mod])) { - UserGroupInformation realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); - user = User.create(realUserUgi); - users.put(userNames[mod], user); - } else { - user = users.get(userNames[mod]); - } + UserGroupInformation realUserUgi; try { + if (!users.containsKey(userNames[mod])) { + if (LoadTestTool.isSecure(conf)) { + realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]); + } else { + realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); + } + user = User.create(realUserUgi); + users.put(userNames[mod], user); + } else { + user = users.get(userNames[mod]); + } Result result = (Result) user.runAs(action); return result; } catch (Exception ie) {
