Repository: sentry Updated Branches: refs/heads/master 58c7c888e -> 1b0596142 (forced update)
SENTRY-1164: fix testCaseSensitivity test failure on a real cluster and also a minor improvements to testConcurrentClients to run locally. (Anne Yu, reviewed by Haohao). Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/1b059614 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/1b059614 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/1b059614 Branch: refs/heads/master Commit: 1b0596142ddc93523c2edd91c8179489e9aea759 Parents: bfb354f Author: Anne Yu <[email protected]> Authored: Tue Mar 29 16:29:50 2016 -0700 Committer: Anne Yu <[email protected]> Committed: Mon Apr 4 14:37:05 2016 -0700 ---------------------------------------------------------------------- .../e2e/dbprovider/TestConcurrentClients.java | 96 ++++++++++++++++---- .../AbstractTestWithStaticConfiguration.java | 4 +- .../sentry/tests/e2e/hive/TestOperations.java | 20 +++- 3 files changed, 97 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/1b059614/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java index bf871f0..13b6e62 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java @@ -16,15 +16,24 @@ */ package org.apache.sentry.tests.e2e.dbprovider; +import org.apache.hadoop.hive.conf.HiveConf; + import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient; +import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; +import org.apache.sentry.provider.db.service.thrift.TSentryRole; import org.apache.sentry.provider.file.PolicyFile; import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration; import org.apache.sentry.tests.e2e.hive.StaticUserGroup; -import static org.junit.Assume.assumeTrue; + +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.*; +import static org.junit.matchers.JUnitMatchers.containsString; + import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,11 +44,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.RandomStringUtils; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertEquals; /** * The test class implements concurrency tests to test: @@ -56,14 +64,19 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration { "sentry.e2e.concurrency.test.tables-per-db", "1")); private final int NUM_OF_PAR = Integer.parseInt(System.getProperty( "sentry.e2e.concurrency.test.partitions-per-tb", "3")); + // number of threads < half of number of tasks, so that there will be + // more than 1 threads for each task to be able to test synchronization private final int NUM_OF_THREADS = Integer.parseInt(System.getProperty( - "sentry.e2e.concurrency.test.threads", "30")); + "sentry.e2e.concurrency.test.threads", "2")); + // since test time out in 10 mins, no more than 600 / 10 = 60 tasks private final int NUM_OF_TASKS = Integer.parseInt(System.getProperty( - "sentry.e2e.concurrency.test.tasks", "100")); + "sentry.e2e.concurrency.test.tasks", "8")); private final Long HS2_CLIENT_TEST_DURATION_MS = Long.parseLong(System.getProperty( - "sentry.e2e.concurrency.test.hs2client.test.time.ms", "10000")); //millis + "sentry.e2e.concurrency.test.hs2client.test.time.ms", "4000")); //millis private final Long SENTRY_CLIENT_TEST_DURATION_MS = Long.parseLong(System.getProperty( - "sentry.e2e.concurrency.test.sentryclient.test.time.ms", "10000")); //millis + "sentry.e2e.concurrency.test.sentryclient.test.time.ms", "4000")); //millis + private final Long EXECUTOR_THREADS_MAX_WAIT_TIME = Long.parseLong(System.getProperty( + "sentry.e2e.concurrency.test.max.wait.time", Integer.toString(NUM_OF_TASKS * 10))); // secs private static Map<String, String> privileges = new HashMap<String, String>(); static { @@ -81,14 +94,13 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration { @BeforeClass public static void setupTestStaticConfiguration() throws Exception { - assumeTrue(Boolean.parseBoolean(System.getProperty("sentry.scaletest.oncluster", "false"))); useSentryService = true; // configure sentry client clientKerberos = true; // need to get client configuration from testing environments AbstractTestWithStaticConfiguration.setupTestStaticConfiguration(); } static String randomString( int len ){ - return RandomStringUtils.random(len, true, false); + return RandomStringUtils.random(len, true, false).toLowerCase(); } private void execStmt(Statement stmt, String sql) throws Exception { @@ -240,8 +252,8 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration { * Privileges are correctly created and updated. * @throws Exception */ - @Test - public void testConccurentHS2Client() throws Exception { + @Test(timeout=600000) // time out in 10 mins + public void testConcurrentHS2Client() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(NUM_OF_THREADS); final TestRuntimeState state = new TestRuntimeState(); @@ -280,7 +292,8 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration { }); } executor.shutdown(); - while (!executor.isTerminated()) { + while (!executor.awaitTermination(EXECUTOR_THREADS_MAX_WAIT_TIME, TimeUnit.SECONDS)) { + LOGGER.info("Awaiting completion of threads."); Thread.sleep(1000); //millisecond } Throwable ex = state.getFirstException(); @@ -292,14 +305,18 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration { * Test when concurrent sentry clients talking to sentry server, threads data are synchronized * @throws Exception */ - @Test + @Test(timeout=600000) // time out in 10 mins public void testConcurrentSentryClient() throws Exception { final String HIVE_KEYTAB_PATH = - System.getProperty("sentry.e2etest.hive.policyOwnerKeytab"); + System.getProperty("sentry.e2etest.hive.policyOwnerKeytab"); final SentryPolicyServiceClient client = getSentryClient("hive", HIVE_KEYTAB_PATH); ExecutorService executor = Executors.newFixedThreadPool(NUM_OF_THREADS); final TestRuntimeState state = new TestRuntimeState(); + String scratchLikeDir = context.getProperty(HiveConf.ConfVars.SCRATCHDIR.varname); + final String uriPrefix = scratchLikeDir.contains("://") ? + scratchLikeDir : (fileSystem.getUri().toString() + scratchLikeDir); + LOGGER.info("uriPrefix = " + uriPrefix); for (int i = 0; i < NUM_OF_TASKS; i ++) { LOGGER.info("Start to test sentry client with task id [" + i + "]"); executor.execute(new Runnable() { @@ -312,16 +329,58 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration { try { String randStr = randomString(5); String test_role = "test_role_" + randStr; + String test_uri = uriPrefix + randStr; + LOGGER.info("Start to test role: " + test_role); Long startTime = System.currentTimeMillis(); Long elapsedTime = 0L; while (Long.compare(elapsedTime, SENTRY_CLIENT_TEST_DURATION_MS) <= 0) { LOGGER.info("Test role " + test_role + " runs " + elapsedTime + " ms."); client.createRole(ADMIN1, test_role); - client.listRoles(ADMIN1); + client.grantRoleToGroup(ADMIN1, ADMINGROUP, test_role); + + // validate role + Set<TSentryRole> sentryRoles = client.listRoles(ADMIN1); + String results = ""; + for (TSentryRole role : sentryRoles) { + results += role.toString() + "|"; + } + LOGGER.info("listRoles = " + results); + assertThat(results, containsString("roleName:" + test_role)); + + // validate privileges + results = ""; client.grantServerPrivilege(ADMIN1, test_role, "server1", false); - client.listAllPrivilegesByRoleName(ADMIN1, test_role); + client.grantURIPrivilege(ADMIN1, test_role, "server1", test_uri); + Set<TSentryPrivilege> sPerms = client.listAllPrivilegesByRoleName(ADMIN1, test_role); + for (TSentryPrivilege sp : sPerms) { + results += sp.toString() + "|"; + } + LOGGER.info("listAllPrivilegesByRoleName = " + results); + assertThat(results, containsString("serverName:server1")); + assertThat(results, containsString("URI:" + test_uri)); + + client.revokeURIPrivilege(ADMIN1, test_role, "server1", test_uri); + client.revokeServerPrivilege(ADMIN1, test_role, "server1", false); + + results = ""; + Set<TSentryPrivilege> rPerms = client.listAllPrivilegesByRoleName(ADMIN1, test_role); + for (TSentryPrivilege rp : rPerms) { + results += rp.toString() + "|"; + } + assertThat(results, not(containsString("URI:" + test_uri))); + assertThat(results, not(containsString("serverName:server1"))); + client.dropRole(ADMIN1, test_role); + + results = ""; + Set<TSentryRole> removedRoles = client.listUserRoles(ADMIN1); + for (TSentryRole role : removedRoles) { + results += role.toString() + "|"; + } + LOGGER.info("listUserRoles = " + results); + assertThat(results, not(containsString(test_role))); + elapsedTime = System.currentTimeMillis() - startTime; } state.setNumSuccess(); @@ -333,7 +392,8 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration { }); } executor.shutdown(); - while (!executor.isTerminated()) { + while (!executor.awaitTermination(EXECUTOR_THREADS_MAX_WAIT_TIME, TimeUnit.SECONDS)) { + LOGGER.info("Awaiting completion of threads."); Thread.sleep(1000); //millisecond } Throwable ex = state.getFirstException(); http://git-wip-us.apache.org/repos/asf/sentry/blob/1b059614/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java index b961757..8515a2b 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java @@ -151,6 +151,7 @@ public abstract class AbstractTestWithStaticConfiguration { private static LoginContext clientLoginContext; protected static SentryPolicyServiceClient client; + private static boolean startSentry = new Boolean(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false")); /** * Get sentry client with authenticated Subject @@ -265,7 +266,6 @@ public abstract class AbstractTestWithStaticConfiguration { policyURI = policyFileLocation.getPath(); } - boolean startSentry = Boolean.valueOf(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false")); if ("true".equalsIgnoreCase(System.getProperty(ENABLE_SENTRY_HA, "false"))) { enableSentryHA = true; } @@ -495,7 +495,7 @@ public abstract class AbstractTestWithStaticConfiguration { */ public static SentryPolicyServiceClient getSentryClient(String clientShortName, String clientKeyTabDir) throws Exception { - if (!useSentryService) { + if (!startSentry) { LOGGER.info("Running on a minicluser env."); return getSentryClient(); } http://git-wip-us.apache.org/repos/asf/sentry/blob/1b059614/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java index 7d3db2b..77106d4 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java @@ -1087,9 +1087,23 @@ public class TestOperations extends AbstractTestWithStaticConfiguration { try { createDb(ADMIN1, DB1); String scratchLikeDir = context.getProperty(HiveConf.ConfVars.SCRATCHDIR.varname); - String extParentDir = dfs.assertCreateDir(scratchLikeDir + "/ABC/hhh").toUri().toString(); - String extTableDir = dfs.assertCreateDir(scratchLikeDir + "/abc/hhh").toUri().toString(); - LOGGER.info("Created extParentDir = " + extParentDir + ", extTableDir = " + extTableDir); + LOGGER.info("scratch like dir = " + scratchLikeDir); + String extParentDir = scratchLikeDir + "/ABC/hhh"; + String extTableDir = scratchLikeDir + "/abc/hhh"; + LOGGER.info("Creating extParentDir = " + extParentDir + ", extTableDir = " + extTableDir); + dfs.assertCreateDir(extParentDir); + dfs.assertCreateDir(extTableDir); + + if (! (extParentDir.toLowerCase().startsWith("hdfs://") + || extParentDir.toLowerCase().startsWith("s3://") + || extParentDir.contains("://"))) { + String scheme = fileSystem.getUri().toString(); + LOGGER.info("scheme = " + scheme); + extParentDir = scheme + extParentDir; + extTableDir = scheme + extTableDir; + LOGGER.info("Add scheme in extParentDir = " + extParentDir + ", extTableDir = " + extTableDir); + } + policyFile .addPermissionsToRole("all_db1", privileges.get("all_db1")) .addPermissionsToRole("all_uri", "server=server1->uri=" + extParentDir)
