Repository: sentry Updated Branches: refs/heads/master afcaa4997 -> a178d7ed8
http://git-wip-us.apache.org/repos/asf/sentry/blob/a178d7ed/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java deleted file mode 100644 index f838005..0000000 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.tests.e2e.dbprovider; - -import static org.junit.Assert.assertEquals; - -import java.sql.Connection; -import java.sql.Statement; - -import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase; -import org.apache.sentry.tests.e2e.hive.SlowE2ETest; - -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * This class covers scenario where HMSFollower takes a snapshot initially when sentry server comes up - * and not subsequently even when HMS event information is cleaned up as there is no out of sync detected. - */ -@SlowE2ETest -public class TestSnapshotCreationWithShorterHMSEventTtl extends TestHDFSIntegrationBase { - - private final static String tableName1 = "tb_1"; - private final static String tableName2 = "tb_2"; - private final static String tableName3 = "tb_3"; - private final static String tableName4 = "tb_4"; - - protected static final String ALL_DB1 = "server=server1->db=db_1", - DB1 = "db_1", - DB2 = "db_2"; - - private Connection connection; - private Statement statement; - - @BeforeClass - public static void setup() throws Exception { - shorterMetaStoreEventDbTtl = true; - TestHDFSIntegrationBase.setup(); - } - - @Before - public void initialize() throws Exception { - super.setUpTempDir(); - admin = "hive"; - connection = hiveServer2.createConnection(admin, admin); - statement = connection.createStatement(); - statement.execute("create role admin_role"); - statement.execute("grant role admin_role to group hive"); - statement.execute("grant all on server server1 to role admin_role"); - } - - @Test - public void BasicSanity() throws Exception { - long latestSnapshotId = 0; - dbNames = new String[]{DB1}; - roles = new String[]{"admin_role", "all_db1", "all_tbl1", "all_tbl2"}; - do { - //Sleep for a sec allowing HMSFollower to create a snapshot - Thread.sleep(1000); - latestSnapshotId = sentryServer.get(0).getCurrentAuthzPathsSnapshotID(); - } while (latestSnapshotId == 0); - - statement.execute("CREATE DATABASE " + DB1); - statement.execute("CREATE DATABASE " + DB2); - statement.execute("create table " + DB1 + "." + tableName1 - + " (under_col int comment 'the under column', value string)"); - statement.execute("create table " + DB1 + "." + tableName2 - + " (under_col int comment 'the under column', value string)"); - - statement.execute("create table " + DB2 + "." + tableName3 - + " (under_col int comment 'the under column', value string)"); - - /* - With shorter TTL HMS would evict the entries in NOTIFICATION_LOG table faster. - Which could be value configured for "hive.metastore.event.db.listener.timetolive" - + 60 sec. As the cleanup happens every 60 sec. Test sleeps for 70 sec to make sure that - cleanup happened. - */ - Thread.sleep(70000); - assertEquals("Another snapshot is created", - latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID()); - statement.execute("create table " + DB2 + "." + tableName4 - + " (under_col int comment 'the under column', value string)"); - - Thread.sleep(maxDelayInFetchingHMSNotifications()); - assertEquals("Another snapshot is created", - latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID()); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/a178d7ed/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java deleted file mode 100644 index 6bc5d11..0000000 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.tests.e2e.dbprovider; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -import java.sql.Connection; -import java.sql.Statement; - -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase; -import org.apache.sentry.tests.e2e.hive.SlowE2ETest; - -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class covers scenario where HMSFollower takes a snapshot initially when sentry server comes up - * and also subsequently when HMS event information is cleaned up and HMSFollower detects that it is out of sync and can - * not recover by fetching delta notifications. - */ -@Ignore -@SlowE2ETest -public class TestSnapshotWithLongerHMSFollowerLongerInterval extends TestHDFSIntegrationBase { - - private static final Logger LOGGER = LoggerFactory - .getLogger(TestSnapshotWithLongerHMSFollowerLongerInterval.class); - - protected static final String DB1 = "db_1", - DB2 = "db_2"; - - private Connection connection; - private Statement statement; - - @BeforeClass - public static void setup() throws Exception { - // Reduces the TTL of the HMS event data so that it is cleaned up faster. - shorterMetaStoreEventDbTtl = true; - // Increases the interval between the fetches the HMSFollower does. - longerHMSFollowerInterval = true; - TestHDFSIntegrationBase.setup(); - } - - @Before - public void initialize() throws Exception { - super.setUpTempDir(); - admin = "hive"; - connection = hiveServer2.createConnection(admin, admin); - statement = connection.createStatement(); - statement.execute("create role admin_role"); - statement.execute("grant role admin_role to group hive"); - statement.execute("grant all on server server1 to role admin_role"); - } - - @Test - public void BasicSanity() throws Exception { - long latestSnapshotId = 0; - Thread th = null; - dbNames = new String[]{DB1}; - roles = new String[]{"admin_role", "all_db1", "all_tbl1", "all_tbl2"}; - do { - //Sleep for a sec allowing HMSFollower to create a snapshot - Thread.sleep(1000); - latestSnapshotId = sentryServer.get(0).getCurrentAuthzPathsSnapshotID(); - } while (latestSnapshotId == 0); - - final AtomicBoolean keepRunning = new AtomicBoolean(true); - th = new Thread() { - @Override - public void run() { - try { - int counter = 1; - while (keepRunning.get()) { - statement.execute("CREATE DATABASE " + "db"+counter); - Thread.sleep(1000L); - counter++; - } - } catch (Exception e) { - LOGGER.info("Could not start Hive Server"); - } - } - }; - th.start(); - - Thread.sleep(130000); - assertEquals("Another snapshot is created, Snapshot ID: ", latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID()); - - Thread.sleep(130000); - long newSnapShot = sentryServer.get(0).getCurrentAuthzPathsSnapshotID(); - assertFalse("Another snapshot should have been created", - (latestSnapshotId == newSnapShot)); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/a178d7ed/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java index dc322b8..4cd00e6 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java @@ -158,8 +158,6 @@ public abstract class TestHDFSIntegrationBase { protected static SentrySrv sentryServer; protected static boolean testSentryHA = false; protected static final long STALE_THRESHOLD = 5000; - protected static Boolean shorterMetaStoreEventDbTtl = false; - protected static Boolean longerHMSFollowerInterval = false; // It is the interval in milliseconds that hdfs uses to get acl from sentry. Default is 500, but // we want it to be low in our tests so that changes reflect soon @@ -542,9 +540,6 @@ public abstract class TestHDFSIntegrationBase { hiveConf.set("hive.metastore.authorization.storage.checks", "true"); hiveConf.set("hive.metastore.uris", "thrift://localhost:" + hmsPort); hiveConf.set("sentry.metastore.service.users", "hive");// queries made by hive user (beeline) skip meta store check - if(shorterMetaStoreEventDbTtl) { - hiveConf.set("hive.metastore.event.db.listener.timetolive", "5s"); - } File confDir = assertCreateDir(new File(baseDir, "etc")); File hiveSite = new File(confDir, "hive-site.xml"); @@ -879,14 +874,7 @@ public abstract class TestHDFSIntegrationBase { "org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); sentryProperties.put("sentry.policy.store.plugins", "org.apache.sentry.hdfs.SentryPlugin"); } - - if(longerHMSFollowerInterval) { - // Idea of increasing HMSFollower interval is to fetching after HMS notifications are evicted. - //This configuration is dependent on hive.metastore.event.db.listener.timetolive. - // HMS runs cleaner every 60 sec. Below configuration should be 60 + TTL + buffer-time - sentryProperties.put(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS, "120000"); - } - for (Map.Entry<String, String> entry : sentryProperties.entrySet()) { + for (Map.Entry<String, String> entry : sentryProperties.entrySet()) { sentryConf.set(entry.getKey(), entry.getValue()); } sentryServer = SentrySrvFactory.create(SentrySrvFactory.SentrySrvType.INTERNAL_SERVER, @@ -902,28 +890,7 @@ public abstract class TestHDFSIntegrationBase { } } - /** - * Method calculates the maximun time it could take to fetch a notification that is inserted to - * NOTIFICATION_LOG table of HMS database. - * @return maximum delay in fetching notification logged by HMS. - */ - protected long maxDelayInFetchingHMSNotifications() { - long interval; - long initDelay; - if (sentryProperties.containsKey(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS)) { - interval = Long.parseLong(sentryProperties.get(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS)); - } else { - interval = ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT; - } - if (sentryProperties.containsKey(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS)) { - initDelay = Long.parseLong(sentryProperties.get(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS)); - } else { - initDelay = ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT; - } - return (interval + initDelay); - } - - @After + @After public void cleanAfterTest() throws Exception { //Clean up database Connection conn; @@ -944,11 +911,7 @@ public abstract class TestHDFSIntegrationBase { stmt = conn.createStatement(); LOGGER.info("About to clear all roles"); for( String role:roles) { - try { - stmt.execute("drop role " + role); - } catch (Exception e) { - LOGGER.info("Exception while dropping role", e.getMessage()); - } + stmt.execute("drop role " + role); } stmt.close(); conn.close();
