Repository: sentry Updated Branches: refs/heads/master 1b71cfb04 -> afcaa4997
http://git-wip-us.apache.org/repos/asf/sentry/blob/afcaa499/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java new file mode 100644 index 0000000..f838005 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.dbprovider; + +import static org.junit.Assert.assertEquals; + +import java.sql.Connection; +import java.sql.Statement; + +import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase; +import org.apache.sentry.tests.e2e.hive.SlowE2ETest; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * This class covers scenario where HMSFollower takes a snapshot initially when sentry server comes up + * and not subsequently even when HMS event information is cleaned up as there is no out of sync detected. + */ +@SlowE2ETest +public class TestSnapshotCreationWithShorterHMSEventTtl extends TestHDFSIntegrationBase { + + private final static String tableName1 = "tb_1"; + private final static String tableName2 = "tb_2"; + private final static String tableName3 = "tb_3"; + private final static String tableName4 = "tb_4"; + + protected static final String ALL_DB1 = "server=server1->db=db_1", + DB1 = "db_1", + DB2 = "db_2"; + + private Connection connection; + private Statement statement; + + @BeforeClass + public static void setup() throws Exception { + shorterMetaStoreEventDbTtl = true; + TestHDFSIntegrationBase.setup(); + } + + @Before + public void initialize() throws Exception { + super.setUpTempDir(); + admin = "hive"; + connection = hiveServer2.createConnection(admin, admin); + statement = connection.createStatement(); + statement.execute("create role admin_role"); + statement.execute("grant role admin_role to group hive"); + statement.execute("grant all on server server1 to role admin_role"); + } + + @Test + public void BasicSanity() throws Exception { + long latestSnapshotId = 0; + dbNames = new String[]{DB1}; + roles = new String[]{"admin_role", "all_db1", "all_tbl1", "all_tbl2"}; + do { + //Sleep for a sec allowing HMSFollower to create a snapshot + Thread.sleep(1000); + latestSnapshotId = sentryServer.get(0).getCurrentAuthzPathsSnapshotID(); + } while (latestSnapshotId == 0); + + statement.execute("CREATE DATABASE " + DB1); + statement.execute("CREATE DATABASE " + DB2); + statement.execute("create table " + DB1 + "." + tableName1 + + " (under_col int comment 'the under column', value string)"); + statement.execute("create table " + DB1 + "." + tableName2 + + " (under_col int comment 'the under column', value string)"); + + statement.execute("create table " + DB2 + "." + tableName3 + + " (under_col int comment 'the under column', value string)"); + + /* + With shorter TTL HMS would evict the entries in NOTIFICATION_LOG table faster. + Which could be value configured for "hive.metastore.event.db.listener.timetolive" + + 60 sec. As the cleanup happens every 60 sec. Test sleeps for 70 sec to make sure that + cleanup happened. + */ + Thread.sleep(70000); + assertEquals("Another snapshot is created", + latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID()); + statement.execute("create table " + DB2 + "." + tableName4 + + " (under_col int comment 'the under column', value string)"); + + Thread.sleep(maxDelayInFetchingHMSNotifications()); + assertEquals("Another snapshot is created", + latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID()); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/afcaa499/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java new file mode 100644 index 0000000..6bc5d11 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.dbprovider; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.sql.Connection; +import java.sql.Statement; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase; +import org.apache.sentry.tests.e2e.hive.SlowE2ETest; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class covers scenario where HMSFollower takes a snapshot initially when sentry server comes up + * and also subsequently when HMS event information is cleaned up and HMSFollower detects that it is out of sync and can + * not recover by fetching delta notifications. + */ +@Ignore +@SlowE2ETest +public class TestSnapshotWithLongerHMSFollowerLongerInterval extends TestHDFSIntegrationBase { + + private static final Logger LOGGER = LoggerFactory + .getLogger(TestSnapshotWithLongerHMSFollowerLongerInterval.class); + + protected static final String DB1 = "db_1", + DB2 = "db_2"; + + private Connection connection; + private Statement statement; + + @BeforeClass + public static void setup() throws Exception { + // Reduces the TTL of the HMS event data so that it is cleaned up faster. + shorterMetaStoreEventDbTtl = true; + // Increases the interval between the fetches the HMSFollower does. + longerHMSFollowerInterval = true; + TestHDFSIntegrationBase.setup(); + } + + @Before + public void initialize() throws Exception { + super.setUpTempDir(); + admin = "hive"; + connection = hiveServer2.createConnection(admin, admin); + statement = connection.createStatement(); + statement.execute("create role admin_role"); + statement.execute("grant role admin_role to group hive"); + statement.execute("grant all on server server1 to role admin_role"); + } + + @Test + public void BasicSanity() throws Exception { + long latestSnapshotId = 0; + Thread th = null; + dbNames = new String[]{DB1}; + roles = new String[]{"admin_role", "all_db1", "all_tbl1", "all_tbl2"}; + do { + //Sleep for a sec allowing HMSFollower to create a snapshot + Thread.sleep(1000); + latestSnapshotId = sentryServer.get(0).getCurrentAuthzPathsSnapshotID(); + } while (latestSnapshotId == 0); + + final AtomicBoolean keepRunning = new AtomicBoolean(true); + th = new Thread() { + @Override + public void run() { + try { + int counter = 1; + while (keepRunning.get()) { + statement.execute("CREATE DATABASE " + "db"+counter); + Thread.sleep(1000L); + counter++; + } + } catch (Exception e) { + LOGGER.info("Could not start Hive Server"); + } + } + }; + th.start(); + + Thread.sleep(130000); + assertEquals("Another snapshot is created, Snapshot ID: ", latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID()); + + Thread.sleep(130000); + long newSnapShot = sentryServer.get(0).getCurrentAuthzPathsSnapshotID(); + assertFalse("Another snapshot should have been created", + (latestSnapshotId == newSnapShot)); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/afcaa499/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java index 4cd00e6..dc322b8 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java @@ -158,6 +158,8 @@ public abstract class TestHDFSIntegrationBase { protected static SentrySrv sentryServer; protected static boolean testSentryHA = false; protected static final long STALE_THRESHOLD = 5000; + protected static Boolean shorterMetaStoreEventDbTtl = false; + protected static Boolean longerHMSFollowerInterval = false; // It is the interval in milliseconds that hdfs uses to get acl from sentry. Default is 500, but // we want it to be low in our tests so that changes reflect soon @@ -540,6 +542,9 @@ public abstract class TestHDFSIntegrationBase { hiveConf.set("hive.metastore.authorization.storage.checks", "true"); hiveConf.set("hive.metastore.uris", "thrift://localhost:" + hmsPort); hiveConf.set("sentry.metastore.service.users", "hive");// queries made by hive user (beeline) skip meta store check + if(shorterMetaStoreEventDbTtl) { + hiveConf.set("hive.metastore.event.db.listener.timetolive", "5s"); + } File confDir = assertCreateDir(new File(baseDir, "etc")); File hiveSite = new File(confDir, "hive-site.xml"); @@ -874,7 +879,14 @@ public abstract class TestHDFSIntegrationBase { "org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); sentryProperties.put("sentry.policy.store.plugins", "org.apache.sentry.hdfs.SentryPlugin"); } - for (Map.Entry<String, String> entry : sentryProperties.entrySet()) { + + if(longerHMSFollowerInterval) { + // Idea of increasing HMSFollower interval is to fetching after HMS notifications are evicted. + //This configuration is dependent on hive.metastore.event.db.listener.timetolive. + // HMS runs cleaner every 60 sec. Below configuration should be 60 + TTL + buffer-time + sentryProperties.put(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS, "120000"); + } + for (Map.Entry<String, String> entry : sentryProperties.entrySet()) { sentryConf.set(entry.getKey(), entry.getValue()); } sentryServer = SentrySrvFactory.create(SentrySrvFactory.SentrySrvType.INTERNAL_SERVER, @@ -890,7 +902,28 @@ public abstract class TestHDFSIntegrationBase { } } - @After + /** + * Method calculates the maximun time it could take to fetch a notification that is inserted to + * NOTIFICATION_LOG table of HMS database. + * @return maximum delay in fetching notification logged by HMS. + */ + protected long maxDelayInFetchingHMSNotifications() { + long interval; + long initDelay; + if (sentryProperties.containsKey(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS)) { + interval = Long.parseLong(sentryProperties.get(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS)); + } else { + interval = ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT; + } + if (sentryProperties.containsKey(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS)) { + initDelay = Long.parseLong(sentryProperties.get(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS)); + } else { + initDelay = ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT; + } + return (interval + initDelay); + } + + @After public void cleanAfterTest() throws Exception { //Clean up database Connection conn; @@ -911,7 +944,11 @@ public abstract class TestHDFSIntegrationBase { stmt = conn.createStatement(); LOGGER.info("About to clear all roles"); for( String role:roles) { - stmt.execute("drop role " + role); + try { + stmt.execute("drop role " + role); + } catch (Exception e) { + LOGGER.info("Exception while dropping role", e.getMessage()); + } } stmt.close(); conn.close();
