http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestAuthzPathCacheOld.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestAuthzPathCacheOld.java b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestAuthzPathCacheOld.java new file mode 100644 index 0000000..ca3ebfe --- /dev/null +++ b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestAuthzPathCacheOld.java @@ -0,0 +1,523 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Table; +//import org.apache.sentry.hdfs.old.AuthzPathCacheOld; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +public class TestAuthzPathCacheOld { + +// @Test +// public void testGetPathElements() { +// String[] as2 = AuthzPathCacheOld.getPathElements(new String("/a/b")); +// String[] as1 = AuthzPathCacheOld.getPathElements(new String("/a/b")); +// Assert.assertArrayEquals(as1, as2); +// +// String[] as = AuthzPathCacheOld.getPathElements(new String("/a/b")); +// Assert.assertArrayEquals(new String[] {"a", "b"}, as); +// +// as = AuthzPathCacheOld.getPathElements(new String("//a/b")); +// Assert.assertArrayEquals(new String[]{"a", "b"}, as); +// +// as = AuthzPathCacheOld.getPathElements(new String("/a//b")); +// Assert.assertArrayEquals(new String[]{"a", "b"}, as); +// +// as = AuthzPathCacheOld.getPathElements(new String("/a/b/")); +// Assert.assertArrayEquals(new String[]{"a", "b"}, as); +// +// as = AuthzPathCacheOld.getPathElements(new String("//a//b//")); +// Assert.assertArrayEquals(new String[]{"a", "b"}, as); +// } +// +// @Test +// public void testGetPathsElements() { +// String[][] as1 = AuthzPathCacheOld.gePathsElements( +// new String[]{new String("/a/b")}); +// String[][] as2 = AuthzPathCacheOld.gePathsElements( +// new String[]{new String("/a/b")}); +// Assert.assertEquals(as1.length, as2.length); +// Assert.assertArrayEquals(as1[0], as2[0]); +// } +// +// @Test +// public void testEntryType() { +// Assert.assertTrue(AuthzPathCacheOld.EntryType.DIR.isRemoveIfDangling()); +// Assert.assertFalse(AuthzPathCacheOld.EntryType.PREFIX.isRemoveIfDangling()); +// Assert.assertTrue( +// AuthzPathCacheOld.EntryType.AUTHZ_OBJECT.isRemoveIfDangling()); +// } +// +// @Test +// public void testRootEntry() { +// AuthzPathCacheOld.Entry root = AuthzPathCacheOld.Entry.createRoot(false); +// root.toString(); +// Assert.assertNull(root.getParent()); +// Assert.assertEquals(AuthzPathCacheOld.EntryType.DIR, root.getType()); +// // NOTE : This was causing some problems during serialization.. so dissabling +//// Assert.assertNull(root.getPathElement()); +// Assert.assertNull(root.getAuthzObj()); +// Assert.assertEquals(Path.SEPARATOR, root.getFullPath()); +// Assert.assertTrue(root.getChildren().isEmpty()); +// root.delete(); +// try { +// root.find(null, true); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// try { +// root.find(new String[0], true); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// try { +// root.find(null, false); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// try { +// root.find(new String[0], false); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// Assert.assertEquals(root, root.find(new String[]{"a"}, true)); +// Assert.assertNull(root.find(new String[]{"a"}, false)); +// Assert.assertNull(root.findPrefixEntry(new String[]{"a"})); +// +// root.delete(); +// } +// +// @Test +// public void testRootPrefixEntry() { +// AuthzPathCacheOld.Entry root = AuthzPathCacheOld.Entry.createRoot(true); +// root.toString(); +// +// Assert.assertEquals(root, root.find(new String[]{"a"}, true)); +// Assert.assertEquals(null, root.find(new String[]{"a"}, false)); +// Assert.assertEquals(root, root.findPrefixEntry(new String[]{"a"})); +// Assert.assertEquals(root, root.findPrefixEntry(new String[]{"a", "b"})); +// +// try { +// root.createPrefix(new String[]{"a"}); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// } +// +// @Test +// public void testImmediatePrefixEntry() { +// AuthzPathCacheOld.Entry root = AuthzPathCacheOld.Entry.createRoot(false); +// AuthzPathCacheOld.Entry entry = root.createPrefix(new String[] {"a"}); +// entry.toString(); +// +// Assert.assertEquals(1, root.getChildren().size()); +// +// Assert.assertEquals(root, entry.getParent()); +// Assert.assertEquals(AuthzPathCacheOld.EntryType.PREFIX, entry.getType()); +// Assert.assertEquals("a", entry.getPathElement()); +// Assert.assertNull(entry.getAuthzObj()); +// Assert.assertEquals(Path.SEPARATOR + "a", entry.getFullPath()); +// Assert.assertTrue(entry.getChildren().isEmpty()); +// +// Assert.assertEquals(entry, root.find(new String[]{"a"}, true)); +// Assert.assertEquals(entry, root.find(new String[]{"a"}, false)); +// Assert.assertEquals(entry, root.findPrefixEntry(new String[]{"a"})); +// Assert.assertEquals(entry, root.findPrefixEntry(new String[]{"a", "b"})); +// +// Assert.assertEquals(entry, root.find(new String[]{"a", "b"}, true)); +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "c"}, true)); +// Assert.assertNull(root.find(new String[]{"a", "b"}, false)); +// +// Assert.assertNull(root.find(new String[]{"b"}, false)); +// Assert.assertNull(root.findPrefixEntry(new String[]{"b"})); +// +// try { +// root.createPrefix(new String[]{"a", "b"}); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// +// try { +// root.createPrefix(new String[]{"a", "b", "c"}); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// +// entry.delete(); +// Assert.assertTrue(root.getChildren().isEmpty()); +// } +// +// @Test +// public void testFurtherPrefixEntry() { +// AuthzPathCacheOld.Entry root = AuthzPathCacheOld.Entry.createRoot(false); +// AuthzPathCacheOld.Entry entry = root.createPrefix(new String[]{"a", "b"}); +// entry.toString(); +// +// Assert.assertEquals(1, root.getChildren().size()); +// +// Assert.assertEquals(root, entry.getParent().getParent()); +// Assert.assertEquals(AuthzPathCacheOld.EntryType.PREFIX, entry.getType()); +// Assert.assertEquals(AuthzPathCacheOld.EntryType.DIR, +// entry.getParent().getType()); +// Assert.assertEquals("b", entry.getPathElement()); +// Assert.assertEquals("a", entry.getParent().getPathElement()); +// Assert.assertNull(entry.getAuthzObj()); +// Assert.assertNull(entry.getParent().getAuthzObj()); +// Assert.assertEquals(Path.SEPARATOR + "a" + Path.SEPARATOR + "b", +// entry.getFullPath()); +// Assert.assertEquals(Path.SEPARATOR + "a", entry.getParent().getFullPath()); +// Assert.assertTrue(entry.getChildren().isEmpty()); +// Assert.assertEquals(1, entry.getParent().getChildren().size()); +// +// Assert.assertEquals(entry, root.find(new String[]{"a", "b"}, true)); +// Assert.assertEquals(entry, root.find(new String[]{"a", "b"}, false)); +// Assert.assertEquals(entry, root.findPrefixEntry(new String[]{"a", "b"})); +// Assert.assertNull(root.findPrefixEntry(new String[]{"a"})); +// +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "c"}, true)); +// Assert.assertNull(root.find(new String[]{"a", "b", "c"}, false)); +// +// try { +// root.createPrefix(new String[]{"a", "b"}); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// +// try { +// root.createPrefix(new String[]{"a", "b", "c"}); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// +// entry.delete(); +// Assert.assertTrue(root.getChildren().isEmpty()); +// } +// +// @Test +// public void testImmediateAuthzEntry() { +// AuthzPathCacheOld.Entry root = AuthzPathCacheOld.Entry.createRoot(false); +// AuthzPathCacheOld.Entry prefix = root.createPrefix(new String[]{"a", "b"}); +// +// AuthzPathCacheOld.Entry entry = root.createAuthzObjPath( +// new String[]{"a", "b", "p1"}, "A"); +// Assert.assertEquals(prefix, entry.getParent()); +// Assert.assertEquals(AuthzPathCacheOld.EntryType.AUTHZ_OBJECT, entry.getType()); +// Assert.assertEquals("p1", entry.getPathElement()); +// Assert.assertEquals("A", entry.getAuthzObj()); +// Assert.assertEquals(Path.SEPARATOR + "a" + Path.SEPARATOR + "b" + +// Path.SEPARATOR + "p1", entry.getFullPath()); +// +// try { +// root.createPrefix(new String[]{"a", "b", "p1", "c"}); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "p1"}, true)); +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "p1"}, false)); +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "p1", "c"}, +// true)); +// Assert.assertNull(root.find(new String[]{"a", "b", "p1", "c"}, false)); +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "p1"})); +// +// root.find(new String[]{"a", "b", "p1"}, true).delete(); +// Assert.assertNull(root.find(new String[]{"a", "b", "p1"}, false)); +// Assert.assertNotNull(root.find(new String[]{"a", "b"}, false)); +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "p1"})); +// +// } +// +// @Test +// public void testFurtherAuthzEntry() { +// AuthzPathCacheOld.Entry root = AuthzPathCacheOld.Entry.createRoot(false); +// AuthzPathCacheOld.Entry prefix = root.createPrefix(new String[]{"a", "b"}); +// +// AuthzPathCacheOld.Entry entry = root.createAuthzObjPath( +// new String[]{"a", "b", "t", "p1"}, "A"); +// Assert.assertEquals(prefix, entry.getParent().getParent()); +// Assert.assertEquals(AuthzPathCacheOld.EntryType.AUTHZ_OBJECT, entry.getType()); +// Assert.assertEquals("p1", entry.getPathElement()); +// Assert.assertEquals("A", entry.getAuthzObj()); +// Assert.assertEquals(Path.SEPARATOR + "a" + Path.SEPARATOR + "b" + +// Path.SEPARATOR + "t" + Path.SEPARATOR + "p1", entry.getFullPath()); +// +// try { +// root.createPrefix(new String[]{"a", "b", "p1", "t", "c"}); +// Assert.fail(); +// } catch (IllegalArgumentException ex) { +// //NOP +// } +// +// AuthzPathCacheOld.Entry ep2 = root.createAuthzObjPath( +// new String[]{"a", "b", "t", "p1", "p2"}, "A"); +// +// Assert.assertEquals(AuthzPathCacheOld.EntryType.AUTHZ_OBJECT, entry.getType()); +// Assert.assertEquals("p1", entry.getPathElement()); +// Assert.assertEquals("A", entry.getAuthzObj()); +// +// Assert.assertEquals(AuthzPathCacheOld.EntryType.AUTHZ_OBJECT, ep2.getType()); +// Assert.assertEquals("p2", ep2.getPathElement()); +// Assert.assertEquals("A", entry.getAuthzObj()); +// +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "t", "p1"}, +// true)); +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "t", "p1"}, +// false)); +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "t", "p1", "c"}, +// true)); +// Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1", "c"}, false)); +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "t", "p1"})); +// +// Assert.assertEquals(ep2, root.find(new String[]{"a", "b", "t", "p1", "p2"}, +// true)); +// Assert.assertEquals(ep2, root.find(new String[]{"a", "b", "t", "p1", "p2"}, +// false)); +// Assert.assertEquals(ep2, root.find(new String[]{"a", "b", "t", "p1", "p2", "c"}, +// true)); +// Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1", "p2", "c"}, false)); +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "t", "p1", "p2"})); +// +// root.find(new String[]{"a", "b", "t", "p1"}, false).delete(); +// +// Assert.assertEquals(entry, root.find(new String[]{"a", "b", "t", "p1"}, +// true)); +// Assert.assertEquals(AuthzPathCacheOld.EntryType.DIR, entry.getType()); +// Assert.assertNull(entry.getAuthzObj()); +// +// Assert.assertNotNull(root.find(new String[]{"a", "b", "t", "p1"}, false)); +// Assert.assertNotNull(root.find(new String[]{"a", "b", "t"}, false)); +// Assert.assertNotNull(root.find(new String[]{"a", "b"}, false)); +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "t", "p1"})); +// +// root.find(new String[]{"a", "b", "t", "p1", "p2"}, false).delete(); +// Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1"}, false)); +// Assert.assertNull(root.find(new String[]{"a", "b", "t"}, false)); +// Assert.assertNotNull(root.find(new String[]{"a", "b"}, false)); +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "t", "p1"})); +// +// } +// +// @Test +// public void testMultipleAuthzEntry() { +// AuthzPathCacheOld.Entry root = AuthzPathCacheOld.Entry.createRoot(false); +// AuthzPathCacheOld.Entry prefix = root.createPrefix(new String[]{"a", "b"}); +// +// AuthzPathCacheOld.Entry e1 = root.createAuthzObjPath( +// new String[]{"a", "b", "t", "p1"}, "A"); +// AuthzPathCacheOld.Entry e2 = root.createAuthzObjPath( +// new String[]{"a", "b", "t", "p2"}, "A"); +// +// +// Assert.assertEquals(e1, root.find(new String[]{"a", "b", "t", "p1"}, true)); +// Assert.assertEquals(e1, root.find(new String[]{"a", "b", "t", "p1"}, +// false)); +// Assert.assertEquals(e1, root.find(new String[]{"a", "b", "t", "p1", "c"}, +// true)); +// Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1", "c"}, false)); +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "t", "p1"})); +// +// Assert.assertEquals(e2, root.find(new String[]{"a", "b", "t", "p2"}, true)); +// Assert.assertEquals(e2, root.find(new String[]{"a", "b", "t", "p2"}, +// false)); +// Assert.assertEquals(e2, root.find(new String[]{"a", "b", "t", "p2", "c"}, +// true)); +// Assert.assertNull(root.find(new String[]{"a", "b", "t", "p2", "c"}, false)); +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "t", "p2"})); +// +// root.find(new String[]{"a", "b", "t", "p1"}, true).delete(); +// Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1"}, false)); +// Assert.assertNotNull(root.find(new String[]{"a", "b", "t"}, false)); +// +// root.find(new String[]{"a", "b", "t", "p2"}, true).delete(); +// Assert.assertNull(root.find(new String[]{"a", "b", "t", "p2"}, false)); +// Assert.assertNull(root.find(new String[]{"a", "b", "t"}, false)); +// Assert.assertNotNull(root.find(new String[]{"a", "b"}, false)); +// +// Assert.assertEquals(prefix, root.findPrefixEntry( +// new String[]{"a", "b", "t", "p3"})); +// } +// +// @Test +// public void testUpdateHandling() throws Exception { +// DummyHMSClient mock = new DummyHMSClient(); +// Database db1 = mock.addDb("db1", "/db1"); +// Table tbl11 = mock.addTable(db1, "tbl11", "/db1/tbl11"); +// mock.addPartition(db1, tbl11, "/db1/tbl11/part111"); +// mock.addPartition(db1, tbl11, "/db1/tbl11/part112"); +// AuthzPathCacheOld AuthzPathUpdater = new AuthzPathCacheOld(mock, new String[]{"/db1"}, 10000); +// +// // Trigger Initial refresh (full dump) +// AuthzPathUpdater.handleUpdateNotification(new PathsUpdate(10, null)); +// waitToCommit(AuthzPathUpdater); +// assertEquals("db1.tbl11", AuthzPathUpdater.findAuthzObject("/db1/tbl11/part111".split("^/")[1].split("/"))); +// assertEquals("db1.tbl11", AuthzPathUpdater.findAuthzObject("/db1/tbl11/part112".split("^/")[1].split("/"))); +// +// // Handle preUpdate from HMS plugin +// PathsUpdate update = new PathsUpdate(11, null); +// update.addPathUpdate("db1.tbl12").addPath("/db1/tbl12").addPath("/db1/tbl12/part121"); +// update.addPathUpdate("db1.tbl11").delPath("/db1/tbl11/part112"); +// +// // Ensure JSON serialization is working : +// assertEquals(PathsUpdate.toJsonString(update), +// PathsUpdate.toJsonString( +// PathsUpdate.fromJsonString( +// PathsUpdate.toJsonString(update)))); +// +// AuthzPathUpdater.handleUpdateNotification(update); +// waitToCommit(AuthzPathUpdater); +// assertNull(AuthzPathUpdater.findAuthzObject("/db1/tbl11/part112".split("^/")[1].split("/"), false)); +// assertEquals("db1.tbl12", AuthzPathUpdater.findAuthzObject("/db1/tbl12/part121".split("^/")[1].split("/"))); +// +// // Add more entries to HMS +// Table tbl13 = mock.addTable(db1, "tbl13", "/db1/tbl13"); +// mock.addPartition(db1, tbl13, "/db1/tbl13/part131"); +// +// // Simulate missed preUpdate (Send empty preUpdate with seqNum 13) +// // On missed preUpdate, refresh again +// AuthzPathUpdater.handleUpdateNotification(new PathsUpdate(13, null)); +// waitToCommit(AuthzPathUpdater); +// assertEquals("db1.tbl13", AuthzPathUpdater.findAuthzObject("/db1/tbl13/part131".split("^/")[1].split("/"))); +// } +// +// @Test +// public void testGetUpdatesFromSrcCache() throws InterruptedException { +// DummyHMSClient mock = new DummyHMSClient(); +// Database db1 = mock.addDb("db1", "/db1"); +// Table tbl11 = mock.addTable(db1, "tbl11", "/db1/tbl11"); +// mock.addPartition(db1, tbl11, "/db1/tbl11/part111"); +// mock.addPartition(db1, tbl11, "/db1/tbl11/part112"); +// +// // This would live in the Sentry Service +// AuthzPathCacheOld srcCache = new AuthzPathCacheOld(mock, new String[]{"/db1"}, 10000); +// +// // Trigger Initial full Image fetch +// srcCache.handleUpdateNotification(new PathsUpdate(10, null)); +// waitToCommit(srcCache); +// +// // This entity would live in the NN plugin : a downstream cache with no updateLog +// AuthzPathCacheOld destCache = new AuthzPathCacheOld(null, new String[]{"/db1"}, 0); +// +// // Adapter to pull updates from upstream cache to downstream Cache +// DummyAdapter<PathsUpdate> adapter = new DummyAdapter<PathsUpdate>(destCache, srcCache); +// adapter.getDestToPullUpdatesFromSrc(); +// waitToCommit(destCache); +// // Check if NN plugin received the updates from Sentry Cache +// assertEquals("db1.tbl11", destCache.findAuthzObject("/db1/tbl11/part111".split("^/")[1].split("/"))); +// assertEquals("db1.tbl11", destCache.findAuthzObject("/db1/tbl11/part112".split("^/")[1].split("/"))); +// +// // Create Upsteram HMS preUpdate +// PathsUpdate update = new PathsUpdate(11, null); +// update.addPathUpdate("db1.tbl12").addPath("/db1/tbl12").addPath("/db1/tbl12/part121"); +// update.addPathUpdate("db1.tbl11").delPath("/db1/tbl11/part112"); +// +// // Send Update to Upstream Cache +// srcCache.handleUpdateNotification(update); +// waitToCommit(srcCache); +// // Pull preUpdate to downstream Cache +// adapter.getDestToPullUpdatesFromSrc(); +// waitToCommit(destCache); +// +// assertNull(srcCache.findAuthzObject("/db1/tbl11/part112".split("^/")[1].split("/"), false)); +// assertNull(destCache.findAuthzObject("/db1/tbl11/part112".split("^/")[1].split("/"), false)); +// assertEquals("db1.tbl11", destCache.findAuthzObject("/db1/tbl11/part112".split("^/")[1].split("/"))); +// assertEquals("db1.tbl12", destCache.findAuthzObject("/db1/tbl12/part121".split("^/")[1].split("/"))); +// } +// +//// @Test(expected = IllegalArgumentException.class) +//// public void testAuthzPathUpdaterRootPrefix() { +//// AuthzPathCacheOld cache = new AuthzPathCacheOld(new String[]{"/", "/b/c"}); +//// } +// +// @Test +// public void testAuthzPathUpdater() { +// AuthzPathCacheOld cache = new AuthzPathCacheOld(null, new String[] { "/a", "/b/c"}, 0); +// Assert.assertTrue(cache.isUnderPrefix("/a".split("^/")[1].split("/"))); +// Assert.assertTrue(cache.isUnderPrefix("/a/x".split("^/")[1].split("/"))); +// Assert.assertTrue(cache.isUnderPrefix("/b/c/".split("^/")[1].split("/"))); +// Assert.assertFalse(cache.isUnderPrefix("/x".split("^/")[1].split("/"))); +// +// Assert.assertNull((cache.findAuthzObject("/a/x".split("^/")[1].split("/")))); +// Assert.assertNull((cache.findAuthzObject("/x".split("^/")[1].split("/")))); +// +// cache.addAuthzObject("T", Arrays.asList("/a/T/p1", "/a/T/p2")); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p2".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1/x".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1/x/x".split("^/")[1].split("/"))); +// Assert.assertNull((cache.findAuthzObject("/a/T/p3".split("^/")[1].split("/")))); +// +// cache.addPathsToAuthzObject("T", Arrays.asList("/a/T/p3")); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p2".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1/x".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1/x/x".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p3".split("^/")[1].split("/"))); +// +// cache.deletePathsFromAuthzObject("T", Arrays.asList("/a/T/p2")); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1".split("^/")[1].split("/"))); +// Assert.assertNull((cache.findAuthzObject("/a/T/p2".split("^/")[1].split("/")))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1/x".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p1/x/x".split("^/")[1].split("/"))); +// Assert.assertEquals("T", cache.findAuthzObject("/a/T/p3".split("^/")[1].split("/"))); +// +// cache.deleteAuthzObject("T"); +// Assert.assertNull((cache.findAuthzObject("/a/T/p1".split("^/")[1].split("/")))); +// Assert.assertNull((cache.findAuthzObject("/a/T/p2".split("^/")[1].split("/")))); +// Assert.assertNull((cache.findAuthzObject("/a/T/p3".split("^/")[1].split("/")))); +// } +// +// private void waitToCommit(AuthzPathCacheOld hmsCache) throws InterruptedException { +// int counter = 0; +// while(!hmsCache.areAllUpdatesCommited()) { +// Thread.sleep(200); +// counter++; +// if (counter > 10000) { +// fail("Updates taking too long to commit !!"); +// } +// } +// } + +}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestAuthzPermCache.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestAuthzPermCache.java b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestAuthzPermCache.java new file mode 100644 index 0000000..f4e569f --- /dev/null +++ b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestAuthzPermCache.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.hadoop.fs.permission.FsAction; +//import org.apache.sentry.hdfs.old.AuthzPermCache; +//import org.apache.sentry.hdfs.old.AuthzPermCache.PrivilegeInfo; +//import org.apache.sentry.hdfs.old.AuthzPermCache.RoleInfo; +import org.junit.Test; + +public class TestAuthzPermCache { + +// @Test +// public void testAuthzAddRemove() throws InterruptedException { +// DummyAuthzSource src = new DummyAuthzSource(); +// AuthzPermCache authzCache = new AuthzPermCache(10000, src, 0); +// src.privs.put("db1.tbl11", new PrivilegeInfo("db1.tbl11").setPermission("r1", FsAction.READ_WRITE)); +// src.privs.put("db1.tbl12", new PrivilegeInfo("db1.tbl12").setPermission("r1", FsAction.READ).setPermission("r2", FsAction.WRITE)); +// src.privs.put("db1.tbl13", new PrivilegeInfo("db1.tbl13").setPermission("r2", FsAction.READ).setPermission("r3", FsAction.WRITE)); +// src.roles.put("r1", new RoleInfo("r1").addGroup("g1")); +// src.roles.put("r2", new RoleInfo("r2").addGroup("g2").addGroup("g1")); +// src.roles.put("r3", new RoleInfo("r3").addGroup("g3").addGroup("g2").addGroup("g1")); +// authzCache.handleUpdateNotification(new PermissionsUpdate(10, false)); +// waitToCommit(authzCache); +// +// assertEquals(FsAction.READ_WRITE, authzCache.getPermissions("db1.tbl11").get("g1")); +// assertEquals(FsAction.READ_WRITE, authzCache.getPermissions("db1.tbl12").get("g1")); +// assertEquals(FsAction.WRITE, authzCache.getPermissions("db1.tbl12").get("g2")); +// assertEquals(FsAction.READ_WRITE, authzCache.getPermissions("db1.tbl13").get("g1")); +// assertEquals(FsAction.READ_WRITE, authzCache.getPermissions("db1.tbl13").get("g2")); +// assertEquals(FsAction.WRITE, authzCache.getPermissions("db1.tbl13").get("g3")); +// } +// +// private void waitToCommit(AuthzPermCache authzCache) throws InterruptedException { +// int counter = 0; +// while(!authzCache.areAllUpdatesCommited()) { +// Thread.sleep(200); +// counter++; +// if (counter > 10000) { +// fail("Updates taking too long to commit !!"); +// } +// } +// } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestHMSPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestHMSPaths.java b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestHMSPaths.java new file mode 100644 index 0000000..29868ae --- /dev/null +++ b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestHMSPaths.java @@ -0,0 +1,357 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import java.util.List; + +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.collect.Lists; + +public class TestHMSPaths { + + @Test + public void testGetPathElements() { + List<String> as2 = HMSPaths.getPathElements(new String("/a/b")); + List<String> as1 = HMSPaths.getPathElements(new String("/a/b")); + Assert.assertEquals(as1, as2); + + List<String> as = HMSPaths.getPathElements(new String("/a/b")); + Assert.assertEquals(Lists.newArrayList("a", "b"), as); + + as = HMSPaths.getPathElements(new String("//a/b")); + Assert.assertEquals(Lists.newArrayList("a", "b"), as); + + as = HMSPaths.getPathElements(new String("/a//b")); + Assert.assertEquals(Lists.newArrayList("a", "b"), as); + + as = HMSPaths.getPathElements(new String("/a/b/")); + Assert.assertEquals(Lists.newArrayList("a", "b"), as); + + as = HMSPaths.getPathElements(new String("//a//b//")); + Assert.assertEquals(Lists.newArrayList("a", "b"), as); + } + + @Test + public void testEntryType() { + Assert.assertTrue(HMSPaths.EntryType.DIR.isRemoveIfDangling()); + Assert.assertFalse(HMSPaths.EntryType.PREFIX.isRemoveIfDangling()); + Assert.assertFalse( + HMSPaths.EntryType.AUTHZ_OBJECT.isRemoveIfDangling()); + } + + @Test + public void testRootEntry() { + HMSPaths.Entry root = HMSPaths.Entry.createRoot(false); + root.toString(); + Assert.assertNull(root.getParent()); + Assert.assertEquals(HMSPaths.EntryType.DIR, root.getType()); + Assert.assertNull(root.getAuthzObj()); + Assert.assertEquals(Path.SEPARATOR, root.getFullPath()); + Assert.assertTrue(root.getChildren().isEmpty()); + root.delete(); + try { + root.find(null, true); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + try { + root.find(new String[0], true); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + try { + root.find(null, false); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + try { + root.find(new String[0], false); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + Assert.assertNull(root.find(new String[]{"a"}, true)); + Assert.assertNull(root.find(new String[]{"a"}, false)); + Assert.assertNull(root.findPrefixEntry(Lists.newArrayList("a"))); + + root.delete(); + } + + @Test + public void testRootPrefixEntry() { + HMSPaths.Entry root = HMSPaths.Entry.createRoot(true); + root.toString(); + + Assert.assertNull(root.find(new String[]{"a"}, true)); + Assert.assertNull(root.find(new String[]{"a"}, false)); + Assert.assertEquals(root, root.findPrefixEntry(Lists.newArrayList("a"))); + Assert.assertEquals(root, root.findPrefixEntry(Lists.newArrayList("a", "b"))); + + try { + root.createPrefix(Lists.newArrayList("a")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + } + + @Test + public void testImmediatePrefixEntry() { + HMSPaths.Entry root = HMSPaths.Entry.createRoot(false); + HMSPaths.Entry entry = root.createPrefix(Lists.newArrayList("a")); + entry.toString(); + + Assert.assertEquals(1, root.getChildren().size()); + + Assert.assertEquals(root, entry.getParent()); + Assert.assertEquals(HMSPaths.EntryType.PREFIX, entry.getType()); + Assert.assertEquals("a", entry.getPathElement()); + Assert.assertNull(entry.getAuthzObj()); + Assert.assertEquals(Path.SEPARATOR + "a", entry.getFullPath()); + Assert.assertTrue(entry.getChildren().isEmpty()); + + Assert.assertEquals(entry, root.findPrefixEntry(Lists.newArrayList("a"))); + Assert.assertEquals(entry, root.findPrefixEntry(Lists.newArrayList("a", "b"))); + + Assert.assertNull(root.find(new String[]{"a", "b"}, false)); + + Assert.assertNull(root.find(new String[]{"b"}, false)); + Assert.assertNull(root.findPrefixEntry(Lists.newArrayList("b"))); + + try { + root.createPrefix(Lists.newArrayList("a", "b")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + + try { + root.createPrefix(Lists.newArrayList("a", "b", "c")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + + entry.delete(); + Assert.assertTrue(root.getChildren().isEmpty()); + } + + @Test + public void testFurtherPrefixEntry() { + HMSPaths.Entry root = HMSPaths.Entry.createRoot(false); + HMSPaths.Entry entry = root.createPrefix(Lists.newArrayList("a", "b")); + entry.toString(); + + Assert.assertEquals(1, root.getChildren().size()); + + Assert.assertEquals(root, entry.getParent().getParent()); + Assert.assertEquals(HMSPaths.EntryType.PREFIX, entry.getType()); + Assert.assertEquals(HMSPaths.EntryType.DIR, + entry.getParent().getType()); + Assert.assertEquals("b", entry.getPathElement()); + Assert.assertEquals("a", entry.getParent().getPathElement()); + Assert.assertNull(entry.getAuthzObj()); + Assert.assertNull(entry.getParent().getAuthzObj()); + Assert.assertEquals(Path.SEPARATOR + "a" + Path.SEPARATOR + "b", + entry.getFullPath()); + Assert.assertEquals(Path.SEPARATOR + "a", entry.getParent().getFullPath()); + Assert.assertTrue(entry.getChildren().isEmpty()); + Assert.assertEquals(1, entry.getParent().getChildren().size()); + + Assert.assertEquals(entry, root.findPrefixEntry(Lists.newArrayList("a", "b"))); + Assert.assertNull(root.findPrefixEntry(Lists.newArrayList("a"))); + + Assert.assertNull(root.find(new String[]{"a", "b", "c"}, false)); + + try { + root.createPrefix(Lists.newArrayList("a", "b")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + + try { + root.createPrefix(Lists.newArrayList("a", "b", "c")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + + entry.delete(); + Assert.assertTrue(root.getChildren().isEmpty()); + } + + @Test + public void testImmediateAuthzEntry() { + HMSPaths.Entry root = HMSPaths.Entry.createRoot(false); + HMSPaths.Entry prefix = root.createPrefix(Lists.newArrayList("a", "b")); + + HMSPaths.Entry entry = root.createAuthzObjPath( + Lists.newArrayList("a", "b", "p1"), "A"); + Assert.assertEquals(prefix, entry.getParent()); + Assert.assertEquals(HMSPaths.EntryType.AUTHZ_OBJECT, entry.getType()); + Assert.assertEquals("p1", entry.getPathElement()); + Assert.assertEquals("A", entry.getAuthzObj()); + Assert.assertEquals(Path.SEPARATOR + "a" + Path.SEPARATOR + "b" + + Path.SEPARATOR + "p1", entry.getFullPath()); + + try { + root.createPrefix(Lists.newArrayList("a", "b", "p1", "c")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + + Assert.assertEquals(entry, root.find(new String[]{"a", "b", "p1"}, true)); + Assert.assertEquals(entry, root.find(new String[]{"a", "b", "p1"}, false)); + Assert.assertEquals(entry, root.find(new String[]{"a", "b", "p1", "c"}, + true)); + Assert.assertNull(root.find(new String[]{"a", "b", "p1", "c"}, false)); + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "p1"))); + + root.find(new String[]{"a", "b", "p1"}, true).delete(); + Assert.assertNull(root.find(new String[]{"a", "b", "p1"}, false)); + Assert.assertNull(root.find(new String[]{"a", "b"}, false)); + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "p1"))); + + } + + @Test + public void testFurtherAuthzEntry() { + HMSPaths.Entry root = HMSPaths.Entry.createRoot(false); + HMSPaths.Entry prefix = root.createPrefix(Lists.newArrayList("a", "b")); + + HMSPaths.Entry entry = root.createAuthzObjPath( + Lists.newArrayList("a", "b", "t", "p1"), "A"); + Assert.assertEquals(prefix, entry.getParent().getParent()); + Assert.assertEquals(HMSPaths.EntryType.AUTHZ_OBJECT, entry.getType()); + Assert.assertEquals("p1", entry.getPathElement()); + Assert.assertEquals("A", entry.getAuthzObj()); + Assert.assertEquals(Path.SEPARATOR + "a" + Path.SEPARATOR + "b" + + Path.SEPARATOR + "t" + Path.SEPARATOR + "p1", entry.getFullPath()); + + try { + root.createPrefix(Lists.newArrayList("a", "b", "p1", "t", "c")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + //NOP + } + + HMSPaths.Entry ep2 = root.createAuthzObjPath( + Lists.newArrayList("a", "b", "t", "p1", "p2"), "A"); + + Assert.assertEquals(HMSPaths.EntryType.AUTHZ_OBJECT, entry.getType()); + Assert.assertEquals("p1", entry.getPathElement()); + Assert.assertEquals("A", entry.getAuthzObj()); + + Assert.assertEquals(HMSPaths.EntryType.AUTHZ_OBJECT, ep2.getType()); + Assert.assertEquals("p2", ep2.getPathElement()); + Assert.assertEquals("A", entry.getAuthzObj()); + + Assert.assertEquals(entry, root.find(new String[]{"a", "b", "t", "p1"}, + true)); + Assert.assertEquals(entry, root.find(new String[]{"a", "b", "t", "p1"}, + false)); + Assert.assertEquals(entry, root.find(new String[]{"a", "b", "t", "p1", "c"}, + true)); + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1", "c"}, false)); + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "t", "p1"))); + + Assert.assertEquals(ep2, root.find(new String[]{"a", "b", "t", "p1", "p2"}, + true)); + Assert.assertEquals(ep2, root.find(new String[]{"a", "b", "t", "p1", "p2"}, + false)); + Assert.assertEquals(ep2, root.find(new String[]{"a", "b", "t", "p1", "p2", "c"}, + true)); + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1", "p2", "c"}, false)); + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "t", "p1", "p2"))); + + root.find(new String[]{"a", "b", "t", "p1"}, false).delete(); + + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1"}, + true)); + Assert.assertEquals(HMSPaths.EntryType.DIR, entry.getType()); + Assert.assertNull(entry.getAuthzObj()); + + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1"}, false)); + Assert.assertNull(root.find(new String[]{"a", "b", "t"}, false)); + Assert.assertNull(root.find(new String[]{"a", "b"}, false)); + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "t", "p1"))); + + Assert.assertNotNull(root.find(new String[]{"a", "b", "t", "p1", "p2"}, false)); + root.find(new String[]{"a", "b", "t", "p1", "p2"}, false).delete(); + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1"}, false)); + Assert.assertNull(root.find(new String[]{"a", "b", "t"}, false)); + Assert.assertNull(root.find(new String[]{"a", "b"}, false)); + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "t", "p1"))); + + } + + @Test + public void testMultipleAuthzEntry() { + HMSPaths.Entry root = HMSPaths.Entry.createRoot(false); + HMSPaths.Entry prefix = root.createPrefix(Lists.newArrayList("a", "b")); + + HMSPaths.Entry e1 = root.createAuthzObjPath( + Lists.newArrayList("a", "b", "t", "p1"), "A"); + HMSPaths.Entry e2 = root.createAuthzObjPath( + Lists.newArrayList("a", "b", "t", "p2"), "A"); + + + Assert.assertEquals(e1, root.find(new String[]{"a", "b", "t", "p1"}, true)); + Assert.assertEquals(e1, root.find(new String[]{"a", "b", "t", "p1"}, + false)); + Assert.assertEquals(e1, root.find(new String[]{"a", "b", "t", "p1", "c"}, + true)); + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1", "c"}, false)); + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "t", "p1"))); + + Assert.assertEquals(e2, root.find(new String[]{"a", "b", "t", "p2"}, true)); + Assert.assertEquals(e2, root.find(new String[]{"a", "b", "t", "p2"}, + false)); + Assert.assertEquals(e2, root.find(new String[]{"a", "b", "t", "p2", "c"}, + true)); + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p2", "c"}, false)); + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "t", "p2"))); + + root.find(new String[]{"a", "b", "t", "p1"}, true).delete(); + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p1"}, false)); + + root.find(new String[]{"a", "b", "t", "p2"}, true).delete(); + Assert.assertNull(root.find(new String[]{"a", "b", "t", "p2"}, false)); + Assert.assertNull(root.find(new String[]{"a", "b", "t"}, false)); + + Assert.assertEquals(prefix, root.findPrefixEntry( + Lists.newArrayList("a", "b", "t", "p3"))); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestHMSPathsFullDump.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestHMSPathsFullDump.java b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestHMSPathsFullDump.java new file mode 100644 index 0000000..b7c7b6d --- /dev/null +++ b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestHMSPathsFullDump.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import junit.framework.Assert; + +import org.apache.sentry.provider.db.service.thrift.TPathsDump; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TCompactProtocol; +import org.junit.Test; + +import com.google.common.collect.Lists; + +public class TestHMSPathsFullDump { + + @Test + public void testDumpAndInitialize() { + HMSPaths hmsPaths = new HMSPaths(new String[] {"/user/hive/warehouse"}); + hmsPaths._addAuthzObject("db1", Lists.newArrayList("/user/hive/warehouse/db1")); + hmsPaths._addAuthzObject("db1.tbl11", Lists.newArrayList("/user/hive/warehouse/db1/tbl11")); + hmsPaths._addPathsToAuthzObject("db1.tbl11", Lists.newArrayList( + "/user/hive/warehouse/db1/tbl11/part111", + "/user/hive/warehouse/db1/tbl11/part112", + "/user/hive/warehouse/db1/tbl11/p1=1/p2=x")); + + Assert.assertEquals("db1", hmsPaths.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1"}, false)); + Assert.assertEquals("db1.tbl11", hmsPaths.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1", "tbl11"}, false)); + Assert.assertEquals("db1.tbl11", hmsPaths.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1", "tbl11", "part111"}, false)); + Assert.assertEquals("db1.tbl11", hmsPaths.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1", "tbl11", "part112"}, false)); + + Assert.assertEquals("db1.tbl11", hmsPaths.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1", "tbl11", "p1=1", "p2=x"}, false)); + Assert.assertEquals("db1.tbl11", hmsPaths.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1", "tbl11", "p1=1"}, true)); + + HMSPathsSerDe serDe = hmsPaths.getPathsDump(); + TPathsDump pathsDump = serDe.createPathsDump(); + HMSPaths hmsPaths2 = serDe.initializeFromDump(pathsDump); + + Assert.assertEquals("db1", hmsPaths2.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1"}, false)); + Assert.assertEquals("db1.tbl11", hmsPaths2.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1", "tbl11"}, false)); + Assert.assertEquals("db1.tbl11", hmsPaths2.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1", "tbl11", "part111"}, false)); + Assert.assertEquals("db1.tbl11", hmsPaths2.findAuthzObject(new String[]{"user", "hive", "warehouse", "db1", "tbl11", "part112"}, false)); + } + + @Test + public void testThrftSerialization() throws TException { + HMSPaths hmsPaths = new HMSPaths(new String[] {"/"}); + String prefix = "/user/hive/warehouse/"; + for (int dbNum = 0; dbNum < 1; dbNum++) { + String dbName = "db" + dbNum; + hmsPaths._addAuthzObject(dbName, Lists.newArrayList(prefix + dbName)); + for (int tblNum = 0; tblNum < 1000000; tblNum++) { + String tblName = "tbl" + tblNum; + hmsPaths._addAuthzObject(dbName + "." + tblName, Lists.newArrayList(prefix + dbName + "/" + tblName)); + for (int partNum = 0; partNum < 1; partNum++) { + String partName = "part" + partNum; + hmsPaths + ._addPathsToAuthzObject( + dbName + "." + tblName, + Lists.newArrayList(prefix + dbName + "/" + tblName + "/" + + partName)); + } + } + } + HMSPathsSerDe serDe = hmsPaths.getPathsDump(); + long t1 = System.currentTimeMillis(); + TPathsDump pathsDump = serDe.createPathsDump(); + byte[] ser = new TSerializer(new TCompactProtocol.Factory()).serialize(pathsDump); + long serTime = System.currentTimeMillis() - t1; + System.out.println("Serialization Time: " + serTime + ", " + ser.length); + + t1 = System.currentTimeMillis(); + TPathsDump tPathsDump = new TPathsDump(); + new TDeserializer(new TCompactProtocol.Factory()).deserialize(tPathsDump, ser); + HMSPaths fromDump = serDe.initializeFromDump(tPathsDump); + System.out.println("Deserialization Time: " + (System.currentTimeMillis() - t1)); + Assert.assertEquals("db9.tbl999", fromDump.findAuthzObject(new String[]{"user", "hive", "warehouse", "db0", "tbl999"}, false)); + Assert.assertEquals("db9.tbl999", fromDump.findAuthzObject(new String[]{"user", "hive", "warehouse", "db0", "tbl999", "part5"}, false)); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestUpdateableAuthzPaths.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestUpdateableAuthzPaths.java b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestUpdateableAuthzPaths.java new file mode 100644 index 0000000..76c849f --- /dev/null +++ b/sentry-hdfs/src/test/java/org/apache/sentry/hdfs/TestUpdateableAuthzPaths.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.hdfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; + +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.sentry.provider.db.service.thrift.TPathChanges; +import org.junit.Test; + +import com.google.common.collect.Lists; + +public class TestUpdateableAuthzPaths { + + @Test + public void testFullUpdate() { + HMSPaths hmsPaths = createBaseHMSPaths(1, 1); + assertEquals("db1", hmsPaths.findAuthzObjectExactMatch(new String[]{"db1"})); + assertEquals("db1.tbl11", hmsPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11"})); + assertEquals("db1.tbl11", hmsPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part111"})); + assertEquals("db1.tbl11", hmsPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part112"})); + + UpdateableAuthzPaths authzPaths = new UpdateableAuthzPaths(hmsPaths); + PathsUpdate update = new PathsUpdate(1, true); + update.getThriftObject().setPathsDump(authzPaths.getPathsDump().createPathsDump()); + + UpdateableAuthzPaths authzPaths2 = new UpdateableAuthzPaths(new String[] {"/"}); + UpdateableAuthzPaths pre = authzPaths2.updateFull(update); + assertFalse(pre == authzPaths2); + authzPaths2 = pre; + + assertEquals("db1", authzPaths2.findAuthzObjectExactMatch(new String[]{"db1"})); + assertEquals("db1.tbl11", authzPaths2.findAuthzObjectExactMatch(new String[]{"db1", "tbl11"})); + assertEquals("db1.tbl11", authzPaths2.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part111"})); + assertEquals("db1.tbl11", authzPaths2.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part112"})); + + // Ensure Full Update wipes old stuff + UpdateableAuthzPaths authzPaths3 = new UpdateableAuthzPaths(createBaseHMSPaths(2, 1)); + update = new PathsUpdate(2, true); + update.getThriftObject().setPathsDump(authzPaths3.getPathsDump().createPathsDump()); + pre = authzPaths2.updateFull(update); + assertFalse(pre == authzPaths2); + authzPaths2 = pre; + + assertNull(authzPaths2.findAuthzObjectExactMatch(new String[]{"db1"})); + assertNull(authzPaths2.findAuthzObjectExactMatch(new String[]{"db1", "tbl11"})); + + assertEquals("db2", authzPaths2.findAuthzObjectExactMatch(new String[]{"db2"})); + assertEquals("db2.tbl21", authzPaths2.findAuthzObjectExactMatch(new String[]{"db2", "tbl21"})); + assertEquals("db2.tbl21", authzPaths2.findAuthzObjectExactMatch(new String[]{"db2", "tbl21", "part211"})); + assertEquals("db2.tbl21", authzPaths2.findAuthzObjectExactMatch(new String[]{"db2", "tbl21", "part212"})); + } + + @Test + public void testPartialUpdateAddPath() { + HMSPaths hmsPaths = createBaseHMSPaths(1, 1); + UpdateableAuthzPaths authzPaths = new UpdateableAuthzPaths(hmsPaths); + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + // Create table + PathsUpdate update = new PathsUpdate(2, false); + TPathChanges pathChange = update.newPathChange("db1.tbl12"); + pathChange.addToAddPaths(PathsUpdate.cleanPath("file:///db1/tbl12")); + authzPaths.updatePartial(Lists.newArrayList(update), lock); + + // Add partition + update = new PathsUpdate(3, false); + pathChange = update.newPathChange("db1.tbl12"); + pathChange.addToAddPaths(PathsUpdate.cleanPath("file:///db1/tbl12/part121")); + authzPaths.updatePartial(Lists.newArrayList(update), lock); + + // Ensure no change in existing Paths + assertEquals("db1", authzPaths.findAuthzObjectExactMatch(new String[]{"db1"})); + assertEquals("db1.tbl11", authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11"})); + assertEquals("db1.tbl11", authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part111"})); + assertEquals("db1.tbl11", authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part112"})); + + // Verify new Paths + assertEquals("db1.tbl12", authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl12"})); + assertEquals("db1.tbl12", authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl12", "part121"})); + } + + @Test + public void testPartialUpdateDelPath() { + HMSPaths hmsPaths = createBaseHMSPaths(1, 1); + UpdateableAuthzPaths authzPaths = new UpdateableAuthzPaths(hmsPaths); + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + assertEquals("db1.tbl11", authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11"})); + assertEquals("db1.tbl11", authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part111"})); + + // Drop partition + PathsUpdate update = new PathsUpdate(2, false); + TPathChanges pathChange = update.newPathChange("db1.tbl11"); + pathChange.addToDelPaths(PathsUpdate.cleanPath("file:///db1/tbl11/part111")); + authzPaths.updatePartial(Lists.newArrayList(update), lock); + + // Verify Paths deleted + assertNull(authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part111"})); + + // Verify rest ok + assertEquals("db1.tbl11", authzPaths.findAuthzObjectExactMatch(new String[]{"db1", "tbl11", "part112"})); + } + + private HMSPaths createBaseHMSPaths(int dbNum, int tblNum) { + String db = "db" + dbNum; + String tbl = "tbl" + dbNum + "" + tblNum; + String fullTbl = db + "." + tbl; + String dbPath = "/" + db; + String tblPath = "/" + db + "/" + tbl; + String partPath = tblPath + "/part" + dbNum + "" + tblNum; + HMSPaths hmsPaths = new HMSPaths(new String[] {"/"}); + hmsPaths._addAuthzObject(db, Lists.newArrayList(dbPath)); + hmsPaths._addAuthzObject(fullTbl, Lists.newArrayList(tblPath)); + hmsPaths._addPathsToAuthzObject(fullTbl, Lists.newArrayList( + partPath + "1", partPath + "2" )); + return hmsPaths; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-hdfs/src/test/resources/hdfs-sentry.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/src/test/resources/hdfs-sentry.xml b/sentry-hdfs/src/test/resources/hdfs-sentry.xml new file mode 100644 index 0000000..c23a431 --- /dev/null +++ b/sentry-hdfs/src/test/resources/hdfs-sentry.xml @@ -0,0 +1,22 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <!-- dummy file that gets rewritten by testcases in target test classpath --> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/0eb6645e/sentry-provider/sentry-provider-db/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/pom.xml b/sentry-provider/sentry-provider-db/pom.xml index b4167e4..b8da31b 100644 --- a/sentry-provider/sentry-provider-db/pom.xml +++ b/sentry-provider/sentry-provider-db/pom.xml @@ -42,6 +42,11 @@ limitations under the License. <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <version>2.5.0</version> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> @@ -72,6 +77,10 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.sentry</groupId> + <artifactId>sentry-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> <artifactId>sentry-core-common</artifactId> </dependency> <dependency> @@ -80,6 +89,10 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.sentry</groupId> + <artifactId>sentry-service-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> <artifactId>sentry-provider-common</artifactId> </dependency> <dependency> @@ -89,6 +102,11 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <version>0.13.1-cdh5.2.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> <artifactId>hive-shims</artifactId> <scope>provided</scope> </dependency> @@ -109,6 +127,11 @@ limitations under the License. <artifactId>ant-contrib</artifactId> </dependency> <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-hdfs-int</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minikdc</artifactId> <scope>test</scope> @@ -138,6 +161,11 @@ limitations under the License. <artifactId>mockito-all</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${hive.version}</version> + </dependency> </dependencies> <build> @@ -189,68 +217,5 @@ limitations under the License. </plugin> </plugins> </build> - <profiles> - <profile> - <id>thriftif</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>generate-thrift-sources</id> - <phase>generate-sources</phase> - <configuration> - <target> - <taskdef name="for" classname="net.sf.antcontrib.logic.ForTask" - classpathref="maven.plugin.classpath" /> - <property name="thrift.args" value="-I ${thrift.home} --gen java:beans,hashcode"/> - <property name="thrift.gen.dir" value="${basedir}/src/gen/thrift"/> - <delete dir="${thrift.gen.dir}"/> - <mkdir dir="${thrift.gen.dir}"/> - <for param="thrift.file"> - <path> - <fileset dir="${basedir}/src/main/resources/" includes="**/*.thrift" /> - </path> - <sequential> - <echo message="Generating Thrift code for @{thrift.file}"/> - <exec executable="${thrift.home}/bin/thrift" failonerror="true" dir="."> - <arg line="${thrift.args} -I ${basedir}/src/main/resources/ -o ${thrift.gen.dir} @{thrift.file} " /> - </exec> - </sequential> - </for> - </target> - </configuration> - <goals> - <goal>run</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <executions> - <execution> - <id>enforce-property</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <requireProperty> - <property>thrift.home</property> - </requireProperty> - </rules> - <fail>true</fail> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> </project>
