Repository: sentry Updated Branches: refs/heads/master d0b12a4ba -> efe2e01b0
SENTRY-1916: Sentry should not store paths outside of the prefix (Alex Kolbasov, reviewed by Vamsee Yarlagadda) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/efe2e01b Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/efe2e01b Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/efe2e01b Branch: refs/heads/master Commit: efe2e01b096b439ad1eee195d4e5604828f57ff3 Parents: d0b12a4 Author: Alexander Kolbasov <[email protected]> Authored: Tue Sep 5 21:15:33 2017 -0700 Committer: Alexander Kolbasov <[email protected]> Committed: Tue Sep 5 21:15:33 2017 -0700 ---------------------------------------------------------------------- .../apache/sentry/hdfs/ServiceConstants.java | 45 ++++++-------------- .../apache/sentry/hdfs/PathImageRetriever.java | 8 ++-- .../org/apache/sentry/hdfs/SentryPlugin.java | 8 +++- .../apache/sentry/hdfs/TestImageRetriever.java | 5 ++- .../tests/e2e/hdfs/TestHDFSIntegration.java | 5 ++- .../tests/e2e/hdfs/TestHDFSIntegrationBase.java | 7 ++- .../e2e/hdfs/TestHDFSIntegrationEnd2End.java | 44 +++++++++++++++++++ 7 files changed, 83 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/efe2e01b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java index 55092ee..0ff717d 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java @@ -15,37 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.sentry.hdfs; - -import java.util.HashMap; -import java.util.Map; - -import javax.security.sasl.Sasl; -import com.google.common.collect.ImmutableMap; +package org.apache.sentry.hdfs; public class ServiceConstants { - private static final ImmutableMap<String, String> SASL_PROPERTIES; - // number used in authz paths and permissions to request initial syncs - public static final long SEQUENCE_NUMBER_UPDATE_UNINITIALIZED = -1L; + static final long SEQUENCE_NUMBER_UPDATE_UNINITIALIZED = -1L; // number used in authz paths and permissions to request initial syncs - public static final long IMAGE_NUMBER_UPDATE_UNINITIALIZED = 0L; - - // number used in authz paths and permissions that specifies an unused image number - public static final long IMAGE_NUMBER_UPDATE_UNUSED = -1L; - - static { - Map<String, String> saslProps = new HashMap<String, String>(); - saslProps.put(Sasl.SERVER_AUTH, "true"); - saslProps.put(Sasl.QOP, "auth-conf"); - SASL_PROPERTIES = ImmutableMap.copyOf(saslProps); - } + static final long IMAGE_NUMBER_UPDATE_UNINITIALIZED = 0L; public static class ServerConfig { - public static final ImmutableMap<String, String> SASL_PROPERTIES = ServiceConstants.SASL_PROPERTIES; /** * This configuration parameter is only meant to be used for testing purposes. */ @@ -60,27 +41,29 @@ public class ServiceConstants { public static final int SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_PART_PER_RPC_DEFAULT = 100; public static final String SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC = "sentry.hdfs.sync.metastore.cache.max-tables-per-rpc"; public static final int SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC_DEFAULT = 100; - public static final String SENTRY_SERVICE_FULL_UPDATE_SIGNAL = "sentry.hdfs.sync.full-update-signal"; + static final String SENTRY_SERVICE_FULL_UPDATE_SIGNAL = "sentry.hdfs.sync.full-update-signal"; + + public static final String SENTRY_HDFS_INTEGRATION_PATH_PREFIXES = "sentry.hdfs.integration.path.prefixes"; + public static final String[] SENTRY_HDFS_INTEGRATION_PATH_PREFIXES_DEFAULT = + new String[]{"/user/hive/warehouse"}; } public static class ClientConfig { - public static final ImmutableMap<String, String> SASL_PROPERTIES = ServiceConstants.SASL_PROPERTIES; public static final String SECURITY_MODE = "sentry.hdfs.service.security.mode"; - public static final String SECURITY_MODE_KERBEROS = "kerberos"; - public static final String SECURITY_USE_UGI_TRANSPORT = "sentry.hdfs.service.security.use.ugi"; + static final String SECURITY_MODE_KERBEROS = "kerberos"; + static final String SECURITY_USE_UGI_TRANSPORT = "sentry.hdfs.service.security.use.ugi"; public static final String PRINCIPAL = "sentry.hdfs.service.server.principal"; public static final String SERVER_RPC_PORT = "sentry.hdfs.service.client.server.rpc-port"; public static final String SERVER_RPC_ADDRESS = "sentry.hdfs.service.client.server.rpc-addresses"; - public static final String USE_COMPACT_TRANSPORT = "sentry.hdfs.service.client.compact.transport"; - public static final boolean USE_COMPACT_TRANSPORT_DEFAULT = false; + static final String USE_COMPACT_TRANSPORT = "sentry.hdfs.service.client.compact.transport"; + static final boolean USE_COMPACT_TRANSPORT_DEFAULT = false; // max message size for thrift messages - public static final String SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE = "sentry.hdfs.thrift.max.message.size"; - public static final long SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT = 100 * 1024 * 1024; + static final String SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE = "sentry.hdfs.thrift.max.message.size"; + static final long SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT = 100 * 1024 * 1024; } - } http://git-wip-us.apache.org/repos/asf/sentry/blob/efe2e01b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java index 69e43fd..1762ee5 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java @@ -40,11 +40,13 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; @ThreadSafe class PathImageRetriever implements ImageRetriever<PathsUpdate> { - private static final String[] root = {"/"}; private final SentryStore sentryStore; + /** List of prefixes managed by Sentry */ + private final String[] prefixes; - PathImageRetriever(SentryStore sentryStore) { + PathImageRetriever(SentryStore sentryStore, String[] prefixes) { this.sentryStore = sentryStore; + this.prefixes = prefixes; } @Override @@ -87,7 +89,7 @@ class PathImageRetriever implements ImageRetriever<PathsUpdate> { // Translate PathsUpdate that contains a full image to TPathsDump for // consumer (NN) to be able to quickly construct UpdateableAuthzPaths // from TPathsDump. - UpdateableAuthzPaths authzPaths = new UpdateableAuthzPaths(root); + UpdateableAuthzPaths authzPaths = new UpdateableAuthzPaths(prefixes); authzPaths.updatePartial(Lists.newArrayList(pathsUpdate), new ReentrantReadWriteLock()); //Setting minimizeSize parameter to false based on interface description http://git-wip-us.apache.org/repos/asf/sentry/blob/efe2e01b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index 551f184..1318082 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -46,6 +46,8 @@ import org.slf4j.LoggerFactory; import static org.apache.sentry.hdfs.ServiceConstants.IMAGE_NUMBER_UPDATE_UNINITIALIZED; import static org.apache.sentry.hdfs.ServiceConstants.SEQUENCE_NUMBER_UPDATE_UNINITIALIZED; +import static org.apache.sentry.hdfs.ServiceConstants.ServerConfig.SENTRY_HDFS_INTEGRATION_PATH_PREFIXES; +import static org.apache.sentry.hdfs.ServiceConstants.ServerConfig.SENTRY_HDFS_INTEGRATION_PATH_PREFIXES_DEFAULT; import static org.apache.sentry.hdfs.Updateable.Update; import static org.apache.sentry.hdfs.service.thrift.sentry_hdfs_serviceConstants.UNUSED_PATH_UPDATE_IMG_NUM; @@ -103,8 +105,12 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen @Override public void initialize(Configuration conf, SentryStore sentryStore) throws SentryPluginException { + // List of paths managed by Sentry + String[] prefixes = + conf.getStrings(SENTRY_HDFS_INTEGRATION_PATH_PREFIXES, + SENTRY_HDFS_INTEGRATION_PATH_PREFIXES_DEFAULT); PermImageRetriever permImageRetriever = new PermImageRetriever(sentryStore); - PathImageRetriever pathImageRetriever = new PathImageRetriever(sentryStore); + PathImageRetriever pathImageRetriever = new PathImageRetriever(sentryStore, prefixes); PermDeltaRetriever permDeltaRetriever = new PermDeltaRetriever(sentryStore); PathDeltaRetriever pathDeltaRetriever = new PathDeltaRetriever(sentryStore); pathsUpdater = new DBUpdateForwarder<>(pathImageRetriever, pathDeltaRetriever); http://git-wip-us.apache.org/repos/asf/sentry/blob/efe2e01b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java index 478ccf9..1bdebb1 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java @@ -36,6 +36,7 @@ import static org.junit.Assert.assertTrue; public class TestImageRetriever { SentryStore sentryStoreMock; + private static final String[] root = {"/"}; @Before public void setUp() { @@ -47,7 +48,7 @@ public class TestImageRetriever { Mockito.when(sentryStoreMock.retrieveFullPathsImage()) .thenReturn(new PathsImage(new HashMap<String, Collection<String>>(), 0, 0)); - PathImageRetriever imageRetriever = new PathImageRetriever(sentryStoreMock); + PathImageRetriever imageRetriever = new PathImageRetriever(sentryStoreMock, root); PathsUpdate pathsUpdate = imageRetriever.retrieveFullImage(); assertEquals(0, pathsUpdate.getImgNum()); @@ -67,7 +68,7 @@ public class TestImageRetriever { Mockito.when(sentryStoreMock.retrieveFullPathsImage()) .thenReturn(new PathsImage(fullPathsImage, 1, 1)); - imageRetriever = new PathImageRetriever(sentryStoreMock); + imageRetriever = new PathImageRetriever(sentryStoreMock, root); pathsUpdate = imageRetriever.retrieveFullImage(); assertEquals(1, pathsUpdate.getImgNum()); http://git-wip-us.apache.org/repos/asf/sentry/blob/efe2e01b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java index acf1cf5..e9ae6a1 100644 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java +++ b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java @@ -131,6 +131,8 @@ public class TestHDFSIntegration { } + private static final String MANAGED_PREFIXES = "/user/hive/warehouse,/tmp/external"; + private static final int NUM_RETRIES = 10; private static final int RETRY_WAIT = 1000; private static final String EXTERNAL_SENTRY_SERVICE = "sentry.e2etest.external.sentry"; @@ -322,7 +324,7 @@ public class TestHDFSIntegration { MiniDFS.PseudoGroupMappingService.class.getName()); Configuration.addDefaultResource("test.xml"); - hadoopConf.set("sentry.authorization-provider.hdfs-path-prefixes", "/user/hive/warehouse,/tmp/external"); + hadoopConf.set("sentry.authorization-provider.hdfs-path-prefixes", MANAGED_PREFIXES); hadoopConf.set("sentry.authorization-provider.cache-refresh-retry-wait.ms", "5000"); hadoopConf.set("sentry.authorization-provider.cache-refresh-interval.ms", String.valueOf(CACHE_REFRESH)); @@ -385,6 +387,7 @@ public class TestHDFSIntegration { @Override public Void run() throws Exception { Configuration sentryConf = new Configuration(false); + sentryConf.set(SENTRY_HDFS_INTEGRATION_PATH_PREFIXES, MANAGED_PREFIXES); Map<String, String> properties = Maps.newHashMap(); properties.put(HiveServerFactory.AUTHZ_PROVIDER_BACKEND, SimpleDBProviderBackend.class.getName()); http://git-wip-us.apache.org/repos/asf/sentry/blob/efe2e01b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java index 63e718c..718b02c 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java @@ -92,6 +92,8 @@ import com.google.common.collect.Maps; import com.google.common.io.Files; import com.google.common.io.Resources; +import static org.apache.sentry.hdfs.ServiceConstants.ServerConfig.SENTRY_HDFS_INTEGRATION_PATH_PREFIXES; + /** * Base abstract class for HDFS Sync integration * (both Non-HA and HA modes) @@ -136,6 +138,8 @@ public abstract class TestHDFSIntegrationBase { } + private static final String MANAGED_PREFIXES = "/user/hive/warehouse,/tmp/external"; + protected static final int NUM_RETRIES = 10; protected static final int RETRY_WAIT = 1000; //ms protected static final String EXTERNAL_SENTRY_SERVICE = "sentry.e2etest.external.sentry"; @@ -687,7 +691,7 @@ public abstract class TestHDFSIntegrationBase { MiniDFS.PseudoGroupMappingService.class.getName()); Configuration.addDefaultResource("test.xml"); - hadoopConf.set("sentry.authorization-provider.hdfs-path-prefixes", "/user/hive/warehouse,/tmp/external"); + hadoopConf.set("sentry.authorization-provider.hdfs-path-prefixes", MANAGED_PREFIXES); hadoopConf.set("sentry.authorization-provider.cache-refresh-retry-wait.ms", "5000"); hadoopConf.set("sentry.authorization-provider.cache-refresh-interval.ms", String.valueOf(CACHE_REFRESH)); @@ -772,6 +776,7 @@ public abstract class TestHDFSIntegrationBase { @Override public Void run() throws Exception { Configuration sentryConf = new Configuration(false); + sentryConf.set(SENTRY_HDFS_INTEGRATION_PATH_PREFIXES, MANAGED_PREFIXES); Map<String, String> properties = Maps.newHashMap(); properties.put(HiveServerFactory.AUTHZ_PROVIDER_BACKEND, SimpleDBProviderBackend.class.getName()); http://git-wip-us.apache.org/repos/asf/sentry/blob/efe2e01b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java index a8d70c1..d4bc97d 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java @@ -46,6 +46,50 @@ public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase { private static String adminRole = "admin_role"; + + @Test + public void testEnd2EndManagedPaths() throws Throwable { + tmpHDFSDir = new Path("/tmp/external"); + dbNames = new String[]{"db1"}; + roles = new String[]{"admin_role", "db_role", "tab_role", "p1_admin"}; + admin = "hive"; + + Connection conn; + Statement stmt; + conn = hiveServer2.createConnection("hive", "hive"); + stmt = conn.createStatement(); + stmt.execute("create role admin_role"); + stmt.execute("grant role admin_role to group hive"); + stmt.execute("grant all on server server1 to role admin_role"); + stmt.execute("create role p1_admin"); + stmt.execute("grant role p1_admin to group hbase"); + stmt.execute("create role db_role"); + stmt.execute("create role tab_role"); + stmt.execute("grant role db_role to group hbase"); + stmt.execute("grant role tab_role to group flume"); + + // Test DB case insensitivity + stmt.execute("create database extdb"); + stmt.execute("grant all on database ExtDb to role p1_admin"); + writeToPath("/tmp/external/ext100", 5, "foo", "bar"); + writeToPath("/tmp/external/ext101", 5, "foo", "bar"); + stmt.execute("use extdb"); + stmt.execute( + "create table ext100 (s string) location \'/tmp/external/ext100\'"); + stmt.execute( + "create table ext101 (s string) location \'/tmp/external/ext101\'"); + verifyQuery(stmt, "ext100", 5); + verifyOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "hbase", true); + stmt.execute("drop table ext100"); + stmt.execute("drop table ext101"); + stmt.execute("use default"); + stmt.execute("drop database extdb"); + + stmt.close(); + conn.close(); + } + + @Test public void testEnd2End() throws Throwable { tmpHDFSDir = new Path("/tmp/external");
