Repository: incubator-sentry Updated Branches: refs/heads/master e4dc61d52 -> de0cc9e7f
SENTRY-477: Sentry service should expose metrics (Sravya Tirukkovalur via Prasad Mujumdar) Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/de0cc9e7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/de0cc9e7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/de0cc9e7 Branch: refs/heads/master Commit: de0cc9e7f62c1a44474b196d521b87d1ff902f0e Parents: e4dc61d Author: Prasad Mujumdar <pras...@cloudera.com> Authored: Wed Oct 29 11:42:28 2014 -0700 Committer: Prasad Mujumdar <pras...@cloudera.com> Committed: Wed Oct 29 11:42:28 2014 -0700 ---------------------------------------------------------------------- pom.xml | 6 +- sentry-provider/sentry-provider-db/pom.xml | 25 ++++ .../db/service/persistent/SentryStore.java | 42 ++++++ ...SentryHealthCheckServletContextListener.java | 35 +++++ .../db/service/thrift/SentryMetrics.java | 135 +++++++++++++++++++ .../SentryMetricsServletContextListener.java | 32 +++++ .../thrift/SentryPolicyStoreProcessor.java | 56 ++++++++ .../db/service/thrift/SentryWebServer.java | 55 ++++++++ .../sentry/service/thrift/SentryService.java | 30 ++++- .../sentry/service/thrift/ServiceConstants.java | 8 ++ .../db/service/persistent/TestSentryStore.java | 65 +++++++++ .../AbstractTestWithStaticConfiguration.java | 2 + 12 files changed, 486 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e172e92..405367f 100644 --- a/pom.xml +++ b/pom.xml @@ -50,8 +50,8 @@ limitations under the License. <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <maven.compile.source>1.6</maven.compile.source> - <maven.compile.target>1.6</maven.compile.target> + <maven.compile.source>1.7</maven.compile.source> + <maven.compile.target>1.7</maven.compile.target> <!-- versions are in alphabetical order --> <ant.contrib.version>1.0b3</ant.contrib.version> <maven.antrun.plugin.version>1.7</maven.antrun.plugin.version> @@ -83,6 +83,8 @@ limitations under the License. <zookeeper.version>3.4.5-cdh5.1.0-SNAPSHOT</zookeeper.version> <pig.version>0.12.0-cdh5.1.0-SNAPSHOT</pig.version> <jackson.version>1.8.8</jackson.version> + <metrics.version>3.1.0</metrics.version> + <jettyVersion>7.6.16.v20140903</jettyVersion> </properties> <dependencyManagement> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/pom.xml b/sentry-provider/sentry-provider-db/pom.xml index b4167e4..fbf831a 100644 --- a/sentry-provider/sentry-provider-db/pom.xml +++ b/sentry-provider/sentry-provider-db/pom.xml @@ -134,6 +134,31 @@ limitations under the License. <artifactId>datanucleus-rdbms</artifactId> </dependency> <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + <version>${metrics.version}</version> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-servlets</artifactId> + <version>${metrics.version}</version> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-jvm</artifactId> + <version>${metrics.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>${jettyVersion}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <version>${jettyVersion}</version> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index 017cf08..f6699d2 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -18,6 +18,7 @@ package org.apache.sentry.provider.db.service.persistent; +import com.codahale.metrics.Gauge; import static org.apache.sentry.provider.common.ProviderConstants.AUTHORIZABLE_JOINER; import static org.apache.sentry.provider.common.ProviderConstants.KV_JOINER; @@ -271,6 +272,47 @@ public class SentryStore { } } + private <T> Long getCount(Class<T> tClass) { + PersistenceManager pm = null; + Long size = new Long(-1); + try { + pm = openTransaction(); + Query query = pm.newQuery(); + query.setClass(tClass); + query.setResult("count(this)"); + size = (Long)query.execute(); + + } finally { + commitTransaction(pm); + } + return size; + } + public Gauge<Long> getRoleCountGauge() { + return new Gauge< Long >() { + @Override + public Long getValue() { + return getCount(MSentryRole.class); + } + }; + } + + public Gauge<Long> getPrivilegeCountGauge() { + return new Gauge< Long >() { + @Override + public Long getValue() { + return getCount(MSentryPrivilege.class); + } + }; + } + public Gauge<Long> getGroupCountGauge() { + return new Gauge< Long >() { + @Override + public Long getValue() { + return getCount(MSentryGroup.class); + } + }; + } + public CommitContext alterSentryRoleGrantPrivilege(String grantorPrincipal, String roleName, TSentryPrivilege privilege) throws SentryUserException { http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryHealthCheckServletContextListener.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryHealthCheckServletContextListener.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryHealthCheckServletContextListener.java new file mode 100644 index 0000000..8822c2e --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryHealthCheckServletContextListener.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.provider.db.service.thrift; + +import com.codahale.metrics.health.HealthCheckRegistry; +import com.codahale.metrics.servlets.HealthCheckServlet; + +/** + * Use this class's registry to register health checks: Can be some tests which make sure Sentry service is healthy + */ +public class SentryHealthCheckServletContextListener extends HealthCheckServlet.ContextListener { + + //This is just a place holder for health check registry, with out this AdminServlet throws out an error + public static final HealthCheckRegistry HEALTH_CHECK_REGISTRY = new HealthCheckRegistry(); + + @Override + protected HealthCheckRegistry getHealthCheckRegistry() { + return HEALTH_CHECK_REGISTRY; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java new file mode 100644 index 0000000..6bd7e2b --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.provider.db.service.thrift; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.MetricSet; +import com.codahale.metrics.Timer; +import com.codahale.metrics.jvm.BufferPoolMetricSet; +import com.codahale.metrics.jvm.GarbageCollectorMetricSet; +import com.codahale.metrics.jvm.MemoryUsageGaugeSet; +import com.codahale.metrics.jvm.ThreadStatesGaugeSet; +import org.apache.sentry.provider.db.service.persistent.SentryStore; + +import java.lang.management.ManagementFactory; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * A singleton class which holds metrics related utility functions as well as the list of metrics + */ +public class SentryMetrics { + private static SentryMetrics sentryMetrics = null; + private boolean reportingInitialized = false; + + public final Timer createRoleTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "create-role")); + public final Timer dropRoleTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "drop-role")); + public final Timer grantRoleTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "grant-role")); + public final Timer revokeRoleTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "revoke-role")); + public final Timer grantTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "grant-privilege")); + public final Timer revokeTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "revoke-privilege")); + + public final Timer dropPrivilegeTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "drop-privilege")); + public final Timer renamePrivilegeTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "rename-privilege")); + + public final Timer listRolesByGroupTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "list-roles-by-group")); + public final Timer listPrivilegesByRoleTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "list-privileges-by-role")); + public final Timer listPrivilegesForProviderTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "list-privileges-for-provider")); + public final Timer listPrivilegesByAuthorizableTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( + MetricRegistry.name(SentryPolicyStoreProcessor.class, "list-privileges-by-authorizable")); + + private SentryMetrics() { + registerMetricSet("gc", new GarbageCollectorMetricSet(), SentryMetricsServletContextListener.METRIC_REGISTRY); + registerMetricSet("buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()), + SentryMetricsServletContextListener.METRIC_REGISTRY); + registerMetricSet("memory", new MemoryUsageGaugeSet(), SentryMetricsServletContextListener.METRIC_REGISTRY); + registerMetricSet("threads", new ThreadStatesGaugeSet(), SentryMetricsServletContextListener.METRIC_REGISTRY); + } + + public static synchronized SentryMetrics getInstance() { + if (sentryMetrics == null) { + sentryMetrics = new SentryMetrics(); + } + return sentryMetrics; + } + + public void addSentryStoreGauges(SentryStore sentryStore) { + addGauge(SentryStore.class, "role_count", sentryStore.getRoleCountGauge()); + addGauge(SentryStore.class, "privilege_count", sentryStore.getPrivilegeCountGauge()); + addGauge(SentryStore.class, "group_count", sentryStore.getGroupCountGauge()); + } + + + /* Should be only called once to initialize the reporters + */ + public synchronized void initReporting(Reporting reporting) { + if(!reportingInitialized) { + switch(reporting) { + case CONSOLE: + final ConsoleReporter consoleReporter = ConsoleReporter.forRegistry(SentryMetricsServletContextListener.METRIC_REGISTRY) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(); + consoleReporter.start(1, TimeUnit.SECONDS); + break; + case JMX: + final JmxReporter jmxReporter = JmxReporter.forRegistry(SentryMetricsServletContextListener.METRIC_REGISTRY) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(); + jmxReporter.start(); + break; + } + } + } + + private <T, V> void addGauge(Class<T> tClass, String gaugeName, Gauge<V> gauge) { + SentryMetricsServletContextListener.METRIC_REGISTRY.register( + MetricRegistry.name(tClass, gaugeName), gauge); + } + + private void registerMetricSet(String prefix, MetricSet metricSet, MetricRegistry registry) { + for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) { + if (entry.getValue() instanceof MetricSet) { + registerMetricSet(prefix + "." + entry.getKey(), (MetricSet) entry.getValue(), registry); + } else { + registry.register(prefix + "." + entry.getKey(), entry.getValue()); + } + } + } + + public enum Reporting { + JMX, + CONSOLE; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetricsServletContextListener.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetricsServletContextListener.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetricsServletContextListener.java new file mode 100644 index 0000000..6692197 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetricsServletContextListener.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.provider.db.service.thrift; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.servlets.MetricsServlet; + +public class SentryMetricsServletContextListener extends MetricsServlet.ContextListener { + + public static final MetricRegistry METRIC_REGISTRY = new MetricRegistry(); + + @Override + protected MetricRegistry getMetricRegistry() { + return METRIC_REGISTRY; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java index d64d019..01077da 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; +import com.codahale.metrics.Timer; import org.apache.hadoop.conf.Configuration; import org.apache.sentry.SentryUserException; import org.apache.sentry.core.model.db.AccessConstants; @@ -67,6 +68,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { private final NotificationHandlerInvoker notificationHandlerInvoker; private final ImmutableSet<String> adminGroups; private boolean isReady; + SentryMetrics sentryMetrics; public SentryPolicyStoreProcessor(String name, Configuration conf) throws Exception { super(); @@ -79,6 +81,24 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { isReady = true; adminGroups = ImmutableSet.copyOf(toTrimedLower(Sets.newHashSet(conf.getStrings( ServerConfig.ADMIN_GROUPS, new String[]{})))); + initReporting(); + } + + private void initReporting() { + String sentryReporting = conf.get(ServerConfig.SENTRY_REPORTING); + if( sentryReporting != null) { + SentryMetrics.Reporting reporting; + try { + reporting = SentryMetrics.Reporting.valueOf(sentryReporting.toUpperCase()); + sentryMetrics = SentryMetrics.getInstance(); + sentryMetrics.addSentryStoreGauges(sentryStore); + sentryMetrics.initReporting(reporting); + + } catch (IllegalArgumentException e) { + LOGGER.warn("Metrics reporting not configured correctly, please set " + ServerConfig.SENTRY_REPORTING + + " to: " + ServerConfig.SENTRY_REPORTING_CONSOLE + "/" + ServerConfig.SENTRY_REPORTING_JMX); + } + } } public void stop() { @@ -148,6 +168,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TCreateSentryRoleResponse create_sentry_role( TCreateSentryRoleRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.createRoleTimer.time(); TCreateSentryRoleResponse response = new TCreateSentryRoleResponse(); try { authorize(request.getRequestorUserName(), @@ -167,6 +188,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error for request: " + request + ", message: " + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } AUDIT_LOGGER.info(JsonLogEntityFactory.getInstance().createJsonLogEntity( @@ -177,6 +200,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TAlterSentryRoleGrantPrivilegeResponse alter_sentry_role_grant_privilege (TAlterSentryRoleGrantPrivilegeRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.grantTimer.time(); TAlterSentryRoleGrantPrivilegeResponse response = new TAlterSentryRoleGrantPrivilegeResponse(); try { @@ -201,6 +225,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error for request: " + request + ", message: " + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } AUDIT_LOGGER.info(JsonLogEntityFactory.getInstance().createJsonLogEntity( @@ -211,6 +237,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TAlterSentryRoleRevokePrivilegeResponse alter_sentry_role_revoke_privilege (TAlterSentryRoleRevokePrivilegeRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.revokeTimer.time(); TAlterSentryRoleRevokePrivilegeResponse response = new TAlterSentryRoleRevokePrivilegeResponse(); try { CommitContext commitContext = sentryStore.alterSentryRoleRevokePrivilege(request.getRequestorUserName(), @@ -237,6 +264,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error for request: " + request + ", message: " + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } AUDIT_LOGGER.info(JsonLogEntityFactory.getInstance().createJsonLogEntity( @@ -247,6 +276,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TDropSentryRoleResponse drop_sentry_role( TDropSentryRoleRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.dropRoleTimer.time(); TDropSentryRoleResponse response = new TDropSentryRoleResponse(); TSentryResponseStatus status; try { @@ -267,6 +297,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error for request: " + request + ", message: " + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } AUDIT_LOGGER.info(JsonLogEntityFactory.getInstance().createJsonLogEntity( @@ -277,6 +309,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TAlterSentryRoleAddGroupsResponse alter_sentry_role_add_groups( TAlterSentryRoleAddGroupsRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.grantRoleTimer.time(); TAlterSentryRoleAddGroupsResponse response = new TAlterSentryRoleAddGroupsResponse(); try { authorize(request.getRequestorUserName(), @@ -297,6 +330,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error for request: " + request + ", message: " + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } AUDIT_LOGGER.info(JsonLogEntityFactory.getInstance().createJsonLogEntity( @@ -307,6 +342,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TAlterSentryRoleDeleteGroupsResponse alter_sentry_role_delete_groups( TAlterSentryRoleDeleteGroupsRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.revokeRoleTimer.time(); TAlterSentryRoleDeleteGroupsResponse response = new TAlterSentryRoleDeleteGroupsResponse(); try { authorize(request.getRequestorUserName(), @@ -327,6 +363,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error adding groups to role: " + request; LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } AUDIT_LOGGER.info(JsonLogEntityFactory.getInstance().createJsonLogEntity( @@ -337,6 +375,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TListSentryRolesResponse list_sentry_roles_by_group( TListSentryRolesRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.listRolesByGroupTimer.time(); TListSentryRolesResponse response = new TListSentryRolesResponse(); TSentryResponseStatus status; Set<TSentryRole> roleSet = new HashSet<TSentryRole>(); @@ -373,6 +412,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error for request: " + request + ", message: " + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } return response; } @@ -380,6 +421,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TListSentryPrivilegesResponse list_sentry_privileges_by_role( TListSentryPrivilegesRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.listPrivilegesByRoleTimer.time(); TListSentryPrivilegesResponse response = new TListSentryPrivilegesResponse(); TSentryResponseStatus status; Set<TSentryPrivilege> privilegeSet = new HashSet<TSentryPrivilege>(); @@ -413,6 +455,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error for request: " + request + ", message: " + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } return response; } @@ -424,6 +468,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TListSentryPrivilegesForProviderResponse list_sentry_privileges_for_provider( TListSentryPrivilegesForProviderRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.listPrivilegesForProviderTimer.time(); TListSentryPrivilegesForProviderResponse response = new TListSentryPrivilegesForProviderResponse(); response.setPrivileges(new HashSet<String>()); try { @@ -449,6 +494,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { String msg = "Unknown error for request: " + request + ", message: " + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } return response; } @@ -495,6 +542,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TDropPrivilegesResponse drop_sentry_privilege( TDropPrivilegesRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.dropPrivilegeTimer.time(); TDropPrivilegesResponse response = new TDropPrivilegesResponse(); try { authorize(request.getRequestorUserName(), adminGroups); @@ -508,6 +556,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } return response; } @@ -515,6 +565,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TRenamePrivilegesResponse rename_sentry_privilege( TRenamePrivilegesRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.renamePrivilegeTimer.time(); TRenamePrivilegesResponse response = new TRenamePrivilegesResponse(); try { authorize(request.getRequestorUserName(), adminGroups); @@ -529,6 +580,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.close(); } return response; } @@ -536,6 +589,7 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { @Override public TListSentryPrivilegesByAuthResponse list_sentry_privileges_by_authorizable( TListSentryPrivilegesByAuthRequest request) throws TException { + final Timer.Context timerContext = sentryMetrics.listPrivilegesByAuthorizableTimer.time(); TListSentryPrivilegesByAuthResponse response = new TListSentryPrivilegesByAuthResponse(); Map<TSentryAuthorizable, TSentryPrivilegeMap> authRoleMap = Maps.newHashMap(); String subject = request.getRequestorUserName(); @@ -586,6 +640,8 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { + e.getMessage(); LOGGER.error(msg, e); response.setStatus(Status.RuntimeError(msg, e)); + } finally { + timerContext.stop(); } return response; } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryWebServer.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryWebServer.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryWebServer.java new file mode 100644 index 0000000..0243c48 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryWebServer.java @@ -0,0 +1,55 @@ +package org.apache.sentry.provider.db.service.thrift; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.codahale.metrics.servlets.AdminServlet; + +import java.util.EventListener; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +import java.util.List; + +public class SentryWebServer { + Server server; + int port; + + public SentryWebServer(List<EventListener> listeners, int port) { + this.port = port; + server = new Server(port); + ServletContextHandler servletContextHandler = new ServletContextHandler(); + ServletHolder servletHolder = new ServletHolder(AdminServlet.class); + servletContextHandler.addServlet(servletHolder, "/*"); + + for(EventListener listener:listeners) { + servletContextHandler.addEventListener(listener); + } + + server.setHandler(servletContextHandler); + } + + public void start() throws Exception{ + server.start(); + } + public void stop() throws Exception{ + server.stop(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java index 40e8a0e..8915b4d 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java @@ -25,7 +25,10 @@ import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.ServerSocket; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.EventListener; import java.util.HashSet; +import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -45,6 +48,9 @@ import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SecurityUtil; import org.apache.sentry.Command; +import org.apache.sentry.provider.db.service.thrift.SentryHealthCheckServletContextListener; +import org.apache.sentry.provider.db.service.thrift.SentryMetricsServletContextListener; +import org.apache.sentry.provider.db.service.thrift.SentryWebServer; import org.apache.sentry.service.thrift.ServiceConstants.ConfUtilties; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.apache.thrift.TMultiplexedProcessor; @@ -55,12 +61,10 @@ import org.apache.thrift.transport.TSaslServerTransport; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TServerTransport; import org.apache.thrift.transport.TTransportFactory; -import org.mortbay.log.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; public class SentryService implements Callable { @@ -83,6 +87,8 @@ public class SentryService implements Callable { private Future future; private TServer thriftServer; private Status status; + private int webServerPort; + private SentryWebServer sentryWebServer; public SentryService(Configuration conf) { this.conf = conf; @@ -133,6 +139,7 @@ public class SentryService implements Callable { + (count++)); } }); + webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT); status = Status.NOT_STARTED; } @@ -210,9 +217,25 @@ public class SentryService implements Callable { .minWorkerThreads(minThreads).maxWorkerThreads(maxThreads); thriftServer = new TThreadPoolServer(args); LOGGER.info("Serving on " + address); + startSentryWebServer(); thriftServer.serve(); } + private void startSentryWebServer() throws Exception{ + List<EventListener> listenerList = new ArrayList<EventListener>(); + listenerList.add(new SentryHealthCheckServletContextListener()); + listenerList.add(new SentryMetricsServletContextListener()); + sentryWebServer = new SentryWebServer(listenerList, webServerPort); + sentryWebServer.start(); + + } + + private void stopSentryWebServer() throws Exception{ + if( sentryWebServer != null) { + sentryWebServer.stop(); + } + } + public InetSocketAddress getAddress() { return address; } @@ -231,7 +254,7 @@ public class SentryService implements Callable { future = serviceExecutor.submit(this); } - public synchronized void stop() { + public synchronized void stop() throws Exception{ if (status == Status.NOT_STARTED) { return; } @@ -241,6 +264,7 @@ public class SentryService implements Callable { thriftServer.stop(); } thriftServer = null; + stopSentryWebServer(); status = Status.NOT_STARTED; LOGGER.info("Stopped..."); } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java index 52eaeed..29a7a1c 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java @@ -24,6 +24,7 @@ import javax.security.sasl.Sasl; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; +import org.apache.sentry.provider.db.service.thrift.SentryMetrics; public class ServiceConstants { @@ -122,6 +123,13 @@ public class ServiceConstants { .put("javax.jdo.option.Multithreaded", "true") .build(); + public static final String SENTRY_WEB_PORT = "sentry.service.web.port"; + public static final int SENTRY_WEB_PORT_DEFAULT = 51000; + public static final String SENTRY_REPORTING = "sentry.service.reporting"; + public static final String SENTRY_REPORTING_JMX = SentryMetrics.Reporting.JMX.name(); //case insensitive + public static final String SENTRY_REPORTING_CONSOLE = SentryMetrics.Reporting.CONSOLE.name();//case insensitive + + } public static class ClientConfig { public static final ImmutableMap<String, String> SASL_PROPERTIES = ServiceConstants.SASL_PROPERTIES; http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index befecf4..2e829d6 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -157,6 +157,7 @@ public class TestSentryStore { // expected } } + @Test public void testCaseSensitiveScope() throws Exception { String roleName = "role1"; @@ -840,6 +841,69 @@ public class TestSentryStore { } } + @Test + public void testSentryRoleSize() throws Exception { + for( long i = 0; i< 5; i++ ) { + assertEquals((Long)i, sentryStore.getRoleCountGauge().getValue()); + sentryStore.createSentryRole("role" + i); + } + } + @Test + public void testSentryPrivilegeSize() throws Exception { + String role1 = "role1"; + String role2 = "role2"; + + sentryStore.createSentryRole(role1); + sentryStore.createSentryRole(role2); + + TSentryPrivilege privilege = new TSentryPrivilege(); + privilege.setPrivilegeScope("TABLE"); + privilege.setServerName("server1"); + privilege.setDbName("db1"); + privilege.setTableName("tb1"); + privilege.setCreateTime(System.currentTimeMillis()); + + String grantor = "g1"; + + assertEquals(new Long(0), sentryStore.getPrivilegeCountGauge().getValue()); + + sentryStore.alterSentryRoleGrantPrivilege(grantor, role1, privilege); + assertEquals(new Long(1), sentryStore.getPrivilegeCountGauge().getValue()); + + sentryStore.alterSentryRoleGrantPrivilege(grantor, role2, privilege); + assertEquals(new Long(1), sentryStore.getPrivilegeCountGauge().getValue()); + + privilege.setTableName("tb2"); + sentryStore.alterSentryRoleGrantPrivilege(grantor, role2, privilege); + assertEquals(new Long(2), sentryStore.getPrivilegeCountGauge().getValue()); + } + + @Test + public void testSentryGroupsSize() throws Exception { + String role1 = "role1"; + String role2 = "role2"; + + sentryStore.createSentryRole(role1); + sentryStore.createSentryRole(role2); + + Set<TSentryGroup> groups = Sets.newHashSet(); + TSentryGroup group = new TSentryGroup(); + group.setGroupName("group1"); + groups.add(group); + + String grantor = "g1"; + + sentryStore.alterSentryRoleAddGroups(grantor, role1, groups); + assertEquals(new Long(1), sentryStore.getGroupCountGauge().getValue()); + + sentryStore.alterSentryRoleAddGroups(grantor, role2, groups); + assertEquals(new Long(1), sentryStore.getGroupCountGauge().getValue()); + + groups.add(new TSentryGroup("group2")); + sentryStore.alterSentryRoleAddGroups(grantor, role2, groups); + assertEquals(new Long(2), sentryStore.getGroupCountGauge().getValue()); + + } protected void addGroupsToUser(String user, String... groupNames) { policyFile.addGroupsToUser(user, groupNames); } @@ -847,4 +911,5 @@ public class TestSentryStore { protected void writePolicyFile() throws Exception { policyFile.write(policyFilePath); } + } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/de0cc9e7/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java index f251ebc..f6b6ad4 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java @@ -360,6 +360,7 @@ public abstract class AbstractTestWithStaticConfiguration { for (Map.Entry<String, String> entry : properties.entrySet()) { sentryConf.set(entry.getKey(), entry.getValue()); } + sentryConf.set(ServerConfig.SENTRY_REPORTING, ServerConfig.SENTRY_REPORTING_CONSOLE); sentryServer = new SentryServiceFactory().create(sentryConf); properties.put(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.getAddress() .getHostName()); @@ -374,6 +375,7 @@ public abstract class AbstractTestWithStaticConfiguration { properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, SentryMetastorePostEventListener.class.getName()); } + } private static void startSentryService() throws Exception {
