Repository: incubator-lens Updated Branches: refs/heads/master 365f5f72b -> 2f3cfe97b
LENS-186 : Add Session lifecycle events and session metrics (Raju Bairishetti via amareshwari) Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/2f3cfe97 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/2f3cfe97 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/2f3cfe97 Branch: refs/heads/master Commit: 2f3cfe97bfe53e1db11d069679b2f58f5fa28fc8 Parents: 365f5f7 Author: Raju Bairishetti <[email protected]> Authored: Thu May 7 14:28:22 2015 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Thu May 7 14:28:22 2015 +0530 ---------------------------------------------------------------------- .../lens/server/api/metrics/MetricsService.java | 28 +++ .../lens/server/api/session/SessionClosed.java | 38 ++++ .../lens/server/api/session/SessionEvent.java | 60 +++++++ .../lens/server/api/session/SessionExpired.java | 38 ++++ .../lens/server/api/session/SessionOpened.java | 47 +++++ .../server/api/session/SessionRestored.java | 38 ++++ .../org/apache/lens/server/LensService.java | 20 +++ .../lens/server/metrics/MetricsServiceImpl.java | 88 ++++++++- .../server/query/QueryExecutionServiceImpl.java | 16 -- .../lens/server/session/HiveSessionService.java | 21 ++- .../lens/server/query/TestEventService.java | 180 +++++++++++++++++++ .../lens/server/session/TestSessionExpiry.java | 7 +- .../server/session/TestSessionResource.java | 43 +++++ src/site/apt/admin/monitoring.apt | 5 + 14 files changed, 610 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java index 71a1f5b..9cfb65d 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java @@ -125,6 +125,14 @@ public interface MetricsService { */ String FINISHED_QUERIES = "finished-queries"; + String OPENED_SESSIONS = "opened-sessions"; + + String CLOSED_SESSIONS = "closed-sessions"; + + String EXPIRED_SESSIONS = "expired-sessions"; + + String ACTIVE_SESSIONS = "active-sessions"; + long getQueuedQueries(); long getRunningQueries(); @@ -157,4 +165,24 @@ public interface MetricsService { * @see MethodMetricsContext */ MethodMetricsContext getMethodMetricsContext(ResourceMethod method, ContainerRequest containerRequest); + + /** + * Specifies the count of opened sessions + */ + long getTotalOpenedSessions(); + + /** + * Specifies the number of sessions closed + */ + long getTotalClosedSessions(); + + /** + * Specifies the number of sessions were idle for more than session timeout + */ + long getTotalExpiredSessions(); + + /** + * Specifies the count of all opened sessions which are not closed after restarted and the restored sessions + */ + int getActiveSessions(); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionClosed.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionClosed.java b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionClosed.java new file mode 100644 index 0000000..6c2e3d6 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionClosed.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.api.session; + +import org.apache.lens.api.LensSessionHandle; + +import lombok.ToString; + +@ToString(callSuper=true) +public class SessionClosed extends SessionEvent { + + /** + * Instantiates a session closed event + * + * @param eventTime the event time + * @param handle the session handle + */ + public SessionClosed(long eventTime, LensSessionHandle handle) { + super(eventTime, handle); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java new file mode 100644 index 0000000..87fa855 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.api.session; + +import java.util.UUID; + +import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.server.api.events.LensEvent; + +import lombok.Getter; +import lombok.ToString; + +@ToString +public abstract class SessionEvent extends LensEvent { + + /** + * The lens session handle. + */ + @Getter + protected final LensSessionHandle sessionHandle; + + /** + * The id. + */ + protected final UUID id = UUID.randomUUID(); + + /** + * Instantiates a new session event + * + * @param eventTime the event time + * @param handle the lens session handle + * @param user name of user + */ + public SessionEvent(long eventTime, LensSessionHandle handle) { + super(eventTime); + this.sessionHandle = handle; + } + + @Override + public String getEventId() { + return id.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionExpired.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionExpired.java b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionExpired.java new file mode 100644 index 0000000..7085b52 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionExpired.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.api.session; + +import org.apache.lens.api.LensSessionHandle; + +import lombok.ToString; + +@ToString(callSuper=true) +public class SessionExpired extends SessionClosed { + + /** + * Instantiates a session expired event + * + * @param eventTime the event time + * @param handle the session handle + */ + public SessionExpired(long eventTime, LensSessionHandle handle) { + super(eventTime, handle); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionOpened.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionOpened.java b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionOpened.java new file mode 100644 index 0000000..75b5dff --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionOpened.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.api.session; + +import org.apache.lens.api.LensSessionHandle; + +import lombok.Getter; +import lombok.ToString; + +@ToString(callSuper=true) +public class SessionOpened extends SessionEvent { + + /** + * The user. + */ + @Getter + private final String user; + + /** + * Instantiates a session opened event + * + * @param eventTime the event time + * @param handle the session handle + * @param user name of user who opened this session + */ + public SessionOpened(long eventTime, LensSessionHandle handle, String user) { + super(eventTime, handle); + this.user = user; + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionRestored.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionRestored.java b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionRestored.java new file mode 100644 index 0000000..04bef23 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionRestored.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.api.session; + +import org.apache.lens.api.LensSessionHandle; + +import lombok.ToString; + +@ToString(callSuper=true) +public class SessionRestored extends SessionEvent { + + /** + * Instantiates a session restored event + * + * @param eventTime the event time + * @param handle the session handle + */ + public SessionRestored(long eventTime, LensSessionHandle handle) { + super(eventTime, handle); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server/src/main/java/org/apache/lens/server/LensService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensService.java b/lens-server/src/main/java/org/apache/lens/server/LensService.java index a35d6f9..ae9af6a 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensService.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensService.java @@ -35,6 +35,8 @@ import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.events.LensEvent; +import org.apache.lens.server.api.events.LensEventService; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.user.UserConfigLoaderFactory; import org.apache.lens.server.util.UtilityMethods; @@ -42,8 +44,10 @@ import org.apache.lens.server.util.UtilityMethods; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; + import org.apache.hive.service.CompositeService; import org.apache.hive.service.auth.AuthenticationProviderFactory; import org.apache.hive.service.auth.HiveAuthFactory; @@ -97,6 +101,10 @@ public abstract class LensService extends CompositeService implements Externaliz return cliService.getHiveConf().get(LensConfConstants.SERVER_DOMAIN); } + public static int getNumberOfSessions() { + return LensService.SESSION_MAP.size(); + } + /** * Open session. * @@ -152,6 +160,18 @@ public abstract class LensService extends CompositeService implements Externaliz return lensSession; } + protected LensEventService getEventService() { + LensEventService eventService = (LensEventService) LensServices.get().getService(LensEventService.NAME); + if (eventService == null) { + throw new NullPointerException("Could not get event service"); + } + return eventService; + } + + protected void notifyEvent(LensEvent event) throws LensException { + getEventService().notifyEvent(event); + } + /** * Restore session from previous instance of lens server. * http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java index 41a2e04..ee24e1f 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java @@ -29,6 +29,7 @@ import java.util.Locale; import java.util.concurrent.TimeUnit; import org.apache.lens.api.query.QueryStatus.Status; +import org.apache.lens.server.LensService; import org.apache.lens.server.LensServices; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.events.AsyncEventListener; @@ -36,6 +37,11 @@ import org.apache.lens.server.api.events.LensEventService; import org.apache.lens.server.api.metrics.*; import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.api.query.StatusChange; +import org.apache.lens.server.api.session.SessionClosed; +import org.apache.lens.server.api.session.SessionEvent; +import org.apache.lens.server.api.session.SessionExpired; +import org.apache.lens.server.api.session.SessionOpened; +import org.apache.lens.server.api.session.SessionService; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.AbstractService; @@ -52,6 +58,7 @@ import com.codahale.metrics.health.HealthCheckRegistry; import com.codahale.metrics.jvm.GarbageCollectorMetricSet; import com.codahale.metrics.jvm.MemoryUsageGaugeSet; import com.codahale.metrics.jvm.ThreadStatesGaugeSet; + import info.ganglia.gmetric4j.gmetric.GMetric; import info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode; import lombok.Getter; @@ -67,6 +74,8 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic /** The query status listener. */ private AsyncEventListener<StatusChange> queryStatusListener; + private AsyncEventListener<SessionEvent> sessionEventListener; + /** The metric registry. */ @Getter private MetricRegistry metricRegistry; @@ -79,6 +88,15 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic @Getter private HealthCheckRegistry healthCheck; + /** The total opened sessions*/ + private Counter totalOpenedSessions; + + /** The total closed sessions*/ + private Counter totalClosedSessions; + + /** The total expired sessions*/ + private Counter totalExpiredSessions; + /** The total accepted queries. */ private Counter totalAcceptedQueries; @@ -94,6 +112,9 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic /** The total cancelled queries. */ private Counter totalCancelledQueries; + /** The opened sessions */ + private Gauge<Integer> activeSessions; + /** The queued queries. */ private Gauge<Long> queuedQueries; @@ -165,6 +186,32 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic } /** + * The listener interface for receiving asyncSession events. The class that is interested in processing a + * asyncSession event implements this interface, and the object created with that class is registered with a + * component using the component's <code>addAsyncSessionEventListener<code> method. When the asyncSessionEvent event + * occurs, that object's appropriate method is invoked. + */ + public class AsyncSessionEventListener extends AsyncEventListener<SessionEvent> { + + /* + * (non-Javadoc) + * + * @see org.apache.lens.server.api.events.AsyncEventListener#process(org.apache.lens.server.api.events.LensEvent) + */ + @Override + public void process(SessionEvent event) { + if (event instanceof SessionOpened) { + totalOpenedSessions.inc(); + } else if (event instanceof SessionExpired) { + totalExpiredSessions.inc(); + totalClosedSessions.inc(); + } else if (event instanceof SessionClosed) { + totalClosedSessions.inc(); + } + } + } + + /** * Instantiates a new metrics service impl. * * @param name the name @@ -188,8 +235,10 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic @Override public synchronized void init(HiveConf hiveConf) { queryStatusListener = new AsyncQueryStatusListener(); + sessionEventListener = new AsyncSessionEventListener(); LensEventService eventService = (LensEventService) LensServices.get().getService(LensEventService.NAME); eventService.addListenerForType(queryStatusListener, StatusChange.class); + eventService.addListenerForType(sessionEventListener, SessionEvent.class); metricRegistry = LensMetricsRegistry.getStaticRegistry(); methodMetricsFactory = new MethodMetricsFactory(metricRegistry); setEnableResourceMethodMetering(hiveConf.getBoolean(LensConfConstants.ENABLE_RESOURCE_METHOD_METERING, false)); @@ -251,6 +300,14 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic * Inits the counters. */ protected void initCounters() { + activeSessions = metricRegistry.register(MetricRegistry.name(SessionService.class, ACTIVE_SESSIONS), + new Gauge<Integer>() { + @Override + public Integer getValue() { + return LensService.getNumberOfSessions(); + } + }); + queuedQueries = metricRegistry.register(MetricRegistry.name(QueryExecutionService.class, QUEUED_QUERIES), new Gauge<Long>() { @Override @@ -288,7 +345,16 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic + FAILED_QUERIES)); totalCancelledQueries = metricRegistry.counter(MetricRegistry.name(QueryExecutionService.class, "total-" - + CANCELLED_QUERIES)); + + CANCELLED_QUERIES)); + + totalOpenedSessions = metricRegistry.counter(MetricRegistry.name(QueryExecutionService.class, "total-" + + OPENED_SESSIONS)); + + totalClosedSessions = metricRegistry.counter(MetricRegistry.name(QueryExecutionService.class, "total-" + + CLOSED_SESSIONS)); + + totalExpiredSessions = metricRegistry.counter(MetricRegistry.name(QueryExecutionService.class, "total-" + + EXPIRED_SESSIONS)); metricRegistry.register("gc", new GarbageCollectorMetricSet()); metricRegistry.register("memory", new MemoryUsageGaugeSet()); @@ -449,6 +515,26 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic return totalFailedQueries.getCount(); } + @Override + public int getActiveSessions() { + return activeSessions.getValue(); + } + + @Override + public long getTotalOpenedSessions() { + return totalOpenedSessions.getCount(); + } + + @Override + public long getTotalClosedSessions() { + return totalClosedSessions.getCount(); + } + + @Override + public long getTotalExpiredSessions() { + return totalExpiredSessions.getCount(); + } + /* * (non-Javadoc) * http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 60f5530..3bf180c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -47,7 +47,6 @@ import org.apache.lens.server.api.driver.*; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.error.LensMultiCauseException; import org.apache.lens.server.api.events.LensEventListener; -import org.apache.lens.server.api.events.LensEventService; import org.apache.lens.server.api.metrics.MethodMetricsContext; import org.apache.lens.server.api.metrics.MethodMetricsFactory; import org.apache.lens.server.api.metrics.MetricsService; @@ -208,11 +207,6 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu private Map<QueryHandle, LensResultSet> resultSets = new HashMap<QueryHandle, LensResultSet>(); /** - * The event service. - */ - private LensEventService eventService; - - /** * The metrics service. */ private MetricsService metricsService; @@ -336,16 +330,6 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu } } - protected LensEventService getEventService() { - if (eventService == null) { - eventService = (LensEventService) LensServices.get().getService(LensEventService.NAME); - if (eventService == null) { - throw new NullPointerException("Could not get event service"); - } - } - return eventService; - } - private MetricsService getMetrics() { if (metricsService == null) { metricsService = (MetricsService) LensServices.get().getService(MetricsService.NAME); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java index eff6d4f..bb0d301 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java @@ -36,6 +36,10 @@ import org.apache.lens.server.LensService; import org.apache.lens.server.LensServices; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.session.SessionClosed; +import org.apache.lens.server.api.session.SessionExpired; +import org.apache.lens.server.api.session.SessionOpened; +import org.apache.lens.server.api.session.SessionRestored; import org.apache.lens.server.api.session.SessionService; import org.apache.lens.server.query.QueryExecutionServiceImpl; import org.apache.lens.server.session.LensSessionImpl.ResourceEntry; @@ -43,11 +47,13 @@ import org.apache.lens.server.session.LensSessionImpl.ResourceEntry; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.processors.SetProcessor; import org.apache.hadoop.hive.ql.session.SessionState; + import org.apache.hive.service.cli.CLIService; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.OperationHandle; @@ -200,6 +206,7 @@ public class HiveSessionService extends LensService implements SessionService { throws LensException { LensSessionHandle sessionid = super.openSession(username, password, configuration); LOG.info("Opened session " + sessionid + " for user " + username); + notifyEvent(new SessionOpened(System.currentTimeMillis(), sessionid, username)); // Set current database if (StringUtils.isNotBlank(database)) { @@ -379,6 +386,7 @@ public class HiveSessionService extends LensService implements SessionService { } } LOG.info("Restored session " + persistInfo.getSessionHandle().getPublicId()); + notifyEvent(new SessionRestored(System.currentTimeMillis(), sessionHandle)); } catch (LensException e) { throw new RuntimeException(e); } @@ -439,6 +447,16 @@ public class HiveSessionService extends LensService implements SessionService { */ @Override public void closeSession(LensSessionHandle sessionHandle) throws LensException { + closeInternal(sessionHandle); + notifyEvent(new SessionClosed(System.currentTimeMillis(), sessionHandle)); + } + + /** + * Close a Lens server session + * @param sessionHandle session handle + * @throws LensException + */ + private void closeInternal(LensSessionHandle sessionHandle) throws LensException { super.closeSession(sessionHandle); // Inform query service LensService svc = LensServices.get().getService(QueryExecutionServiceImpl.NAME); @@ -492,9 +510,10 @@ public class HiveSessionService extends LensService implements SessionService { for (LensSessionHandle sessionHandle : sessionsToRemove) { try { long lastAccessTime = getSession(sessionHandle).getLastAccessTime(); - closeSession(sessionHandle); + closeInternal(sessionHandle); LOG.info("Closed inactive session " + sessionHandle.getPublicId() + " last accessed at " + new Date(lastAccessTime)); + notifyEvent(new SessionExpired(System.currentTimeMillis(), sessionHandle)); } catch (ClientErrorException nfe) { // Do nothing } catch (LensException e) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java index 5e62aed..a77378b 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java @@ -24,6 +24,7 @@ import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryStatus; import org.apache.lens.server.EventServiceImpl; @@ -35,6 +36,10 @@ import org.apache.lens.server.api.events.LensEvent; import org.apache.lens.server.api.events.LensEventListener; import org.apache.lens.server.api.events.LensEventService; import org.apache.lens.server.api.query.*; +import org.apache.lens.server.api.session.SessionClosed; +import org.apache.lens.server.api.session.SessionExpired; +import org.apache.lens.server.api.session.SessionOpened; +import org.apache.lens.server.api.session.SessionRestored; import org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger; import org.apache.lens.server.stats.event.query.QueryExecutionStatistics; @@ -67,6 +72,18 @@ public class TestEventService { /** The ended listener. */ MockEndedListener endedListener; + /** the session opened listener */ + MockerSessionOpened sessionOpenedListener; + + /** the session closed listener */ + MockerSessionClosed sessionClosedListener; + + /** the session expired listener */ + MockerSessionExpired sessionExpiredListner; + + /** the session restored listener */ + MockerSessionRestored sessionRestoredListener; + /** The latch. */ CountDownLatch latch; @@ -173,6 +190,90 @@ public class TestEventService { } /** + * The Class MockerSessionOpened. + */ + class MockerSessionOpened implements LensEventListener<SessionOpened> { + + /** The processed. */ + boolean processed = false; + + /* + * (non-Javadoc) + * + * @see org.apache.lens.server.api.events.LensEventListener#onEvent(org.apache.lens.server.api.events.LensEvent) + */ + @Override + public void onEvent(SessionOpened event) throws LensException { + processed = true; + latch.countDown(); + LOG.info("Session opened: " + event); + } + } + + /** + * The Class MockerSessionClosed. + */ + class MockerSessionClosed implements LensEventListener<SessionClosed> { + + /** The processed. */ + boolean processed = false; + + /* + * (non-Javadoc) + * + * @see org.apache.lens.server.api.events.LensEventListener#onEvent(org.apache.lens.server.api.events.LensEvent) + */ + @Override + public void onEvent(SessionClosed event) throws LensException { + processed = true; + latch.countDown(); + LOG.info("Session closed: " + event); + } + } + + /** + * The Class MockerSessionExpired. + */ + class MockerSessionExpired implements LensEventListener<SessionExpired> { + + /** The processed. */ + boolean processed = false; + + /* + * (non-Javadoc) + * + * @see org.apache.lens.server.api.events.LensEventListener#onEvent(org.apache.lens.server.api.events.LensEvent) + */ + @Override + public void onEvent(SessionExpired event) throws LensException { + processed = true; + latch.countDown(); + LOG.info("Session expired: " + event); + } + } + + /** + * The Class MockerSessionRestored. + */ + class MockerSessionRestored implements LensEventListener<SessionRestored> { + + /** The processed. */ + boolean processed = false; + + /* + * (non-Javadoc) + * + * @see org.apache.lens.server.api.events.LensEventListener#onEvent(org.apache.lens.server.api.events.LensEvent) + */ + @Override + public void onEvent(SessionRestored event) throws LensException { + processed = true; + latch.countDown(); + LOG.info("Session restored: " + event); + } + } + + /** * Setup. * * @throws Exception the exception @@ -200,11 +301,21 @@ public class TestEventService { service.addListenerForType(failedListener, QueryFailed.class); queuePositionChangeListener = new MockQueuePositionChange(); service.addListenerForType(queuePositionChangeListener, QueuePositionChange.class); + sessionOpenedListener = new MockerSessionOpened(); + service.addListenerForType(sessionOpenedListener, SessionOpened.class); + sessionClosedListener = new MockerSessionClosed(); + service.addListenerForType(sessionClosedListener, SessionClosed.class); + sessionExpiredListner = new MockerSessionExpired(); + service.addListenerForType(sessionExpiredListner, SessionExpired.class); + sessionRestoredListener = new MockerSessionRestored(); + service.addListenerForType(sessionRestoredListener, SessionRestored.class); assertTrue(service.getListeners(LensEvent.class).contains(genericEventListener)); assertTrue(service.getListeners(QueryFailed.class).contains(failedListener)); assertTrue(service.getListeners(QueryEnded.class).contains(endedListener)); assertTrue(service.getListeners(QueuePositionChange.class).contains(queuePositionChangeListener)); + assertTrue(service.getListeners(SessionOpened.class).contains(sessionOpenedListener)); + assertTrue(service.getListeners(SessionClosed.class).contains(sessionClosedListener)); } /** @@ -230,6 +341,75 @@ public class TestEventService { } /** + * Reset session listeners. + */ + private void resetSessionListeners() { + genericEventListener.processed = false; + sessionOpenedListener.processed = false; + sessionClosedListener.processed = false; + sessionExpiredListner.processed = false; + sessionRestoredListener.processed = false; + } + + /** + * Test handle event. + * + * @throws Exception + * the exception + */ + @Test + public void testSesionHandleEvent() throws Exception { + LensSessionHandle sessionHandle = new LensSessionHandle(UUID.randomUUID(), UUID.randomUUID()); + String user = "user"; + long now = System.currentTimeMillis(); + SessionOpened sessionOpenedEvent = new SessionOpened(now, sessionHandle, user); + SessionClosed sessionClosedEvent = new SessionClosed(now, sessionHandle); + SessionRestored sessionRestored = new SessionRestored(now, sessionHandle); + SessionExpired sessionExpired = new SessionExpired(now, sessionHandle); + + try { + latch = new CountDownLatch(3); + LOG.info("Sending session opened event: " + sessionOpenedEvent); + service.notifyEvent(sessionOpenedEvent); + LOG.info("Sending session restored event: " + sessionRestored); + service.notifyEvent(sessionRestored); + latch.await(5, TimeUnit.SECONDS); + assertTrue(genericEventListener.processed); + assertTrue(sessionOpenedListener.processed); + assertTrue(sessionRestoredListener.processed); + resetSessionListeners(); + + LensEvent genEvent = new LensEvent(now) { + @Override + public String getEventId() { + return "TEST_EVENT"; + } + }; + + latch = new CountDownLatch(2); + LOG.info("Sending generic event " + genEvent.getEventId()); + service.notifyEvent(genEvent); + latch.await(5, TimeUnit.SECONDS); + assertTrue(genericEventListener.processed); + resetSessionListeners(); + + latch = new CountDownLatch(3); + LOG.info("Sending session closed event " + sessionClosedEvent); + service.notifyEvent(sessionClosedEvent); + LOG.info("Sending session expired event " + sessionExpired); + service.notifyEvent(sessionExpired); + latch.await(5, TimeUnit.SECONDS); + assertTrue(sessionClosedListener.processed); + assertTrue(sessionExpiredListner.processed); + assertFalse(sessionOpenedListener.processed); + assertFalse(sessionRestoredListener.processed); + + } catch (LensException e) { + fail(e.getMessage()); + } + } + + /** * Test handle event. * * @throws Exception the exception http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java index c1cb9da..f494fd5 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java @@ -24,7 +24,9 @@ import java.util.HashMap; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.server.LensServerConf; +import org.apache.lens.server.LensServices; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.metrics.MetricsService; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.cli.CLIService; @@ -51,6 +53,7 @@ public class TestSessionExpiry { HiveSessionService lensService = new HiveSessionService(cliService); lensService.init(conf); lensService.start(); + MetricsService metricSvc = (MetricsService) LensServices.get().getService(MetricsService.NAME); try { LensSessionHandle sessionHandle = lensService.openSession("foo", "bar", new HashMap<String, String>()); @@ -59,9 +62,11 @@ public class TestSessionExpiry { session.setLastAccessTime(session.getLastAccessTime() - 2000 * conf.getLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, LensConfConstants.SESSION_TIMEOUT_SECONDS_DEFAULT)); assertFalse(session.isActive()); - // run the expiry thread lensService.getSessionExpiryRunnable().run(); + assertTrue(metricSvc.getTotalExpiredSessions() >= 1); + assertTrue(metricSvc.getTotalClosedSessions() >= 1); + try { lensService.getSession(sessionHandle); // should throw exception since session should be expired by now http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java index a8cbd67..5f2cbfd 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java @@ -41,12 +41,14 @@ import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.LensServices; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.metrics.MetricsService; import org.apache.lens.server.api.session.SessionService; import org.apache.lens.server.common.LenServerTestException; import org.apache.lens.server.common.LensServerTestFileUtils; import org.apache.lens.server.common.TestResourceFile; import org.apache.commons.io.FileUtils; + import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -67,6 +69,10 @@ import org.testng.annotations.Test; @Test(groups = "unit-test") public class TestSessionResource extends LensJerseyTest { + + /** The metrics svc. */ + MetricsService metricsSvc; + /* * (non-Javadoc) * @@ -74,6 +80,7 @@ public class TestSessionResource extends LensJerseyTest { */ @BeforeTest public void setUp() throws Exception { + metricsSvc = (MetricsService) LensServices.get().getService(MetricsService.NAME); super.setUp(); } @@ -470,4 +477,40 @@ public class TestSessionResource extends LensJerseyTest { // PASS } } + + private FormDataMultiPart getMultiFormData(String username, String password) { + final FormDataMultiPart mp = new FormDataMultiPart(); + + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("username").build(), username)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("password").build(), password)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionconf").fileName("sessionconf").build(), + new LensConf(), MediaType.APPLICATION_XML_TYPE)); + return mp; + } + + @Test + public void testSessionEvents() { + final WebTarget target = target().path("session"); + FormDataMultiPart mp = getMultiFormData("foo", "bar"); + + LensSessionHandle lensSessionHandle = target.request().post( + Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), LensSessionHandle.class); + Assert.assertTrue(lensSessionHandle != null); + Assert.assertTrue(metricsSvc.getTotalOpenedSessions() >= 1); + Assert.assertTrue(metricsSvc.getActiveSessions() >= 1); + + LensSessionHandle lensSessionHandle1 = target.request().post( + Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), LensSessionHandle.class); + Assert.assertTrue(lensSessionHandle1 != null); + Assert.assertTrue(metricsSvc.getTotalOpenedSessions() >= 2); + Assert.assertTrue(metricsSvc.getActiveSessions() >= 2); + + APIResult result = target.queryParam("sessionid", lensSessionHandle).request().delete(APIResult.class); + Assert.assertTrue(metricsSvc.getTotalOpenedSessions() >= 1); + Assert.assertTrue(metricsSvc.getTotalClosedSessions() >= 1); + Assert.assertTrue(metricsSvc.getActiveSessions() >= 1); + + result = target.queryParam("sessionid", lensSessionHandle1).request().delete(APIResult.class); + Assert.assertTrue(metricsSvc.getTotalClosedSessions() >= 2); + } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2f3cfe97/src/site/apt/admin/monitoring.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/monitoring.apt b/src/site/apt/admin/monitoring.apt index 41c7951..3ad2f92 100644 --- a/src/site/apt/admin/monitoring.apt +++ b/src/site/apt/admin/monitoring.apt @@ -39,6 +39,11 @@ Lens server monitoring * Number of result formatting error + * Total number of opened sessions from the server start/restart + + * Total number of closed sessions + + * Number of active sessions Lens server also emits following metrics for other services
