Repository: lens Updated Branches: refs/heads/master 020b65167 -> 01d0d846d
LENS-904: Session close should not result in queued query failures Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/01d0d846 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/01d0d846 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/01d0d846 Branch: refs/heads/master Commit: 01d0d846d2bf7eb6b767be016584ede9855701ad Parents: 020b651 Author: Rajat Khandelwal <pro...@apache.org> Authored: Thu Jun 9 13:07:22 2016 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Thu Jun 9 13:07:22 2016 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/lens-errors.conf | 7 + .../org/apache/lens/client/TestLensClient.java | 11 +- .../org/apache/lens/driver/hive/HiveDriver.java | 2 +- .../lens/server/api/SessionValidator.java | 26 +++ .../api/metastore/CubeMetastoreService.java | 3 +- .../server/api/query/QueryExecutionService.java | 4 +- .../org/apache/lens/server/BaseLensService.java | 52 +++-- .../lens/server/error/LensServerErrorCode.java | 3 +- .../server/metastore/MetastoreResource.java | 10 +- .../server/query/QueryExecutionServiceImpl.java | 16 +- .../lens/server/query/QueryServiceResource.java | 9 +- .../lens/server/session/HiveSessionService.java | 11 +- .../lens/server/session/LensSessionImpl.java | 41 +++- .../org/apache/lens/server/LensJerseyTest.java | 24 +- .../apache/lens/server/TestServerRestart.java | 102 ++------- .../lens/server/common/RestAPITestUtil.java | 3 +- .../lens/server/query/TestQueryConstraints.java | 16 +- .../TestQueryIndependenceFromSessionClose.java | 225 +++++++++++++++++++ .../lens/server/query/TestQueryService.java | 4 +- .../server/query/TestQueryServiceDuplicate.java | 16 +- 20 files changed, 403 insertions(+), 182 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-api/src/main/resources/lens-errors.conf ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index 395d63b..14ed167 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -26,6 +26,7 @@ BAD_REQUEST = 400 NOT_FOUND = 404 UNAUTHORIZED = 401 +GONE = 410 TOO_MANY_REQUESTS = 429 INTERNAL_SERVER_ERROR = 500 @@ -101,6 +102,12 @@ lensServerErrors = [ httpStatusCode = ${TOO_MANY_REQUESTS} errorMsg = "Too many opened sessions for [%s] user. Session limit [%d] is already reached" } + + { + errorCode = 2005 + httpStatusCode = ${GONE} + errorMsg = "Session %s was closed" + } ] # lensCubeErrors: Defined for lens-cube module http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java b/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java index c49b5e8..ea464d5 100644 --- a/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java +++ b/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java @@ -39,8 +39,7 @@ import org.apache.lens.client.exceptions.LensAPIException; import org.apache.lens.client.exceptions.LensClientIOException; import org.apache.lens.client.resultset.ResultSet; import org.apache.lens.server.LensAllApplicationJerseyTest; - -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.lens.server.api.util.LensUtil; import org.testng.Assert; import org.testng.annotations.*; @@ -61,11 +60,9 @@ public class TestLensClient extends LensAllApplicationJerseyTest { } @Override - public HiveConf getServerConf() { - HiveConf conf = super.getServerConf(); - //Use MockQueryExecutionServiceImpl as QueryExecutionService for client tests - conf.set("lens.server.query.service.impl", "org.apache.lens.server.MockQueryExecutionServiceImpl"); - return conf; + public Map<String, String> getServerConfOverWrites() { + return LensUtil.getHashMap("lens.server.query.service.impl", + "org.apache.lens.server.MockQueryExecutionServiceImpl"); } @BeforeTest http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 04d059d..88de651 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -115,7 +115,7 @@ public class HiveDriver extends AbstractLensDriver { /** The hive handles. */ @Getter - private Map<QueryHandle, OperationHandle> hiveHandles = new ConcurrentHashMap<QueryHandle, OperationHandle>(); + private final Map<QueryHandle, OperationHandle> hiveHandles = new ConcurrentHashMap<QueryHandle, OperationHandle>(); /** The orphaned hive sessions. */ private ConcurrentLinkedQueue<SessionHandle> orphanedHiveSessions; http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java new file mode 100644 index 0000000..1185274 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api; + +import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.server.api.error.LensException; + +public interface SessionValidator { + void validateSession(LensSessionHandle handle) throws LensException; +} http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index 3e9f286..28b9d22 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.metastore.*; import org.apache.lens.server.api.LensService; +import org.apache.lens.server.api.SessionValidator; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -31,7 +32,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; /** * Server api for OLAP Cube Metastore. */ -public interface CubeMetastoreService extends LensService { +public interface CubeMetastoreService extends LensService, SessionValidator { /** The constant NAME */ String NAME = "metastore"; http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java index 15ed222..d10ad09 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java @@ -25,13 +25,15 @@ import javax.ws.rs.core.Response; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.query.*; +import org.apache.lens.server.api.LensService; +import org.apache.lens.server.api.SessionValidator; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.cost.QueryCost; /** * The Interface QueryExecutionService. */ -public interface QueryExecutionService { +public interface QueryExecutionService extends LensService, SessionValidator { /** * The Constant NAME. http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java index b96cdf0..0b4720e 100644 --- a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java +++ b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java @@ -18,6 +18,9 @@ */ package org.apache.lens.server; +import static org.apache.lens.server.error.LensServerErrorCode.SESSION_CLOSED; +import static org.apache.lens.server.error.LensServerErrorCode.SESSION_ID_NOT_PROVIDED; + import java.io.*; import java.util.HashMap; import java.util.Map; @@ -27,17 +30,20 @@ import javax.ws.rs.BadRequestException; import javax.ws.rs.ClientErrorException; import javax.ws.rs.NotAuthorizedException; import javax.ws.rs.NotFoundException; +import javax.ws.rs.core.Response; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.util.PathValidator; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.LensService; +import org.apache.lens.server.api.SessionValidator; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.events.LensEvent; import org.apache.lens.server.api.events.LensEventService; -import org.apache.lens.server.api.health.HealthStatus; +import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.error.LensServerErrorCode; +import org.apache.lens.server.query.QueryExecutionServiceImpl; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.user.UserConfigLoaderFactory; import org.apache.lens.server.util.UtilityMethods; @@ -62,7 +68,8 @@ import lombok.extern.slf4j.Slf4j; * The Class LensService. */ @Slf4j -public abstract class BaseLensService extends CompositeService implements Externalizable, LensService { +public abstract class BaseLensService extends CompositeService implements Externalizable, LensService, + SessionValidator { /** The cli service. */ private final CLIService cliService; @@ -290,11 +297,21 @@ public abstract class BaseLensService extends CompositeService implements Extern */ public void closeSession(LensSessionHandle sessionHandle) throws LensException { try { - String userName = getSession(sessionHandle).getLoggedInUser(); - cliService.closeSession(getHiveSessionHandle(sessionHandle)); - String publicId = sessionHandle.getPublicId().toString(); - SESSION_MAP.remove(publicId); - decrementSessionCountForUser(sessionHandle, userName); + LensSessionImpl session = getSession(sessionHandle); + if (session.activeOperationsPresent()) { + session.markForClose(); + } else { + cliService.closeSession(getHiveSessionHandle(sessionHandle)); + SESSION_MAP.remove(sessionHandle.getPublicId().toString()); + } + decrementSessionCountForUser(sessionHandle, session.getLoggedInUser()); + if (!SESSION_MAP.containsKey(sessionHandle.getPublicId().toString())) { + // Inform query service + BaseLensService svc = LensServices.get().getService(QueryExecutionService.NAME); + if (svc instanceof QueryExecutionServiceImpl) { + ((QueryExecutionServiceImpl) svc).closeDriverSessions(sessionHandle); + } + } } catch (HiveSQLException e) { throw new LensException(e); } @@ -331,13 +348,13 @@ public abstract class BaseLensService extends CompositeService implements Extern if (sessionHandle == null) { throw new ClientErrorException("Session is null", 400); } - try { return ((LensSessionImpl) getSessionManager().getSession(getHiveSessionHandle(sessionHandle))); } catch (HiveSQLException exc) { log.warn("Session {} not found", sessionHandle.getPublicId(), exc); // throw resource gone exception (410) - throw new ClientErrorException("Session " + sessionHandle.getPublicId() + " is invalid " + sessionHandle, 410); + throw new ClientErrorException("Session " + sessionHandle.getPublicId() + " is invalid " + sessionHandle, + Response.Status.GONE, exc); } } @@ -480,12 +497,6 @@ public abstract class BaseLensService extends CompositeService implements Extern public void writeExternal(ObjectOutput out) throws IOException { } - /** - * Returns the health status of the service. - * - * @return - */ - public abstract HealthStatus getHealthStatus(); /** * Method that uses PathValidator to get appropriate path. @@ -516,4 +527,15 @@ public abstract class BaseLensService extends CompositeService implements Extern } return pathValidator.removePrefixBeforeURI(path); } + + @Override + public void validateSession(LensSessionHandle handle) throws LensException { + if (handle == null) { + throw new LensException(SESSION_ID_NOT_PROVIDED.getLensErrorInfo()); + } + if (!getSession(handle).isActive()) { + throw new LensException(SESSION_CLOSED.getLensErrorInfo(), handle); + } + } } + http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java b/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java index 2443fec..b9b6b21 100644 --- a/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java +++ b/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java @@ -25,7 +25,8 @@ public enum LensServerErrorCode { SESSION_ID_NOT_PROVIDED(2001, 0), NULL_OR_EMPTY_OR_BLANK_QUERY(2002, 0), UNSUPPORTED_QUERY_SUBMIT_OPERATION(2003, 0), - TOO_MANY_OPEN_SESSIONS(2004, 0); + TOO_MANY_OPEN_SESSIONS(2004, 0), + SESSION_CLOSED(2005, 0); public LensErrorInfo getLensErrorInfo() { return this.errorInfo; http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 4a6d779..57b1836 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -32,6 +32,7 @@ import org.apache.lens.api.DateTime; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.StringList; import org.apache.lens.api.metastore.*; +import org.apache.lens.api.result.LensErrorTO; import org.apache.lens.server.LensServices; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.CubeMetastoreService; @@ -57,10 +58,13 @@ public class MetastoreResource { public static CubeMetastoreService getSvc() { return LensServices.get().getService(CubeMetastoreService.NAME); } - private static void checkSessionId(LensSessionHandle sessionHandle) { - if (sessionHandle == null) { - throw new BadRequestException("Invalid session handle"); + try { + getSvc().validateSession(sessionHandle); + } catch (LensException e) { + LensErrorTO to = e.buildLensErrorTO(LensServices.get().getErrorCollection()); + throw new ClientErrorException(to.getMessage(), + LensServices.get().getErrorCollection().getLensError(e.getErrorCode()).getHttpStatusCode().getStatusCode()); } } http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index fde432f..39984d4 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -170,7 +170,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE /** * The all queries. */ - protected ConcurrentMap<QueryHandle, QueryContext> allQueries = new ConcurrentHashMap<QueryHandle, QueryContext>(); + protected final ConcurrentMap<QueryHandle, QueryContext> allQueries = new ConcurrentHashMap<>(); /** * The conf. @@ -1300,8 +1300,9 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE * @see org.apache.hive.service.CompositeService#start() */ public synchronized void start() { - final List<QueryContext> allRestoredQueuedQueries = new LinkedList<QueryContext>(); synchronized (allQueries) { + // populate the query queues + final List<QueryContext> allRestoredQueuedQueries = new LinkedList<QueryContext>(); for (QueryContext ctx : allQueries.values()) { // recover query configurations from session try { @@ -2933,17 +2934,6 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE } } - /* - * (non-Javadoc) - * - * @see org.apache.lens.server.LensService#closeSession(org.apache.lens.api.LensSessionHandle) - */ - public void closeSession(LensSessionHandle sessionHandle) throws LensException { - super.closeSession(sessionHandle); - // Call driver session close in case some one closes sessions directly on query service - closeDriverSessions(sessionHandle); - } - // Used in test code Collection<LensDriver> getDrivers() { return drivers.values(); http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java index c70689b..ec6efe2 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java @@ -19,7 +19,6 @@ package org.apache.lens.server.query; import static org.apache.lens.server.error.LensServerErrorCode.NULL_OR_EMPTY_OR_BLANK_QUERY; -import static org.apache.lens.server.error.LensServerErrorCode.SESSION_ID_NOT_PROVIDED; import java.util.List; @@ -67,15 +66,15 @@ public class QueryServiceResource { * @param sessionHandle the session handle */ private void checkSessionId(final LensSessionHandle sessionHandle) { - if (sessionHandle == null) { + try { + validateSessionId(sessionHandle); + } catch (LensException e) { throw new BadRequestException("Invalid session handle"); } } private void validateSessionId(final LensSessionHandle sessionHandle) throws LensException { - if (sessionHandle == null) { - throw new LensException(SESSION_ID_NOT_PROVIDED.getLensErrorInfo()); - } + queryServer.validateSession(sessionHandle); } private SubmitOp checkAndGetQuerySubmitOperation(final String operation) throws UnSupportedQuerySubmitOpException { http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java index 6c5e52d..8909098 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java @@ -33,13 +33,10 @@ import javax.ws.rs.WebApplicationException; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.server.BaseLensService; -import org.apache.lens.server.LensServices; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.health.HealthStatus; -import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.api.session.*; -import org.apache.lens.server.query.QueryExecutionServiceImpl; import org.apache.lens.server.session.LensSessionImpl.ResourceEntry; import org.apache.commons.lang3.StringUtils; @@ -376,6 +373,7 @@ public class HiveSessionService extends BaseLensService implements SessionServic session.getLensSessionPersistInfo().setConfig(persistInfo.getConfig()); session.getLensSessionPersistInfo().setResources(persistInfo.getResources()); session.setCurrentDatabase(persistInfo.getDatabase()); + session.getLensSessionPersistInfo().setMarkedForClose(persistInfo.isMarkedForClose()); // Add resources for restored sessions for (LensSessionImpl.ResourceEntry resourceEntry : session.getResources()) { @@ -399,7 +397,7 @@ public class HiveSessionService extends BaseLensService implements SessionServic throw new RuntimeException(e); } } - log.info("Session service restoed " + restorableSessions.size() + " sessions"); + log.info("Session service restored " + restorableSessions.size() + " sessions"); } private int getSessionExpiryInterval() { @@ -481,11 +479,6 @@ public class HiveSessionService extends BaseLensService implements SessionServic */ private void closeInternal(LensSessionHandle sessionHandle) throws LensException { super.closeSession(sessionHandle); - // Inform query service - BaseLensService svc = LensServices.get().getService(QueryExecutionService.NAME); - if (svc instanceof QueryExecutionServiceImpl) { - ((QueryExecutionServiceImpl) svc).closeDriverSessions(sessionHandle); - } } /** http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java index 04812a6..40124ee 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java @@ -85,7 +85,6 @@ public class LensSessionImpl extends HiveSessionImpl { /** The conf. */ private Configuration conf = createDefaultConf(); - /** * List of queries which are submitted in this session. */ @@ -255,6 +254,7 @@ public class LensSessionImpl extends HiveSessionImpl { ClassLoader classLoader = getClassLoader(getCurrentDatabase()); Thread.currentThread().setContextClassLoader(classLoader); SessionState.getSessionConf().setClassLoader(classLoader); + setActive(); } /* @@ -263,6 +263,7 @@ public class LensSessionImpl extends HiveSessionImpl { * @see org.apache.hive.service.cli.session.HiveSessionImpl#release() */ public synchronized void release() { + setActive(); this.release(true); } @Override @@ -274,8 +275,12 @@ public class LensSessionImpl extends HiveSessionImpl { } public boolean isActive() { - long inactiveAge = System.currentTimeMillis() - lastAccessTime; - return inactiveAge < sessionTimeout; + return System.currentTimeMillis() - lastAccessTime < sessionTimeout + && (!persistInfo.markedForClose|| activeOperationsPresent()); + } + + public void setActive() { + setLastAccessTime(System.currentTimeMillis()); } /** @@ -466,7 +471,7 @@ public class LensSessionImpl extends HiveSessionImpl { /** * Return resources which are added statically to the database - * @return DatabaseResources + * @return db resources */ public Collection<ResourceEntry> getDBResources(String database) { synchronized (failedDBResources) { @@ -494,13 +499,17 @@ public class LensSessionImpl extends HiveSessionImpl { } /** - * Get effective class loader for this session - * @return current classloader + * @return effective class loader for this session */ public ClassLoader getClassLoader() { return getClassLoader(getCurrentDatabase()); } + public void markForClose() { + log.info("Marking session {} for close. Operations on this session will be rejected", this); + persistInfo.markedForClose = true; + } + /** * The Class ResourceEntry. */ @@ -558,8 +567,7 @@ public class LensSessionImpl extends HiveSessionImpl { } /** - * Returns the value of restoreCount for the resource - * @return restore count + * @return the value of restoreCount for the resource */ public int getRestoreCount() { return restoreCount.get(); @@ -617,6 +625,9 @@ public class LensSessionImpl extends HiveSessionImpl { /** The last access time. */ private long lastAccessTime; + /** Whether it's marked for close */ + private boolean markedForClose; + public void setSessionConf(Map<String, String> sessionConf) { UtilityMethods.mergeMaps(config, sessionConf, true); } @@ -645,6 +656,7 @@ public class LensSessionImpl extends HiveSessionImpl { out.writeUTF(config.get(key)); } out.writeLong(lastAccessTime); + out.writeBoolean(markedForClose); } /* @@ -675,18 +687,27 @@ public class LensSessionImpl extends HiveSessionImpl { config.put(key, val); } lastAccessTime = in.readLong(); + markedForClose = in.readBoolean(); } } public void addToActiveQueries(QueryHandle queryHandle) { + log.info("Adding {} to active queries for session {}", queryHandle, this); synchronized (this.activeQueries) { - this.activeQueries.add(queryHandle); + activeQueries.add(queryHandle); } } public void removeFromActiveQueries(QueryHandle queryHandle) { + log.info("Removing {} from active queries for session {}", queryHandle, this); + synchronized (this.activeQueries) { + activeQueries.remove(queryHandle); + } + } + + public boolean activeOperationsPresent() { synchronized (this.activeQueries) { - this.activeQueries.remove(queryHandle); + return !activeQueries.isEmpty(); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java index b5d5482..d5e975a 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.ServerSocket; import java.net.URI; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; import javax.ws.rs.client.Entity; @@ -127,8 +128,20 @@ public abstract class LensJerseyTest extends JerseyTest { config.register(LensJAXBContextResolver.class); } - public HiveConf getServerConf() { - return LensServerConf.getHiveConf(); + public final HiveConf getServerConf() { + HiveConf serverConf = LensServerConf.getHiveConf(); + Map<String, String> overWrites = getServerConfOverWrites(); + if (overWrites != null) { + serverConf = new HiveConf(serverConf); + for (Map.Entry<String, String> overWrite : overWrites.entrySet()) { + serverConf.set(overWrite.getKey(), overWrite.getValue()); + } + } + return serverConf; + } + + public Map<String, String> getServerConfOverWrites() { + return null; } /** @@ -212,7 +225,7 @@ public abstract class LensJerseyTest extends JerseyTest { /** * Restart lens server. */ - public void restartLensServer() { + protected void restartLensServer() { HiveConf h = getServerConf(); restartLensServer(h, false); } @@ -263,11 +276,6 @@ public abstract class LensJerseyTest extends JerseyTest { } public static Entity getEntityForString(String o, MediaType mt) { - if (mt.equals(MediaType.APPLICATION_JSON_TYPE)) { - return Entity.json(o); - } else if (mt.equals(MediaType.APPLICATION_XML_TYPE)) { - return Entity.xml(o); - } return Entity.entity(o, mt); } } http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java index f3d72f4..db46da8 100644 --- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java +++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java @@ -21,7 +21,7 @@ package org.apache.lens.server; import static org.apache.lens.server.LensServerTestUtil.createTable; import static org.apache.lens.server.LensServerTestUtil.loadData; import static org.apache.lens.server.api.user.MockDriverQueryHook.*; -import static org.apache.lens.server.common.RestAPITestUtil.execute; +import static org.apache.lens.server.common.RestAPITestUtil.*; import static org.testng.Assert.*; @@ -43,6 +43,7 @@ import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.api.session.SessionService; +import org.apache.lens.server.api.util.LensUtil; import org.apache.lens.server.common.TestResourceFile; import org.apache.lens.server.query.QueryExecutionServiceImpl; import org.apache.lens.server.query.TestQueryService; @@ -50,7 +51,6 @@ import org.apache.lens.server.session.HiveSessionService; import org.apache.lens.server.session.LensSessionImpl; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.Service; import org.glassfish.jersey.media.multipart.FormDataBodyPart; @@ -90,17 +90,10 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { } @Override - public HiveConf getServerConf() { - HiveConf conf = super.getServerConf(); - conf.set("lens.server.state.persistence.interval.millis", "1000"); - return conf; + public Map<String, String> getServerConfOverWrites() { + return LensUtil.getHashMap("lens.server.state.persistence.interval.millis", "1000"); } - /* - * (non-Javadoc) - * - * @see org.glassfish.jersey.test.JerseyTest#tearDown() - */ @AfterTest public void tearDown() throws Exception { super.tearDown(); @@ -158,8 +151,6 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { log.info("Loaded data"); // test post execute op - final WebTarget target = target().path("queryapi/queries"); - List<QueryHandle> launchedQueries = new ArrayList<>(); final int NUM_QUERIES = 10; @@ -178,20 +169,9 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { isQuerySubmitterPaused = true; } - final FormDataMultiPart mp = new FormDataMultiPart(); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - defaultMT)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), - "select COUNT(ID) from test_server_restart")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), - new LensConf(), defaultMT)); - final QueryHandle handle = target.request(defaultMT).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); - - Assert.assertNotNull(handle); - LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT) - .get(LensQuery.class); + final QueryHandle handle = executeAndGetHandle(target(), Optional.of(lensSessionId), + Optional.of("select COUNT(ID) from test_server_restart"), Optional.<LensConf>absent(), defaultMT); + LensQuery ctx = getLensQuery(target(), lensSessionId, handle, defaultMT); log.info("{} submitted query {} state: {}", i, handle, ctx.getStatus().getStatus()); launchedQueries.add(handle); if (i == (NUM_QUERIES-1)) { @@ -217,19 +197,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { for (QueryHandle handle : launchedQueries) { log.info("Polling query {}", handle); try { - LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT) - .get(LensQuery.class); - QueryStatus stat = ctx.getStatus(); - while (!stat.finished()) { - log.info("Polling query {} Status:{}", handle, stat); - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT) - .get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, "Expected to be successful " + handle); - PersistentQueryResult resultset = target.path(handle.toString()).path("resultset") - .queryParam("sessionid", lensSessionId).request(defaultMT).get(PersistentQueryResult.class); + PersistentQueryResult resultset = getLensQueryResult(target(), lensSessionId, handle, defaultMT); List<String> rows = TestQueryService.readResultSet(resultset, handle, true); assertEquals(rows.size(), 1); assertEquals(rows.get(0), "" + NROWS); @@ -306,31 +274,14 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { log.info("Hive Server restart test"); // test post execute op - final WebTarget target = target().path("queryapi/queries"); - - // Submit query, restart HS2, submit another query - FormDataMultiPart mp = new FormDataMultiPart(); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - defaultMT)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), - "select COUNT(ID) from test_hive_server_restart")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), - defaultMT)); - QueryHandle handle = target.request(defaultMT) - .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); - Assert.assertNotNull(handle); + QueryHandle handle = executeAndGetHandle(target(), Optional.of(lensSessionId), + Optional.of("select COUNT(ID) from test_hive_server_restart"), Optional.<LensConf>absent(), defaultMT); // wait for query to move out of QUEUED state - LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT) - .get(LensQuery.class); - QueryStatus stat = ctx.getStatus(); - while (stat.queued()) { - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT) - .get(LensQuery.class); - stat = ctx.getStatus(); + LensQuery ctx = getLensQuery(target(), lensSessionId, handle, defaultMT); + while (ctx.getStatus().queued()) { + ctx = getLensQuery(target(), lensSessionId, handle, defaultMT); Thread.sleep(1000); } @@ -363,19 +314,10 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { verifyParamOnRestart(lensSessionId); // Poll for first query, we should not get any exception - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT) - .get(LensQuery.class); - stat = ctx.getStatus(); - while (!stat.finished()) { - log.info("Polling query {} Status:{}", handle, stat); - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT) - .get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } + ctx = waitForQueryToFinish(target(), lensSessionId, handle, defaultMT); - Assert.assertTrue(stat.finished()); - log.info("Previous query status: {}", stat.getStatusMessage()); + Assert.assertTrue(ctx.getStatus().finished()); + log.info("Previous query status: {}", ctx.getStatus().getStatusMessage()); // After hive server restart, first few queries fail with Invalid Operation Handle followed by // Invalid Session Handle. Ideal behaviour is to fail with Invalid Session Handle immediately. @@ -391,16 +333,8 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { handle = response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); // Poll for second query, this should finish successfully - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT).get(LensQuery.class); - stat = ctx.getStatus(); - while (!stat.finished()) { - log.info("Post restart polling query {} Status:{}", handle, stat); - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(defaultMT) - .get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - log.info("Final status for {}: {}", handle, stat.getStatus()); + ctx = waitForQueryToFinish(target(), lensSessionId, handle, defaultMT); + log.info("Final status for {}: {}", handle, ctx.getStatus().getStatus()); // Now we can expect that session resources have been added back exactly once for (int i = 0; i < sessionResources.size(); i++) { http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java index 2c42631..57786e6 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java @@ -248,7 +248,8 @@ public class RestAPITestUtil { } public static PersistentQueryResult getLensQueryResult(final WebTarget target, - final LensSessionHandle lensSessionHandle, final QueryHandle handle, MediaType mt) { + final LensSessionHandle lensSessionHandle, final QueryHandle handle, MediaType mt) throws InterruptedException { + waitForQueryToFinish(target, lensSessionHandle, handle, QueryStatus.Status.SUCCESSFUL, mt); return target.path("queryapi/queries").path(handle.toString()).path("resultset") .queryParam("sessionid", lensSessionHandle).request(mt).get(PersistentQueryResult.class); } http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java index 8493d85..6ed3770 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java @@ -31,7 +31,6 @@ import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.driver.hive.HiveDriver; import org.apache.lens.server.LensJerseyTest; -import org.apache.lens.server.LensServerConf; import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.LensServices; import org.apache.lens.server.api.LensConfConstants; @@ -41,11 +40,11 @@ import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.metrics.MetricsService; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.QueryExecutionService; +import org.apache.lens.server.api.util.LensUtil; import org.apache.lens.server.common.RestAPITestUtil; import org.apache.lens.server.common.TestResourceFile; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.glassfish.jersey.test.TestProperties; @@ -66,7 +65,6 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Test(groups = "two-working-drivers", dependsOnGroups = "filter-test") public class TestQueryConstraints extends LensJerseyTest { - private HiveConf serverConf; public static class RoundRobinSelector implements DriverSelector { int counter = 0; @@ -108,15 +106,9 @@ public class TestQueryConstraints extends LensJerseyTest { } @Override - public HiveConf getServerConf() { - if (serverConf == null) { - serverConf = new HiveConf(super.getServerConf()); - // Lets test only mockHive. updating lens server conf for same - serverConf.set(LensConfConstants.DRIVER_TYPES_AND_CLASSES, "mockHive:" + HiveDriver.class.getName()); - serverConf.set("lens.server.driver.selector.class", RoundRobinSelector.class.getName()); - LensServerConf.getConfForDrivers().addResource(serverConf); - } - return serverConf; + public Map<String, String> getServerConfOverWrites() { + return LensUtil.getHashMap(LensConfConstants.DRIVER_TYPES_AND_CLASSES, "mockHive:" + HiveDriver.class.getName(), + LensConfConstants.DRIVER_SELECTOR_CLASS, RoundRobinSelector.class.getName()); } /* http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java new file mode 100644 index 0000000..24cbcaa --- /dev/null +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java @@ -0,0 +1,225 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.query; + +import static org.testng.Assert.*; + +import java.util.Map; + +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.lens.api.LensConf; +import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.api.result.LensAPIResult; +import org.apache.lens.driver.hive.HiveDriver; +import org.apache.lens.server.LensJerseyTest; +import org.apache.lens.server.LensServerTestUtil; +import org.apache.lens.server.LensServices; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.QueryExecutionService; +import org.apache.lens.server.api.util.LensUtil; +import org.apache.lens.server.common.RestAPITestUtil; +import org.apache.lens.server.common.TestResourceFile; +import org.apache.lens.server.error.LensServerErrorCode; + +import org.glassfish.jersey.test.TestProperties; +import org.testng.annotations.*; + +import com.google.common.base.Optional; +import lombok.extern.slf4j.Slf4j; + +/** + * The Class TestQueryService. + */ +@Slf4j +@Test(groups = "post-restart", dependsOnGroups = "restart-test") +public class TestQueryIndependenceFromSessionClose extends LensJerseyTest { + /** The query service. */ + QueryExecutionServiceImpl queryService; + + /** The lens session id. */ + LensSessionHandle lensSessionId; + private LensConf conf; + + /* + * (non-Javadoc) + * + * @see org.glassfish.jersey.test.JerseyTest#setUp() + */ + @BeforeClass + public void setUp() throws Exception { + super.setUp(); + queryService = LensServices.get().getService(QueryExecutionService.NAME); + lensSessionId = getSession(); + createTable(TEST_TABLE); + loadData(TEST_TABLE, TestResourceFile.TEST_DATA2_FILE.getValue()); + conf = new LensConf(); + conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); + conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, + TestQueryService.DeferredPersistentResultFormatter.class.getName()); + conf.addProperty("deferPersistenceByMillis", 5000); // defer persistence for 5 secs + } + + @Override + public Map<String, String> getServerConfOverWrites() { + return LensUtil.getHashMap("lens.server.total.query.cost.ceiling.per.user", "1", "lens.server.drivers", + "hive:org.apache.lens.driver.hive.HiveDriver"); + } + + private LensSessionHandle getSession() throws LensException { + queryService = LensServices.get().getService(QueryExecutionService.NAME); + return queryService.openSession("foo", "bar", null); + } + + private void closeSession(LensSessionHandle session) throws LensException { + queryService.closeSession(session); + } + + /* + * (non-Javadoc) + * + * @see org.glassfish.jersey.test.JerseyTest#tearDown() + */ + @AfterClass + public void tearDown() throws Exception { + dropTable(TEST_TABLE); + queryService.closeSession(lensSessionId); + for (LensDriver driver : queryService.getDrivers()) { + if (driver instanceof HiveDriver) { + assertFalse(((HiveDriver) driver).hasLensSession(lensSessionId)); + } + } + super.tearDown(); + } + + @Override + protected void restartLensServer() { + queryService = null; + super.restartLensServer(); + queryService = LensServices.get().getService(QueryExecutionService.NAME); + } + + /* + * (non-Javadoc) + * + * @see org.glassfish.jersey.test.JerseyTest#configure() + */ + @Override + protected Application configure() { + enable(TestProperties.LOG_TRAFFIC); + enable(TestProperties.DUMP_ENTITY); + return new TestQueryService.QueryServiceTestApp(); + } + + /** The test table. */ + public static final String TEST_TABLE = "TEST_TABLE_INDEPENDENCE"; + + /** + * Creates the table. + * + * @param tblName the tbl name + * @throws InterruptedException the interrupted exception + */ + private void createTable(String tblName) throws InterruptedException { + LensServerTestUtil.createTable(tblName, target(), lensSessionId, defaultMT); + } + + /** + * Load data. + * + * @param tblName the tbl name + * @param testDataFile the test data file + * @throws InterruptedException the interrupted exception + */ + private void loadData(String tblName, final String testDataFile) throws InterruptedException { + LensServerTestUtil.loadDataFromClasspath(tblName, testDataFile, target(), lensSessionId, defaultMT); + } + + /** + * Drop table. + * + * @param tblName the tbl name + * @throws InterruptedException the interrupted exception + */ + private void dropTable(String tblName) throws InterruptedException { + LensServerTestUtil.dropTable(tblName, target(), lensSessionId, defaultMT); + } + + @DataProvider + public Object[][] restartDataProvider() { + return new Object[][]{ + {true, true}, + {true, false}, + {false, true}, + {false, false}, + }; + } + + @Test(dataProvider = "restartDataProvider") + public void testQueryAliveOnSessionClose(boolean restartBeforeFinish, boolean restartAfterFinish) + throws LensException, InterruptedException { + MediaType mt = MediaType.APPLICATION_XML_TYPE; + LensSessionHandle sesssionHandle = getSession(); + QueryHandle queryHandle1 = RestAPITestUtil.executeAndGetHandle(target(), + Optional.of(sesssionHandle), Optional.of("select * from " + TEST_TABLE), Optional.of(conf), mt); + QueryHandle queryHandle2 = RestAPITestUtil.executeAndGetHandle(target(), + Optional.of(sesssionHandle), Optional.of("select * from " + TEST_TABLE), Optional.of(conf), mt); + assertNotEquals(queryHandle1, queryHandle2); + // Second query should be queued + assertEquals(queryService.getQueryContext(queryHandle2).getStatus().getStatus(), QueryStatus.Status.QUEUED); + closeSession(sesssionHandle); + // Session not 'truly' closed + assertNotNull(queryService.getSession(sesssionHandle)); + // Just 'marked' for closing + assertTrue(queryService.getSession(sesssionHandle).getLensSessionPersistInfo().isMarkedForClose()); + if (restartBeforeFinish) { + restartLensServer(); + } + assertTrue(queryService.getSession(sesssionHandle).getLensSessionPersistInfo().isMarkedForClose()); + assertTrue(queryService.getSession(sesssionHandle).isActive()); + RestAPITestUtil.waitForQueryToFinish(target(), lensSessionId, queryHandle2, QueryStatus.Status.SUCCESSFUL, mt); + RestAPITestUtil.waitForQueryToFinish(target(), lensSessionId, queryHandle1, QueryStatus.Status.SUCCESSFUL, mt); + // Session should not be active + assertFalse(queryService.getSession(sesssionHandle).isActive()); + if (restartAfterFinish) { + restartLensServer(); + } + assertTrue(queryService.getSession(sesssionHandle).getLensSessionPersistInfo().isMarkedForClose()); + // Now, session is not active anymore + assertFalse(queryService.getSession(sesssionHandle).isActive()); + // It should not be possible to submit queries now + Response response = RestAPITestUtil.postQuery(target(), Optional.of(sesssionHandle), + Optional.of("select * from " + TEST_TABLE), Optional.of("execute"), Optional.of(conf), mt); + assertEquals(response.getStatus(), 410); + LensAPIResult apiResult = response.readEntity(LensAPIResult.class); + assertEquals(apiResult.getErrorCode(), LensServerErrorCode.SESSION_CLOSED.getLensErrorInfo().getErrorCode()); + } + + @AfterMethod + private void waitForPurge() throws InterruptedException { + waitForPurge(0, queryService.finishedQueries); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index a85cf98..6af4225 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -1357,8 +1357,8 @@ public class TestQueryService extends LensJerseyTest { } } /** - * Data provider for test case {@link testExecuteWithTimeoutAndPreFetchAndServerPersistence} - * @return + * Data provider for test case + * {@link #testExecuteWithTimeoutAndPreFetchAndServerPersistence(long, int, boolean, long)} */ @DataProvider public Object[][] executeWithTimeoutAndPreFetechAndServerPersistenceDP() { http://git-wip-us.apache.org/repos/asf/lens/blob/01d0d846/lens-server/src/test/java/org/apache/lens/server/query/TestQueryServiceDuplicate.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryServiceDuplicate.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryServiceDuplicate.java index 45bdfbe..a6ded7e 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryServiceDuplicate.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryServiceDuplicate.java @@ -40,11 +40,10 @@ import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.metrics.MetricsService; import org.apache.lens.server.api.query.QueryExecutionService; +import org.apache.lens.server.api.util.LensUtil; import org.apache.lens.server.common.TestResourceFile; import org.apache.lens.server.query.TestQueryService.QueryServiceTestApp; -import org.apache.hadoop.hive.conf.HiveConf; - import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; @@ -78,19 +77,17 @@ public class TestQueryServiceDuplicate extends LensJerseyTest { super.setUp(); } - @Override - public HiveConf getServerConf() { - HiveConf serverConf = new HiveConf(super.getServerConf()); - serverConf.setBoolean(LensConfConstants.SERVER_DUPLICATE_QUERY_ALLOWED, false); - return serverConf; - } - @AfterTest public void tearDown() throws Exception { super.tearDown(); } @Override + public Map<String, String> getServerConfOverWrites() { + return LensUtil.getHashMap(LensConfConstants.SERVER_DUPLICATE_QUERY_ALLOWED, String.valueOf(false)); + } + + @Override protected Application configure() { enable(TestProperties.LOG_TRAFFIC); enable(TestProperties.DUMP_ENTITY); @@ -275,6 +272,7 @@ public class TestQueryServiceDuplicate extends LensJerseyTest { target.path(handle8.toString()).queryParam("sessionid", lensSessionId1).request(mt).delete(APIResult.class); } finally { queryService.pauseQuerySubmitter(false); + // cleanup dropTable(TEST_TABLE); queryService.closeSession(lensSessionId); for (LensDriver driver : queryService.getDrivers()) {
