Repository: lens Updated Branches: refs/heads/master a4b4b85d8 -> 5c25e68fe
LENS-1155 : Seeing INTERNAL_SERVER_ERROR for JDBC queries due to connection unavailability Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5c25e68f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5c25e68f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5c25e68f Branch: refs/heads/master Commit: 5c25e68fe71670df6c72d3b8dc08fcbcad6011c4 Parents: a4b4b85 Author: SushilMohanty <[email protected]> Authored: Wed May 25 16:53:22 2016 +0530 Committer: sushilmohanty <[email protected]> Committed: Wed May 25 16:53:22 2016 +0530 ---------------------------------------------------------------------- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 3 +- .../jdbc/MaxJDBCConnectionCheckConstraint.java | 48 ++++++++++++++++++++ ...MaxJDBCConnectionCheckConstraintFactory.java | 38 ++++++++++++++++ .../src/main/resources/jdbcdriver-default.xml | 3 +- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 37 +++++++++++++++ .../apache/lens/server/api/util/LensUtil.java | 2 +- 6 files changed, 128 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 044a19c..8047302 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -94,6 +94,7 @@ public class JDBCDriver extends AbstractLensDriver { private ExecutorService asyncQueryPool; /** The query context map. */ + @Getter private ConcurrentHashMap<QueryHandle, JdbcQueryContext> queryContextMap; /** The conf. */ @@ -458,7 +459,7 @@ public class JDBCDriver extends AbstractLensDriver { final int maxPoolSize = parseInt(this.conf.get(JDBC_POOL_MAX_SIZE.getConfigKey())); final int maxConcurrentQueries = parseInt(this.conf.get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY)); - checkState(maxPoolSize == maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:" + checkState(maxPoolSize >= maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:" + maxConcurrentQueries); queryContextMap = new ConcurrentHashMap<>(); http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java new file mode 100644 index 0000000..82b5647 --- /dev/null +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.driver.jdbc; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection; +import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MaxJDBCConnectionCheckConstraint implements QueryLaunchingConstraint { + + private final int poolMaxSize; + + public MaxJDBCConnectionCheckConstraint(final int poolMaxSize) { + this.poolMaxSize = poolMaxSize; + } + + @Override + public boolean allowsLaunchOf(final QueryContext candidateQuery, + EstimatedImmutableQueryCollection launchedQueries) { + final LensDriver selectedDriver = candidateQuery.getSelectedDriver(); + final boolean canLaunch = (selectedDriver instanceof JDBCDriver) + && (((JDBCDriver) selectedDriver).getQueryContextMap().size() < poolMaxSize); + + log.debug("canLaunch:{}", canLaunch); + return canLaunch; + } +} + http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraintFactory.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraintFactory.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraintFactory.java new file mode 100644 index 0000000..477e197 --- /dev/null +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraintFactory.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.driver.jdbc; + +import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.ConnectionPoolProperties.JDBC_POOL_MAX_SIZE; + +import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory; +import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; + +import org.apache.hadoop.conf.Configuration; + +public class MaxJDBCConnectionCheckConstraintFactory implements + ConfigBasedObjectCreationFactory<QueryLaunchingConstraint> { + + @Override + public MaxJDBCConnectionCheckConstraint create(Configuration conf) { + final int poolMaxSize = Integer.parseInt(conf.get(JDBC_POOL_MAX_SIZE.getConfigKey())); + + return new MaxJDBCConnectionCheckConstraint(poolMaxSize); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml index b5b6164..b446f7a 100644 --- a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml +++ b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml @@ -228,7 +228,8 @@ <property> <name>lens.driver.jdbc.query.launching.constraint.factories</name> - <value>org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory</value> + <value>org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory, + org.apache.lens.driver.jdbc.MaxJDBCConnectionCheckConstraintFactory</value> <description>Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index 67f6c1f..491aa69 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -411,6 +411,43 @@ public class TestJdbcDriver { } } + @Test + public void testJDBCMaxConnectionConstraintCheck() throws Exception { + close(); + // Create table execute_test + createTable("max_connection_test"); + // Insert some data into table + insertData("max_connection_test"); + + MaxJDBCConnectionCheckConstraintFactory factory = new MaxJDBCConnectionCheckConstraintFactory(); + MaxJDBCConnectionCheckConstraint constraint = factory.create(driver.getConf()); + + // check constraint in driver + assertTrue(driver.getQueryConstraints().toString().contains("MaxJDBCConnectionCheckConstraint")); + + String query; + QueryContext context = createQueryContext("SELECT * FROM max_connection_test"); + + for (int i = 1; i <= JDBC_POOL_MAX_SIZE.getDefaultValue(); i++) { + query = "SELECT " + i + " FROM max_connection_test"; + context = createQueryContext(query); + driver.executeAsync(context); + } + + //pool max size is same as number of query context hold on driver + assertEquals(driver.getQueryContextMap().size(), JDBC_POOL_MAX_SIZE.getDefaultValue()); + + //new query shouldn't be allowed + QueryContext newcontext = createQueryContext("SELECT 123 FROM max_connection_test"); + assertFalse(constraint.allowsLaunchOf(newcontext, null)); + + //close one query and launch the previous query again + driver.closeQuery(context.getQueryHandle()); + assertTrue(constraint.allowsLaunchOf(newcontext, null)); + close(); + } + + /** * Data provider for test case {@link #testExecuteWithPreFetch()} * @return http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java index a0691a5..8261d8a 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java @@ -85,7 +85,7 @@ public final class LensUtil { for (String factoryName : factoryNames) { if (StringUtils.isNotBlank(factoryName)) { - final T implementation = getImplementation(factoryName, conf); + final T implementation = getImplementation(factoryName.trim(), conf); implSet.add(implementation); } }
