LENS-743: Query retry framework for retrying upon transient failures
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/38ab6c60 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/38ab6c60 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/38ab6c60 Branch: refs/heads/master Commit: 38ab6c6082b6221502daac979551e8c5fca72241 Parents: 182f6dc Author: Rajat Khandelwal <pro...@apache.org> Authored: Mon Oct 17 11:13:34 2016 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Mon Oct 17 11:13:34 2016 +0530 ---------------------------------------------------------------------- checkstyle/src/main/resources/checkstyle.xml | 2 +- .../apache/lens/api/query/FailedAttempt.java | 68 ++++++ .../org/apache/lens/api/query/LensQuery.java | 6 + .../org/apache/lens/api/query/QueryStatus.java | 34 ++- .../lens/api/jaxb/YAMLToStringStrategyTest.java | 14 +- .../org.apache.lens.api.query.LensQuery.xml | 5 - .../org.apache.lens.api.query.QueryStatus.xml | 5 - .../org/apache/lens/driver/es/ESDriver.java | 29 +-- lens-driver-es/src/test/resources/hive-site.xml | 5 + .../org/apache/lens/driver/hive/HiveDriver.java | 65 +----- .../apache/lens/driver/hive/TestHiveDriver.java | 2 +- .../lens/driver/hive/TestRemoteHiveDriver.java | 2 +- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 69 ++---- .../driver/jdbc/JDBCDriverConfConstants.java | 6 - .../jdbc/MaxJDBCConnectionCheckConstraint.java | 16 +- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 4 +- .../regression/core/constants/DriverConfig.java | 4 +- .../lens/server/api/LensConfConstants.java | 21 +- .../server/api/common/BackOffRetryHandler.java | 71 ------- .../lens/server/api/common/FailureContext.java | 43 ---- ...FibonacciExponentialBackOffRetryHandler.java | 77 ------- .../common/OperationRetryHandlerFactory.java | 41 ---- .../server/api/driver/AbstractLensDriver.java | 56 ++++- .../server/api/driver/DriverConfiguration.java | 60 ++++++ .../server/api/driver/DriverQueryStatus.java | 48 ++--- .../lens/server/api/driver/LensDriver.java | 3 + .../server/api/query/AbstractQueryContext.java | 2 +- .../api/query/DriverSelectorQueryContext.java | 17 +- .../server/api/query/FinishedLensQuery.java | 15 +- .../lens/server/api/query/PriorityChange.java | 40 ---- .../lens/server/api/query/QueryAccepted.java | 39 ---- .../lens/server/api/query/QueryCancelled.java | 51 ----- .../lens/server/api/query/QueryClosed.java | 50 ----- .../lens/server/api/query/QueryContext.java | 68 ++++-- .../lens/server/api/query/QueryEnded.java | 87 -------- .../lens/server/api/query/QueryEvent.java | 93 -------- .../lens/server/api/query/QueryExecuted.java | 41 ---- .../lens/server/api/query/QueryFailed.java | 50 ----- .../lens/server/api/query/QueryLaunched.java | 41 ---- .../lens/server/api/query/QueryQueued.java | 59 ------ .../lens/server/api/query/QueryRejected.java | 39 ---- .../lens/server/api/query/QueryRunning.java | 41 ---- .../lens/server/api/query/QuerySuccess.java | 47 ---- .../server/api/query/QueuePositionChange.java | 39 ---- .../lens/server/api/query/StatusChange.java | 52 ----- .../api/query/StatusUpdateFailureContext.java | 45 ++++ .../query/comparators/ChainedComparator.java | 41 ++++ .../query/comparators/FIFOQueryComparator.java | 29 +++ .../comparators/MoreRetriesFirstComparator.java | 30 +++ .../api/query/comparators/QueryComparator.java | 28 +++ .../query/comparators/QueryCostComparator.java | 30 +++ .../comparators/QueryPriorityComparator.java | 29 +++ .../MaxConcurrentDriverQueriesConstraint.java | 64 ++++-- .../constraint/QueryLaunchingConstraint.java | 4 +- .../server/api/query/events/PriorityChange.java | 40 ++++ .../server/api/query/events/QueryAccepted.java | 39 ++++ .../server/api/query/events/QueryCancelled.java | 52 +++++ .../server/api/query/events/QueryClosed.java | 51 +++++ .../server/api/query/events/QueryEnded.java | 88 ++++++++ .../server/api/query/events/QueryEvent.java | 93 ++++++++ .../server/api/query/events/QueryExecuted.java | 41 ++++ .../server/api/query/events/QueryFailed.java | 51 +++++ .../server/api/query/events/QueryLaunched.java | 41 ++++ .../server/api/query/events/QueryQueued.java | 59 ++++++ .../api/query/events/QueryQueuedForRetry.java | 41 ++++ .../server/api/query/events/QueryRejected.java | 39 ++++ .../server/api/query/events/QueryRunning.java | 41 ++++ .../server/api/query/events/QuerySuccess.java | 48 +++++ .../api/query/events/QueuePositionChange.java | 39 ++++ .../server/api/query/events/StatusChange.java | 52 +++++ .../server/api/retry/BackOffRetryHandler.java | 74 +++++++ .../api/retry/ChainedRetryPolicyDecider.java | 72 +++++++ .../api/retry/DefaultRetryPolicyDecider.java | 27 +++ .../lens/server/api/retry/FailureContext.java | 38 ++++ ...FibonacciExponentialBackOffRetryHandler.java | 78 +++++++ .../server/api/retry/ImmediateRetryHandler.java | 46 ++++ .../lens/server/api/retry/NoRetryHandler.java | 35 +++ .../api/retry/OperationRetryHandlerFactory.java | 42 ++++ .../server/api/retry/RetryPolicyDecider.java | 25 +++ .../apache/lens/server/api/util/LensUtil.java | 34 +-- .../TestExponentialBackOffRetryHandler.java | 52 ----- .../lens/server/api/driver/MockDriver.java | 4 +- .../lens/server/api/query/TestQueryContext.java | 3 +- .../comparators/ChainedComparatorTest.java | 78 +++++++ ...axConcurrentDriverQueriesConstraintTest.java | 43 +++- .../TestExponentialBackOffRetryHandler.java | 55 +++++ lens-server/pom.xml | 2 +- .../lens/server/metrics/MetricsServiceImpl.java | 2 +- .../lens/server/query/FIFOQueryComparator.java | 33 --- .../apache/lens/server/query/LensServerDAO.java | 98 +++++++-- .../lens/server/query/QueryComparator.java | 28 --- .../lens/server/query/QueryCostComparator.java | 39 ---- .../lens/server/query/QueryEndHttpNotifier.java | 4 +- .../lens/server/query/QueryEndNotifier.java | 2 +- .../server/query/QueryEventHttpNotifier.java | 4 +- .../server/query/QueryExecutionServiceImpl.java | 152 ++++++++++--- .../QueryExecutionStatisticsGenerator.java | 2 +- .../server/query/QueryPriorityComparator.java | 38 ---- .../lens/server/query/ResultFormatter.java | 1 + ...DefaultQueryLaunchingConstraintsChecker.java | 19 +- .../RetryPolicyToConstraingAdapter.java | 40 ++++ .../TotalQueryCostCeilingConstraint.java | 12 +- .../apache/lens/server/rewrite/RewriteUtil.java | 2 +- .../scheduler/SchedulerQueryEventListener.java | 2 +- .../server/scheduler/SchedulerServiceImpl.java | 2 +- .../query/QueryContextComparatorTest.java | 20 +- .../lens/server/query/TestEventService.java | 2 +- .../apache/lens/server/query/TestLensDAO.java | 23 +- .../TestQueryIndependenceFromSessionClose.java | 5 + .../query/TestQueryNotifictaionResource.java | 4 +- .../lens/server/query/TestQueryService.java | 72 ++++--- .../server/query/collect/QueryCollectUtil.java | 4 +- ...ultQueryLaunchingConstraintsCheckerTest.java | 18 +- .../ThreadSafeEstimatedQueryCollectionTest.java | 11 +- .../TotalQueryCostCeilingConstraintTest.java | 18 +- .../query/retry/MockDriverForRetries.java | 82 +++++++ .../query/retry/MockRetryPolicyDecider.java | 35 +++ .../lens/server/query/retry/QueryRetryTest.java | 212 +++++++++++++++++++ .../retry/TestServerRetryPolicyDecider.java | 32 +++ .../scheduler/util/SchedulerTestUtils.java | 2 +- .../failing-query-driver-site.xml | 32 +++ .../retry/double_failure/driver-site.xml | 31 +++ .../retry/single_failure/driver-site.xml | 31 +++ .../retry/triple_failure/driver-site.xml | 31 +++ 124 files changed, 2988 insertions(+), 1607 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/checkstyle/src/main/resources/checkstyle.xml ---------------------------------------------------------------------- diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml index 17e55ea..634234d 100644 --- a/checkstyle/src/main/resources/checkstyle.xml +++ b/checkstyle/src/main/resources/checkstyle.xml @@ -62,7 +62,7 @@ </module> <module name="FileLength"> - <property name="max" value="3500"/> + <property name="max" value="4000"/> </module> <module name="FileTabCharacter"/> http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java b/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java new file mode 100644 index 0000000..5cf2dc3 --- /dev/null +++ b/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.api.query; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import lombok.*; + + +@XmlRootElement +@AllArgsConstructor +/** + * Instantiates a new query status. + */ +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@EqualsAndHashCode +public class FailedAttempt implements Comparable<FailedAttempt>, Serializable { + @XmlElement + @Getter + private String driverName; + @XmlElement + @Getter + private double progress; + @XmlElement + @Getter + private String progressMessage; + @XmlElement + @Getter + private String errorMessage; + @XmlElement + @Getter + private Long driverStartTime; + @XmlElement + @Getter + private Long driverFinishTime; + + @Override + public int compareTo(FailedAttempt o) { + if (o == null) { + return 1; + } + int startCompare = driverStartTime.compareTo(o.getDriverStartTime()); + if (startCompare != 0) { + return startCompare; + } + return driverFinishTime.compareTo(o.getDriverFinishTime()); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java index 7d079a9..d7b8963 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java @@ -21,6 +21,8 @@ */ package org.apache.lens.api.query; +import java.util.List; + import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; @@ -215,6 +217,10 @@ public class LensQuery extends ToYAMLString { @Getter private String queryName; + @XmlElement + @Getter + private List<FailedAttempt> failedAttempts; + /** * @return error code in case of query failures */ http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java index daedbf9..3c6ecf7 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java @@ -62,11 +62,11 @@ import lombok.*; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class QueryStatus extends ToYAMLString implements Serializable { + /** * The Constant serialVersionUID. */ private static final long serialVersionUID = 1L; - /** * The Enum Status. */ @@ -106,6 +106,12 @@ public class QueryStatus extends ToYAMLString implements Serializable { EXECUTED, /** + * This state is when depending on retry policy, either the query moves to QUEUED (in case retries are to be done), + * or to FAILED. + */ + FAILING, + + /** * The successful. * At this point all operations related to the query are finished successfully by driver and server. */ @@ -177,7 +183,7 @@ public class QueryStatus extends ToYAMLString implements Serializable { @Setter private String errorMessage; - @XmlElement + @Getter private LensErrorTO lensErrorTO; public boolean finished() { @@ -205,6 +211,10 @@ public class QueryStatus extends ToYAMLString implements Serializable { return status.equals(Status.FAILED); } + public boolean failing() { + return status.equals(Status.FAILING); + } + public boolean cancelled() { return status.equals(Status.CANCELED); } @@ -232,7 +242,7 @@ public class QueryStatus extends ToYAMLString implements Serializable { case QUEUED: switch (newState) { case LAUNCHED: - case FAILED: + case FAILING: case CANCELED: return true; } @@ -242,7 +252,7 @@ public class QueryStatus extends ToYAMLString implements Serializable { case LAUNCHED: case RUNNING: case CANCELED: - case FAILED: + case FAILING: case EXECUTED: return true; } @@ -251,7 +261,7 @@ public class QueryStatus extends ToYAMLString implements Serializable { switch (newState) { case RUNNING: case CANCELED: - case FAILED: + case FAILING: case EXECUTED: return true; } @@ -260,17 +270,25 @@ public class QueryStatus extends ToYAMLString implements Serializable { switch (newState) { case EXECUTED: case SUCCESSFUL: - case FAILED: + case FAILING: case CANCELED: return true; } break; + case FAILING: + switch(newState) { + case QUEUED: + case FAILED: + return true; + } + break; case FAILED: case CANCELED: case SUCCESSFUL: if (Status.CLOSED.equals(newState)) { return true; } + break; default: // fall-through } @@ -294,4 +312,8 @@ public class QueryStatus extends ToYAMLString implements Serializable { public String getLensErrorTOErrorMsg() { return (this.lensErrorTO != null) ? this.lensErrorTO.getMessage() : null; } + + public static QueryStatus getQueuedStatus() { + return new QueryStatus(0.0, null, Status.QUEUED, "Query is queued", false, null, null, null); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java ---------------------------------------------------------------------- diff --git a/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java b/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java index 04d9ad6..eb568a7 100644 --- a/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java +++ b/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java @@ -147,11 +147,15 @@ public class YAMLToStringStrategyTest { return name.endsWith("xml"); } })) { - Class<?> clazz = Class.forName(fn.substring(0, fn.length() - 4)); - Object unmarshalled = clazz.cast(new LensJAXBContext(clazz) - .createUnmarshaller().unmarshal(getClass().getResourceAsStream("/toString/" + fn))); - String toString = readYAML("/toString/" + fn.replaceAll("xml$", "yaml")); - dataList.add(new ToStringTestData(fn, unmarshalled, toString)); + try { + Class<?> clazz = Class.forName(fn.substring(0, fn.length() - 4)); + Object unmarshalled = clazz.cast(new LensJAXBContext(clazz) + .createUnmarshaller().unmarshal(getClass().getResourceAsStream("/toString/" + fn))); + String toString = readYAML("/toString/" + fn.replaceAll("xml$", "yaml")); + dataList.add(new ToStringTestData(fn, unmarshalled, toString)); + } catch (Throwable th) { + dataList.add(new ToStringTestData(fn, th)); + } } return dataList; } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml ---------------------------------------------------------------------- diff --git a/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml b/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml index a681bca..fc609d9 100644 --- a/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml +++ b/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml @@ -34,11 +34,6 @@ <status>RUNNING</status> <statusMessage>query running</statusMessage> <isResultSetAvailable>false</isResultSetAvailable> - <lensErrorTO> - <code>9999</code> - <message>error message</message> - <stackTrace>random stack trace</stackTrace> - </lensErrorTO> <errorMessage>error message</errorMessage> <progressMessage>progress message</progressMessage> <queueNumber>11</queueNumber> http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml ---------------------------------------------------------------------- diff --git a/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml b/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml index 70f9c93..b1957d7 100644 --- a/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml +++ b/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml @@ -24,11 +24,6 @@ <status>RUNNING</status> <statusMessage>query running</statusMessage> <isResultSetAvailable>false</isResultSetAvailable> - <lensErrorTO> - <code>9999</code> - <message>error message</message> - <stackTrace>random stack trace</stackTrace> - </lensErrorTO> <errorMessage>error message</errorMessage> <progressMessage>progress message</progressMessage> <queueNumber>11</queueNumber> http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java index fceabea..e735c0c 100644 --- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java +++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java @@ -42,8 +42,6 @@ import org.apache.lens.server.api.events.LensEventListener; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; -import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy; -import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; import org.apache.lens.server.api.query.cost.QueryCost; @@ -60,9 +58,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.antlr.runtime.CommonToken; import org.antlr.runtime.tree.Tree; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; @@ -76,7 +72,6 @@ public class ESDriver extends AbstractLensDriver { private static final double STREAMING_PARTITION_COST = 0; private static final QueryCost ES_DRIVER_COST = new FactPartitionBasedQueryCost(STREAMING_PARTITION_COST); - private Configuration conf; private ESClient esClient; private ExecutorService asyncQueryPool; private ESDriverConfig config; @@ -89,11 +84,6 @@ public class ESDriver extends AbstractLensDriver { private final Map<QueryHandle, QueryContext> handleContextMap = Maps.newConcurrentMap(); @Override - public Configuration getConf() { - return conf; - } - - @Override public QueryCost estimate(AbstractQueryContext qctx) { return ES_DRIVER_COST; } @@ -232,16 +222,6 @@ public class ESDriver extends AbstractLensDriver { } - @Override - public ImmutableSet<QueryLaunchingConstraint> getQueryConstraints() { - return ImmutableSet.copyOf(Sets.<QueryLaunchingConstraint>newHashSet()); - } - - @Override - public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() { - return ImmutableSet.copyOf(Sets.<WaitingQueriesSelectionPolicy>newHashSet()); - } - private ESQuery rewrite(AbstractQueryContext context) throws LensException { final String key = keyFor(context); if (rewrittenQueriesCache.containsKey(key)) { @@ -299,18 +279,15 @@ public class ESDriver extends AbstractLensDriver { @Override public void configure(Configuration conf, String driverType, String driverName) throws LensException { super.configure(conf, driverType, driverName); - this.conf = new Configuration(conf); - this.conf.addResource("esdriver-default.xml"); - this.conf.addResource(getDriverResourcePath("esdriver-site.xml")); - config = new ESDriverConfig(this.conf); + config = new ESDriverConfig(getConf()); Class klass; try { - klass = Class.forName(this.conf.get(ESDriverConfig.CLIENT_CLASS_KEY)); + klass = Class.forName(getConf().get(ESDriverConfig.CLIENT_CLASS_KEY)); if (klass != null) { log.debug("Picked up class {}", klass); if (ESClient.class.isAssignableFrom(klass)) { final Constructor constructor = klass.getConstructor(ESDriverConfig.class, Configuration.class); - esClient = (ESClient) constructor.newInstance(config, this.conf); + esClient = (ESClient) constructor.newInstance(config, getConf()); log.debug("Successfully instantiated es client of type {}", klass); } } else { http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-es/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/test/resources/hive-site.xml b/lens-driver-es/src/test/resources/hive-site.xml index e49c876..f2f4b9b 100644 --- a/lens-driver-es/src/test/resources/hive-site.xml +++ b/lens-driver-es/src/test/resources/hive-site.xml @@ -50,4 +50,9 @@ <value>true</value> </property> + <property> + <name>hive.metastore.schema.verification</name> + <value>false</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 0d8810f..f307ccc 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -19,7 +19,6 @@ package org.apache.lens.driver.hive; import static org.apache.lens.server.api.error.LensDriverErrorCode.*; -import static org.apache.lens.server.api.util.LensUtil.getImplementations; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -45,8 +44,6 @@ import org.apache.lens.server.api.events.LensEventListener; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; -import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy; -import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.query.cost.QueryCostCalculator; @@ -70,7 +67,6 @@ import org.apache.hive.service.rpc.thrift.TSessionHandle; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableSet; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -83,8 +79,6 @@ public class HiveDriver extends AbstractLensDriver { /** The Constant HIVE_CONNECTION_CLASS. */ public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class"; - public static final String HIVE_QUERY_HOOK_CLASS = "lens.driver.hive.query.hook.class"; - /** The Constant HS2_CONNECTION_EXPIRY_DELAY. */ public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.hs2.connection.expiry.delay"; @@ -101,15 +95,6 @@ public class HiveDriver extends AbstractLensDriver { public static final String HS2_PRIORITY_DEFAULT_RANGES = "VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW"; public static final String SESSION_KEY_DELIMITER = "."; - public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY - = "lens.driver.hive.query.launching.constraint.factories"; - - private static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY - = "lens.driver.hive.waiting.queries.selection.policy.factories"; - - /** The driver conf- which will merged with query conf */ - private Configuration driverConf; - /** The HiveConf - used for connecting to hive server and metastore */ private HiveConf hiveConf; @@ -153,17 +138,12 @@ public class HiveDriver extends AbstractLensDriver { QueryPriorityDecider queryPriorityDecider; // package-local. Test case can change. boolean whetherCalculatePriority; - private DriverQueryHook queryHook; private static final Map<String, String> SESSION_CONF = new HashMap<String, String>() { { put(HiveConf.ConfVars.HIVE_SERVER2_CLOSE_SESSION_ON_DISCONNECT.varname, "false"); } }; - @Getter - protected ImmutableSet<QueryLaunchingConstraint> queryConstraints; - private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies; - private String sessionDbKey(String sessionHandle, String database) { return sessionHandle + SESSION_KEY_DELIMITER + database; } @@ -340,11 +320,6 @@ public class HiveDriver extends AbstractLensDriver { log.info("Hive driver inited"); } - @Override - public Configuration getConf() { - return driverConf; - } - /* * (non-Javadoc) * @@ -353,23 +328,16 @@ public class HiveDriver extends AbstractLensDriver { @Override public void configure(Configuration conf, String driverType, String driverName) throws LensException { super.configure(conf, driverType, driverName); - this.driverConf = new Configuration(conf); - String driverConfPath = getDriverResourcePath("hivedriver-site.xml"); - this.driverConf.addResource("hivedriver-default.xml"); - this.driverConf.addResource(driverConfPath); - // resources have to be added separately on hiveConf again because new HiveConf() overrides hive.* properties - // from HiveConf this.hiveConf = new HiveConf(conf, HiveDriver.class); - this.hiveConf.addResource("hivedriver-default.xml"); - this.hiveConf.addResource(driverConfPath); + this.hiveConf.addResource(getConf()); - connectionClass = this.driverConf.getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, + connectionClass = getConf().getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class); isEmbedded = (connectionClass.getName().equals(EmbeddedThriftConnection.class.getName())); - connectionExpiryTimeout = this.driverConf.getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY); - whetherCalculatePriority = this.driverConf.getBoolean(HS2_CALCULATE_PRIORITY, true); - Class<? extends QueryCostCalculator> queryCostCalculatorClass = this.driverConf.getClass(HS2_COST_CALCULATOR, + connectionExpiryTimeout = getConf().getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY); + whetherCalculatePriority = getConf().getBoolean(HS2_CALCULATE_PRIORITY, true); + Class<? extends QueryCostCalculator> queryCostCalculatorClass = getConf().getClass(HS2_COST_CALCULATOR, FactPartitionBasedQueryCostCalculator.class, QueryCostCalculator.class); try { queryCostCalculator = queryCostCalculatorClass.newInstance(); @@ -377,18 +345,9 @@ public class HiveDriver extends AbstractLensDriver { throw new LensException("Can't instantiate query cost calculator of class: " + queryCostCalculatorClass, e); } queryPriorityDecider = new CostRangePriorityDecider( - new CostToPriorityRangeConf(driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES)) + new CostToPriorityRangeConf(getConf().get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES)) ); - try { - queryHook = driverConf.getClass( - HIVE_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class - ).newInstance(); - queryHook.setDriver(this); - } catch (InstantiationException | IllegalAccessException e) { - throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e); - } - queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, driverConf); - selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, driverConf); + log.info("Hive driver {} configured successfully", getFullyQualifiedName()); } @@ -809,11 +768,6 @@ public class HiveDriver extends AbstractLensDriver { } @Override - public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() { - return selectionPolicies; - } - - @Override public Priority decidePriority(AbstractQueryContext ctx) { return decidePriority(ctx, queryPriorityDecider); } @@ -1385,9 +1339,4 @@ public class HiveDriver extends AbstractLensDriver { public boolean hasLensSession(LensSessionHandle session) { return lensToHiveSession.containsKey(session.getPublicId().toString()); } - - @Override - public DriverQueryHook getQueryHook() { - return queryHook; - } } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java index daf01ac..43b33f3 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java @@ -118,7 +118,7 @@ public class TestHiveDriver { protected void createDriver() throws LensException { driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml"); driverConf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class); - driverConf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class); + driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class); driverConf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); driver = new HiveDriver(); http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java index 1acbb13..961ec4e 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java @@ -148,7 +148,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml"); driver = new HiveDriver(); driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); - driverConf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class); + driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class); driver.configure(driverConf, "hive", "hive1"); drivers = Lists.<LensDriver>newArrayList(driver); System.out.println("TestRemoteHiveDriver created"); http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index e41077c..82e0231 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -23,7 +23,6 @@ import static java.util.Arrays.asList; import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.*; import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.ConnectionPoolProperties.*; -import static org.apache.lens.server.api.util.LensUtil.getImplementations; import static com.google.common.base.Preconditions.checkState; @@ -51,9 +50,7 @@ import org.apache.lens.server.api.metrics.MethodMetricsFactory; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; -import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy; import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory; -import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.query.rewrite.QueryRewriter; @@ -68,8 +65,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; -import com.google.common.collect.ImmutableSet; - import lombok.*; import lombok.extern.slf4j.Slf4j; @@ -95,20 +90,12 @@ public class JDBCDriver extends AbstractLensDriver { @Getter private ConcurrentHashMap<QueryHandle, JdbcQueryContext> queryContextMap; - /** The conf. */ - private Configuration conf; - /** Configuration for estimate connection pool */ private Configuration estimateConf; /** Estimate connection provider */ private ConnectionProvider estimateConnectionProvider; private LogSegregationContext logSegregationContext; - private DriverQueryHook queryHook; - - @Getter - private ImmutableSet<QueryLaunchingConstraint> queryConstraints; - private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies; private boolean isStatementCancelSupported; /** @@ -321,6 +308,7 @@ public class JDBCDriver extends AbstractLensDriver { queryContext.getLensContext().getDriverStatus().setDriverFinishTime(System.currentTimeMillis()); } } + return result; } @@ -383,14 +371,6 @@ public class JDBCDriver extends AbstractLensDriver { } } - /** - * Get driver configuration - */ - @Override - public Configuration getConf() { - return conf; - } - /* * (non-Javadoc) * @@ -399,18 +379,7 @@ public class JDBCDriver extends AbstractLensDriver { @Override public void configure(Configuration conf, String driverType, String driverName) throws LensException { super.configure(conf, driverType, driverName); - this.conf = new Configuration(conf); - this.conf.addResource("jdbcdriver-default.xml"); - this.conf.addResource(getDriverResourcePath("jdbcdriver-site.xml")); - init(conf); - try { - queryHook = this.conf.getClass( - JDBC_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class - ).newInstance(); - queryHook.setDriver(this); - } catch (InstantiationException | IllegalAccessException e) { - throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e); - } + init(); configured = true; log.info("JDBC Driver {} configured", getFullyQualifiedName()); } @@ -418,14 +387,12 @@ public class JDBCDriver extends AbstractLensDriver { /** * Inits the. * - * @param conf the conf * @throws LensException the lens exception */ - protected void init(Configuration conf) throws LensException { - - final int maxPoolSize = parseInt(this.conf.get(JDBC_POOL_MAX_SIZE.getConfigKey())); + public void init() throws LensException { + final int maxPoolSize = parseInt(getConf().get(JDBC_POOL_MAX_SIZE.getConfigKey())); final int maxConcurrentQueries - = parseInt(this.conf.get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY)); + = parseInt(getConf().get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY)); checkState(maxPoolSize >= maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:" + maxConcurrentQueries); @@ -439,7 +406,7 @@ public class JDBCDriver extends AbstractLensDriver { } }); - Class<? extends ConnectionProvider> cpClass = conf.getClass(JDBC_CONNECTION_PROVIDER, + Class<? extends ConnectionProvider> cpClass = getConf().getClass(JDBC_CONNECTION_PROVIDER, DataSourceConnectionProvider.class, ConnectionProvider.class); try { connectionProvider = cpClass.newInstance(); @@ -449,9 +416,8 @@ public class JDBCDriver extends AbstractLensDriver { throw new LensException(e); } this.logSegregationContext = new MappedDiagnosticLogSegregationContext(); - this.queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, this.conf); - this.selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, this.conf); - this.isStatementCancelSupported = conf.getBoolean(STATEMENT_CANCEL_SUPPORTED, DEFAULT_STATEMENT_CANCEL_SUPPORTED); + this.isStatementCancelSupported = getConf().getBoolean(STATEMENT_CANCEL_SUPPORTED, + DEFAULT_STATEMENT_CANCEL_SUPPORTED); } /** @@ -469,7 +435,7 @@ public class JDBCDriver extends AbstractLensDriver { try { // Add here to cover the path when the queries are executed it does not // use the driver conf - return connectionProvider.getConnection(conf); + return connectionProvider.getConnection(getConf()); } catch (SQLException e) { throw new LensException(e); } @@ -483,7 +449,7 @@ public class JDBCDriver extends AbstractLensDriver { */ protected QueryRewriter getQueryRewriter() throws LensException { QueryRewriter rewriter; - Class<? extends QueryRewriter> queryRewriterClass = conf.getClass(JDBC_QUERY_REWRITER_CLASS, + Class<? extends QueryRewriter> queryRewriterClass = getConf().getClass(JDBC_QUERY_REWRITER_CLASS, DummyQueryRewriter.class, QueryRewriter.class); try { rewriter = queryRewriterClass.newInstance(); @@ -492,7 +458,7 @@ public class JDBCDriver extends AbstractLensDriver { log.error("{} Unable to create rewriter object", getFullyQualifiedName(), e); throw new LensException(e); } - rewriter.init(conf); + rewriter.init(getConf()); return rewriter; } @@ -674,7 +640,7 @@ public class JDBCDriver extends AbstractLensDriver { // Get connection config used by estimate pool. protected final Configuration getEstimateConnectionConf() { if (estimateConf == null) { - Configuration tmpConf = new Configuration(conf); + Configuration tmpConf = new Configuration(getConf()); // Override JDBC settings in estimate conf, if set by user explicitly. Otherwise fall back to default JDBC pool // config for (String key : asList(JDBC_CONNECTION_PROPERTIES, JDBC_DB_URI, JDBC_DRIVER_CLASS, JDBC_USER, JDBC_PASSWORD, @@ -1059,11 +1025,6 @@ public class JDBCDriver extends AbstractLensDriver { } - @Override - public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() { - return this.selectionPolicies; - } - /* * (non-Javadoc) * @@ -1084,12 +1045,6 @@ public class JDBCDriver extends AbstractLensDriver { public void writeExternal(ObjectOutput arg0) throws IOException { // TODO Auto-generated method stub } - - @Override - public DriverQueryHook getQueryHook() { - return queryHook; - } - @Override public StatusUpdateMethod getStatusUpdateMethod() { return StatusUpdateMethod.PUSH; http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java index 3c34eb1..f4e0451 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java @@ -38,7 +38,6 @@ public final class JDBCDriverConfConstants { /** The Constant JDBC_QUERY_REWRITER_CLASS. */ public static final String JDBC_QUERY_REWRITER_CLASS = JDBC_DRIVER_PFX + "query.rewriter"; - public static final String JDBC_QUERY_HOOK_CLASS = JDBC_DRIVER_PFX + "query.hook.class"; /** The Constant JDBC_DRIVER_CLASS. */ public static final String JDBC_DRIVER_CLASS = JDBC_DRIVER_PFX + "driver.class"; @@ -104,11 +103,6 @@ public final class JDBCDriverConfConstants { public static final String JDBC_FETCH_SIZE = JDBC_DRIVER_PFX + "fetch.size"; public static final int DEFAULT_JDBC_FETCH_SIZE = 1000; - public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = JDBC_DRIVER_PFX - + "query.launching.constraint.factories"; - - public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY = JDBC_DRIVER_PFX - + "waiting.queries.selection.policy.factories"; public static final String REGEX_REPLACEMENT_VALUES = JDBC_DRIVER_PFX + "regex.replacement.values"; public static final String STATEMENT_CANCEL_SUPPORTED = JDBC_DRIVER_PFX + "statement.cancel.supported"; public static final boolean DEFAULT_STATEMENT_CANCEL_SUPPORTED = true; http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java index 82b5647..f293ea8 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java @@ -35,14 +35,18 @@ public class MaxJDBCConnectionCheckConstraint implements QueryLaunchingConstrain } @Override - public boolean allowsLaunchOf(final QueryContext candidateQuery, + public String allowsLaunchOf(final QueryContext candidateQuery, EstimatedImmutableQueryCollection launchedQueries) { final LensDriver selectedDriver = candidateQuery.getSelectedDriver(); - final boolean canLaunch = (selectedDriver instanceof JDBCDriver) - && (((JDBCDriver) selectedDriver).getQueryContextMap().size() < poolMaxSize); - - log.debug("canLaunch:{}", canLaunch); - return canLaunch; + if (!(selectedDriver instanceof JDBCDriver)) { + return "driver isn't jdbc driver"; + } + int runningQueries = ((JDBCDriver) selectedDriver).getQueryContextMap().size(); + if (runningQueries >= poolMaxSize) { + return runningQueries + "/" + poolMaxSize + " queries running on driver " + + candidateQuery.getSelectedDriver().getFullyQualifiedName(); + } + return null; } } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index 2ad7f76..a402f91 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -446,11 +446,11 @@ public class TestJdbcDriver { //new query shouldn't be allowed QueryContext newcontext = createQueryContext("SELECT 123 FROM max_connection_test"); - assertFalse(constraint.allowsLaunchOf(newcontext, null)); + assertNotNull(constraint.allowsLaunchOf(newcontext, null)); //close one query and launch the previous query again driver.closeQuery(context.getQueryHandle()); - assertTrue(constraint.allowsLaunchOf(newcontext, null)); + assertNull(constraint.allowsLaunchOf(newcontext, null)); close(); } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java index d80fba9..cff8e91 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java @@ -19,8 +19,8 @@ package org.apache.lens.regression.core.constants; -import org.apache.lens.driver.hive.HiveDriver; import org.apache.lens.driver.jdbc.JDBCDriverConfConstants; +import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory; import org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory; @@ -38,7 +38,7 @@ public class DriverConfig { MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY; public static final String JDBC_POOL_SIZE = JDBCDriverConfConstants.ConnectionPoolProperties. JDBC_POOL_MAX_SIZE.getConfigKey(); - public static final String HIVE_CONSTRAINT_FACTORIES = HiveDriver.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY; + public static final String HIVE_CONSTRAINT_FACTORIES = LensConfConstants.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX; public static final String MAX_CONCURRENT_CONSTRAINT_FACTORY = MaxConcurrentDriverQueriesConstraintFactory http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 8cf617b..3ae59c6 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -98,7 +98,7 @@ public final class LensConfConstants { public static final String MAX_SESSIONS_PER_USER = SERVER_PFX + "max.sessions.per.user"; - public static final String QUERY_COMPARATOR_CLASS = SERVER_PFX + "query.comparator.class"; + public static final String QUERY_COMPARATOR_CLASSES = SERVER_PFX + "query.comparator.classes"; public static final Integer DEFAULT_MAX_SESSIONS_PER_USER = 10; @@ -913,8 +913,9 @@ public final class LensConfConstants { /** * Key to get the implementations of query constraint factories. */ + public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX = "query.launching.constraint.factories"; public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = SERVER_PFX - + "query.launching.constraint.factories"; + + QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX; /** * Key to get the total query cost ceiling per user. @@ -925,8 +926,10 @@ public final class LensConfConstants { /** * Key to get the implementations of waiting queries selection policy factories. */ + public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX = + "waiting.queries.selection.policy.factories"; public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY = SERVER_PFX - + "waiting.queries.selection.policy.factories"; + + WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX; /** * Key denoting the dialect class property of saved query service. @@ -954,6 +957,18 @@ public final class LensConfConstants { public static final String DRIVER_WEIGHT = DRIVER_PFX + "weight"; /** + * Key for specifying Retry policy class + */ + public static final String RETRY_POLICY_CLASSES_SFX = "query.retry.policy.classes"; + + public static final String QUERY_RETRY_POLICY_CLASSES = SERVER_PFX + RETRY_POLICY_CLASSES_SFX; + + /** + * Driver hook property + */ + public static final String DRIVER_HOOK_CLASS_SFX = "query.hook.class"; + + /** * Default driver weight */ public static final int DEFAULT_DRIVER_WEIGHT = 1; http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java deleted file mode 100644 index 17bfba0..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lens.server.api.common; - -/** - * A backoff retry handler. - * - * This allows a backoff on any call, so provides methods whether we can try the operation now, - * whats next time when operation can be performed and whether operation has exhausted all retries. - * - * Callers of this would do the following : - * - * if (handler.canTryOpNow(FailureContext)) { - * try { - * tryCallerOperation(); - * FailureContext.clear(); - * } catch (any Transient Exception) { - * FailureContext.updateFailure(); - * if (!handler.hasExhaustedRetries(FailureContext)) { - * // will be tried later again - * } - * throw exception; - * } - * } - */ -public interface BackOffRetryHandler { - - /** - * To know whether operation can be done now. - * - * @param failContext FailureContext holding failures till now. - * - * @return true if operation can be done now, false otherwise. - */ - boolean canTryOpNow(FailureContext failContext); - - /** - * Get the time when the operation can be done next. - * - * @param failContext FailureContext holding failures till now. - * - * @return Next operation time in millis since epoch - */ - long getOperationNextTime(FailureContext failContext); - - /** - * Has the operation exhausted all its retries - * - * @param failContext FailureContext holding failures till now. - * - * @return true if all retries have exhausted, false otherwise. - */ - boolean hasExhaustedRetries(FailureContext failContext); -} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java deleted file mode 100644 index 70a34b0..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lens.server.api.common; - -import lombok.Getter; - -/** - * Failure context captures last failure time and number of failures. - */ -public class FailureContext { - - @Getter - private long lastFailedTime = 0; - @Getter - private int failCount = 0; - - public synchronized void updateFailure() { - lastFailedTime = System.currentTimeMillis(); - failCount++; - } - - public synchronized void clear() { - lastFailedTime = 0; - failCount = 0; - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java deleted file mode 100644 index e7fb8ce..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lens.server.api.common; - -import static com.google.common.base.Preconditions.checkArgument; - -/** - * A exponential backoff retry handler. - * - * It allows the the failures to be retried at a next update time, which can increase exponentially. - * - */ -public class FibonacciExponentialBackOffRetryHandler implements BackOffRetryHandler { - final int[] fibonacci; - final long maxDelay; - final long waitMillis; - - public FibonacciExponentialBackOffRetryHandler(int numRetries, long maxDelay, long waitMillis) { - checkArgument(numRetries > 2); - fibonacci = new int[numRetries]; - fibonacci[0] = 1; - fibonacci[1] = 1; - for(int i = 2; i < numRetries; ++i) { - fibonacci[i] = fibonacci[i-1] + fibonacci[i-2]; - } - this.maxDelay = maxDelay; - this.waitMillis = waitMillis; - } - - public boolean canTryOpNow(FailureContext failContext) { - synchronized (failContext) { - if (failContext.getFailCount() != 0) { - long now = System.currentTimeMillis(); - if (now < getOperationNextTime(failContext)) { - return false; - } - } - return true; - } - } - - public long getOperationNextTime(FailureContext failContext) { - synchronized (failContext) { - if (failContext.getFailCount() >= fibonacci.length) { - return failContext.getLastFailedTime() + maxDelay; - } - long delay = Math.min(maxDelay, fibonacci[failContext.getFailCount()] * waitMillis); - return failContext.getLastFailedTime() + delay; - } - } - - public boolean hasExhaustedRetries(FailureContext failContext) { - synchronized (failContext) { - if (failContext.getFailCount() >= fibonacci.length) { - return true; - } - return false; - } - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java deleted file mode 100644 index 88fbe43..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lens.server.api.common; - -/** - * Factory which creates operation retry handler - */ -public class OperationRetryHandlerFactory { - private OperationRetryHandlerFactory() { - } - - /** - * Create exponential backoff handler - * - * @param numRetries Number of exponential backoff retries - * @param maxDelay Maximum delay an operation can wait for next - * @param waitMillis Number of millis that would grow exponentially incase of failures - * - * @return BackOffRetryHandler - */ - public static BackOffRetryHandler createExponentialBackOffHandler(int numRetries, long maxDelay, long waitMillis) { - return new FibonacciExponentialBackOffRetryHandler(numRetries, maxDelay, waitMillis); - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java index 365a619..8f30aa0 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java @@ -18,15 +18,23 @@ */ package org.apache.lens.server.api.driver; +import static org.apache.lens.server.api.LensConfConstants.*; +import static org.apache.lens.server.api.util.LensUtil.getImplementations; + import org.apache.lens.api.Priority; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy; +import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; +import org.apache.lens.server.api.retry.ChainedRetryPolicyDecider; +import org.apache.lens.server.api.retry.RetryPolicyDecider; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import com.google.common.collect.ImmutableSet; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -47,15 +55,48 @@ public abstract class AbstractLensDriver implements LensDriver { @Getter private String fullyQualifiedName = null; - private DriverQueryHook noOpDriverQueryHook = new NoOpDriverQueryHook(); + @Getter + private Configuration conf; + + @Getter + private ImmutableSet<QueryLaunchingConstraint> queryConstraints; + @Getter + private ImmutableSet<WaitingQueriesSelectionPolicy> waitingQuerySelectionPolicies; + @Getter + RetryPolicyDecider<QueryContext> retryPolicyDecider; + @Getter + private DriverQueryHook queryHook; @Override public void configure(Configuration conf, String driverType, String driverName) throws LensException { if (StringUtils.isBlank(driverType) || StringUtils.isBlank(driverName)) { throw new LensException("Driver Type and Name can not be null or empty"); } - fullyQualifiedName = new StringBuilder(driverType).append(SEPARATOR).append(driverName).toString(); - noOpDriverQueryHook.setDriver(this); + fullyQualifiedName = driverType + SEPARATOR + driverName; + this.conf = new DriverConfiguration(conf, driverType, getClass()); + this.conf.addResource(getClass().getSimpleName().toLowerCase() + "-default.xml"); + this.conf.addResource(getDriverResourcePath(getClass().getSimpleName().toLowerCase() + "-site.xml")); + + this.queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX, getConf()); + this.waitingQuerySelectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX, getConf()); + + loadRetryPolicyDecider(); + loadQueryHook(); + } + + protected void loadQueryHook() throws LensException { + try { + queryHook = getConf().getClass( + DRIVER_HOOK_CLASS_SFX, NoOpDriverQueryHook.class, DriverQueryHook.class + ).newInstance(); + queryHook.setDriver(this); + } catch (InstantiationException | IllegalAccessException e) { + throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e); + } + } + + protected void loadRetryPolicyDecider() throws LensException { + this.retryPolicyDecider = ChainedRetryPolicyDecider.from(getConf(), RETRY_POLICY_CLASSES_SFX); } /** @@ -96,8 +137,8 @@ public abstract class AbstractLensDriver implements LensDriver { * @return */ protected String getDriverResourcePath(String resourceName) { - return new StringBuilder(LensConfConstants.DRIVERS_BASE_DIR).append(SEPARATOR).append(getFullyQualifiedName()) - .append(SEPARATOR).append(resourceName).toString(); + return LensConfConstants.DRIVERS_BASE_DIR + SEPARATOR + getFullyQualifiedName() + + SEPARATOR + resourceName; } @Override @@ -106,11 +147,6 @@ public abstract class AbstractLensDriver implements LensDriver { } @Override - public DriverQueryHook getQueryHook() { - return noOpDriverQueryHook; - } - - @Override public StatusUpdateMethod getStatusUpdateMethod() { return StatusUpdateMethod.PULL; } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java new file mode 100644 index 0000000..69a1a0b --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.driver; + + +import static org.apache.lens.server.api.LensConfConstants.DRIVER_PFX; + +import org.apache.hadoop.conf.Configuration; + +public class DriverConfiguration extends Configuration { + private final String driverClassType; + private String driverType; + private final Class<? extends AbstractLensDriver> driverClass; + + public DriverConfiguration(Configuration conf, String driverType, Class<? extends AbstractLensDriver> driverClass) { + super(conf); + this.driverType = driverType; + this.driverClass = driverClass; + this.driverClassType = driverClass.getSimpleName().toLowerCase().replaceAll("driver$", ""); + } + + @Override + public String[] getStrings(String name) { + for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name, + DRIVER_PFX + name, name, }) { + String[] s = super.getStrings(key); + if (s != null) { + return s; + } + } + return null; + } + + @Override + public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) { + for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name, + DRIVER_PFX + name, name, }) { + if (getTrimmed(key) != null) { + return super.getClass(key, defaultValue, xface); + } + } + return defaultValue; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java index fc24fc6..b5c5dcd 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java @@ -22,12 +22,12 @@ import java.io.Serializable; import org.apache.lens.api.query.QueryStatus; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; /** * The Class DriverQueryStatus. */ +@Data public class DriverQueryStatus implements Serializable { /** @@ -35,6 +35,9 @@ public class DriverQueryStatus implements Serializable { */ private static final long serialVersionUID = 1L; + public boolean failed() { + return state == DriverQueryState.FAILED; + } /** @@ -96,59 +99,56 @@ public class DriverQueryStatus implements Serializable { /** * The progress. */ - @Getter - @Setter - private double progress = 0.0f; + private double progress; /** * The state. */ - @Getter - @Setter - private DriverQueryState state = DriverQueryState.NEW; + private DriverQueryState state; /** * The status message. */ - @Getter - @Setter private String statusMessage; /** * The is result set available. */ - @Getter - @Setter - private boolean isResultSetAvailable = false; + private boolean isResultSetAvailable; /** * The progress message. */ - @Getter - @Setter private String progressMessage; /** * The error message. */ - @Getter - @Setter private String errorMessage; /** * The driver start time. */ - @Getter - @Setter - private Long driverStartTime = 0L; + private Long driverStartTime; /** * The driver finish time. */ - @Getter - @Setter - private Long driverFinishTime = 0L; + private Long driverFinishTime; + { + clear(); + } + public void clear() { + progress = 0.0f; + state = DriverQueryState.NEW; + statusMessage = null; + isResultSetAvailable = false; + progressMessage = null; + errorMessage = null; + driverStartTime = 0L; + driverFinishTime = 0L; + } /** * To query status. * @@ -169,7 +169,7 @@ public class DriverQueryStatus implements Serializable { qstate = QueryStatus.Status.EXECUTED; break; case FAILED: - qstate = QueryStatus.Status.FAILED; + qstate = QueryStatus.Status.FAILING; break; case CANCELED: qstate = QueryStatus.Status.CANCELED; http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java index e472de0..1462239 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java @@ -31,6 +31,7 @@ import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy; import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; import org.apache.lens.server.api.query.cost.QueryCost; +import org.apache.lens.server.api.retry.RetryPolicyDecider; import org.apache.hadoop.conf.Configuration; @@ -232,4 +233,6 @@ public interface LensDriver extends Externalizable { * @return The method of status update supported by this driver. */ StatusUpdateMethod getStatusUpdateMethod(); + + RetryPolicyDecider<QueryContext> getRetryPolicyDecider(); } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java index e160f58..c6a872d 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java @@ -202,7 +202,7 @@ public abstract class AbstractQueryContext implements Serializable { public Map<LensDriver, DriverEstimateRunnable> getDriverEstimateRunnables() throws LensException { Map<LensDriver, DriverEstimateRunnable> estimateRunnables = new HashMap<LensDriver, DriverEstimateRunnable>(); - for (LensDriver driver : driverContext.getDrivers()) { + for (LensDriver driver : driverContext.getEligibleDrivers()) { estimateRunnables.put(driver, new DriverEstimateRunnable(this, driver)); } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java index 5ff59bd..8e431d1 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java @@ -50,8 +50,8 @@ public class DriverSelectorQueryContext { */ @Getter @Setter - protected Map<LensDriver, DriverQueryContext> driverQueryContextMap = new HashMap<LensDriver, - DriverQueryContext>(); + protected Map<LensDriver, DriverQueryContext> driverQueryContextMap = new HashMap<>(); + private Set<LensDriver> blackListedDrivers = Sets.newHashSet(); public DriverSelectorQueryContext(final String userQuery, final Configuration queryConf, final Collection<LensDriver> drivers) { @@ -77,6 +77,10 @@ public class DriverSelectorQueryContext { } } + public void blacklist(LensDriver selectedDriver) { + blackListedDrivers.add(selectedDriver); + } + public static class DriverQueryContext { @Getter @@ -264,13 +268,16 @@ public class DriverSelectorQueryContext { public Collection<LensDriver> getDrivers() { return driverQueryContextMap.keySet(); } + public Collection<LensDriver> getEligibleDrivers() { + return Sets.difference(driverQueryContextMap.keySet(), blackListedDrivers); + } public Collection<LensDriver> getDriversWithValidQueryCost() { final Set<LensDriver> eligibleDrivers = Sets.newLinkedHashSet(); - for (Map.Entry<LensDriver, DriverQueryContext> driverToDriverContext : this.driverQueryContextMap.entrySet()) { - if (driverToDriverContext.getValue().driverCost != null) { - eligibleDrivers.add(driverToDriverContext.getKey()); + for (LensDriver driver: getEligibleDrivers()) { + if (driverQueryContextMap.get(driver).driverCost != null) { + eligibleDrivers.add(driver); } } return Collections.unmodifiableCollection(eligibleDrivers); http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java index b58fcf9..d88944b 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java @@ -19,10 +19,11 @@ package org.apache.lens.server.api.query; import java.util.Collection; -import java.util.Iterator; +import java.util.List; import org.apache.lens.api.LensConf; import org.apache.lens.api.Priority; +import org.apache.lens.api.query.FailedAttempt; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryStatus; import org.apache.lens.server.api.driver.LensDriver; @@ -46,7 +47,7 @@ import lombok.ToString; * * @see java.lang.Object#hashCode() */ -@EqualsAndHashCode(exclude = {"selectedDriver", "conf"}) +@EqualsAndHashCode(exclude = {"selectedDriver", "conf", "failedAttempts"}) /* * (non-Javadoc) * @@ -185,6 +186,10 @@ public class FinishedLensQuery { @Setter private String driverQuery; + @Getter + @Setter + private List<FailedAttempt> failedAttempts; + /** * Instantiates a new finished lens query. */ @@ -222,6 +227,7 @@ public class FinishedLensQuery { this.priority = ctx.getPriority().toString(); } this.conf = ctx.getLensConf(); + this.failedAttempts = ctx.getFailedAttempts(); } public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) { @@ -249,13 +255,12 @@ public class FinishedLensQuery { if (getPriority() != null) { qctx.setPriority(Priority.valueOf(getPriority())); } + qctx.setFailedAttempts(getFailedAttempts()); return qctx; } private LensDriver getDriverFromName(Collection<LensDriver> drivers) { - Iterator<LensDriver> iterator = drivers.iterator(); - while (iterator.hasNext()) { - LensDriver driver = iterator.next(); + for (LensDriver driver : drivers) { if (driverName.equals(driver.getFullyQualifiedName())) { return driver; } http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java deleted file mode 100644 index eaf3fee..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server.api.query; - -import org.apache.lens.api.Priority; -import org.apache.lens.api.query.QueryHandle; - -/** - * Event fired when query priority changes. - */ -public class PriorityChange extends QueryEvent<Priority> { - - /** - * Instantiates a new priority change. - * - * @param eventTime the event time - * @param prev the prev - * @param current the current - * @param handle the handle - */ - public PriorityChange(long eventTime, Priority prev, Priority current, QueryHandle handle) { - super(eventTime, prev, current, handle); - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java deleted file mode 100644 index 8980b61..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server.api.query; - -import org.apache.lens.api.query.QueryHandle; - -/** - * The Class QueryAccepted. - */ -public class QueryAccepted extends QueryEvent<String> { - - /** - * Instantiates a new query accepted. - * - * @param eventTime the event time - * @param prev the prev - * @param current the current - * @param handle the handle - */ - public QueryAccepted(long eventTime, String prev, String current, QueryHandle handle) { - super(eventTime, prev, current, handle); - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java deleted file mode 100644 index a473a47..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server.api.query; - -import org.apache.lens.api.query.QueryHandle; -import org.apache.lens.api.query.QueryStatus; - -/** - * Event fired when query is cancelled. - */ -public class QueryCancelled extends QueryEnded { - - /** - * Instantiates a new query cancelled. - * - * @param ctx the query context - * @param eventTime the event time - * @param prev the prev - * @param current the current - * @param handle the handle - * @param user the user - * @param cause the cause - */ - public QueryCancelled(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current, - QueryHandle handle, - String user, String cause) { - super(ctx, eventTime, prev, current, handle, user, cause); - checkCurrentState(QueryStatus.Status.CANCELED); - } - - public QueryCancelled(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) { - // TODO: correct username. put who cancelled it, not the submitter. Similar for others - this(ctx, ctx.getEndTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause); - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java deleted file mode 100644 index 3837087..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server.api.query; - -import org.apache.lens.api.query.QueryHandle; -import org.apache.lens.api.query.QueryStatus; - -/** - * Event fired when a query is closed. - */ -public class QueryClosed extends QueryEnded { - - /** - * Instantiates a new query closed. - * - * @param ctx the query context - * @param eventTime the event time - * @param prev the prev - * @param current the current - * @param handle the handle - * @param user the user - * @param cause the cause - */ - public QueryClosed(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current, - QueryHandle handle, - String user, String cause) { - super(ctx, eventTime, prev, current, handle, user, cause); - checkCurrentState(QueryStatus.Status.CLOSED); - } - - public QueryClosed(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) { - this(ctx, ctx.getClosedTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause); - } -}