[2/4] lens git commit: LENS-743: Query retry framework for retrying upon transient failures
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
new file mode 100644
index 000..6169744
--- /dev/null
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+
+/**
+ * The Class StatusChange.
+ */
+public abstract class StatusChange extends QueryEvent {
+
+ /**
+ * Instantiates a new status change.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handlethe handle
+ */
+ public StatusChange(long eventTime, QueryStatus.Status prev,
QueryStatus.Status current, QueryHandle handle) {
+super(eventTime, prev, current, handle);
+ }
+
+ /**
+ * Check current state.
+ *
+ * @param status the status
+ */
+ protected void checkCurrentState(QueryStatus.Status status) {
+if (currentValue != status) {
+ throw new IllegalStateException("Invalid query state: " + currentValue +
" query:" + queryHandle.toString());
+}
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
new file mode 100644
index 000..5ea5710
--- /dev/null
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.retry;
+
+import java.io.Serializable;
+
+/**
+ * A backoff retry handler.
+ *
+ * This allows a backoff on any call, so provides methods whether we can try
the operation now,
+ * whats next time when operation can be performed and whether operation has
exhausted all retries.
+ * Callers of this can do the following :
+ *
+ * if (handler.canTryOpNow(FailureContext)) {
+ *try {
+ * tryCallerOperation();
+ * FailureContext.clear();
+ *} catch (any Transient Exception) {
+ * FailureContext.updateFailure();
+ * if (!handler.hasExhaustedRetries(FailureContext)) {
+ *// will be tried later again
+ * }
+ * throw exception;
+ *}
+ * }
+ *
+ * Note that this is only one of the possible use cases, other complex use
cases are in retry framework.
+ */
+public interface BackOffRetryHandler extends
Serializable {
+
+ /**
+ * To know whether operation can be done now.
+ *
+ * @param failContext FailureContext holding failures till now.
+ *
+ * @return true if operation can be done now, false otherwise.
+ */
+ boolean canTryOpNow(FC failContext);
+
+ /**
+ * Get the time when the operation can be done next.
+ *
+ * @param failContext FC
[1/4] lens git commit: LENS-743: Query retry framework for retrying upon transient failures
Repository: lens
Updated Branches:
refs/heads/master 182f6dcac -> 38ab6c608
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
index c683a2c..00130d0 100644
---
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
@@ -41,8 +41,8 @@ import org.apache.lens.server.api.LensErrorInfo;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.SchedulerAlarmEvent;
import org.apache.lens.server.api.health.HealthStatus;
-import org.apache.lens.server.api.query.QueryEnded;
import org.apache.lens.server.api.query.QueryExecutionService;
+import org.apache.lens.server.api.query.events.QueryEnded;
import org.apache.lens.server.api.scheduler.SchedulerService;
import org.apache.lens.server.error.LensSchedulerErrorCode;
import org.apache.lens.server.session.LensSessionImpl;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
b/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
index 46adb7b..20243f4 100644
---
a/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
+++
b/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
@@ -27,16 +27,26 @@ import java.util.Comparator;
import org.apache.lens.api.Priority;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.comparators.ChainedComparator;
+import org.apache.lens.server.api.query.comparators.FIFOQueryComparator;
+import org.apache.lens.server.api.query.comparators.QueryCostComparator;
+import org.apache.lens.server.api.query.comparators.QueryPriorityComparator;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+import com.google.common.collect.Lists;
+
public class QueryContextComparatorTest {
private final Comparator priorityComparator = new
QueryPriorityComparator();
private final Comparator costComparator = new
QueryCostComparator();
-
+ private final Comparator fifoComparator = new
FIFOQueryComparator();
+ private final Comparator priorityAndFifoComparator
+= new ChainedComparator<>(Lists.newArrayList(priorityComparator,
fifoComparator));
+ private final Comparator costAndFifoComparator
+= new ChainedComparator<>(Lists.newArrayList(costComparator,
fifoComparator));
@DataProvider
@@ -61,7 +71,7 @@ public class QueryContextComparatorTest {
when(query2.getSelectedDriverQueryCost()).thenReturn(qcO2);
when(qcO1.compareTo(qcO2)).thenReturn(resultOfQueryCostCompare);
-assertEquals(costComparator.compare(query1, query2), expectedResult);
+assertEquals(costAndFifoComparator.compare(query1, query2),
expectedResult);
}
@Test
@@ -73,7 +83,7 @@ public class QueryContextComparatorTest {
QueryContext query2 = mock(QueryContext.class);
when(query2.getPriority()).thenReturn(Priority.LOW); // Ordinal = 3
-assertEquals(priorityComparator.compare(query1, query2), -2);
+assertEquals(priorityAndFifoComparator.compare(query1, query2), -2);
}
@@ -114,8 +124,8 @@ public class QueryContextComparatorTest {
// Cost and Priority both are same, hence the comparison should happen
// on query submission time
-assertEquals(priorityComparator.compare(query1, query2), expectedResult);
-assertEquals(costComparator.compare(query1, query2), expectedResult);
+assertEquals(priorityAndFifoComparator.compare(query1, query2),
expectedResult);
+assertEquals(costAndFifoComparator.compare(query1, query2),
expectedResult);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
index 526accc..b906776 100644
---
a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
+++
b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
@@ -39,7 +39,7 @@ import org.apache.lens.server.api.events.AsyncEventListener;
[3/4] lens git commit: LENS-743: Query retry framework for retrying upon transient failures
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index d0662f4..63d3539 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -27,16 +27,17 @@ import java.util.*;
import java.util.concurrent.Future;
import org.apache.lens.api.LensConf;
+import org.apache.lens.api.query.FailedAttempt;
import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.query.QueryStatus.Status;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.common.BackOffRetryHandler;
-import org.apache.lens.server.api.common.FailureContext;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.retry.BackOffRetryHandler;
+import org.apache.lens.server.api.retry.FailureContext;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -53,7 +54,7 @@ import lombok.extern.slf4j.Slf4j;
* The Class QueryContext.
*/
@Slf4j
-public class QueryContext extends AbstractQueryContext {
+public class QueryContext extends AbstractQueryContext implements
FailureContext {
/**
* The Constant serialVersionUID.
@@ -191,7 +192,7 @@ public class QueryContext extends AbstractQueryContext {
@Setter
private byte[] queryConfHash;
- transient FailureContext statusUpdateFailures = new FailureContext();
+ transient StatusUpdateFailureContext statusUpdateFailures = new
StatusUpdateFailureContext();
@Getter
@Setter
@@ -200,7 +201,18 @@ public class QueryContext extends AbstractQueryContext {
@Getter
@Setter
private transient Future queryLauncher;
+
private final List
driverStatusUpdateListeners = Lists.newArrayList();
+ @Getter
+ @Setter
+ List failedAttempts = Lists.newArrayList();
+
+ @Getter
+ @Setter
+ private BackOffRetryHandler driverRetryPolicy;
+ @Getter
+ @Setter
+ private BackOffRetryHandler serverRetryPolicy;
/**
* Creates context from query
@@ -224,7 +236,7 @@ public class QueryContext extends AbstractQueryContext {
*/
public QueryContext(PreparedQueryContext prepared, String user, LensConf
qconf, Configuration conf) {
this(prepared.getUserQuery(), user, qconf, mergeConf(prepared.getConf(),
conf), prepared.getDriverContext()
-.getDriverQueryContextMap().keySet(),
prepared.getDriverContext().getSelectedDriver(), true);
+ .getDriverQueryContextMap().keySet(),
prepared.getDriverContext().getSelectedDriver(), true);
setDriverContext(prepared.getDriverContext());
setSelectedDriverQuery(prepared.getSelectedDriverQuery());
setSelectedDriverQueryCost(prepared.getSelectedDriverQueryCost());
@@ -241,7 +253,7 @@ public class QueryContext extends AbstractQueryContext {
* @param selectedDriver SelectedDriver
*/
QueryContext(String userQuery, String user, LensConf qconf, Configuration
conf, Collection drivers,
- LensDriver selectedDriver, boolean mergeDriverConf) {
+LensDriver selectedDriver, boolean mergeDriverConf) {
this(userQuery, user, qconf, conf, drivers, selectedDriver,
System.currentTimeMillis(), mergeDriverConf);
}
@@ -257,7 +269,7 @@ public class QueryContext extends AbstractQueryContext {
* @param submissionTime the submission time
*/
QueryContext(String userQuery, String user, LensConf qconf, Configuration
conf, Collection drivers,
- LensDriver selectedDriver, long submissionTime, boolean mergeDriverConf)
{
+LensDriver selectedDriver, long submissionTime, boolean mergeDriverConf) {
super(userQuery, user, qconf, conf, drivers, mergeDriverConf);
this.submissionTime = submissionTime;
this.queryHandle = new QueryHandle(UUID.randomUUID());
@@ -265,9 +277,9 @@ public class QueryContext extends AbstractQueryContext {
this.lensConf = qconf;
this.conf = conf;
this.isPersistent =
conf.getBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_SET,
-LensConfConstants.DEFAULT_PERSISTENT_RESULT_SET);
+ LensConfConstants.DEFAULT_PERSISTENT_RESULT_SET);
this.isDriverPersistent =
conf.getBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER,
-LensConfConstants.DEFAULT_DRIVER_PERSISTENT_RESULT_SET);
+ LensConfConstants.DEFAULT_DRIVER_PERSISTENT_RESULT_SET);
this.userQuery = userQuery;
if (selectedDriver != null) {
