abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1953
Change subject: [ASTERIXDB-2049][ING] Fix hang in Start Feed
......................................................................
[ASTERIXDB-2049][ING] Fix hang in Start Feed
- user model changes: no
- storage format changes: no
- interface changes: no
details:
- The hang is caused by one runtime finishes and unregisters
before another runtime registers. When that happens, the number
of registered runtimes never reaches the total number of runtimes
and so the start feed statement doesn't complete.
- To avoid the situation described above, we use different counters
for registration and deregistration.
Change-Id: I0019f5634009bf924fb37acc78eb796842eef492
---
M
asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
A
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActionSubscriber.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
A
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/RuntimeRegistration.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestNodeControllerActor.java
M
hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
M
hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
9 files changed, 252 insertions(+), 22 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/53/1953/1
diff --git
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index c0717b9..eaaa85c 100644
---
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -153,7 +153,8 @@
ActiveRuntimeId runtimeId = (ActiveRuntimeId) message.getPayload();
IActiveRuntime runtime = runtimes.get(runtimeId);
if (runtime == null) {
- LOGGER.warning("Request to stop a runtime that is not registered "
+ runtimeId);
+ LOGGER.warning("Request to stop runtime: " + runtimeId
+ + " that is not registered. Could be that it finished
running locally ");
} else {
executor.execute(() -> {
try {
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index c6f41bf..79b1cd5 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -93,6 +93,7 @@
protected String stats;
protected boolean isFetchingStats;
protected int numRegistered;
+ protected int numDeRegistered;
protected volatile Future<Void> recoveryTask;
protected volatile boolean cancelRecovery;
protected volatile boolean suspended = false;
@@ -123,6 +124,7 @@
this.runtimeName = runtimeName;
this.locations = locations;
this.numRegistered = 0;
+ this.numDeRegistered = 0;
this.handler =
(ActiveNotificationHandler)
metadataProvider.getApplicationContext().getActiveNotificationHandler();
handler.registerListener(this);
@@ -177,13 +179,17 @@
setState(ActivityState.RUNNING);
}
} else if (message.getEvent() == Event.RUNTIME_DEREGISTERED) {
- numRegistered--;
+ numDeRegistered++;
}
}
@SuppressWarnings("unchecked")
protected void finish(ActiveEvent event) throws HyracksDataException {
LOGGER.log(level, "the job " + jobId + " finished");
+ if (numRegistered != numDeRegistered) {
+ LOGGER.log(Level.WARNING, "the job " + jobId + " finished with
registrations = " + numRegistered
+ + " and deregistrations = " + numDeRegistered);
+ }
jobId = null;
Pair<JobStatus, List<Exception>> status = (Pair<JobStatus,
List<Exception>>) event.getEventObject();
JobStatus jobStatus = status.getLeft();
@@ -202,8 +208,9 @@
}
protected void start(ActiveEvent event) {
- this.jobId = event.getJobId();
+ jobId = event.getJobId();
numRegistered = 0;
+ numDeRegistered = 0;
}
@Override
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActionSubscriber.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActionSubscriber.java
new file mode 100644
index 0000000..111f984
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActionSubscriber.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.active;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ActionSubscriber {
+ private final List<Action> actions = new ArrayList<>();
+ private boolean stop = false;
+
+ public synchronized void beforeSchedule(Action a) {
+ actions.add(a);
+ notifyAll();
+ }
+
+ public synchronized void beforeExecute() throws InterruptedException {
+ while (stop) {
+ wait();
+ }
+ }
+
+ synchronized void stop() {
+ stop = true;
+ }
+
+ synchronized void resume() {
+ stop = false;
+ notifyAll();
+ }
+
+ public List<Action> getActions() {
+ return actions;
+ }
+
+ public synchronized Action get(int i) throws InterruptedException {
+ while (actions.size() <= i) {
+ wait();
+ }
+ return actions.get(i);
+ }
+}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
index d38a363..c5465b1 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
@@ -159,6 +159,36 @@
}
@Test
+ public void testStartWhenOneNodeFinishesBeforeOtherNodeStarts() throws
Exception {
+ Assert.assertEquals(ActivityState.STOPPED, listener.getState());
+ listener.onStart(Behavior.SUCCEED);
+ listener.onStop(Behavior.SUCCEED);
+ ActionSubscriber fastSubscriber = new ActionSubscriber();
+ nodeControllers[0].subscribe(fastSubscriber);
+ ActionSubscriber slowSubscriber = new ActionSubscriber();
+ slowSubscriber.stop();
+ nodeControllers[1].subscribe(slowSubscriber);
+ Action startActivityAction = users[0].startActivity(listener);
+ RuntimeRegistration registration = (RuntimeRegistration)
fastSubscriber.get(0);
+ registration.sync();
+ registration.deregister();
+ Action deregistration = fastSubscriber.get(1);
+ deregistration.sync();
+ // Node 0 has completed registration and deregistration.. unblock node
1
+ slowSubscriber.resume();
+ registration = (RuntimeRegistration) slowSubscriber.get(0);
+ registration.sync();
+ // now that node 1 is unblocked and completed registration, ensure
that start has completed
+ startActivityAction.sync();
+ assertSuccess(startActivityAction);
+ Assert.assertEquals(ActivityState.RUNNING, listener.getState());
+ Action stopAction = users[0].stopActivity(listener);
+ stopAction.sync();
+ assertSuccess(stopAction);
+ Assert.assertEquals(ActivityState.STOPPED, listener.getState());
+ }
+
+ @Test
public void testStopWhenStopSucceed() throws Exception {
testStartWhenStartSucceed();
Assert.assertEquals(ActivityState.RUNNING, listener.getState());
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/RuntimeRegistration.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/RuntimeRegistration.java
new file mode 100644
index 0000000..0c4b806
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/RuntimeRegistration.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.active;
+
+import java.util.Objects;
+
+import org.apache.asterix.active.ActiveEvent;
+import org.apache.asterix.active.ActiveEvent.Kind;
+import org.apache.asterix.active.ActiveRuntimeId;
+import org.apache.asterix.active.EntityId;
+import org.apache.asterix.active.message.ActivePartitionMessage;
+import org.apache.asterix.active.message.ActivePartitionMessage.Event;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.hyracks.api.job.JobId;
+
+public class RuntimeRegistration extends Action {
+
+ private final TestNodeControllerActor nc;
+ private final JobId jobId;
+ private final EntityId entityId;
+ private final int partition;
+
+ public RuntimeRegistration(TestNodeControllerActor nc, JobId jobId,
EntityId entityId, int partition) {
+ this.nc = nc;
+ this.jobId = jobId;
+ this.entityId = entityId;
+ this.partition = partition;
+ }
+
+ @Override
+ protected void doExecute(MetadataProvider mdProvider) throws Exception {
+ for (ActionSubscriber subscriber : nc.getSubscribers()) {
+ subscriber.beforeExecute();
+ }
+ ActiveEvent event = new ActiveEvent(jobId, Kind.PARTITION_EVENT,
entityId, new ActivePartitionMessage(
+ new ActiveRuntimeId(entityId, nc.getId(), partition), jobId,
Event.RUNTIME_REGISTERED, null));
+ nc.getClusterController().activeEvent(event);
+ }
+
+ public Action deregister() {
+ return nc.doDeRegisterRuntime(jobId, entityId, partition);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(jobId, entityId, partition);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof RuntimeRegistration)) {
+ return false;
+ }
+ RuntimeRegistration o = (RuntimeRegistration) obj;
+ return Objects.equals(jobId, o.jobId) && Objects.equals(entityId,
o.entityId) && partition == o.partition;
+ }
+}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
index d896995..905df72 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
@@ -122,7 +122,7 @@
Collections.singletonList(new
HyracksDataException("RuntimeFailure")));
} else {
for (int i = 0; i < nodeControllers.length; i++) {
- TestNodeControllerActor nodeController = nodeControllers[0];
+ TestNodeControllerActor nodeController = nodeControllers[i];
nodeController.registerRuntime(jobId, entityId, i);
}
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestNodeControllerActor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestNodeControllerActor.java
index 99499a3..a40bf86 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestNodeControllerActor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestNodeControllerActor.java
@@ -18,6 +18,12 @@
*/
package org.apache.asterix.test.active;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+
import org.apache.asterix.active.ActiveEvent;
import org.apache.asterix.active.ActiveEvent.Kind;
import org.apache.asterix.active.ActiveRuntimeId;
@@ -28,9 +34,12 @@
import org.apache.hyracks.api.job.JobId;
public class TestNodeControllerActor extends Actor {
+ private static final Logger LOGGER =
Logger.getLogger(TestNodeControllerActor.class.getName());
private final String id;
private final TestClusterControllerActor clusterController;
+ private final Set<RuntimeRegistration> registrations = new HashSet<>();
+ private final List<ActionSubscriber> subscribers = new ArrayList<>();
public TestNodeControllerActor(String name, TestClusterControllerActor
clusterController) {
super("NC: " + name, null);
@@ -39,28 +48,76 @@
}
public Action registerRuntime(JobId jobId, EntityId entityId, int
partition) {
- Action registration = new Action() {
- @Override
- protected void doExecute(MetadataProvider actorMdProvider) throws
Exception {
- ActiveEvent event = new ActiveEvent(jobId,
Kind.PARTITION_EVENT, entityId, new ActivePartitionMessage(
- new ActiveRuntimeId(entityId, id, partition), jobId,
Event.RUNTIME_REGISTERED, null));
- clusterController.activeEvent(event);
- }
- };
+ RuntimeRegistration registration = new RuntimeRegistration(this,
jobId, entityId, partition);
+ for (ActionSubscriber subscriber : subscribers) {
+ subscriber.beforeSchedule(registration);
+ }
+ registrations.add(registration);
add(registration);
return registration;
}
public Action deRegisterRuntime(JobId jobId, EntityId entityId, int
partition) {
- Action registration = new Action() {
+ RuntimeRegistration registration = new RuntimeRegistration(this,
jobId, entityId, partition);
+ if (registrations.remove(registration)) {
+ return registration.deregister();
+ } else {
+ LOGGER.warning("Request to stop runtime: " + new
ActiveRuntimeId(entityId, "Test", partition)
+ + " that is not registered. Could be that it finished
running locally ");
+ return new Action() {
+ @Override
+ protected void doExecute(MetadataProvider mdProvider) throws
Exception {
+ }
+
+ @Override
+ public void sync() throws InterruptedException {
+ return;
+ }
+
+ @Override
+ public boolean isDone() {
+ return true;
+ }
+ };
+ }
+ }
+
+ public Action doDeRegisterRuntime(JobId jobId, EntityId entityId, int
partition) {
+ Action deregistration = new Action() {
@Override
protected void doExecute(MetadataProvider actorMdProvider) throws
Exception {
+ for (ActionSubscriber subscriber : subscribers) {
+ subscriber.beforeExecute();
+ }
ActiveEvent event = new ActiveEvent(jobId,
Kind.PARTITION_EVENT, entityId, new ActivePartitionMessage(
new ActiveRuntimeId(entityId, id, partition), jobId,
Event.RUNTIME_DEREGISTERED, null));
clusterController.activeEvent(event);
}
};
- add(registration);
- return registration;
+ for (ActionSubscriber subscriber : subscribers) {
+ subscriber.beforeSchedule(deregistration);
+ }
+ add(deregistration);
+ return deregistration;
+ }
+
+ public void subscribe(ActionSubscriber subscriber) {
+ subscribers.add(subscriber);
+ }
+
+ public void unsubscribe() {
+ subscribers.clear();
+ }
+
+ public List<ActionSubscriber> getSubscribers() {
+ return subscribers;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public TestClusterControllerActor getClusterController() {
+ return clusterController;
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index e190bfa..6ceafc6 100644
---
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -84,7 +84,8 @@
executor = new ThreadPoolExecutor(numExecutorThreads,
numExecutorThreads, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(requestQueueSize),
runnable -> new Thread(runnable, "HttpExecutor(port:" + port +
")-" + threadId.getAndIncrement()));
- long directMemoryBudget = numExecutorThreads * (long)
HIGH_WRITE_BUFFER_WATER_MARK;
+ long directMemoryBudget = numExecutorThreads * (long)
HIGH_WRITE_BUFFER_WATER_MARK
+ + numExecutorThreads *
HttpServerInitializer.RESPONSE_CHUNK_SIZE;
LOGGER.log(Level.INFO, "The direct memory budget for this server is "
+ directMemoryBudget + " bytes");
}
@@ -258,8 +259,8 @@
return b && (path.length() == cpl || '/' == path.charAt(cpl));
}
- protected HttpServerHandler createHttpHandler(int chunkSize) {
- return new HttpServerHandler(this, chunkSize);
+ protected HttpServerHandler<HttpServer> createHttpHandler(int chunkSize) {
+ return new HttpServerHandler<>(this, chunkSize);
}
public ExecutorService getExecutor() {
diff --git
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
index 4f8655f..a32da39 100644
---
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
+++
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
@@ -27,10 +27,10 @@
public class HttpServerInitializer extends ChannelInitializer<SocketChannel> {
- private static final int MAX_REQUEST_CHUNK_SIZE = 262144;
- private static final int MAX_REQUEST_HEADER_SIZE = 262144;
- private static final int MAX_REQUEST_INITIAL_LINE_LENGTH = 131072;
- private static final int RESPONSE_CHUNK_SIZE = 4096;
+ public static final int MAX_REQUEST_CHUNK_SIZE = 262144;
+ public static final int MAX_REQUEST_HEADER_SIZE = 262144;
+ public static final int MAX_REQUEST_INITIAL_LINE_LENGTH = 131072;
+ public static final int RESPONSE_CHUNK_SIZE = 4096;
private HttpServer server;
public HttpServerInitializer(HttpServer server) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/1953
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0019f5634009bf924fb37acc78eb796842eef492
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
