Michael Kublin has uploaded a new change for review. Change subject: engine: Introducing a queue for failovers event ......................................................................
engine: Introducing a queue for failovers event The following patch will introduce a queue for failovers event. Idea is to use a follwoing queue inorder to ensure a correct order of the following operations: reconstruct, domain recovery, domain monitoring, etc... The appropriate event should be submitted to queue and will be executed accorrding to internal logic of queue. Because of execution of events is done in separate thread no deadlock should occurred, also no need for excplicit additional locking These is a first patch it will introduce a queue and all stuff should work as before, but using of queue with small change will allow: to reject unneeded events when some othe event is running, ensure order, ensure appropriate release of quartz threads, fix a races Change-Id: Ic1224feacdcdaaaf0b59d26105805ba7ef2a2fff Bug-Url: https://bugzilla.redhat.com/?????? Signed-off-by: Michael Kublin <[email protected]> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java A backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java A backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/Event.java A backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventQueue.java A backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventResult.java A backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java M backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/BeanType.java M backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/EngineEJBUtilsStrategy.java M backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/threadpool/ThreadPoolUtil.java D backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IIrsBroker.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IrsBrokerCommand.java 12 files changed, 437 insertions(+), 186 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/38/9838/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java index 0fe80e0..f099d6c 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsEventListener.java @@ -40,6 +40,8 @@ import org.ovirt.engine.core.common.config.Config; import org.ovirt.engine.core.common.config.ConfigValues; import org.ovirt.engine.core.common.errors.VdcBllErrors; +import org.ovirt.engine.core.common.eventqueue.EventResult; +import org.ovirt.engine.core.common.eventqueue.EventType; import org.ovirt.engine.core.common.vdscommands.SetVmTicketVDSCommandParameters; import org.ovirt.engine.core.common.vdscommands.StartSpiceVDSCommandParameters; import org.ovirt.engine.core.common.vdscommands.VDSCommandType; @@ -71,22 +73,24 @@ } @Override - public void storageDomainNotOperational(Guid storageDomainId, Guid storagePoolId) { + public EventResult storageDomainNotOperational(Guid storageDomainId, Guid storagePoolId) { StorageDomainPoolParametersBase parameters = new StorageDomainPoolParametersBase(storageDomainId, storagePoolId); parameters.setIsInternal(true); parameters.setInactive(true); - Backend.getInstance().runInternalAction(VdcActionType.DeactivateStorageDomain, + boolean isSucceeded = Backend.getInstance().runInternalAction(VdcActionType.DeactivateStorageDomain, parameters, - ExecutionHandler.createInternalJobContext()); + ExecutionHandler.createInternalJobContext()).getSucceeded(); + return new EventResult(isSucceeded, EventType.DOMAINNOTOPERATIONAL); } @Override - public void masterDomainNotOperational(Guid storageDomainId, Guid storagePoolId) { + public EventResult masterDomainNotOperational(Guid storageDomainId, Guid storagePoolId) { VdcActionParametersBase parameters = new ReconstructMasterParameters(storagePoolId, storageDomainId, true); - Backend.getInstance().runInternalAction(VdcActionType.ReconstructMasterDomain, + boolean isSucceeded = Backend.getInstance().runInternalAction(VdcActionType.ReconstructMasterDomain, parameters, - ExecutionHandler.createInternalJobContext()); + ExecutionHandler.createInternalJobContext()).getSucceeded(); + return new EventResult(isSucceeded, EventType.RECONSTRUCT); } @Override diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java new file mode 100644 index 0000000..c2181c0 --- /dev/null +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java @@ -0,0 +1,183 @@ +package org.ovirt.engine.core.bll.eventqueue; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.locks.ReentrantLock; + +import javax.ejb.DependsOn; +import javax.ejb.Local; +import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; + +import org.ovirt.engine.core.common.eventqueue.Event; +import org.ovirt.engine.core.common.eventqueue.EventQueue; +import org.ovirt.engine.core.common.eventqueue.EventResult; +import org.ovirt.engine.core.common.eventqueue.EventType; +import org.ovirt.engine.core.common.utils.Pair; +import org.ovirt.engine.core.compat.Guid; +import org.ovirt.engine.core.utils.log.Log; +import org.ovirt.engine.core.utils.log.LogFactory; +import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil; + +@Stateless(name = "EventQueue") +@TransactionAttribute(TransactionAttributeType.SUPPORTS) +@Local(EventQueue.class) +@DependsOn("Backend") +public class EventQueueMonitor implements EventQueue { + + private static Log log = LogFactory.getLog(EventQueueMonitor.class); + + private static final ConcurrentMap<Guid, ReentrantLock> poolsLockMap = new ConcurrentHashMap<Guid, ReentrantLock>(); + private static final Map<Guid, Queue<Pair<Event, FutureTask<EventResult>>>> poolsEventsMap = + new HashMap<Guid, Queue<Pair<Event, FutureTask<EventResult>>>>(); + private static final Map<Guid, Event> poolCurrentEventMap = new HashMap<Guid, Event>(); + + @Override + public void submitEventAsync(Event event, Callable<EventResult> callable) { + submitTaskInternal(event, callable); + } + + @Override + public void submitEventSync(Event event, Callable<EventResult> callable) { + FutureTask<EventResult> task = submitTaskInternal(event, callable); + if (task != null) { + try { + task.get(); + } catch (Exception e) { + log.errorFormat("Failed at submitEventSync, for pool {0} with exception {1}", + event.getStoragePoolId(), + e.getMessage()); + } + } + } + + private FutureTask<EventResult> submitTaskInternal(Event event, + Callable<EventResult> callable) { + FutureTask<EventResult> task = null; + Guid storagePoolId = event.getStoragePoolId(); + ReentrantLock lock = getPoolLock(storagePoolId); + lock.lock(); + try { + Event currentEvent = poolCurrentEventMap.get(storagePoolId); + if (currentEvent != null) { + switch (currentEvent.getEventType()) { + case RECONSTRUCT: + log.debugFormat("Current event was skiped because of reconstruct is running now for pool {0}, event {1}", + storagePoolId, + event); + break; + default: + task = new FutureTask<EventResult>(callable); + getEventQueue(storagePoolId).add(new Pair<Event, FutureTask<EventResult>>(event, task)); + break; + } + } else { + task = new FutureTask<EventResult>(callable); + getEventQueue(storagePoolId).add(new Pair<Event, FutureTask<EventResult>>(event, task)); + poolCurrentEventMap.put(storagePoolId, event); + ThreadPoolUtil.execute(new InternalEventQueuRunnable(storagePoolId, + lock, + poolsEventsMap, + poolCurrentEventMap)); + } + } finally { + lock.unlock(); + } + return task; + } + + private Queue<Pair<Event, FutureTask<EventResult>>> getEventQueue(Guid storagePoolId) { + Queue<Pair<Event, FutureTask<EventResult>>> queue = poolsEventsMap.get(storagePoolId); + if (queue == null) { + queue = new LinkedList<Pair<Event, FutureTask<EventResult>>>(); + poolsEventsMap.put(storagePoolId, queue); + } + return queue; + } + + private ReentrantLock getPoolLock(Guid poolId) { + if (!poolsLockMap.containsKey(poolId)) { + poolsLockMap.putIfAbsent(poolId, new ReentrantLock()); + } + return poolsLockMap.get(poolId); + } + + private static class InternalEventQueuRunnable implements Runnable { + + private Guid storagePoolId; + private ReentrantLock lock; + private Map<Guid, Event> poolCurrentEventMap; + private Map<Guid, Queue<Pair<Event, FutureTask<EventResult>>>> poolsEventsMap; + + public InternalEventQueuRunnable(Guid storagePoolId, + ReentrantLock lock, + Map<Guid, Queue<Pair<Event, FutureTask<EventResult>>>> poolsEventsMap, + Map<Guid, Event> poolCurrentEventMap) { + this.storagePoolId = storagePoolId; + this.lock = lock; + this.poolsEventsMap = poolsEventsMap; + this.poolCurrentEventMap = poolCurrentEventMap; + } + + @Override + public void run() { + while (true) { + Pair<Event, FutureTask<EventResult>> pair; + lock.lock(); + try { + pair = poolsEventsMap.get(storagePoolId).poll(); + if (pair != null) { + poolCurrentEventMap.put(storagePoolId, pair.getFirst()); + } else { + poolCurrentEventMap.remove(storagePoolId); + poolsEventsMap.remove(storagePoolId); + } + } finally { + lock.unlock(); + } + if (pair != null) { + Future<EventResult> futureResult = ThreadPoolUtil.execute(pair.getSecond()); + EventResult result; + try { + result = futureResult.get(); + if (result == null) { + result = pair.getSecond().get(); + if (result != null && result.getEventType() == EventType.RECONSTRUCT) { + log.infoFormat("Finished reconstruct for pool {0}. Clearing all event queue", + storagePoolId); + lock.lock(); + try { + for (Pair<Event, FutureTask<EventResult>> task : poolsEventsMap.get(storagePoolId)) { + log.infoFormat("The following task {0} was cancelled, because of recosntruct was run before", + task.getFirst()); + task.getSecond().cancel(true); + } + poolCurrentEventMap.remove(storagePoolId); + poolsEventsMap.remove(storagePoolId); + break; + } finally { + lock.unlock(); + } + } + } + } catch (Exception e) { + log.errorFormat("Exception during process of events for pool {0}, error is {1}", + storagePoolId, + e.getMessage()); + } + } else { + log.debugFormat("All task for event query were executed pool {0}", storagePoolId); + break; + } + } + } + } +} diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java index 4c19477..570aa35 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/IVdsEventListener.java @@ -3,6 +3,7 @@ import java.util.Map; import org.ovirt.engine.core.common.AuditLogType; import org.ovirt.engine.core.common.errors.VdcBllErrors; +import org.ovirt.engine.core.common.eventqueue.EventResult; import org.ovirt.engine.core.compat.Guid; import org.ovirt.engine.core.compat.TransactionScopeOption; @@ -18,9 +19,9 @@ void vdsMovedToMaintanance(Guid vdsId); // BLL - void storageDomainNotOperational(Guid storageDomainId, Guid storagePoolId); // BLL + EventResult storageDomainNotOperational(Guid storageDomainId, Guid storagePoolId); // BLL - void masterDomainNotOperational(Guid storageDomainId, Guid storagePoolId); // BLL + EventResult masterDomainNotOperational(Guid storageDomainId, Guid storagePoolId); // BLL /** * Temoporary patch. Vitaly todo: fix it diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/Event.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/Event.java new file mode 100644 index 0000000..01d7c4e --- /dev/null +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/Event.java @@ -0,0 +1,79 @@ +package org.ovirt.engine.core.common.eventqueue; + +import org.ovirt.engine.core.compat.Guid; + +public class Event { + + private Guid storagePoolId; + private Guid domainId; + private EventType eventType; + + public Event(Guid storagePoolId, Guid domainId, EventType eventType) { + this.storagePoolId = storagePoolId; + this.domainId = domainId; + this.eventType = eventType; + } + + public Guid getStoragePoolId() { + return storagePoolId; + } + + public void setStoragePoolId(Guid storagePoolId) { + this.storagePoolId = storagePoolId; + } + + public EventType getEventType() { + return eventType; + } + + public void setEventType(EventType eventType) { + this.eventType = eventType; + } + + public Guid getDomainId() { + return domainId; + } + + public void setDomainId(Guid domainId) { + this.domainId = domainId; + } + + @Override + public String toString() { + return "Event [storagePoolId=" + storagePoolId + ", domainId=" + domainId + ", eventType=" + eventType + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((domainId == null) ? 0 : domainId.hashCode()); + result = prime * result + ((eventType == null) ? 0 : eventType.hashCode()); + result = prime * result + ((storagePoolId == null) ? 0 : storagePoolId.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Event other = (Event) obj; + if (domainId == null) { + if (other.domainId != null) + return false; + } else if (!domainId.equals(other.domainId)) + return false; + if (eventType != other.eventType) + return false; + if (storagePoolId == null) { + if (other.storagePoolId != null) + return false; + } else if (!storagePoolId.equals(other.storagePoolId)) + return false; + return true; + } +} diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventQueue.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventQueue.java new file mode 100644 index 0000000..2b26adc --- /dev/null +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventQueue.java @@ -0,0 +1,24 @@ +package org.ovirt.engine.core.common.eventqueue; + +import java.util.concurrent.Callable; + +public interface EventQueue { + + /** + * The following method should allow to submit asynchronous event + * Event will be submitted to queue and will be executed when after that + * @param event - description of event + * @param callable - a code which should be run + */ + void submitEventAsync(Event event, Callable<EventResult> callable); + + /** + * The following method is used in order to submit synchronous event + * Event will submitted and a thread will be stuck until event will be + * executed or aborted + * @param event + * @param callable + */ + void submitEventSync(Event event, Callable<EventResult> callable); + +} diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventResult.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventResult.java new file mode 100644 index 0000000..bfb8b82 --- /dev/null +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventResult.java @@ -0,0 +1,29 @@ +package org.ovirt.engine.core.common.eventqueue; + +public class EventResult { + + private boolean success; + private EventType eventType; + + public EventResult(boolean success, EventType eventType) { + this.success = success; + this.eventType = eventType; + } + + public EventType getEventType() { + return eventType; + } + + public void setEventType(EventType eventType) { + this.eventType = eventType; + } + + public boolean isSuccess() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + +} diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java new file mode 100644 index 0000000..33af831 --- /dev/null +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java @@ -0,0 +1,8 @@ +package org.ovirt.engine.core.common.eventqueue; + +public enum EventType { + RECONSTRUCT, + DOMAINFAILOVER, + DOMAINNOTOPERATIONAL, + DOMAINMONITORING; +} diff --git a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/BeanType.java b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/BeanType.java index a92f739..8203e2d 100644 --- a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/BeanType.java +++ b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/BeanType.java @@ -10,6 +10,7 @@ SCHEDULER, // SchedulerUtil USERS_DOMAINS_CACHE, VDS_EVENT_LISTENER, - LOCK_MANAGER + LOCK_MANAGER, + EVENTQUEUE_MANAGER; } diff --git a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/EngineEJBUtilsStrategy.java b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/EngineEJBUtilsStrategy.java index 5cd0552..f4333ad 100644 --- a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/EngineEJBUtilsStrategy.java +++ b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/ejb/EngineEJBUtilsStrategy.java @@ -19,6 +19,7 @@ ENGINE_CONTEXT_PREFIX.concat("engine-bll/UsersDomainsCacheManagerService")); addBeanJNDIName(BeanType.VDS_EVENT_LISTENER, ENGINE_CONTEXT_PREFIX.concat("engine-bll/VdsEventListener")); addBeanJNDIName(BeanType.LOCK_MANAGER, ENGINE_CONTEXT_PREFIX.concat("engine-bll/LockManager")); + addBeanJNDIName(BeanType.EVENTQUEUE_MANAGER, ENGINE_CONTEXT_PREFIX.concat("engine-bll/EventQueue")); } @Override diff --git a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/threadpool/ThreadPoolUtil.java b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/threadpool/ThreadPoolUtil.java index 771dca4..4be7426 100644 --- a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/threadpool/ThreadPoolUtil.java +++ b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/threadpool/ThreadPoolUtil.java @@ -1,6 +1,8 @@ package org.ovirt.engine.core.utils.threadpool; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -79,4 +81,14 @@ throw e; } } + + @SuppressWarnings("unchecked") + public static <V> Future<V> execute(FutureTask<V> command) { + try { + return (Future<V>) es.submit(command); + } catch (RejectedExecutionException e) { + log.warn("The thread pool is out of limit. A submitted task was rejected"); + throw e; + } + } } diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IIrsBroker.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IIrsBroker.java deleted file mode 100644 index 66453cd..0000000 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IIrsBroker.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.ovirt.engine.core.vdsbroker.irsbroker; - -import org.ovirt.engine.core.compat.*; -import org.ovirt.engine.core.common.businessentities.*; - -public interface IIrsBroker { - - /** - * create a new Image - * - * @param imageName - * string Name - * @param imageSizeInBytes - * int number of bytes - * @param imageType - * int, for future use - * @return Guid <exception>VdcDAL.IrsBroker.IRSProtocolException <exception>VdcDAL.IrsBroker.IRSNetworkException - * <exception>VdcDAL.IrsBrokerIRSErrorException. <exception>System.Exception - */ - Guid create(String imageName, long imageSizeInBytes, int imageType); - - /** - * Lists all images on IRS - * - * @return Guid[] <exception>VdcDAL.IrsBroker.IRSProtocolException <exception>VdcDAL.IrsBroker.IRSNetworkException - * <exception>VdcDAL.IrsBrokerIRSErrorException. <exception>System.Exception - */ - Guid[] listImageIds(); - - /** - * Destroy an Image - * - * @param imageId - * Guid ImageId <exception>VdcDAL.IrsBroker.IRSProtocolException - * <exception>VdcDAL.IrsBroker.IRSNetworkException <exception>VdcDAL.IrsBrokerIRSErrorException. - * <exception>System.Exception - */ - void destroy(Guid imageId); - - /** - * Gets Information about the Image - * - * @param imageId - * Guid - * @return DislImage <exception>VdcDAL.IrsBroker.IRSProtocolException - * <exception>VdcDAL.IrsBroker.IRSNetworkException <exception>VdcDAL.IrsBrokerIRSErrorException. - * <exception>System.Exception - */ - DiskImage getImageInfo(Guid imageId); - - /** - * Copy an Image - * - * @param srcImageId - * @return Guid - imageId of the copied image <exception>VdcDAL.IrsBroker.IRSProtocolException - * <exception>VdcDAL.IrsBroker.IRSNetworkException <exception>VdcDAL.IrsBrokerIRSErrorException. - * <exception>System.Exception - */ - Guid copyImage(Guid srcImageId); - - /** - * Create a snapshot of an image - * - * @param srcImageId - * Guid - * @return Guid - the id of the new created snapshot <exception>VdcDAL.IrsBroker.IRSProtocolException - * <exception>VdcDAL.IrsBroker.IRSNetworkException <exception>VdcDAL.IrsBrokerIRSErrorException. - * <exception>System.Exception - */ - Guid createSnapshot(Guid srcImageId); - - /** - * Gets the parent image - * - * @param imageId - * Guid - the child image - * @return Guid - the image id of the parent image <exception>VdcDAL.IrsBroker.IRSProtocolException - * <exception>VdcDAL.IrsBroker.IRSNetworkException <exception>VdcDAL.IrsBrokerIRSErrorException. - * <exception>System.Exception - */ - Guid getParent(Guid imageId); - - /** - * Merge 2 snapshots into one - * - * @param imageId1 - * Guid - id of first mage - * @param imageId2 - * Guid - id of second image <exception>VdcDAL.IrsBroker.IRSProtocolException - * <exception>VdcDAL.IrsBroker.IRSNetworkException <exception>VdcDAL.IrsBrokerIRSErrorException. - * <exception>System.Exception - */ - void mergeSnapshots(Guid imageId1, Guid imageId2); - - /** - * Sets the image description - * - * @param imageId - * Guid Image id - * @param descr - * string description <exception>VdcDAL.IrsBroker.IRSProtocolException - * <exception>VdcDAL.IrsBroker.IRSNetworkException <exception>VdcDAL.IrsBrokerIRSErrorException. - * <exception>System.Exception - */ - void setImageDescr(Guid imageId, String descr); - -} diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IrsBrokerCommand.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IrsBrokerCommand.java index c0b2804..6ae4eb4 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IrsBrokerCommand.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/irsbroker/IrsBrokerCommand.java @@ -9,10 +9,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.commons.httpclient.HttpClient; import org.apache.commons.lang.exception.ExceptionUtils; import org.ovirt.engine.core.common.AuditLogType; @@ -38,6 +37,10 @@ import org.ovirt.engine.core.common.config.ConfigValues; import org.ovirt.engine.core.common.errors.VDSError; import org.ovirt.engine.core.common.errors.VdcBllErrors; +import org.ovirt.engine.core.common.eventqueue.Event; +import org.ovirt.engine.core.common.eventqueue.EventQueue; +import org.ovirt.engine.core.common.eventqueue.EventResult; +import org.ovirt.engine.core.common.eventqueue.EventType; import org.ovirt.engine.core.common.utils.Pair; import org.ovirt.engine.core.common.vdscommands.ConnectStoragePoolVDSCommandParameters; import org.ovirt.engine.core.common.vdscommands.DisconnectStoragePoolVDSCommandParameters; @@ -56,6 +59,9 @@ import org.ovirt.engine.core.dal.dbbroker.DbFacade; import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector; import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase; +import org.ovirt.engine.core.utils.ejb.BeanProxyType; +import org.ovirt.engine.core.utils.ejb.BeanType; +import org.ovirt.engine.core.utils.ejb.EjbUtils; import org.ovirt.engine.core.utils.log.Log; import org.ovirt.engine.core.utils.log.LogFactory; import org.ovirt.engine.core.utils.log.Logged; @@ -432,29 +438,32 @@ * @param logMessage * The log message to write in the log. */ - private void reconstructMasterDomainNotInSync(Guid storagePoolId, - Guid masterDomainId, - String exceptionMessage, - String logMessage) { - if (duringReconstructMaster.compareAndSet(false, true)) { - try { - log.warnFormat(logMessage); + private void reconstructMasterDomainNotInSync(final Guid storagePoolId, + final Guid masterDomainId, + final String exceptionMessage, + final String logMessage) { - AuditLogableBase logable = new AuditLogableBase(mCurrentVdsId); - logable.setStorageDomainId(masterDomainId); - AuditLogDirector.log(logable, AuditLogType.SYSTEM_MASTER_DOMAIN_NOT_IN_SYNC); + ((EventQueue) EjbUtils.findBean(BeanType.EVENTQUEUE_MANAGER, BeanProxyType.LOCAL)).submitEventSync(new Event(_storagePoolId, + masterDomainId, + EventType.RECONSTRUCT), + new Callable<EventResult>() { + @Override + public EventResult call() { + EventResult result = null; + log.warnFormat(logMessage); - ResourceManager.getInstance() - .getEventListener() - .masterDomainNotOperational(masterDomainId, storagePoolId); - } finally { - duringReconstructMaster.set(false); - } - throw new IRSNoMasterDomainException(exceptionMessage); - } else { - log.debug("The pool is during reconstruct master, skipping handling problematic domains " - + _storagePoolId); - } + AuditLogableBase logable = new AuditLogableBase(mCurrentVdsId); + logable.setStorageDomainId(masterDomainId); + AuditLogDirector.log(logable, AuditLogType.SYSTEM_MASTER_DOMAIN_NOT_IN_SYNC); + + result = ResourceManager.getInstance() + .getEventListener() + .masterDomainNotOperational(masterDomainId, storagePoolId); + return result; + + } + }); + throw new IRSNoMasterDomainException(exceptionMessage); } public java.util.HashSet<Guid> getTriedVdssList() { @@ -993,7 +1002,6 @@ private final Map<Guid, HashSet<Guid>> _vdssInProblem = new HashMap<Guid, HashSet<Guid>>(); private final Map<Guid, HashSet<Guid>> _domainsInProblem = new ConcurrentHashMap<Guid, HashSet<Guid>>(); private final Map<Guid, String> _timers = new HashMap<Guid, String>(); - private AtomicBoolean duringReconstructMaster = new AtomicBoolean(false); private final Object _lockObject = new Object(); public void UpdateVdsDomainsData(final Guid vdsId, final String vdsName, @@ -1072,18 +1080,24 @@ } } - if (domainsInProblems != null) { - synchronized (_lockObject) { - // during reconstruct master we do not want to update - // domains status - if (duringReconstructMaster.get()) { - log.debug("The pool is during reconstruct master, skipping handling problematic domains " - + _storagePoolId); - return; - } + updateDomainInProblem(vdsId, vdsName, domainsInProblems); + } - updateProblematicVdsData(vdsId, vdsName, domainsInProblems); - } + private void updateDomainInProblem(final Guid vdsId, final String vdsName, final Set<Guid> domainsInProblems) { + if (domainsInProblems != null) { + ((EventQueue) EjbUtils.findBean(BeanType.EVENTQUEUE_MANAGER, BeanProxyType.LOCAL)).submitEventSync(new Event(_storagePoolId, + null, + EventType.DOMAINMONITORING), + new Callable<EventResult>() { + @Override + public EventResult call() { + EventResult result = new EventResult(true, EventType.DOMAINMONITORING); + synchronized (_lockObject) { + updateProblematicVdsData(vdsId, vdsName, domainsInProblems); + } + return result; + } + }); } } @@ -1205,14 +1219,24 @@ } @OnTimerMethodAnnotation("OnTimer") - public void OnTimer(Guid domainId) { - synchronized (_lockObject) { - if (_domainsInProblem.containsKey(domainId)) { - log.info("starting ProcessDomainRecovery for domain " + getDomainIdTuple(domainId)); - ProcessDomainRecovery(domainId); - } - _timers.remove(domainId); - } + public void OnTimer(final Guid domainId) { + ((EventQueue) EjbUtils.findBean(BeanType.EVENTQUEUE_MANAGER, BeanProxyType.LOCAL)).submitEventAsync(new Event(_storagePoolId, + domainId, + EventType.DOMAINFAILOVER), + new Callable<EventResult>() { + @Override + public EventResult call() { + EventResult result = null; + synchronized (_lockObject) { + if (_domainsInProblem.containsKey(domainId)) { + log.info("starting ProcessDomainRecovery for domain " + getDomainIdTuple(domainId)); + result = ProcessDomainRecovery(domainId); + } + _timers.remove(domainId); + } + return result; + } + }); } private void UpdateDomainInProblemData(Guid domainId, Guid vdsId, String vdsName) { @@ -1228,7 +1252,8 @@ _vdssInProblem.get(vdsId).add(domainId); } - private void ProcessDomainRecovery(final Guid domainId) { + private EventResult ProcessDomainRecovery(final Guid domainId) { + EventResult result = null; // build a list of all the hosts in status UP in // Pool. List<Guid> vdssInPool = new ArrayList<Guid>(); @@ -1311,19 +1336,11 @@ if (storageDomain.getstorage_domain_type() != StorageDomainType.Master) { log.warnFormat("domain {0} was reported by all hosts in status UP as problematic. Moving the Domain to NonOperational.", domainIdTuple); - ResourceManager.getInstance() + result = ResourceManager.getInstance() .getEventListener().storageDomainNotOperational(domainId, _storagePoolId); - } else if (duringReconstructMaster.compareAndSet(false, true)) { - try { - ResourceManager.getInstance() - .getEventListener().masterDomainNotOperational(domainId, _storagePoolId); - } finally { - duringReconstructMaster.set(false); - } } else { - log.warnFormat("domain {0} was reported by all hosts in status UP as problematic. But not moving the Domain to NonOperational. Because of is reconstract now", - domainIdTuple); - return; + result = ResourceManager.getInstance() + .getEventListener().masterDomainNotOperational(domainId, _storagePoolId); } } @@ -1331,6 +1348,7 @@ // _domainsInProblem clearDomainFromCache(domainId); ClearTimer(domainId); + return result; } /** @@ -1374,14 +1392,12 @@ _timers.clear(); _domainsInProblem.clear(); _vdssInProblem.clear(); - duringReconstructMaster.set(false); } } public void clearPoolTimers() { synchronized (_lockObject) { log.info("clear domain error-timers for pool " + _storagePoolId); - duringReconstructMaster.set(true); for (String jobId : _timers.values()) { try { SchedulerUtilQuartzImpl.getInstance().deleteJob(jobId); @@ -1622,19 +1638,19 @@ } if (masterDomain != null) { - if (getCurrentIrsProxyData().duringReconstructMaster.compareAndSet(false, true)) { - try { - ResourceManager.getInstance() - .getEventListener() - .masterDomainNotOperational(masterDomain.getId(), - getParameters().getStoragePoolId()); - } finally { - getCurrentIrsProxyData().duringReconstructMaster.set(false); - } - } else { - log.debug("The pool is during reconstruct master, skipping handling problematic domains " - + getParameters().getStoragePoolId()); - } + final Guid masterDomainId = masterDomain.getId(); + ((EventQueue) EjbUtils.findBean(BeanType.EVENTQUEUE_MANAGER, BeanProxyType.LOCAL)).submitEventAsync(new Event(getParameters().getStoragePoolId(), + masterDomainId, + EventType.RECONSTRUCT), + new Callable<EventResult>() { + @Override + public EventResult call() { + return ResourceManager.getInstance() + .getEventListener() + .masterDomainNotOperational(masterDomainId, + getParameters().getStoragePoolId()); + } + }); } else { log.errorFormat( "IrsBroker::IRSNoMasterDomainException:: Could not find master domain for pool {0} !!!", -- To view, visit http://gerrit.ovirt.org/9838 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ic1224feacdcdaaaf0b59d26105805ba7ef2a2fff Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Michael Kublin <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
