Shireesh Anjal has uploaded a new change for review. Change subject: gluster: [WIP] Sync gluster service statuses ......................................................................
gluster: [WIP] Sync gluster service statuses Change-Id: I312cacf698cb3420e30d96c42a92959b257c4abd Signed-off-by: Shireesh Anjal <[email protected]> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJob.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJobsManager.java A backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterServiceSyncJob.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/gluster/GlusterServiceStatus.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/constants/gluster/GlusterConstants.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/gluster/GlusterFeatureSupported.java M backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/gluster/GlusterAuditLogUtil.java 9 files changed, 326 insertions(+), 1 deletion(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/44/14644/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJob.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJob.java index 8d4da77..8d242f2 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJob.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJob.java @@ -23,8 +23,11 @@ import org.ovirt.engine.core.dao.VdsStaticDAO; import org.ovirt.engine.core.dao.VdsStatisticsDAO; import org.ovirt.engine.core.dao.gluster.GlusterBrickDao; +import org.ovirt.engine.core.dao.gluster.GlusterClusterServiceDao; import org.ovirt.engine.core.dao.gluster.GlusterHooksDao; import org.ovirt.engine.core.dao.gluster.GlusterOptionDao; +import org.ovirt.engine.core.dao.gluster.GlusterServerServiceDao; +import org.ovirt.engine.core.dao.gluster.GlusterServiceDao; import org.ovirt.engine.core.dao.gluster.GlusterVolumeDao; import org.ovirt.engine.core.dao.network.InterfaceDao; import org.ovirt.engine.core.utils.lock.EngineLock; @@ -102,6 +105,19 @@ protected GlusterHooksDao getHooksDao() { return DbFacade.getInstance().getGlusterHooksDao(); } + + protected GlusterServiceDao getGlusterServiceDao() { + return DbFacade.getInstance().getGlusterServiceDao(); + } + + protected GlusterServerServiceDao getGlusterServerServiceDao() { + return DbFacade.getInstance().getGlusterServerServiceDao(); + } + + protected GlusterClusterServiceDao getGlusterClusterServiceDao() { + return DbFacade.getInstance().getGlusterClusterServiceDao(); + } + /** * Acquires a lock on the cluster with given id and locking group {@link LockingGroup#GLUSTER} * diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJobsManager.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJobsManager.java index b908a69..6feb2d7 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJobsManager.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterJobsManager.java @@ -48,6 +48,13 @@ getRefreshRate(ConfigValues.GlusterRefreshRateHooks), TimeUnit.SECONDS); + scheduler.scheduleAFixedDelayJob(GlusterSyncJob.getInstance(), + "refreshGlusterServices", + new Class[0], + new Object[0], + getRefreshRate(ConfigValues.GlusterRefreshRateLight), + getRefreshRate(ConfigValues.GlusterRefreshRateLight), + TimeUnit.SECONDS); } private static boolean glusterModeSupported() { diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterServiceSyncJob.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterServiceSyncJob.java new file mode 100644 index 0000000..da536bf --- /dev/null +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/gluster/GlusterServiceSyncJob.java @@ -0,0 +1,279 @@ +package org.ovirt.engine.core.bll.gluster; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.Callable; + +import org.ovirt.engine.core.common.AuditLogType; +import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.VDSGroup; +import org.ovirt.engine.core.common.businessentities.gluster.GlusterClusterService; +import org.ovirt.engine.core.common.businessentities.gluster.GlusterServerService; +import org.ovirt.engine.core.common.businessentities.gluster.GlusterService; +import org.ovirt.engine.core.common.businessentities.gluster.GlusterServiceStatus; +import org.ovirt.engine.core.common.businessentities.gluster.ServiceType; +import org.ovirt.engine.core.common.constants.gluster.GlusterConstants; +import org.ovirt.engine.core.common.gluster.GlusterFeatureSupported; +import org.ovirt.engine.core.common.vdscommands.VDSCommandType; +import org.ovirt.engine.core.common.vdscommands.VDSReturnValue; +import org.ovirt.engine.core.common.vdscommands.gluster.GlusterServicesListVDSParameters; +import org.ovirt.engine.core.compat.Guid; +import org.ovirt.engine.core.dal.dbbroker.DbFacade; +import org.ovirt.engine.core.utils.log.Log; +import org.ovirt.engine.core.utils.log.LogFactory; +import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil; +import org.ovirt.engine.core.utils.timer.OnTimerMethodAnnotation; + +public class GlusterServiceSyncJob extends GlusterJob { + private static final Log log = LogFactory.getLog(GlusterHookSyncJob.class); + private static final GlusterServiceSyncJob instance = new GlusterServiceSyncJob(); + private final Map<ServiceType, GlusterService> serviceTypeMap = new HashMap<ServiceType, GlusterService>(); + private final Map<String, GlusterService> serviceNameMap = new HashMap<String, GlusterService>(); + + private GlusterServiceSyncJob() { + populateServiceMaps(); + } + + public static GlusterServiceSyncJob getInstance() { + return instance; + } + + @OnTimerMethodAnnotation("refreshGlusterServices") + public void refreshGlusterServices() { + List<VDSGroup> clusters = getClusterDao().getAll(); + + for (VDSGroup cluster : clusters) { + if (supportsGlusterServicesFeature(cluster)) { + try { + log.debugFormat("Syncing gluster related services for cluster {0}", cluster.getname()); + refreshClusterServices(cluster, ThreadPoolUtil.invokeAll(createTaskList(cluster))); + } catch (Exception e) { + log.errorFormat("Error while refreshing service statuses of cluster {0}!", + cluster.getname(), + e); + } + } + } + } + + private List<Callable<Map<String, GlusterServiceStatus>>> createTaskList(VDSGroup cluster) { + List<VDS> upServers = getClusterUtils().getAllUpServers(cluster.getId()); + + List<Callable<Map<String, GlusterServiceStatus>>> taskList = + new ArrayList<Callable<Map<String, GlusterServiceStatus>>>(); + for (final VDS upServer : upServers) { + taskList.add(new Callable<Map<String, GlusterServiceStatus>>() { + /** + * Fetches and updates status of all services of the given server, <br> + * and returns a map having key = service name and value = service status + */ + @Override + public Map<String, GlusterServiceStatus> call() throws Exception { + return refreshServerServices(upServer); + } + }); + } + return taskList; + } + + /** + * Analyses statuses of services from all servers of the cluster, and updates the status of the cluster level + * service type accordingly. + * + * @param cluster + * Cluster being processed + * @param serviceStatusMaps + * List of service name to status maps from each (UP) server of the cluster + */ + private void refreshClusterServices(VDSGroup cluster, List<Map<String, GlusterServiceStatus>> serviceStatusMaps) { + Map<ServiceType, GlusterClusterService> clusterServiceMap = getClusterServiceMap(cluster); + for (Entry<String, GlusterServiceStatus> entry : mergeServiceStatusMaps(serviceStatusMaps).entrySet()) { + String serviceName = entry.getKey(); + GlusterServiceStatus status = entry.getValue(); + ServiceType type = serviceNameMap.get(serviceName).getServiceType(); + + /** + * TODO: This is not as simple as it seems. We've got server level statuses (RUNNING/STOPPED/MIXED). Now we + * need to identify cluster level status for the "service type". Following code will *not* work. + */ + GlusterClusterService clusterService = clusterServiceMap.get(type); + if (clusterService == null) { + mapServiceTypeToCluster(cluster, type, status); + } else { + if (clusterService.getStatus() != status) { + updateClusterServiceStatus(clusterService, status); + } + } + } + } + + private Map<String, GlusterServiceStatus> mergeServiceStatusMaps(List<Map<String, GlusterServiceStatus>> serviceStatusMaps) { + Map<String, GlusterServiceStatus> serviceStatusMap = new HashMap<String, GlusterServiceStatus>(); + for (Map<String, GlusterServiceStatus> pairResult : serviceStatusMaps) { + for (Entry<String, GlusterServiceStatus> entry : pairResult.entrySet()) { + String serviceName = entry.getKey(); + GlusterServiceStatus status = entry.getValue(); + GlusterServiceStatus alreadyFoundStatus = serviceStatusMap.get(serviceName); + if (alreadyFoundStatus == null) { + serviceStatusMap.put(serviceName, status); + } else if (alreadyFoundStatus != status && alreadyFoundStatus != GlusterServiceStatus.MIXED) { + serviceStatusMap.put(serviceName, GlusterServiceStatus.MIXED); + } + } + } + return serviceStatusMap; + } + + private Map<ServiceType, GlusterClusterService> getClusterServiceMap(VDSGroup cluster) { + List<GlusterClusterService> clusterServices = getGlusterClusterServiceDao().getByClusterId(cluster.getId()); + if (clusterServices == null) { + clusterServices = new ArrayList<GlusterClusterService>(); + } + + Map<ServiceType, GlusterClusterService> clusterServiceMap = new HashMap<ServiceType, GlusterClusterService>(); + for (GlusterClusterService clusterService : clusterServices) { + clusterServiceMap.put(clusterService.getServiceType(), clusterService); + } + return clusterServiceMap; + } + + /** + * Refreshes statuses of services on given server, and returns a map of service name to it's status + * + * @param server + * The server whose services statuses are to be refreshed + * @return map of service name to it's status + */ + @SuppressWarnings("unchecked") + private Map<String, GlusterServiceStatus> refreshServerServices(VDS server) { + Map<String, GlusterServiceStatus> serviceStatusMap = new HashMap<String, GlusterServiceStatus>(); + + acquireLock(server.getId()); + try { + + Map<Guid, GlusterServerService> existingServicesMap = getExistingServices(server); + List<GlusterServerService> servicesToUpdate = new ArrayList<GlusterServerService>(); + + VDSReturnValue returnValue = runVdsCommand(VDSCommandType.GlusterServicesList, + new GlusterServicesListVDSParameters(server.getId(), serviceNameMap.keySet())); + + if (!returnValue.getSucceeded()) { + log.errorFormat("Couldn't fetch services statuses from server {0}, error: {1}! " + + "Updating statuses of all services on this server as UNKNOWN.", + server.getHostName(), + returnValue.getVdsError().getMessage()); + return updateStatusToUnknown(existingServicesMap.values()); + } + + for (GlusterServerService fetchedService : (List<GlusterServerService>) returnValue.getReturnValue()) { + serviceStatusMap.put(fetchedService.getServiceName(), fetchedService.getStatus()); + GlusterServerService existingService = existingServicesMap.get(fetchedService.getId()); + if (existingService == null) { + insertServerService(server, fetchedService); + } else { + if (existingService.getStatus() != fetchedService.getStatus()) { + log.infoFormat("Status of service {0} on server {1} changed from {2} to {3}. Updating in engine now.", + fetchedService.getServiceName(), + server.getHostName(), + existingService.getStatus().name(), + fetchedService.getStatus().name()); + existingService.setStatus(fetchedService.getStatus()); + servicesToUpdate.add(existingService); + } + } + } + if (servicesToUpdate.size() > 0) { + getGlusterServerServiceDao().updateAll(servicesToUpdate); + } + + return serviceStatusMap; + } finally { + releaseLock(server.getId()); + } + } + + private Map<String, GlusterServiceStatus> updateStatusToUnknown(Collection<GlusterServerService> existingServices) { + Map<String, GlusterServiceStatus> serviceStatusMap = new HashMap<String, GlusterServiceStatus>(); + + List<GlusterServerService> servicesToUpdate = new ArrayList<GlusterServerService>(); + for (GlusterServerService existingService : existingServices) { + existingService.setStatus(GlusterServiceStatus.UNKNOWN); + servicesToUpdate.add(existingService); + serviceStatusMap.put(existingService.getServiceName(), existingService.getStatus()); + } + + getGlusterServerServiceDao().updateAll(servicesToUpdate); + return serviceStatusMap; + } + + private Map<Guid, GlusterServerService> getExistingServices(VDS server) { + List<GlusterServerService> existingServices = getGlusterServerServiceDao().getByServerId(server.getId()); + Map<Guid, GlusterServerService> existingServicesMap = new HashMap<Guid, GlusterServerService>(); + if (existingServices != null) { + for (GlusterServerService service : existingServices) { + existingServicesMap.put(service.getId(), service); + } + } + return existingServicesMap; + } + + private void insertServerService(VDS server, GlusterServerService fetchedService) { + log.infoFormat("Found new service {0} on server {1} that didn't exist in engine. Adding now.", + fetchedService.getServiceName(), + server.getHostName()); + fetchedService.setId(Guid.NewGuid()); + getGlusterServerServiceDao().save(fetchedService); + } + + @SuppressWarnings("serial") + private void updateClusterServiceStatus(final GlusterClusterService clusterService, + final GlusterServiceStatus newStatus) { + clusterService.setStatus(newStatus); + getGlusterClusterServiceDao().update(clusterService); + log.infoFormat("Updated status of Service type {0} on cluster {1} to {2}.", + clusterService.getServiceType(), + clusterService.getClusterId(), + newStatus); + logUtil.logAuditMessage(clusterService.getClusterId(), + null, + null, + AuditLogType.GLUSTER_CLUSTER_SERVICE_STATUS_CHANGED, + new HashMap<String, String>() { + { + put(GlusterConstants.OLD_STATUS, clusterService.getStatus().name()); + put(GlusterConstants.NEW_STATUS, newStatus.name()); + } + }); + } + + private void mapServiceTypeToCluster(VDSGroup cluster, ServiceType serviceType, GlusterServiceStatus status) { + GlusterClusterService clusterService = new GlusterClusterService(); + clusterService.setClusterId(Guid.NewGuid()); + clusterService.setServiceType(serviceType); + clusterService.setStatus(status); + + getGlusterClusterServiceDao().save(clusterService); + + log.infoFormat("Service type {0} not mapped to cluster {1}. Added it now.", + serviceType, + cluster.getname()); + logUtil.logClusterMessage(cluster.getId(), AuditLogType.GLUSTER_SERVICE_TYPE_ADDED_TO_CLUSTER); + } + + private boolean supportsGlusterServicesFeature(VDSGroup cluster) { + return cluster.supportsGlusterService() + && GlusterFeatureSupported.glusterServices(cluster.getcompatibility_version()); + } + + private void populateServiceMaps() { + List<GlusterService> services = DbFacade.getInstance().getGlusterServiceDao().getAll(); + for (GlusterService service : services) { + serviceNameMap.put(service.getServiceName(), service); + serviceTypeMap.put(service.getServiceType(), service); + } + } +} diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java index 1ebd08d..d9535f1 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java @@ -249,6 +249,9 @@ GLUSTER_HOOK_CONFLICT_DETECTED(4049), GLUSTER_HOOK_ADDED(4050), GLUSTER_HOOK_REMOVED(4051), + GLUSTER_SERVICES_LIST_FAILED(4052), + GLUSTER_SERVICE_TYPE_ADDED_TO_CLUSTER(4053), + GLUSTER_CLUSTER_SERVICE_STATUS_CHANGED(4054), USER_VDS_RESTART(41), USER_FAILED_VDS_RESTART(107), diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/gluster/GlusterServiceStatus.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/gluster/GlusterServiceStatus.java index 953484e..056824b 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/gluster/GlusterServiceStatus.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/gluster/GlusterServiceStatus.java @@ -10,5 +10,6 @@ ERROR, NOT_INSTALLED, MIXED, // cluster-wide status, few up, few down + UNKNOWN, // Couldn't fetch status ; } diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java index cd57c1c..eea0e6a 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java @@ -1315,6 +1315,10 @@ @DefaultValueAttribute("600") GlusterRefreshRateHooks(425), + @TypeConverterAttribute(Boolean.class) + @DefaultValueAttribute("true") + GlusterServicesEnabled(426), + @TypeConverterAttribute(String.class) @DefaultValueAttribute("Auto") ClientConsoleModeDefault(501), diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/constants/gluster/GlusterConstants.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/constants/gluster/GlusterConstants.java index 781f913..eb9762e 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/constants/gluster/GlusterConstants.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/constants/gluster/GlusterConstants.java @@ -22,6 +22,8 @@ public static final String OPTION_VALUE = "value"; public static final String OPTION_OLD_VALUE = "oldvalue"; public static final String OPTION_NEW_VALUE = "newvalue"; + public static final String OLD_STATUS = "oldstatus"; + public static final String NEW_STATUS = "newstatus"; public static final String HOOK_NAME = "glusterhookname"; public static final String FAILURE_MESSAGE = "failuremessage"; diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/gluster/GlusterFeatureSupported.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/gluster/GlusterFeatureSupported.java index 615ae61..ee1d6f2 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/gluster/GlusterFeatureSupported.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/gluster/GlusterFeatureSupported.java @@ -31,11 +31,20 @@ /** * * @param version - * Compatibility version to check for. + * Compatibility version to check for. * @return <code>true</code> if gluster hooks management feature is enabled, <code>false</code> if it's not. */ public static boolean glusterHooks(Version version) { return supportedInConfig(ConfigValues.GlusterHooksEnabled, version); } + /** + * + * @param version + * Compatibility version to check for. + * @return <code>true</code> if gluster services management feature is enabled, <code>false</code> if it's not. + */ + public static boolean glusterServices(Version version) { + return supportedInConfig(ConfigValues.GlusterServicesEnabled, version); + } } diff --git a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/gluster/GlusterAuditLogUtil.java b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/gluster/GlusterAuditLogUtil.java index b3e1578..f0eb63a 100644 --- a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/gluster/GlusterAuditLogUtil.java +++ b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/gluster/GlusterAuditLogUtil.java @@ -33,6 +33,10 @@ logAuditMessage(null, null, server, logType, Collections.<String, String> emptyMap()); } + public void logClusterMessage(final Guid clusterId, final AuditLogType logType) { + logAuditMessage(clusterId, null, null, logType, Collections.<String, String> emptyMap()); + } + public void logAuditMessage(final Guid clusterId, final GlusterVolumeEntity volume, final VDS server, -- To view, visit http://gerrit.ovirt.org/14644 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I312cacf698cb3420e30d96c42a92959b257c4abd Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Shireesh Anjal <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
