Copilot commented on code in PR #13478: URL: https://github.com/apache/cloudstack/pull/13478#discussion_r4456055070
########## server/src/main/java/com/cloud/alert/ResourceAlertServiceImpl.java: ########## @@ -0,0 +1,641 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.alert; + +import org.apache.cloudstack.alert.ResourceAlertService; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.command.user.resourcealert.CreateResourceAlertRuleCmd; +import org.apache.cloudstack.api.command.user.resourcealert.DeleteResourceAlertRuleCmd; +import org.apache.cloudstack.api.command.user.resourcealert.ListResourceAlertRulesCmd; +import org.apache.cloudstack.api.command.user.resourcealert.ListResourceAlertsCmd; +import org.apache.cloudstack.api.command.user.resourcealert.UpdateResourceAlertRuleCmd; +import org.apache.cloudstack.api.response.ResourceAlertResponse; +import org.apache.cloudstack.api.response.ResourceAlertRuleResponse; +import com.cloud.alert.dao.ResourceAlertDao; +import com.cloud.alert.dao.ResourceAlertRuleDao; +import com.cloud.api.query.dao.ResourceAlertRuleJoinDao; +import com.cloud.alert.dao.ResourceAlertRuleWebhookDao; +import com.cloud.api.query.vo.ResourceAlertRuleJoinVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.webhook.WebhookHelper; +import org.apache.commons.lang3.EnumUtils; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.projects.Project; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.UserVmDao; + +import org.apache.cloudstack.context.CallContext; + +public class ResourceAlertServiceImpl extends ManagerBase implements ResourceAlertService { + + static final int OWNER_LOCK_WAIT_SECONDS = 30; + + @Inject + AccountManager accountManager; + @Inject + ResourceAlertRuleDao ruleDao; + @Inject + ResourceAlertRuleJoinDao ruleJoinDao; + @Inject + ResourceAlertDao alertDao; + @Inject + ResourceAlertRuleWebhookDao ruleWebhookDao; + @Inject + UserVmDao userVmDao; + @Inject + VolumeDao volumeDao; + @Inject + HostDao hostDao; + @Inject + PrimaryDataStoreDao storagePoolDao; + + @Override + @ActionEvent(eventType = EventTypes.EVENT_RESOURCE_ALERT_RULE_CREATE, eventDescription = "creating resource alert rule") + public ResourceAlertRuleResponse createResourceAlertRule(CreateResourceAlertRuleCmd cmd) { + ResourceAlertRule.ResourceType resourceType = parseResourceType(cmd.getResourceType()); + AlertCondition condition = parseCondition(cmd.getCondition()); + AlertSeverity severity = parseSeverity(cmd.getSeverity()); + ResourceAlertMetric metric = parseMetric(cmd.getMetric(), resourceType); + + validateThreshold(metric, cmd.getThreshold()); + validateResetInterval(cmd.getResetInterval()); + int resetInterval = cmd.getResetInterval() != null ? cmd.getResetInterval() : ResourceAlertManagerImpl.DEFAULT_RESET_INTERVAL.value(); + boolean email = cmd.getEmail() != null && cmd.getEmail(); + + Account caller = CallContext.current().getCallingAccount(); + checkInfrastructureAccess(caller, resourceType); + checkEmailAccess(caller, email); + Account owner = accountManager.finalizeOwner(caller, cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); + + long domainId = owner.getDomainId(); + + InternalIdentity resource = findResourceOrFail(resourceType, cmd.getResourceId()); + checkResourceReportsMetrics(resource); + if (resource instanceof ControlledEntity) { + accountManager.checkAccess(owner, null, false, (ControlledEntity) resource); + } + Long resourceId = resource != null ? resource.getId() : null; + List<Long> webhookIds = resolveWebhookIds(owner, cmd.getWebhookIds()); + + // Held across management servers so parallel creates can't go over the limit or reuse a name. + GlobalLock lock = getOwnerLock(owner.getId()); + if (!lock.lock(OWNER_LOCK_WAIT_SECONDS)) { + throw new CloudRuntimeException("Unable to create the resource alert rule as another rule is being created for the account, please try again"); + } + ResourceAlertRuleVO rule; + try { + int limit = ResourceAlertManagerImpl.RULES_PER_ACCOUNT_LIMIT.valueIn(owner.getId()); + if (limit >= 0 && ruleDao.countActiveByAccountId(owner.getId()) >= limit) { + throw new InvalidParameterValueException( + "Account has reached the maximum of " + limit + " resource alert rules"); + } + String name = validateName(owner.getId(), cmd.getName(), null); + + rule = new ResourceAlertRuleVO( + name, resourceType, resourceId, + owner.getId(), domainId, + metric.name(), condition, cmd.getThreshold(), severity, + cmd.getMessage(), email, resetInterval); + ruleDao.persist(rule); + if (!webhookIds.isEmpty()) { + ruleWebhookDao.replaceWebhooksForRule(rule.getId(), webhookIds); + } + } finally { + lock.unlock(); + lock.releaseRef(); + } + CallContext.current().setEventResourceId(rule.getId()); + CallContext.current().setEventDetails("Rule: " + rule.getName()); + return toRuleResponse(ruleJoinDao.findById(rule.getId())); + } + + @Override + public ListResponse<ResourceAlertRuleResponse> listResourceAlertRules(ListResourceAlertRulesCmd cmd) { + ResourceAlertRule.ResourceType resourceType = StringUtils.isNotBlank(cmd.getResourceType()) ? + parseResourceType(cmd.getResourceType()) : null; + Long resourceId = null; + if (StringUtils.isNotBlank(cmd.getResourceId())) { + Pair<ResourceAlertRule.ResourceType, Long> resource = findRuleResourceOrFail(resourceType, cmd.getResourceId()); + resourceType = resource.first(); + resourceId = resource.second(); + } + String metric = StringUtils.isNotBlank(cmd.getMetric()) ? parseMetricName(cmd.getMetric()) : null; + AlertSeverity severity = StringUtils.isNotBlank(cmd.getSeverity()) ? parseSeverity(cmd.getSeverity()) : null; + + Account caller = CallContext.current().getCallingAccount(); + List<Long> permittedAccounts = new ArrayList<>(); + Ternary<Long, Boolean, Project.ListProjectResourcesCriteria> domainIdRecursiveListProject = + new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); + accountManager.buildACLSearchParameters(caller, cmd.getId(), cmd.getAccountName(), cmd.getProjectId(), + permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); Review Comment: This API authorizes ResourceAdmin, but `AccountManagerImpl.buildACLSearchParameters()` only supplies the implicit domain restriction for `DOMAIN_ADMIN`, not `RESOURCE_DOMAIN_ADMIN`, when `listall=true`. A ResourceAdmin in a non-root private-zone domain can therefore list rules belonging to other domains and root admins; the resulting criteria only exclude project accounts. Explicitly apply that role's domain-tree scope rather than relying on the helper to populate it. This issue also appears on line 577 of the same file. ########## server/src/main/java/com/cloud/alert/ResourceAlertManagerImpl.java: ########## @@ -0,0 +1,736 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.alert; + +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import com.cloud.alert.dao.ResourceAlertDao; +import com.cloud.alert.dao.ResourceAlertRuleDao; +import com.cloud.alert.dao.ResourceAlertRuleWebhookDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.cloudstack.utils.mailing.MailAddress; +import org.apache.cloudstack.utils.mailing.SMTPMailProperties; +import org.apache.cloudstack.utils.mailing.SMTPMailSender; +import org.apache.cloudstack.webhook.WebhookHelper; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; + +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.event.AlertGenerator; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.host.Host; +import com.cloud.host.HostStats; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.resource.ResourceState; +import com.cloud.server.ResourceTag; +import com.cloud.server.StatsCollector; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageStats; +import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeStats; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.tags.dao.ResourceTagDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.GlobalLock; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmStats; +import com.cloud.vm.dao.UserVmDao; +import com.google.gson.JsonObject; + +public class ResourceAlertManagerImpl extends ManagerBase implements ResourceAlertManager, Configurable { + + static final String ALERT_EVENT_TYPE = "RESOURCE.ALERT"; + + static final ConfigKey<Integer> EVAL_INTERVAL = new ConfigKey<>("Advanced", Integer.class, + "resource.alert.evaluation.interval", "60", + "Interval in seconds between resource alert rule evaluations", false); + + public static final ConfigKey<Integer> RULES_PER_ACCOUNT_LIMIT = new ConfigKey<>("Advanced", Integer.class, + "resource.alert.per.user.limit", "20", + "Maximum number of resource alert rules an account can own; -1 = unlimited", true, ConfigKey.Scope.Account); + + static final ConfigKey<Integer> HISTORY_RETENTION_DAYS = new ConfigKey<>("Advanced", Integer.class, + "resource.alert.history.retention.days", "30", + "Number of days to keep fired resource alerts; -1 keeps them forever", true); + + public static final ConfigKey<Integer> DEFAULT_RESET_INTERVAL = new ConfigKey<>("Advanced", Integer.class, + "resource.alert.repeat.interval.default", "600", + "Default minimum seconds between repeat firings of a resource alert rule, used when a rule does not set one", true); + + @Inject ResourceAlertRuleDao ruleDao; + @Inject ResourceAlertDao alertDao; + @Inject ResourceAlertRuleWebhookDao ruleWebhookDao; + @Inject UserVmDao userVmDao; + @Inject HostDao hostDao; + @Inject PrimaryDataStoreDao storagePoolDao; + @Inject VolumeDao volumeDao; + @Inject StatsCollector statsCollector; + @Inject ConfigurationDao configDao; + @Inject ResourceTagDao resourceTagDao; + @Inject AccountDao accountDao; + @Inject AccountManager accountManager; + @Inject DomainDao domainDao; + @Inject ManagementServerHostDao managementServerHostDao; + + private ScheduledExecutorService executor; + static final int EMAIL_QUEUE_SIZE = 100; + static final long COOLDOWN_TOLERANCE_MILLIS = 5000L; + + // One sender and a bounded queue, so a slow mail server or an alert storm can't pile up threads. + ExecutorService emailExecutor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(EMAIL_QUEUE_SIZE), r -> { + Thread t = new Thread(r, "ResourceAlertEmailSender"); + t.setDaemon(true); + return t; + }, (r, e) -> logger.warn("Dropping resource alert email as {} emails are already waiting to be sent", EMAIL_QUEUE_SIZE)); + + private final Map<Long, VmStats> vmStatsCache = new HashMap<>(); + + private SMTPMailSender mailSender; + private String[] emailRecipients; + private String senderAddress; + + @Override + public boolean configure(String name, Map<String, Object> params) throws ConfigurationException { + String emailList = configDao.getValue("alert.email.addresses"); + if (StringUtils.isNotBlank(emailList)) { + emailRecipients = emailList.split(","); + } + senderAddress = configDao.getValue("alert.email.sender"); + + Map<String, String> smtpConfigs = new HashMap<>(); + for (String key : new String[]{ + "alert.smtp.host", "alert.smtp.port", "alert.smtp.useAuth", + "alert.smtp.username", "alert.smtp.password", "alert.smtp.useStartTLS", + "alert.smtp.enabledSecurityProtocols", "alert.smtp.timeout", "alert.smtp.connectiontimeout"}) { + String val = configDao.getValue(key); + if (val != null) smtpConfigs.put(key, val); + } + mailSender = new SMTPMailSender(smtpConfigs, "alert.smtp"); + + return super.configure(name, params); + } + + @Override + public boolean start() { + int interval = getEvaluationInterval(); + executor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "ResourceAlertEvaluator"); + t.setDaemon(true); + return t; + }); + executor.scheduleAtFixedRate(new EvaluationTask(), interval, interval, TimeUnit.SECONDS); + return true; + } + + Integer configuredEvaluationInterval() { + return EVAL_INTERVAL.value(); + } + + int getEvaluationInterval() { + Integer interval = configuredEvaluationInterval(); + if (interval == null || interval < 1) { + logger.warn("Invalid {} value {}, using {} seconds", EVAL_INTERVAL.key(), interval, EVAL_INTERVAL.defaultValue()); + return Integer.parseInt(EVAL_INTERVAL.defaultValue()); + } + return interval; + } + + @Override + public boolean stop() { + if (executor != null) { + executor.shutdown(); + } + emailExecutor.shutdown(); + return true; + } + + @Override + public void evaluateRules() { + vmStatsCache.clear(); + try { + List<ResourceAlertRuleVO> rules = ruleDao.listActive(); + for (ResourceAlertRuleVO rule : rules) { + // One broken rule must not stop the others from being checked. + try { + if (isOrphaned(rule)) { + logger.info("Removing resource alert rule {} as its owner or resource is gone, or the owner lost access to the resource", rule.getUuid()); + alertDao.removeByAlertRuleId(rule.getId()); + ruleDao.remove(rule.getId()); + continue; + } + if (ResourceAlertRule.State.Disabled.equals(rule.getState())) { + continue; + } + evaluateRule(rule); + } catch (Exception e) { + logger.warn("Failed to evaluate resource alert rule {}", rule.getUuid(), e); + } + } + } finally { + vmStatsCache.clear(); + } + } + + // Several rules can watch the same VM, so its stats are read once per run. + private VmStats getVmStats(long vmId) { + if (!vmStatsCache.containsKey(vmId)) { + vmStatsCache.put(vmId, statsCollector.getRecentVmStats(vmId)); + } + return vmStatsCache.get(vmId); + } + + boolean isOrphaned(ResourceAlertRuleVO rule) { + AccountVO owner = accountDao.findById(rule.getAccountId()); + if (owner == null) { + return true; + } + Long resourceId = rule.getResourceId(); + if (resourceId == null) { + return false; + } + switch (rule.getResourceType()) { + case VirtualMachine: + return !ownerCanAccess(owner, userVmDao.findById(resourceId)); + case Volume: + return !ownerCanAccess(owner, volumeDao.findById(resourceId)); + case Host: + return hostDao.findById(resourceId) == null; + case StoragePool: + return storagePoolDao.findById(resourceId) == null; + default: + return false; + } + } + + // A VM can move to another account or its owner can leave a project, so access is checked on every run. + private boolean ownerCanAccess(Account owner, ControlledEntity resource) { + if (resource == null) { + return false; + } + try { + accountManager.checkAccess(owner, null, false, resource); + return true; + } catch (PermissionDeniedException e) { + return false; + } + } + + // Stopped VMs and hosts or pools out of service report stale or zero stats. + boolean isInService(ResourceAlertRule.ResourceType type, long resourceId) { + switch (type) { + case VirtualMachine: { + UserVmVO vm = userVmDao.findById(resourceId); + return vm != null && VirtualMachine.State.Running.equals(vm.getState()); + } + case Volume: { + VolumeVO volume = volumeDao.findById(resourceId); + return volume != null && Volume.State.Ready.equals(volume.getState()); + } + case Host: { + HostVO host = hostDao.findById(resourceId); + return host != null && Status.Up.equals(host.getStatus()) && !ResourceState.isMaintenanceState(host.getResourceState()); + } + case StoragePool: { + StoragePoolVO pool = storagePoolDao.findById(resourceId); + return pool != null && StoragePoolStatus.Up.equals(pool.getStatus()); + } + default: + return false; + } + } + + // Every management server collects stats for all hosts, so only one may evaluate or alerts fire once per server. + boolean isEvaluatingServer() { + ManagementServerHostVO msHost = managementServerHostDao.findOneByLongestRuntime(); + return msHost != null && msHost.getMsid() == ManagementServerNode.getManagementServerId(); + } + + void removeExpiredAlerts() { + int days = HISTORY_RETENTION_DAYS.value(); + if (days < 0) { + return; + } + int removed = alertDao.removeOlderThan(new Date(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(days))); + if (removed > 0) { + logger.debug("Removed {} resource alerts older than {} days", removed, days); + } + } + + void removeStaleWebhookLinks() { + int removed = ruleWebhookDao.removeLinksToRemovedWebhooks(); + if (removed > 0) { + logger.debug("Removed {} resource alert rule links to deleted webhooks", removed); + } + } + + class EvaluationTask extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock lock = GlobalLock.getInternLock("ResourceAlertEvaluation"); + try { + if (!lock.lock(5)) { + return; + } + try { + if (isEvaluatingServer()) { + evaluateRules(); + removeExpiredAlerts(); + removeStaleWebhookLinks(); + } + } finally { + lock.unlock(); + } + } catch (Exception e) { + logger.warn("Failed to evaluate resource alert rules", e); + } finally { + lock.releaseRef(); + } + } + } + + private void evaluateRule(ResourceAlertRuleVO rule) { + ResourceAlertMetric metric = ResourceAlertMetric.valueOf(rule.getMetric()); + boolean isGeneric = rule.getResourceId() == null; + for (Long resourceId : getResourceIds(rule)) { + try { + if (checksServiceState(rule) && !isInService(rule.getResourceType(), resourceId)) continue; + if (isGeneric) { + if (isOptedOut(rule.getResourceType(), resourceId)) continue; + if (ruleDao.existsSpecificRule(rule.getResourceType(), rule.getMetric(), resourceId, rule.getAccountId())) continue; + } + Double value = getMetricValue(rule.getResourceType(), metric, resourceId); + if (value == null || value < 0) { + continue; + } + if (rule.getCondition().evaluate(value, rule.getThreshold()) + && canFire(rule.getId(), resourceId, rule.getResetInterval())) { + fireAlert(rule, resourceId, value); + } + } catch (Exception e) { + logger.warn("Failed to evaluate resource alert rule {} for resource {}", rule.getUuid(), resourceId, e); + } + } + } + + private boolean isOptedOut(ResourceAlertRule.ResourceType type, long resourceId) { + ResourceTag.ResourceObjectType objType = null; + if (type == ResourceAlertRule.ResourceType.VirtualMachine) { + objType = ResourceTag.ResourceObjectType.UserVm; + } else if (type == ResourceAlertRule.ResourceType.Volume) { + objType = ResourceTag.ResourceObjectType.Volume; + } + if (objType == null) return false; + ResourceTag tag = resourceTagDao.findByKey(resourceId, objType, "resource.alert.opt.out"); + return tag != null && "true".equalsIgnoreCase(tag.getValue()); + } + + // All-resources VM and volume rules already list only running VMs and ready volumes. + private boolean checksServiceState(ResourceAlertRuleVO rule) { + return rule.getResourceId() != null + || ResourceAlertRule.ResourceType.Host.equals(rule.getResourceType()) + || ResourceAlertRule.ResourceType.StoragePool.equals(rule.getResourceType()); + } + + private List<Long> getResourceIds(ResourceAlertRuleVO rule) { + if (rule.getResourceId() != null) { + return Collections.singletonList(rule.getResourceId()); + } + switch (rule.getResourceType()) { + case VirtualMachine: { + Pair<Long, List<Long>> scope = getGenericRuleScope(rule); + return scope == null ? Collections.emptyList() : userVmDao.listIdsByAccountOrDomainsAndState( + scope.first(), scope.second(), VirtualMachine.State.Running); + } + case Volume: { + Pair<Long, List<Long>> scope = getGenericRuleScope(rule); + return scope == null ? Collections.emptyList() : volumeDao.listUserVolumeIdsByAccountOrDomainsAndState( + scope.first(), scope.second(), Volume.State.Ready); + } + case Host: + return hostDao.listAll().stream() + .filter(h -> Host.Type.Routing.equals(h.getType())) + .map(h -> h.getId()) + .collect(Collectors.toList()); + case StoragePool: + return storagePoolDao.listAll().stream() + .map(p -> p.getId()) + .collect(Collectors.toList()); + default: + return Collections.emptyList(); + } + } + + // Root admin rules cover the whole cloud, domain admin rules their domain tree, other rules their own account. + Pair<Long, List<Long>> getGenericRuleScope(ResourceAlertRule rule) { + Account owner = accountDao.findById(rule.getAccountId()); + if (owner == null) { + return null; + } + if (Account.Type.ADMIN.equals(owner.getType())) { + return new Pair<>(null, null); + } + if (Account.Type.DOMAIN_ADMIN.equals(owner.getType()) || Account.Type.RESOURCE_DOMAIN_ADMIN.equals(owner.getType())) { + return new Pair<>(null, domainDao.getDomainAndChildrenIds(owner.getDomainId())); + } + return new Pair<>(owner.getId(), null); + } + + private Double getMetricValue(ResourceAlertRule.ResourceType type, ResourceAlertMetric metric, long resourceId) { + if (!metric.appliesTo(type)) { + return null; + } + switch (metric) { + case CPU_UTILIZATION: + if (type == ResourceAlertRule.ResourceType.VirtualMachine) { + VmStats s = getVmStats(resourceId); + return s != null ? s.getCPUUtilization() : null; + } + if (type == ResourceAlertRule.ResourceType.Host) { + HostStats s = statsCollector.getHostStats(resourceId); + return s != null ? s.getCpuUtilization() : null; + } + break; + case MEMORY_UTILIZATION: + if (type == ResourceAlertRule.ResourceType.VirtualMachine) { + VmStats s = getVmStats(resourceId); + if (s == null) return null; + double total = s.getMemoryKBs(); + double free = s.getIntFreeMemoryKBs(); + // free is -1 when VM has no balloon driver + if (total <= 0 || free < 0) return null; + return (1.0 - free / total) * 100.0; + } + if (type == ResourceAlertRule.ResourceType.Host) { + HostStats s = statsCollector.getHostStats(resourceId); + if (s == null) return null; + double total = s.getTotalMemoryKBs(); + double free = s.getFreeMemoryKBs(); + if (total <= 0) return null; + return ((total - free) / total) * 100.0; + } + break; + case DISK_READ_IOPS: { + VmStats s = getVmStats(resourceId); + return s != null ? s.getDiskReadIOs() : null; + } + case DISK_WRITE_IOPS: { + VmStats s = getVmStats(resourceId); + return s != null ? s.getDiskWriteIOs() : null; + } + case DISK_READ_KBPS: { + VmStats s = getVmStats(resourceId); + return s != null ? s.getDiskReadKBs() : null; + } + case DISK_WRITE_KBPS: { + VmStats s = getVmStats(resourceId); + return s != null ? s.getDiskWriteKBs() : null; + } + case NETWORK_READ_KBPS: { + if (type == ResourceAlertRule.ResourceType.Host) { + HostStats s = statsCollector.getHostStats(resourceId); + return s != null ? s.getNetworkReadKBs() : null; Review Comment: For KVM hosts, `LibvirtGetHostStatsCommandWrapper` fills this field from the bridge's cumulative `rx_bytes` counter, and `HostCollector` caches it unchanged. Comparing it directly with a KB/s threshold therefore measures traffic since interface creation: once the counter exceeds the threshold, the rule keeps firing even on an idle host. Calculate the received-byte delta divided by elapsed sample time before applying the threshold. ########## server/src/main/java/com/cloud/alert/ResourceAlertServiceImpl.java: ########## @@ -0,0 +1,641 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.alert; + +import org.apache.cloudstack.alert.ResourceAlertService; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.command.user.resourcealert.CreateResourceAlertRuleCmd; +import org.apache.cloudstack.api.command.user.resourcealert.DeleteResourceAlertRuleCmd; +import org.apache.cloudstack.api.command.user.resourcealert.ListResourceAlertRulesCmd; +import org.apache.cloudstack.api.command.user.resourcealert.ListResourceAlertsCmd; +import org.apache.cloudstack.api.command.user.resourcealert.UpdateResourceAlertRuleCmd; +import org.apache.cloudstack.api.response.ResourceAlertResponse; +import org.apache.cloudstack.api.response.ResourceAlertRuleResponse; +import com.cloud.alert.dao.ResourceAlertDao; +import com.cloud.alert.dao.ResourceAlertRuleDao; +import com.cloud.api.query.dao.ResourceAlertRuleJoinDao; +import com.cloud.alert.dao.ResourceAlertRuleWebhookDao; +import com.cloud.api.query.vo.ResourceAlertRuleJoinVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.webhook.WebhookHelper; +import org.apache.commons.lang3.EnumUtils; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.projects.Project; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.UserVmDao; + +import org.apache.cloudstack.context.CallContext; + +public class ResourceAlertServiceImpl extends ManagerBase implements ResourceAlertService { + + static final int OWNER_LOCK_WAIT_SECONDS = 30; + + @Inject + AccountManager accountManager; + @Inject + ResourceAlertRuleDao ruleDao; + @Inject + ResourceAlertRuleJoinDao ruleJoinDao; + @Inject + ResourceAlertDao alertDao; + @Inject + ResourceAlertRuleWebhookDao ruleWebhookDao; + @Inject + UserVmDao userVmDao; + @Inject + VolumeDao volumeDao; + @Inject + HostDao hostDao; + @Inject + PrimaryDataStoreDao storagePoolDao; + + @Override + @ActionEvent(eventType = EventTypes.EVENT_RESOURCE_ALERT_RULE_CREATE, eventDescription = "creating resource alert rule") + public ResourceAlertRuleResponse createResourceAlertRule(CreateResourceAlertRuleCmd cmd) { + ResourceAlertRule.ResourceType resourceType = parseResourceType(cmd.getResourceType()); + AlertCondition condition = parseCondition(cmd.getCondition()); + AlertSeverity severity = parseSeverity(cmd.getSeverity()); + ResourceAlertMetric metric = parseMetric(cmd.getMetric(), resourceType); + + validateThreshold(metric, cmd.getThreshold()); + validateResetInterval(cmd.getResetInterval()); + int resetInterval = cmd.getResetInterval() != null ? cmd.getResetInterval() : ResourceAlertManagerImpl.DEFAULT_RESET_INTERVAL.value(); + boolean email = cmd.getEmail() != null && cmd.getEmail(); + + Account caller = CallContext.current().getCallingAccount(); + checkInfrastructureAccess(caller, resourceType); + checkEmailAccess(caller, email); + Account owner = accountManager.finalizeOwner(caller, cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); + + long domainId = owner.getDomainId(); + + InternalIdentity resource = findResourceOrFail(resourceType, cmd.getResourceId()); + checkResourceReportsMetrics(resource); + if (resource instanceof ControlledEntity) { + accountManager.checkAccess(owner, null, false, (ControlledEntity) resource); + } + Long resourceId = resource != null ? resource.getId() : null; + List<Long> webhookIds = resolveWebhookIds(owner, cmd.getWebhookIds()); + + // Held across management servers so parallel creates can't go over the limit or reuse a name. + GlobalLock lock = getOwnerLock(owner.getId()); + if (!lock.lock(OWNER_LOCK_WAIT_SECONDS)) { + throw new CloudRuntimeException("Unable to create the resource alert rule as another rule is being created for the account, please try again"); + } + ResourceAlertRuleVO rule; + try { + int limit = ResourceAlertManagerImpl.RULES_PER_ACCOUNT_LIMIT.valueIn(owner.getId()); + if (limit >= 0 && ruleDao.countActiveByAccountId(owner.getId()) >= limit) { + throw new InvalidParameterValueException( + "Account has reached the maximum of " + limit + " resource alert rules"); + } + String name = validateName(owner.getId(), cmd.getName(), null); + + rule = new ResourceAlertRuleVO( + name, resourceType, resourceId, + owner.getId(), domainId, + metric.name(), condition, cmd.getThreshold(), severity, + cmd.getMessage(), email, resetInterval); + ruleDao.persist(rule); + if (!webhookIds.isEmpty()) { + ruleWebhookDao.replaceWebhooksForRule(rule.getId(), webhookIds); + } + } finally { + lock.unlock(); + lock.releaseRef(); + } + CallContext.current().setEventResourceId(rule.getId()); + CallContext.current().setEventDetails("Rule: " + rule.getName()); + return toRuleResponse(ruleJoinDao.findById(rule.getId())); + } + + @Override + public ListResponse<ResourceAlertRuleResponse> listResourceAlertRules(ListResourceAlertRulesCmd cmd) { + ResourceAlertRule.ResourceType resourceType = StringUtils.isNotBlank(cmd.getResourceType()) ? + parseResourceType(cmd.getResourceType()) : null; + Long resourceId = null; + if (StringUtils.isNotBlank(cmd.getResourceId())) { + Pair<ResourceAlertRule.ResourceType, Long> resource = findRuleResourceOrFail(resourceType, cmd.getResourceId()); + resourceType = resource.first(); + resourceId = resource.second(); + } + String metric = StringUtils.isNotBlank(cmd.getMetric()) ? parseMetricName(cmd.getMetric()) : null; + AlertSeverity severity = StringUtils.isNotBlank(cmd.getSeverity()) ? parseSeverity(cmd.getSeverity()) : null; + + Account caller = CallContext.current().getCallingAccount(); + List<Long> permittedAccounts = new ArrayList<>(); + Ternary<Long, Boolean, Project.ListProjectResourcesCriteria> domainIdRecursiveListProject = + new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); + accountManager.buildACLSearchParameters(caller, cmd.getId(), cmd.getAccountName(), cmd.getProjectId(), + permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); + SearchBuilder<ResourceAlertRuleJoinVO> sb = createAclSearchBuilder(domainIdRecursiveListProject, permittedAccounts); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("keyword", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("resourceType", sb.entity().getResourceType(), SearchCriteria.Op.EQ); + sb.and("resourceId", sb.entity().getResourceId(), SearchCriteria.Op.EQ); + sb.and("metric", sb.entity().getMetric(), SearchCriteria.Op.EQ); + sb.and("severity", sb.entity().getSeverity(), SearchCriteria.Op.EQ); + SearchCriteria<ResourceAlertRuleJoinVO> sc = createAclSearchCriteria(sb, domainIdRecursiveListProject, permittedAccounts); + if (cmd.getId() != null) sc.setParameters("id", cmd.getId()); + if (StringUtils.isNotBlank(cmd.getRuleName())) sc.setParameters("name", cmd.getRuleName()); + if (StringUtils.isNotBlank(cmd.getKeyword())) sc.setParameters("keyword", "%" + cmd.getKeyword() + "%"); + if (resourceType != null) sc.setParameters("resourceType", resourceType); + if (resourceId != null) sc.setParameters("resourceId", resourceId); + if (metric != null) sc.setParameters("metric", metric); + if (severity != null) sc.setParameters("severity", severity); + + Filter filter = new Filter(ResourceAlertRuleJoinVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + Pair<List<ResourceAlertRuleJoinVO>, Integer> rules = ruleJoinDao.searchAndCount(sc, filter); + + List<ResourceAlertRuleResponse> responses = rules.first().stream() + .map(this::toRuleResponse) + .collect(Collectors.toList()); + + ListResponse<ResourceAlertRuleResponse> response = new ListResponse<>(); + response.setResponses(responses, rules.second()); + return response; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_RESOURCE_ALERT_RULE_UPDATE, eventDescription = "updating resource alert rule") + public ResourceAlertRuleResponse updateResourceAlertRule(UpdateResourceAlertRuleCmd cmd) { + ResourceAlertRuleVO rule = findRuleForCaller(cmd.getId()); + checkEmailAccess(CallContext.current().getCallingAccount(), Boolean.TRUE.equals(cmd.getEmail())); + + if (cmd.getName() != null) rule.setName(validateName(rule.getAccountId(), cmd.getName(), rule.getId())); + if (StringUtils.isNotBlank(cmd.getCondition())) rule.setCondition(parseCondition(cmd.getCondition())); + if (cmd.getThreshold() != null) { + validateThreshold(ResourceAlertMetric.valueOf(rule.getMetric()), cmd.getThreshold()); + rule.setThreshold(cmd.getThreshold()); + } + if (StringUtils.isNotBlank(cmd.getSeverity())) rule.setSeverity(parseSeverity(cmd.getSeverity())); + if (cmd.getMessage() != null) rule.setMessage(cmd.getMessage()); + if (cmd.getEmail() != null) rule.setEmail(cmd.getEmail()); + if (cmd.getResetInterval() != null) { + validateResetInterval(cmd.getResetInterval()); + rule.setResetInterval(cmd.getResetInterval()); + } + if (StringUtils.isNotBlank(cmd.getState())) rule.setState(parseState(cmd.getState())); + rule.setUpdated(new Date()); + + if (cmd.isCleanupWebhooks()) { + ruleWebhookDao.replaceWebhooksForRule(rule.getId(), new ArrayList<>()); + } else if (cmd.getWebhookIds() != null) { + Account owner = accountManager.getAccount(rule.getAccountId()); + ruleWebhookDao.replaceWebhooksForRule(rule.getId(), resolveWebhookIds(owner, cmd.getWebhookIds())); + } + ruleDao.update(rule.getId(), rule); + return toRuleResponse(ruleJoinDao.findById(rule.getId())); + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_RESOURCE_ALERT_RULE_DELETE, eventDescription = "deleting resource alert rule") + public boolean deleteResourceAlertRule(DeleteResourceAlertRuleCmd cmd) { + findRuleForCaller(cmd.getId()); + alertDao.removeByAlertRuleId(cmd.getId()); + return ruleDao.remove(cmd.getId()); Review Comment: Deletion is not synchronized with evaluation. An evaluator can load the rule in `listActive()`, then this call deletes its history and soft-deletes the rule, after which the evaluator still inserts and delivers another alert from its cached rule. The foreign key permits that insertion because the rule row still exists, leaving history behind after a successful delete. Serialize deletion with evaluation and recheck the rule's active state under the same lock before persisting an alert. ########## server/src/main/java/com/cloud/alert/ResourceAlertServiceImpl.java: ########## @@ -0,0 +1,641 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.alert; + +import org.apache.cloudstack.alert.ResourceAlertService; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.command.user.resourcealert.CreateResourceAlertRuleCmd; +import org.apache.cloudstack.api.command.user.resourcealert.DeleteResourceAlertRuleCmd; +import org.apache.cloudstack.api.command.user.resourcealert.ListResourceAlertRulesCmd; +import org.apache.cloudstack.api.command.user.resourcealert.ListResourceAlertsCmd; +import org.apache.cloudstack.api.command.user.resourcealert.UpdateResourceAlertRuleCmd; +import org.apache.cloudstack.api.response.ResourceAlertResponse; +import org.apache.cloudstack.api.response.ResourceAlertRuleResponse; +import com.cloud.alert.dao.ResourceAlertDao; +import com.cloud.alert.dao.ResourceAlertRuleDao; +import com.cloud.api.query.dao.ResourceAlertRuleJoinDao; +import com.cloud.alert.dao.ResourceAlertRuleWebhookDao; +import com.cloud.api.query.vo.ResourceAlertRuleJoinVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.webhook.WebhookHelper; +import org.apache.commons.lang3.EnumUtils; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.projects.Project; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.UserVmDao; + +import org.apache.cloudstack.context.CallContext; + +public class ResourceAlertServiceImpl extends ManagerBase implements ResourceAlertService { + + static final int OWNER_LOCK_WAIT_SECONDS = 30; + + @Inject + AccountManager accountManager; + @Inject + ResourceAlertRuleDao ruleDao; + @Inject + ResourceAlertRuleJoinDao ruleJoinDao; + @Inject + ResourceAlertDao alertDao; + @Inject + ResourceAlertRuleWebhookDao ruleWebhookDao; + @Inject + UserVmDao userVmDao; + @Inject + VolumeDao volumeDao; + @Inject + HostDao hostDao; + @Inject + PrimaryDataStoreDao storagePoolDao; + + @Override + @ActionEvent(eventType = EventTypes.EVENT_RESOURCE_ALERT_RULE_CREATE, eventDescription = "creating resource alert rule") + public ResourceAlertRuleResponse createResourceAlertRule(CreateResourceAlertRuleCmd cmd) { + ResourceAlertRule.ResourceType resourceType = parseResourceType(cmd.getResourceType()); + AlertCondition condition = parseCondition(cmd.getCondition()); + AlertSeverity severity = parseSeverity(cmd.getSeverity()); + ResourceAlertMetric metric = parseMetric(cmd.getMetric(), resourceType); + + validateThreshold(metric, cmd.getThreshold()); + validateResetInterval(cmd.getResetInterval()); + int resetInterval = cmd.getResetInterval() != null ? cmd.getResetInterval() : ResourceAlertManagerImpl.DEFAULT_RESET_INTERVAL.value(); + boolean email = cmd.getEmail() != null && cmd.getEmail(); + + Account caller = CallContext.current().getCallingAccount(); + checkInfrastructureAccess(caller, resourceType); + checkEmailAccess(caller, email); + Account owner = accountManager.finalizeOwner(caller, cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); + + long domainId = owner.getDomainId(); + + InternalIdentity resource = findResourceOrFail(resourceType, cmd.getResourceId()); + checkResourceReportsMetrics(resource); + if (resource instanceof ControlledEntity) { + accountManager.checkAccess(owner, null, false, (ControlledEntity) resource); + } + Long resourceId = resource != null ? resource.getId() : null; + List<Long> webhookIds = resolveWebhookIds(owner, cmd.getWebhookIds()); + + // Held across management servers so parallel creates can't go over the limit or reuse a name. + GlobalLock lock = getOwnerLock(owner.getId()); + if (!lock.lock(OWNER_LOCK_WAIT_SECONDS)) { + throw new CloudRuntimeException("Unable to create the resource alert rule as another rule is being created for the account, please try again"); + } + ResourceAlertRuleVO rule; + try { + int limit = ResourceAlertManagerImpl.RULES_PER_ACCOUNT_LIMIT.valueIn(owner.getId()); + if (limit >= 0 && ruleDao.countActiveByAccountId(owner.getId()) >= limit) { + throw new InvalidParameterValueException( + "Account has reached the maximum of " + limit + " resource alert rules"); + } + String name = validateName(owner.getId(), cmd.getName(), null); + + rule = new ResourceAlertRuleVO( + name, resourceType, resourceId, + owner.getId(), domainId, + metric.name(), condition, cmd.getThreshold(), severity, + cmd.getMessage(), email, resetInterval); + ruleDao.persist(rule); + if (!webhookIds.isEmpty()) { + ruleWebhookDao.replaceWebhooksForRule(rule.getId(), webhookIds); + } + } finally { + lock.unlock(); + lock.releaseRef(); + } + CallContext.current().setEventResourceId(rule.getId()); + CallContext.current().setEventDetails("Rule: " + rule.getName()); + return toRuleResponse(ruleJoinDao.findById(rule.getId())); + } + + @Override + public ListResponse<ResourceAlertRuleResponse> listResourceAlertRules(ListResourceAlertRulesCmd cmd) { + ResourceAlertRule.ResourceType resourceType = StringUtils.isNotBlank(cmd.getResourceType()) ? + parseResourceType(cmd.getResourceType()) : null; + Long resourceId = null; + if (StringUtils.isNotBlank(cmd.getResourceId())) { + Pair<ResourceAlertRule.ResourceType, Long> resource = findRuleResourceOrFail(resourceType, cmd.getResourceId()); + resourceType = resource.first(); + resourceId = resource.second(); + } + String metric = StringUtils.isNotBlank(cmd.getMetric()) ? parseMetricName(cmd.getMetric()) : null; + AlertSeverity severity = StringUtils.isNotBlank(cmd.getSeverity()) ? parseSeverity(cmd.getSeverity()) : null; + + Account caller = CallContext.current().getCallingAccount(); + List<Long> permittedAccounts = new ArrayList<>(); + Ternary<Long, Boolean, Project.ListProjectResourcesCriteria> domainIdRecursiveListProject = + new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); + accountManager.buildACLSearchParameters(caller, cmd.getId(), cmd.getAccountName(), cmd.getProjectId(), + permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); + SearchBuilder<ResourceAlertRuleJoinVO> sb = createAclSearchBuilder(domainIdRecursiveListProject, permittedAccounts); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("keyword", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("resourceType", sb.entity().getResourceType(), SearchCriteria.Op.EQ); + sb.and("resourceId", sb.entity().getResourceId(), SearchCriteria.Op.EQ); + sb.and("metric", sb.entity().getMetric(), SearchCriteria.Op.EQ); + sb.and("severity", sb.entity().getSeverity(), SearchCriteria.Op.EQ); + SearchCriteria<ResourceAlertRuleJoinVO> sc = createAclSearchCriteria(sb, domainIdRecursiveListProject, permittedAccounts); + if (cmd.getId() != null) sc.setParameters("id", cmd.getId()); + if (StringUtils.isNotBlank(cmd.getRuleName())) sc.setParameters("name", cmd.getRuleName()); + if (StringUtils.isNotBlank(cmd.getKeyword())) sc.setParameters("keyword", "%" + cmd.getKeyword() + "%"); + if (resourceType != null) sc.setParameters("resourceType", resourceType); + if (resourceId != null) sc.setParameters("resourceId", resourceId); + if (metric != null) sc.setParameters("metric", metric); + if (severity != null) sc.setParameters("severity", severity); + + Filter filter = new Filter(ResourceAlertRuleJoinVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + Pair<List<ResourceAlertRuleJoinVO>, Integer> rules = ruleJoinDao.searchAndCount(sc, filter); + + List<ResourceAlertRuleResponse> responses = rules.first().stream() + .map(this::toRuleResponse) + .collect(Collectors.toList()); + + ListResponse<ResourceAlertRuleResponse> response = new ListResponse<>(); + response.setResponses(responses, rules.second()); + return response; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_RESOURCE_ALERT_RULE_UPDATE, eventDescription = "updating resource alert rule") + public ResourceAlertRuleResponse updateResourceAlertRule(UpdateResourceAlertRuleCmd cmd) { + ResourceAlertRuleVO rule = findRuleForCaller(cmd.getId()); + checkEmailAccess(CallContext.current().getCallingAccount(), Boolean.TRUE.equals(cmd.getEmail())); + + if (cmd.getName() != null) rule.setName(validateName(rule.getAccountId(), cmd.getName(), rule.getId())); Review Comment: Renaming does not acquire the owner lock used by creation. Two updates, or a rename concurrent with a create, can both pass `validateName()` before either persists and leave two active rules with the same owner/name; the schema has no unique constraint to prevent this. Hold the same owner lock across name validation and the corresponding update so the uniqueness check also holds for renames. ########## server/src/main/java/com/cloud/server/StatsCollector.java: ########## @@ -1327,6 +1327,24 @@ protected void runInContext() { } } + /** + * Gets the newest stats collected from a given VM, ignoring stats older than three collection intervals. + * + * @param vmId the specific VM. + * @return the newest stats, or null if there are none or they are too old. + */ + public VmStats getRecentVmStats(long vmId) { + VmStatsVO latest = vmStatsDao.findLatestByVmId(vmId); + if (latest == null) { + return null; + } + long maxAge = 3 * Math.max(vmStatsInterval, ONE_MINUTE_IN_MILLISCONDS); + if (System.currentTimeMillis() - latest.getTimestamp().getTime() > maxAge) { + return null; + } + return gson.fromJson(latest.getVmStatsData(), VmStatsEntry.class); Review Comment: The new evaluator treats the disk/network fields returned here as IOPS and KB/s, but KVM's `calculateVmMetrics()` fills them with totals since the previous sample, and `persistVirtualMachineStats()` stores them unchanged. For example, 100 IOPS over a 60-second collection interval is evaluated as 6000 IOPS, producing false threshold alerts. Normalize these fields using the actual sampling duration before evaluating rate rules, accounting for hypervisors such as VMware that already return rates. ########## ui/src/views/resourcealert/CreateResourceAlertRule.vue: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +<template> + <a-spin :spinning="loading"> + <a-form + class="form" + layout="vertical" + ref="formRef" + :model="form" + :rules="rules" + @finish="handleSubmit" + v-ctrl-enter="handleSubmit"> + + <a-form-item name="name" ref="name"> + <template #label>{{ $t('label.name') }}</template> + <a-input v-focus="true" v-model:value="form.name" /> + </a-form-item> + + <a-form-item name="domainid" ref="domainid" v-if="canPickOwner"> + <template #label>{{ $t('label.domain') }}</template> + <a-select + v-model:value="form.domainid" + allowClear + showSearch + optionFilterProp="label" + :loading="domainsLoading" + :placeholder="$t('label.resource.alert.owner.self')" + @change="onDomainChange"> + <a-select-option v-for="d in domains" :key="d.id" :value="d.id" :label="d.path || d.name">{{ d.path || d.name }}</a-select-option> + </a-select> + </a-form-item> + + <a-form-item name="account" ref="account" v-if="canPickOwner && form.domainid"> + <template #label>{{ $t('label.account') }}</template> + <a-select + v-model:value="form.account" + showSearch + optionFilterProp="label" + :loading="accountsLoading" + @change="onOwnerChange"> + <a-select-option v-for="a in accounts" :key="a.id" :value="a.name" :label="a.name">{{ a.name }}</a-select-option> + </a-select> + </a-form-item> + + <a-form-item name="resourcetype" ref="resourcetype"> + <template #label>{{ $t('label.resourcetype') }}</template> + <a-select v-model:value="form.resourcetype" @change="onResourceTypeChange"> + <a-select-option v-for="rt in resourceTypes" :key="rt" :value="rt">{{ resourceTypeLabels[rt] || rt }}</a-select-option> + </a-select> + </a-form-item> + + <a-form-item name="resourceid" ref="resourceid"> + <template #label>{{ $t('label.resource') }}</template> + <a-select + v-model:value="form.resourceid" + :disabled="!form.resourcetype" + showSearch + optionFilterProp="label" + :loading="resourcesLoading"> + <a-select-option value="" :label="$t('label.resource.alert.all.resources')">{{ $t('label.resource.alert.all.resources') }}</a-select-option> + <a-select-option v-for="r in resources" :key="r.id" :value="r.id" :label="r.displayname || r.name">{{ r.displayname || r.name }}</a-select-option> + </a-select> + </a-form-item> + + <a-form-item name="metric" ref="metric"> + <template #label>{{ $t('label.metric') }}</template> + <a-select v-model:value="form.metric" :disabled="!form.resourcetype"> + <a-select-option v-for="m in availableMetrics" :key="m" :value="m">{{ metricLabels[m] || m }}</a-select-option> + </a-select> + </a-form-item> + + <a-form-item name="condition" ref="condition"> + <template #label>{{ $t('label.condition') }}</template> + <a-select v-model:value="form.condition"> + <a-select-option v-for="c in conditions" :key="c" :value="c">{{ conditionLabels[c] || c }}</a-select-option> + </a-select> + </a-form-item> + + <a-form-item name="threshold" ref="threshold"> + <template #label>{{ $t('label.threshold') }}</template> + <a-input-number v-model:value="form.threshold" :min="0" style="width: 100%" /> + </a-form-item> + + <a-form-item name="severity" ref="severity"> + <template #label>{{ $t('label.severity') }}</template> + <a-select v-model:value="form.severity"> + <a-select-option v-for="s in severities" :key="s" :value="s">{{ severityLabels[s] || s }}</a-select-option> + </a-select> + </a-form-item> + + <a-form-item name="message" ref="message"> + <template #label>{{ $t('label.message') }}</template> + <a-input v-model:value="form.message" /> + </a-form-item> + + <a-form-item name="email" ref="email" v-if="isRootAdmin"> + <template #label>{{ $t('label.email') }}</template> + <a-switch v-model:checked="form.email" /> + </a-form-item> + + <a-form-item name="resetinterval" ref="resetinterval"> + <template #label>{{ $t('label.resetinterval') }}</template> + <a-input-number v-model:value="form.resetinterval" :min="0" :placeholder="$t('label.resource.alert.cooldown.default')" style="width: 100%" /> + </a-form-item> + + <a-form-item name="webhookids" ref="webhookids" v-if="'listWebhooks' in $store.getters.apis"> + <template #label>{{ $t('label.webhooks') }}</template> + <a-select + v-model:value="form.webhookids" + mode="multiple" + :loading="webhooksLoading" + :disabled="!webhooksLoading && webhooks.length === 0 && form.webhookids.length === 0" + :placeholder="!webhooksLoading && webhooks.length === 0 ? $t('label.resource.alert.no.webhooks') : undefined" + optionFilterProp="label" + :filterOption="(input, option) => option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0"> + <a-select-option v-for="wh in webhooks" :key="wh.id" :value="wh.id" :label="wh.name">{{ wh.name }}</a-select-option> + </a-select> + </a-form-item> + + <div :span="24" class="action-button"> + <a-button @click="() => { this.$emit('close-action') }">{{ $t('label.cancel') }}</a-button> + <a-button type="primary" ref="submit" :loading="loading" @click="handleSubmit">{{ $t('label.ok') }}</a-button> + </div> + </a-form> + </a-spin> +</template> + +<script> +import { getAPI, postAPI } from '@/api' +import { METRICS_BY_TYPE, CONDITIONS, SEVERITIES, RESOURCE_TYPE_LABELS, METRIC_LABELS, CONDITION_LABELS, SEVERITY_LABELS } from './resourceAlertOptions' + +export default { + name: 'CreateResourceAlertRule', + props: { + resource: { + type: Object, + default: () => ({}) + }, + presetResourceType: { + type: String, + default: '' + } + }, + data () { + return { + loading: false, + webhooks: [], + webhooksLoading: false, + resources: [], + resourcesLoading: false, + domains: [], + domainsLoading: false, + accounts: [], + accountsLoading: false, + form: { + name: '', + domainid: undefined, + account: undefined, + resourcetype: undefined, + resourceid: '', + metric: undefined, + condition: undefined, + threshold: undefined, + severity: undefined, + message: '', + email: false, + resetinterval: undefined, + webhookids: [] + }, + rules: { + name: [{ required: true, message: this.$t('label.required') }], + resourcetype: [{ required: true, message: this.$t('label.required') }], + metric: [{ required: true, message: this.$t('label.required') }], + condition: [{ required: true, message: this.$t('label.required') }], + threshold: [{ required: true, message: this.$t('label.required') }], + severity: [{ required: true, message: this.$t('label.required') }] + }, + + conditions: CONDITIONS, + severities: SEVERITIES, + resourceTypeLabels: RESOURCE_TYPE_LABELS, + metricLabels: METRIC_LABELS, + conditionLabels: CONDITION_LABELS, + severityLabels: SEVERITY_LABELS + } + }, + computed: { + isAdminOrDomainAdmin () { + return ['Admin', 'DomainAdmin'].includes(this.$store.getters.userInfo.roletype) + }, + // In project view the project owns the rule, so there is no owner to pick + canPickOwner () { + return this.isAdminOrDomainAdmin && !this.$store.getters.project?.id + }, + ownerParams () { + return this.form.domainid && this.form.account ? { domainid: this.form.domainid, account: this.form.account } : {} + }, + isRootAdmin () { + return this.$store.getters.userInfo.roletype === 'Admin' + }, + resourceTypes () { + return this.isRootAdmin ? ['VirtualMachine', 'Host', 'Volume', 'StoragePool'] : ['VirtualMachine', 'Volume'] + }, + availableMetrics () { + return METRICS_BY_TYPE[this.form.resourcetype] || [] + } + }, + created () { + this.fetchWebhooks() + if (this.canPickOwner) this.fetchDomains() + this.presetResource() + }, + watch: { + 'resource.id' () { + this.presetResource() + } + }, + methods: { + // Opened from a resource's own page: pick its type and the resource itself + presetResource () { + if (!this.presetResourceType || !this.resource?.id) return + this.form.resourcetype = this.presetResourceType + this.form.metric = undefined + this.fetchResources().then(() => { + this.form.resourceid = this.resource.id + }) + }, + fetchWebhooks () { + if (!('listWebhooks' in this.$store.getters.apis)) return + this.webhooksLoading = true + getAPI('listWebhooks', { listall: true, ...this.ownerParams }).then(json => { + this.webhooks = json?.listwebhooksresponse?.webhook || [] + }).finally(() => { + this.webhooksLoading = false + }) + }, + fetchDomains () { + this.domainsLoading = true + getAPI('listDomains', { listall: true, details: 'min' }).then(json => { + this.domains = json?.listdomainsresponse?.domain || [] + }).finally(() => { + this.domainsLoading = false + }) + }, + onDomainChange () { + this.form.account = undefined + this.accounts = [] + if (this.form.domainid) { + this.accountsLoading = true + getAPI('listAccounts', { domainid: this.form.domainid, details: 'min' }).then(json => { + this.accounts = json?.listaccountsresponse?.account || [] + }).finally(() => { + this.accountsLoading = false + }) + } + this.onOwnerChange() + }, + onOwnerChange () { + this.form.webhookids = [] + this.fetchWebhooks() + this.fetchResources() + }, + fetchResources () { + this.form.resourceid = '' + this.resources = [] + const sources = { + VirtualMachine: ['listVirtualMachines', 'listvirtualmachinesresponse', 'virtualmachine', { listall: true, details: 'min', ...this.ownerParams }], + Volume: ['listVolumes', 'listvolumesresponse', 'volume', { listall: true, ...this.ownerParams }], + Host: ['listHosts', 'listhostsresponse', 'host', { type: 'Routing' }], + StoragePool: ['listStoragePools', 'liststoragepoolsresponse', 'storagepool', {}] + } + const source = sources[this.form.resourcetype] + if (!source) return Promise.resolve() + const [api, responseKey, itemKey, params] = source + this.resourcesLoading = true + const hideExtensions = ['VirtualMachine', 'Host'].includes(this.form.resourcetype) + return getAPI(api, params).then(json => { Review Comment: This loads only the first API page, while the resource selector searches only the returned options. `BaseListCmd` defaults to `default.page.size` (500), so resources beyond that limit cannot be selected when creating a rule from Monitoring. Request the complete list or implement server-side search/pagination. ########## ui/src/components/view/SearchView.vue: ########## @@ -929,6 +942,40 @@ export default { } }) }, + fetchResourceAlertFieldData (arrayField) { + const setOpts = (name, opts) => { + const index = this.fields.findIndex(item => item.name === name) + if (index > -1) { + this.fields[index].opts = opts + this.fields[index].loading = false + } + } + const toOpts = (labels) => Object.entries(labels).map(([id, name]) => ({ id, name })) + if (arrayField.includes('metric')) setOpts('metric', toOpts(METRIC_LABELS)) + if (arrayField.includes('severity')) setOpts('severity', toOpts(SEVERITY_LABELS)) + if (!arrayField.includes('resourceid')) return + const resourceIndex = this.fields.findIndex(item => item.name === 'resourceid') + this.fields[resourceIndex].loading = true + const sources = [ + ['VirtualMachine', 'listVirtualMachines', 'listvirtualmachinesresponse', 'virtualmachine', { listall: true, details: 'min' }], + ['Volume', 'listVolumes', 'listvolumesresponse', 'volume', { listall: true }] + ] + if (this.$store.getters.userInfo.roletype === 'Admin') { + sources.push(['Host', 'listHosts', 'listhostsresponse', 'host', { type: 'Routing' }]) + sources.push(['StoragePool', 'listStoragePools', 'liststoragepoolsresponse', 'storagepool', {}]) + } + Promise.all(sources.filter(([, api]) => api in this.$store.getters.apis).map(([type, api, responseKey, itemKey, params]) => + getAPI(api, params).then(json => (json?.[responseKey]?.[itemKey] || []).map(r => ({ Review Comment: Each resource lookup is limited to the first API page, but this filter offers only client-side search over those results. Once a resource type exceeds `default.page.size` (500 by default), rules for the remaining resources cannot be filtered by resource through this control. Fetch all pages, request an unlimited page, or add server-side resource search. ########## api/src/main/java/org/apache/cloudstack/api/command/user/resourcealert/ListResourceAlertRulesCmd.java: ########## @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.resourcealert; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import com.cloud.alert.ResourceAlertRule; +import org.apache.cloudstack.alert.ResourceAlertService; +import org.apache.cloudstack.api.response.ResourceAlertRuleResponse; + +@APICommand(name = "listResourceAlertRules", + description = "Lists resource alert rules", + responseObject = ResourceAlertRuleResponse.class, + entityType = {ResourceAlertRule.class}, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "24.0.0") +public class ListResourceAlertRulesCmd extends BaseListProjectAndAccountResourcesCmd { + + @Inject + ResourceAlertService resourceAlertService; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = ResourceAlertRuleResponse.class, + description = "the ID of the alert rule") + private Long id; + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, + description = "filter by resource type: VirtualMachine, Volume, Host, StoragePool") + private String resourceType; + + @Parameter(name = ApiConstants.RESOURCE_ID, type = CommandType.STRING, + description = "filter by UUID of a specific resource") + private String resourceId; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, + description = "filter by rule name") + private String name; + + @Parameter(name = ApiConstants.METRIC, type = CommandType.STRING, + description = "filter by metric, for example CpuUtilization") Review Comment: The documented metric example is rejected by this endpoint: `parseMetricName()` uppercases it to `CPUUTILIZATION`, whereas the accepted enum is `CPU_UTILIZATION`. Use a valid value in the API description so clients following the generated documentation can successfully filter rules. ########## server/src/main/java/com/cloud/alert/ResourceAlertManagerImpl.java: ########## @@ -0,0 +1,736 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.alert; + +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import com.cloud.alert.dao.ResourceAlertDao; +import com.cloud.alert.dao.ResourceAlertRuleDao; +import com.cloud.alert.dao.ResourceAlertRuleWebhookDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.cloudstack.utils.mailing.MailAddress; +import org.apache.cloudstack.utils.mailing.SMTPMailProperties; +import org.apache.cloudstack.utils.mailing.SMTPMailSender; +import org.apache.cloudstack.webhook.WebhookHelper; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; + +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.event.AlertGenerator; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.host.Host; +import com.cloud.host.HostStats; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.resource.ResourceState; +import com.cloud.server.ResourceTag; +import com.cloud.server.StatsCollector; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageStats; +import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeStats; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.tags.dao.ResourceTagDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.GlobalLock; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmStats; +import com.cloud.vm.dao.UserVmDao; +import com.google.gson.JsonObject; + +public class ResourceAlertManagerImpl extends ManagerBase implements ResourceAlertManager, Configurable { + + static final String ALERT_EVENT_TYPE = "RESOURCE.ALERT"; + + static final ConfigKey<Integer> EVAL_INTERVAL = new ConfigKey<>("Advanced", Integer.class, + "resource.alert.evaluation.interval", "60", + "Interval in seconds between resource alert rule evaluations", false); + + public static final ConfigKey<Integer> RULES_PER_ACCOUNT_LIMIT = new ConfigKey<>("Advanced", Integer.class, + "resource.alert.per.user.limit", "20", + "Maximum number of resource alert rules an account can own; -1 = unlimited", true, ConfigKey.Scope.Account); + + static final ConfigKey<Integer> HISTORY_RETENTION_DAYS = new ConfigKey<>("Advanced", Integer.class, + "resource.alert.history.retention.days", "30", + "Number of days to keep fired resource alerts; -1 keeps them forever", true); + + public static final ConfigKey<Integer> DEFAULT_RESET_INTERVAL = new ConfigKey<>("Advanced", Integer.class, + "resource.alert.repeat.interval.default", "600", + "Default minimum seconds between repeat firings of a resource alert rule, used when a rule does not set one", true); + + @Inject ResourceAlertRuleDao ruleDao; + @Inject ResourceAlertDao alertDao; + @Inject ResourceAlertRuleWebhookDao ruleWebhookDao; + @Inject UserVmDao userVmDao; + @Inject HostDao hostDao; + @Inject PrimaryDataStoreDao storagePoolDao; + @Inject VolumeDao volumeDao; + @Inject StatsCollector statsCollector; + @Inject ConfigurationDao configDao; + @Inject ResourceTagDao resourceTagDao; + @Inject AccountDao accountDao; + @Inject AccountManager accountManager; + @Inject DomainDao domainDao; + @Inject ManagementServerHostDao managementServerHostDao; + + private ScheduledExecutorService executor; + static final int EMAIL_QUEUE_SIZE = 100; + static final long COOLDOWN_TOLERANCE_MILLIS = 5000L; + + // One sender and a bounded queue, so a slow mail server or an alert storm can't pile up threads. + ExecutorService emailExecutor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(EMAIL_QUEUE_SIZE), r -> { + Thread t = new Thread(r, "ResourceAlertEmailSender"); + t.setDaemon(true); + return t; + }, (r, e) -> logger.warn("Dropping resource alert email as {} emails are already waiting to be sent", EMAIL_QUEUE_SIZE)); + + private final Map<Long, VmStats> vmStatsCache = new HashMap<>(); + + private SMTPMailSender mailSender; + private String[] emailRecipients; + private String senderAddress; + + @Override + public boolean configure(String name, Map<String, Object> params) throws ConfigurationException { + String emailList = configDao.getValue("alert.email.addresses"); + if (StringUtils.isNotBlank(emailList)) { + emailRecipients = emailList.split(","); + } + senderAddress = configDao.getValue("alert.email.sender"); + + Map<String, String> smtpConfigs = new HashMap<>(); + for (String key : new String[]{ + "alert.smtp.host", "alert.smtp.port", "alert.smtp.useAuth", + "alert.smtp.username", "alert.smtp.password", "alert.smtp.useStartTLS", + "alert.smtp.enabledSecurityProtocols", "alert.smtp.timeout", "alert.smtp.connectiontimeout"}) { + String val = configDao.getValue(key); + if (val != null) smtpConfigs.put(key, val); + } + mailSender = new SMTPMailSender(smtpConfigs, "alert.smtp"); + + return super.configure(name, params); + } + + @Override + public boolean start() { + int interval = getEvaluationInterval(); + executor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "ResourceAlertEvaluator"); + t.setDaemon(true); + return t; + }); + executor.scheduleAtFixedRate(new EvaluationTask(), interval, interval, TimeUnit.SECONDS); + return true; + } + + Integer configuredEvaluationInterval() { + return EVAL_INTERVAL.value(); + } + + int getEvaluationInterval() { + Integer interval = configuredEvaluationInterval(); + if (interval == null || interval < 1) { + logger.warn("Invalid {} value {}, using {} seconds", EVAL_INTERVAL.key(), interval, EVAL_INTERVAL.defaultValue()); + return Integer.parseInt(EVAL_INTERVAL.defaultValue()); + } + return interval; + } + + @Override + public boolean stop() { + if (executor != null) { + executor.shutdown(); + } + emailExecutor.shutdown(); + return true; + } + + @Override + public void evaluateRules() { + vmStatsCache.clear(); + try { + List<ResourceAlertRuleVO> rules = ruleDao.listActive(); + for (ResourceAlertRuleVO rule : rules) { + // One broken rule must not stop the others from being checked. + try { + if (isOrphaned(rule)) { + logger.info("Removing resource alert rule {} as its owner or resource is gone, or the owner lost access to the resource", rule.getUuid()); + alertDao.removeByAlertRuleId(rule.getId()); + ruleDao.remove(rule.getId()); + continue; + } + if (ResourceAlertRule.State.Disabled.equals(rule.getState())) { + continue; + } + evaluateRule(rule); + } catch (Exception e) { + logger.warn("Failed to evaluate resource alert rule {}", rule.getUuid(), e); + } + } + } finally { + vmStatsCache.clear(); + } + } + + // Several rules can watch the same VM, so its stats are read once per run. + private VmStats getVmStats(long vmId) { + if (!vmStatsCache.containsKey(vmId)) { + vmStatsCache.put(vmId, statsCollector.getRecentVmStats(vmId)); + } + return vmStatsCache.get(vmId); + } + + boolean isOrphaned(ResourceAlertRuleVO rule) { + AccountVO owner = accountDao.findById(rule.getAccountId()); + if (owner == null) { + return true; + } + Long resourceId = rule.getResourceId(); + if (resourceId == null) { + return false; + } + switch (rule.getResourceType()) { + case VirtualMachine: + return !ownerCanAccess(owner, userVmDao.findById(resourceId)); + case Volume: + return !ownerCanAccess(owner, volumeDao.findById(resourceId)); + case Host: + return hostDao.findById(resourceId) == null; + case StoragePool: + return storagePoolDao.findById(resourceId) == null; + default: + return false; + } + } + + // A VM can move to another account or its owner can leave a project, so access is checked on every run. + private boolean ownerCanAccess(Account owner, ControlledEntity resource) { + if (resource == null) { + return false; + } + try { + accountManager.checkAccess(owner, null, false, resource); + return true; + } catch (PermissionDeniedException e) { + return false; + } + } + + // Stopped VMs and hosts or pools out of service report stale or zero stats. + boolean isInService(ResourceAlertRule.ResourceType type, long resourceId) { + switch (type) { + case VirtualMachine: { + UserVmVO vm = userVmDao.findById(resourceId); + return vm != null && VirtualMachine.State.Running.equals(vm.getState()); + } + case Volume: { + VolumeVO volume = volumeDao.findById(resourceId); + return volume != null && Volume.State.Ready.equals(volume.getState()); + } + case Host: { + HostVO host = hostDao.findById(resourceId); + return host != null && Status.Up.equals(host.getStatus()) && !ResourceState.isMaintenanceState(host.getResourceState()); + } + case StoragePool: { + StoragePoolVO pool = storagePoolDao.findById(resourceId); + return pool != null && StoragePoolStatus.Up.equals(pool.getStatus()); + } + default: + return false; + } + } + + // Every management server collects stats for all hosts, so only one may evaluate or alerts fire once per server. + boolean isEvaluatingServer() { + ManagementServerHostVO msHost = managementServerHostDao.findOneByLongestRuntime(); + return msHost != null && msHost.getMsid() == ManagementServerNode.getManagementServerId(); + } + + void removeExpiredAlerts() { + int days = HISTORY_RETENTION_DAYS.value(); + if (days < 0) { + return; + } + int removed = alertDao.removeOlderThan(new Date(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(days))); + if (removed > 0) { + logger.debug("Removed {} resource alerts older than {} days", removed, days); + } + } + + void removeStaleWebhookLinks() { + int removed = ruleWebhookDao.removeLinksToRemovedWebhooks(); + if (removed > 0) { + logger.debug("Removed {} resource alert rule links to deleted webhooks", removed); + } + } + + class EvaluationTask extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock lock = GlobalLock.getInternLock("ResourceAlertEvaluation"); + try { + if (!lock.lock(5)) { + return; + } + try { + if (isEvaluatingServer()) { + evaluateRules(); + removeExpiredAlerts(); + removeStaleWebhookLinks(); + } + } finally { + lock.unlock(); + } + } catch (Exception e) { + logger.warn("Failed to evaluate resource alert rules", e); + } finally { + lock.releaseRef(); + } + } + } + + private void evaluateRule(ResourceAlertRuleVO rule) { + ResourceAlertMetric metric = ResourceAlertMetric.valueOf(rule.getMetric()); + boolean isGeneric = rule.getResourceId() == null; + for (Long resourceId : getResourceIds(rule)) { + try { + if (checksServiceState(rule) && !isInService(rule.getResourceType(), resourceId)) continue; + if (isGeneric) { + if (isOptedOut(rule.getResourceType(), resourceId)) continue; + if (ruleDao.existsSpecificRule(rule.getResourceType(), rule.getMetric(), resourceId, rule.getAccountId())) continue; + } + Double value = getMetricValue(rule.getResourceType(), metric, resourceId); + if (value == null || value < 0) { + continue; + } + if (rule.getCondition().evaluate(value, rule.getThreshold()) + && canFire(rule.getId(), resourceId, rule.getResetInterval())) { + fireAlert(rule, resourceId, value); + } + } catch (Exception e) { + logger.warn("Failed to evaluate resource alert rule {} for resource {}", rule.getUuid(), resourceId, e); + } + } + } + + private boolean isOptedOut(ResourceAlertRule.ResourceType type, long resourceId) { + ResourceTag.ResourceObjectType objType = null; + if (type == ResourceAlertRule.ResourceType.VirtualMachine) { + objType = ResourceTag.ResourceObjectType.UserVm; + } else if (type == ResourceAlertRule.ResourceType.Volume) { + objType = ResourceTag.ResourceObjectType.Volume; + } + if (objType == null) return false; + ResourceTag tag = resourceTagDao.findByKey(resourceId, objType, "resource.alert.opt.out"); + return tag != null && "true".equalsIgnoreCase(tag.getValue()); + } + + // All-resources VM and volume rules already list only running VMs and ready volumes. + private boolean checksServiceState(ResourceAlertRuleVO rule) { + return rule.getResourceId() != null + || ResourceAlertRule.ResourceType.Host.equals(rule.getResourceType()) + || ResourceAlertRule.ResourceType.StoragePool.equals(rule.getResourceType()); + } + + private List<Long> getResourceIds(ResourceAlertRuleVO rule) { + if (rule.getResourceId() != null) { + return Collections.singletonList(rule.getResourceId()); + } + switch (rule.getResourceType()) { + case VirtualMachine: { + Pair<Long, List<Long>> scope = getGenericRuleScope(rule); + return scope == null ? Collections.emptyList() : userVmDao.listIdsByAccountOrDomainsAndState( + scope.first(), scope.second(), VirtualMachine.State.Running); + } + case Volume: { + Pair<Long, List<Long>> scope = getGenericRuleScope(rule); + return scope == null ? Collections.emptyList() : volumeDao.listUserVolumeIdsByAccountOrDomainsAndState( + scope.first(), scope.second(), Volume.State.Ready); + } + case Host: + return hostDao.listAll().stream() + .filter(h -> Host.Type.Routing.equals(h.getType())) + .map(h -> h.getId()) + .collect(Collectors.toList()); + case StoragePool: + return storagePoolDao.listAll().stream() + .map(p -> p.getId()) + .collect(Collectors.toList()); + default: + return Collections.emptyList(); + } + } + + // Root admin rules cover the whole cloud, domain admin rules their domain tree, other rules their own account. + Pair<Long, List<Long>> getGenericRuleScope(ResourceAlertRule rule) { + Account owner = accountDao.findById(rule.getAccountId()); + if (owner == null) { + return null; + } + if (Account.Type.ADMIN.equals(owner.getType())) { + return new Pair<>(null, null); + } + if (Account.Type.DOMAIN_ADMIN.equals(owner.getType()) || Account.Type.RESOURCE_DOMAIN_ADMIN.equals(owner.getType())) { + return new Pair<>(null, domainDao.getDomainAndChildrenIds(owner.getDomainId())); + } + return new Pair<>(owner.getId(), null); + } + + private Double getMetricValue(ResourceAlertRule.ResourceType type, ResourceAlertMetric metric, long resourceId) { + if (!metric.appliesTo(type)) { + return null; + } + switch (metric) { + case CPU_UTILIZATION: + if (type == ResourceAlertRule.ResourceType.VirtualMachine) { + VmStats s = getVmStats(resourceId); + return s != null ? s.getCPUUtilization() : null; + } + if (type == ResourceAlertRule.ResourceType.Host) { + HostStats s = statsCollector.getHostStats(resourceId); + return s != null ? s.getCpuUtilization() : null; + } + break; + case MEMORY_UTILIZATION: + if (type == ResourceAlertRule.ResourceType.VirtualMachine) { + VmStats s = getVmStats(resourceId); + if (s == null) return null; + double total = s.getMemoryKBs(); + double free = s.getIntFreeMemoryKBs(); + // free is -1 when VM has no balloon driver + if (total <= 0 || free < 0) return null; + return (1.0 - free / total) * 100.0; + } + if (type == ResourceAlertRule.ResourceType.Host) { + HostStats s = statsCollector.getHostStats(resourceId); + if (s == null) return null; + double total = s.getTotalMemoryKBs(); + double free = s.getFreeMemoryKBs(); + if (total <= 0) return null; + return ((total - free) / total) * 100.0; + } + break; + case DISK_READ_IOPS: { + VmStats s = getVmStats(resourceId); + return s != null ? s.getDiskReadIOs() : null; + } + case DISK_WRITE_IOPS: { + VmStats s = getVmStats(resourceId); + return s != null ? s.getDiskWriteIOs() : null; + } + case DISK_READ_KBPS: { + VmStats s = getVmStats(resourceId); + return s != null ? s.getDiskReadKBs() : null; + } + case DISK_WRITE_KBPS: { + VmStats s = getVmStats(resourceId); + return s != null ? s.getDiskWriteKBs() : null; + } + case NETWORK_READ_KBPS: { + if (type == ResourceAlertRule.ResourceType.Host) { + HostStats s = statsCollector.getHostStats(resourceId); + return s != null ? s.getNetworkReadKBs() : null; + } + VmStats s = getVmStats(resourceId); + return s != null ? s.getNetworkReadKBs() : null; + } + case NETWORK_WRITE_KBPS: { + if (type == ResourceAlertRule.ResourceType.Host) { + HostStats s = statsCollector.getHostStats(resourceId); + return s != null ? s.getNetworkWriteKBs() : null; + } + VmStats s = getVmStats(resourceId); + return s != null ? s.getNetworkWriteKBs() : null; + } + case STORAGE_USED_IOPS: { + // only reported by storage drivers that track IOPS + StorageStats pool = statsCollector.getStoragePoolStats(resourceId); + return pool != null && pool.getUsedIops() != null ? pool.getUsedIops().doubleValue() : null; + } + case VOLUME_USED_GB: { + VolumeStats s = getVolumeStats(resourceId); + return s != null ? s.getPhysicalSize() / (1024.0 * 1024.0 * 1024.0) : null; + } + case VOLUME_UTILIZATION: { + VolumeStats s = getVolumeStats(resourceId); + if (s == null || s.getVirtualSize() <= 0) return null; + return ((double) s.getPhysicalSize() / s.getVirtualSize()) * 100.0; + } + case LOAD_AVERAGE: { + HostStats s = statsCollector.getHostStats(resourceId); + return s != null ? s.getLoadAverage() : null; + } + case STORAGE_UTILIZATION: { + StorageStats pool = statsCollector.getStoragePoolStats(resourceId); + if (pool == null || pool.getCapacityBytes() <= 0) return null; + return ((double) pool.getByteUsed() / pool.getCapacityBytes()) * 100.0; + } + default: + break; + } + return null; + } + + // Stats are keyed by path, except OVA volumes which are keyed by chain info. + private VolumeStats getVolumeStats(long volumeId) { + VolumeVO vol = volumeDao.findById(volumeId); + if (vol == null) return null; + String locator = Storage.ImageFormat.OVA.equals(vol.getFormat()) ? vol.getChainInfo() : vol.getPath(); + return locator != null ? statsCollector.getVolumeStats(locator) : null; + } + + // Alert times are stored without milliseconds and runs drift a little, so a cooldown equal to + // the check interval would otherwise skip every other check. + boolean canFire(long ruleId, Long resourceId, int resetInterval) { + ResourceAlertVO last = alertDao.findLastFiredForRule(ruleId, resourceId); + if (last == null) return true; Review Comment: Cooldown state is lost when history retention removes the last alert. For example, with `resource.alert.history.retention.days=0`, each evaluation deletes its freshly fired alert and the next evaluation takes this `last == null` branch, so the default 600-second cooldown becomes an alert every minute. The same problem occurs whenever a rule's cooldown exceeds the retention period. Preserve per-rule/resource last-fired state independently of history cleanup, or retain the cooldown marker until it expires. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
