bhouse-nexthop commented on code in PR #14207:
URL: https://github.com/apache/cloudstack/pull/14207#discussion_r4208608966
##########
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java:
##########
@@ -178,23 +207,63 @@ private void processReport(long hostId, Map<Long,
VirtualMachine.PowerState> tra
logger.debug("Done with process of VM state report. host: {}", () ->
hostCache.get(hostId));
}
- public Map<Long, VirtualMachine.PowerState>
convertVmStateReport(Map<String, HostVmStateReportEntry> states) {
+ public Map<Long, VirtualMachine.PowerState> convertVmStateReport(long
hostId, Map<String, HostVmStateReportEntry> states) {
final HashMap<Long, VirtualMachine.PowerState> map = new HashMap<>();
if (MapUtils.isEmpty(states)) {
+ reportUnknownInstances(hostId, new HashSet<>());
return map;
}
+ Set<String> unknownInstanceNames = new HashSet<>();
Map<String, Long> nameIdMap =
_instanceDao.getNameIdMapForVmInstanceNames(states.keySet());
for (Map.Entry<String, HostVmStateReportEntry> entry :
states.entrySet()) {
Long id = nameIdMap.get(entry.getKey());
if (id != null) {
map.put(id, entry.getValue().getState());
} else {
+ unknownInstanceNames.add(entry.getKey());
logger.debug("Unable to find matched VM in CloudStack DB.
name: {} powerstate: {}", entry.getKey(), entry.getValue());
}
}
+ reportUnknownInstances(hostId, unknownInstanceNames);
return map;
}
+ /**
+ * A host reporting an instance that CloudStack has no record of means
something is running there unmanaged,
+ * usually left behind by a deploy or an expunge that did not reach the
host. Previously this produced one debug
+ * line per instance per report and nothing else, so it could go unnoticed
indefinitely.
+ *
+ * Logged at warn level, and only when the set of unknown instances on a
host changes, so a standing condition
+ * does not repeat on every report.
+ *
+ * @return true when the set changed and was reported, false when there
was nothing new to say.
+ */
+ protected boolean reportUnknownInstances(long hostId, Set<String>
unknownInstanceNames) {
+ Set<String> previous = unknownInstancesByHost.get(hostId);
+ if (unknownInstanceNames.equals(previous) ||
(CollectionUtils.isEmpty(unknownInstanceNames) && previous == null)) {
+ return false;
+ }
+ if (unknownInstanceNames.isEmpty()) {
+ unknownInstancesByHost.remove(hostId);
+ logger.info("No unknown instances are reported anymore. host: {}",
() -> hostCache.get(hostId));
+ return true;
+ }
+ unknownInstancesByHost.put(hostId, unknownInstanceNames);
+ HostVO host = hostCache.get(hostId);
+ String names = String.join(", ", unknownInstanceNames);
+ logger.warn("Host reports {} instance(s) that do not exist in
CloudStack DB, they are running unmanaged. " +
+ "host: {}, instances: [{}]",
+ unknownInstanceNames.size(), host, names);
+ if (host != null) {
+ _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC,
host.getDataCenterId(), host.getPodId(),
Review Comment:
Yes, it would have. `VmwareResource.getHostVmStateReport()` adds every
non-template VM on the host in any power state, keyed by the
`cloud.vm.internal.name` custom field when set and the vSphere name otherwise.
Nothing there filters worker VMs (`VmwareManagerImpl.composeWorkerName()`, a
UUID without dashes, or `cloud.uuid-<uuid>` on VVOL per
`VmwareResource.getWorkerName()`). The dedup in `reportUnknownInstances()`
compares the whole set per host, so every worker appearing or going away
counted as a change and raised an alert.
Fixed in 41630b480c. A name is only collected for the warn/alert when the
host reports it `PowerOn` and it has a CloudStack instance name:
`i-<account>-<id>-<suffix>` (this includes the `vm.instancename.flag` form), or
`r|s|v|b|l|NS-<id>-<suffix>` for system VMs. Worker VMs, vCLS, the vCenter
appliance and anything else not created by CloudStack, plus powered-off
leftovers, stay at debug as they did before this PR. New tests in
`VirtualMachinePowerStateSyncImplTest` cover those names, and check that a
worker VM coming and going next to a standing unknown instance leaves the
reported set unchanged.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
