andrijapanicsb commented on code in PR #14223:
URL: https://github.com/apache/cloudstack/pull/14223#discussion_r4394971237
##########
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java:
##########
@@ -376,24 +377,171 @@ private void
scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType,
List<KubernetesClusterVmMapVO> vmList =
kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(),
nodeType);
final long tobeScaledVMCount = Math.min(vmList.size(), size);
for (long i = 0; i < tobeScaledVMCount; i++) {
- KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i);
- UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId());
- boolean result = false;
- try {
- result = userVmManager.upgradeVirtualMachine(userVM.getId(),
serviceOffering.getId(), new HashMap<String, String>());
- } catch (RuntimeException | ResourceUnavailableException |
ManagementServerException | VirtualMachineMigrationException e) {
- logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s",
kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e);
- }
- if (!result) {
- logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s",
kubernetesCluster.getName(),
userVM.getDisplayName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
- }
+ new NodeOfferingResize(vmList.get((int) i), nodeType,
serviceOffering).execute();
if (System.currentTimeMillis() > scaleTimeoutTime) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, scaling action timed out",
kubernetesCluster.getName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
}
}
kubernetesCluster = updateKubernetesClusterEntryForNodeType(null,
nodeType, serviceOffering, updateNodeOffering, updateClusterOffering);
}
+ private final class NodeOfferingResize {
Review Comment:
Done, it's now `KubernetesClusterNodeOfferingResizer`, with only the
dependencies it needs.
##########
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java:
##########
@@ -376,24 +377,171 @@ private void
scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType,
List<KubernetesClusterVmMapVO> vmList =
kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(),
nodeType);
final long tobeScaledVMCount = Math.min(vmList.size(), size);
for (long i = 0; i < tobeScaledVMCount; i++) {
- KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i);
- UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId());
- boolean result = false;
- try {
- result = userVmManager.upgradeVirtualMachine(userVM.getId(),
serviceOffering.getId(), new HashMap<String, String>());
- } catch (RuntimeException | ResourceUnavailableException |
ManagementServerException | VirtualMachineMigrationException e) {
- logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s",
kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e);
- }
- if (!result) {
- logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s",
kubernetesCluster.getName(),
userVM.getDisplayName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
- }
+ new NodeOfferingResize(vmList.get((int) i), nodeType,
serviceOffering).execute();
if (System.currentTimeMillis() > scaleTimeoutTime) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, scaling action timed out",
kubernetesCluster.getName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
}
}
kubernetesCluster = updateKubernetesClusterEntryForNodeType(null,
nodeType, serviceOffering, updateNodeOffering, updateClusterOffering);
}
+ private final class NodeOfferingResize {
+ private final KubernetesClusterVmMapVO vmMap;
+ private final KubernetesClusterNodeType nodeType;
+ private final ServiceOffering targetOffering;
+ private final ServiceOffering oldOffering;
+ private UserVmVO vm;
+ private NodeAccess access;
+ private NodeCapacitySnapshot before;
+ private boolean reconcileKubernetes;
+ private boolean cksCordonCompleted;
+ private boolean resizeSucceeded;
+
+ private NodeOfferingResize(KubernetesClusterVmMapVO vmMap,
KubernetesClusterNodeType nodeType, ServiceOffering targetOffering) {
+ this.vmMap = vmMap;
+ this.nodeType = nodeType;
+ this.targetOffering = targetOffering;
+ vm = userVmDao.findById(vmMap.getVmId());
+ if (vm == null) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to find cluster VM %s",
+ kubernetesCluster.getName(), vmMap.getVmId()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+ }
+ oldOffering =
serviceOfferingDao.findById(vm.getServiceOfferingId());
+ resizeSucceeded = Objects.equals(targetOffering.getId(),
vm.getServiceOfferingId());
+ }
+
+ private void execute() {
+ validateLiveResize();
+ try {
+ prepareReconciliation();
+ resizeVm();
+ reconcileCapacity();
+ } catch (ResourceUnavailableException | ManagementServerException
| VirtualMachineMigrationException | CloudRuntimeException e) {
+ restoreAfterFailedResize();
+ reportFailure(e);
+ }
+ }
+
+ private void validateLiveResize() {
+ if
(KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering,
targetOffering)
+ && KubernetesCluster.State.Running == originalState &&
ETCD == nodeType
+ && KubernetesCluster.ClusterType.CloudManaged ==
kubernetesCluster.getClusterType()) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s cannot live-resize dedicated etcd VM %s; " +
+ "etcd health reconciliation is not implemented",
kubernetesCluster.getName(), vm.getDisplayName()),
+ kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
+ }
Review Comment:
Thanks for catching this. Looking at it again, the check itself was the
problem. On main, dedicated etcd VMs are live-resized like the other VMs, and
since they aren't Kubernetes nodes there's nothing to reconcile on them. So I
removed the check: etcd VMs get exactly the same resize as on main, and a
request that includes etcd is no longer rejected halfway through.
##########
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java:
##########
@@ -376,24 +377,171 @@ private void
scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType,
List<KubernetesClusterVmMapVO> vmList =
kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(),
nodeType);
final long tobeScaledVMCount = Math.min(vmList.size(), size);
for (long i = 0; i < tobeScaledVMCount; i++) {
- KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i);
- UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId());
- boolean result = false;
- try {
- result = userVmManager.upgradeVirtualMachine(userVM.getId(),
serviceOffering.getId(), new HashMap<String, String>());
- } catch (RuntimeException | ResourceUnavailableException |
ManagementServerException | VirtualMachineMigrationException e) {
- logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s",
kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e);
- }
- if (!result) {
- logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s",
kubernetesCluster.getName(),
userVM.getDisplayName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
- }
+ new NodeOfferingResize(vmList.get((int) i), nodeType,
serviceOffering).execute();
if (System.currentTimeMillis() > scaleTimeoutTime) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, scaling action timed out",
kubernetesCluster.getName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
}
}
kubernetesCluster = updateKubernetesClusterEntryForNodeType(null,
nodeType, serviceOffering, updateNodeOffering, updateClusterOffering);
}
+ private final class NodeOfferingResize {
+ private final KubernetesClusterVmMapVO vmMap;
+ private final KubernetesClusterNodeType nodeType;
+ private final ServiceOffering targetOffering;
+ private final ServiceOffering oldOffering;
+ private UserVmVO vm;
+ private NodeAccess access;
+ private NodeCapacitySnapshot before;
+ private boolean reconcileKubernetes;
+ private boolean cksCordonCompleted;
+ private boolean resizeSucceeded;
+
+ private NodeOfferingResize(KubernetesClusterVmMapVO vmMap,
KubernetesClusterNodeType nodeType, ServiceOffering targetOffering) {
+ this.vmMap = vmMap;
+ this.nodeType = nodeType;
+ this.targetOffering = targetOffering;
+ vm = userVmDao.findById(vmMap.getVmId());
+ if (vm == null) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to find cluster VM %s",
+ kubernetesCluster.getName(), vmMap.getVmId()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+ }
+ oldOffering =
serviceOfferingDao.findById(vm.getServiceOfferingId());
+ resizeSucceeded = Objects.equals(targetOffering.getId(),
vm.getServiceOfferingId());
+ }
+
+ private void execute() {
+ validateLiveResize();
+ try {
+ prepareReconciliation();
+ resizeVm();
+ reconcileCapacity();
+ } catch (ResourceUnavailableException | ManagementServerException
| VirtualMachineMigrationException | CloudRuntimeException e) {
+ restoreAfterFailedResize();
+ reportFailure(e);
+ }
+ }
+
+ private void validateLiveResize() {
+ if
(KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering,
targetOffering)
+ && KubernetesCluster.State.Running == originalState &&
ETCD == nodeType
+ && KubernetesCluster.ClusterType.CloudManaged ==
kubernetesCluster.getClusterType()) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s cannot live-resize dedicated etcd VM %s; " +
+ "etcd health reconciliation is not implemented",
kubernetesCluster.getName(), vm.getDisplayName()),
+ kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
+ }
+ }
+
+ private void prepareReconciliation() {
+ reconcileKubernetes = shouldReconcileNodeCapacity(vmMap, vm,
oldOffering, targetOffering, nodeType);
Review Comment:
Done, there's one log line per node with the reason (external node, not
CloudManaged, not KVM, cluster not Running, not a control or worker VM, no
CPU/RAM change, or already on the target offering).
##########
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java:
##########
@@ -376,24 +377,171 @@ private void
scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType,
List<KubernetesClusterVmMapVO> vmList =
kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(),
nodeType);
final long tobeScaledVMCount = Math.min(vmList.size(), size);
for (long i = 0; i < tobeScaledVMCount; i++) {
- KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i);
- UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId());
- boolean result = false;
- try {
- result = userVmManager.upgradeVirtualMachine(userVM.getId(),
serviceOffering.getId(), new HashMap<String, String>());
- } catch (RuntimeException | ResourceUnavailableException |
ManagementServerException | VirtualMachineMigrationException e) {
- logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s",
kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e);
- }
- if (!result) {
- logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s",
kubernetesCluster.getName(),
userVM.getDisplayName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
- }
+ new NodeOfferingResize(vmList.get((int) i), nodeType,
serviceOffering).execute();
if (System.currentTimeMillis() > scaleTimeoutTime) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, scaling action timed out",
kubernetesCluster.getName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
}
}
kubernetesCluster = updateKubernetesClusterEntryForNodeType(null,
nodeType, serviceOffering, updateNodeOffering, updateClusterOffering);
}
+ private final class NodeOfferingResize {
+ private final KubernetesClusterVmMapVO vmMap;
+ private final KubernetesClusterNodeType nodeType;
+ private final ServiceOffering targetOffering;
+ private final ServiceOffering oldOffering;
+ private UserVmVO vm;
+ private NodeAccess access;
+ private NodeCapacitySnapshot before;
+ private boolean reconcileKubernetes;
+ private boolean cksCordonCompleted;
+ private boolean resizeSucceeded;
+
+ private NodeOfferingResize(KubernetesClusterVmMapVO vmMap,
KubernetesClusterNodeType nodeType, ServiceOffering targetOffering) {
+ this.vmMap = vmMap;
+ this.nodeType = nodeType;
+ this.targetOffering = targetOffering;
+ vm = userVmDao.findById(vmMap.getVmId());
+ if (vm == null) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to find cluster VM %s",
+ kubernetesCluster.getName(), vmMap.getVmId()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+ }
+ oldOffering =
serviceOfferingDao.findById(vm.getServiceOfferingId());
+ resizeSucceeded = Objects.equals(targetOffering.getId(),
vm.getServiceOfferingId());
+ }
+
+ private void execute() {
+ validateLiveResize();
+ try {
+ prepareReconciliation();
+ resizeVm();
+ reconcileCapacity();
+ } catch (ResourceUnavailableException | ManagementServerException
| VirtualMachineMigrationException | CloudRuntimeException e) {
+ restoreAfterFailedResize();
+ reportFailure(e);
+ }
+ }
+
+ private void validateLiveResize() {
+ if
(KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering,
targetOffering)
+ && KubernetesCluster.State.Running == originalState &&
ETCD == nodeType
+ && KubernetesCluster.ClusterType.CloudManaged ==
kubernetesCluster.getClusterType()) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s cannot live-resize dedicated etcd VM %s; " +
+ "etcd health reconciliation is not implemented",
kubernetesCluster.getName(), vm.getDisplayName()),
+ kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
+ }
+ }
+
+ private void prepareReconciliation() {
+ reconcileKubernetes = shouldReconcileNodeCapacity(vmMap, vm,
oldOffering, targetOffering, nodeType);
+ if (reconcileKubernetes || isReconciliationRetryCandidate(vmMap,
vm, targetOffering, nodeType)) {
+ access = resolveNodeAccess(vm);
+ before =
kubernetesClusterNodeCapacityReconciler.captureBefore(kubernetesCluster, vm,
access);
Review Comment:
Agreed that it shouldn't need SSH to the node. I didn't want to skip on an
error though: if that node still had unfinished work, skipping would leave it
with the old capacity and nobody would notice. So I changed how it's detected.
The node now gets a small "pending" annotation when its resize starts, and it's
removed only once the node is verified. For a node that's already on the target
offering, only the Node object is read through the control plane: no annotation
means it's skipped without any SSH to the node, and with the annotation it's
finished first. This also fixed a case I had missed, where a node cordoned by
an operator would never have been retried.
##########
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java:
##########
@@ -376,24 +377,171 @@ private void
scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType,
List<KubernetesClusterVmMapVO> vmList =
kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(),
nodeType);
final long tobeScaledVMCount = Math.min(vmList.size(), size);
for (long i = 0; i < tobeScaledVMCount; i++) {
- KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i);
- UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId());
- boolean result = false;
- try {
- result = userVmManager.upgradeVirtualMachine(userVM.getId(),
serviceOffering.getId(), new HashMap<String, String>());
- } catch (RuntimeException | ResourceUnavailableException |
ManagementServerException | VirtualMachineMigrationException e) {
- logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s",
kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e);
- }
- if (!result) {
- logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, unable to scale cluster VM : %s",
kubernetesCluster.getName(),
userVM.getDisplayName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
- }
+ new NodeOfferingResize(vmList.get((int) i), nodeType,
serviceOffering).execute();
if (System.currentTimeMillis() > scaleTimeoutTime) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling
Kubernetes cluster : %s failed, scaling action timed out",
kubernetesCluster.getName()),kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
}
}
kubernetesCluster = updateKubernetesClusterEntryForNodeType(null,
nodeType, serviceOffering, updateNodeOffering, updateClusterOffering);
}
+ private final class NodeOfferingResize {
+ private final KubernetesClusterVmMapVO vmMap;
+ private final KubernetesClusterNodeType nodeType;
+ private final ServiceOffering targetOffering;
+ private final ServiceOffering oldOffering;
+ private UserVmVO vm;
+ private NodeAccess access;
+ private NodeCapacitySnapshot before;
+ private boolean reconcileKubernetes;
+ private boolean cksCordonCompleted;
+ private boolean resizeSucceeded;
+
+ private NodeOfferingResize(KubernetesClusterVmMapVO vmMap,
KubernetesClusterNodeType nodeType, ServiceOffering targetOffering) {
+ this.vmMap = vmMap;
+ this.nodeType = nodeType;
+ this.targetOffering = targetOffering;
+ vm = userVmDao.findById(vmMap.getVmId());
+ if (vm == null) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s failed, unable to find cluster VM %s",
+ kubernetesCluster.getName(), vmMap.getVmId()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+ }
+ oldOffering =
serviceOfferingDao.findById(vm.getServiceOfferingId());
+ resizeSucceeded = Objects.equals(targetOffering.getId(),
vm.getServiceOfferingId());
+ }
+
+ private void execute() {
+ validateLiveResize();
+ try {
+ prepareReconciliation();
+ resizeVm();
+ reconcileCapacity();
+ } catch (ResourceUnavailableException | ManagementServerException
| VirtualMachineMigrationException | CloudRuntimeException e) {
+ restoreAfterFailedResize();
+ reportFailure(e);
+ }
+ }
+
+ private void validateLiveResize() {
+ if
(KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering,
targetOffering)
+ && KubernetesCluster.State.Running == originalState &&
ETCD == nodeType
+ && KubernetesCluster.ClusterType.CloudManaged ==
kubernetesCluster.getClusterType()) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Scaling
Kubernetes cluster : %s cannot live-resize dedicated etcd VM %s; " +
+ "etcd health reconciliation is not implemented",
kubernetesCluster.getName(), vm.getDisplayName()),
+ kubernetesCluster.getId(),
KubernetesCluster.Event.OperationFailed);
+ }
+ }
+
+ private void prepareReconciliation() {
+ reconcileKubernetes = shouldReconcileNodeCapacity(vmMap, vm,
oldOffering, targetOffering, nodeType);
+ if (reconcileKubernetes || isReconciliationRetryCandidate(vmMap,
vm, targetOffering, nodeType)) {
+ access = resolveNodeAccess(vm);
+ before =
kubernetesClusterNodeCapacityReconciler.captureBefore(kubernetesCluster, vm,
access);
+ reconcileKubernetes = reconcileKubernetes ||
before.isCloudStackResizeCordon();
+ }
+ if (reconcileKubernetes) {
+
kubernetesClusterNodeCapacityReconciler.cordonIfNeeded(kubernetesCluster, vm,
before, access, scaleTimeoutTime);
+ cksCordonCompleted = !before.isUnschedulable() ||
before.isCloudStackResizeCordon();
+ }
+ }
+
+ private void resizeVm() throws ResourceUnavailableException,
ManagementServerException, VirtualMachineMigrationException {
+ if (resizeSucceeded) {
+ return;
+ }
+ if (!userVmManager.upgradeVirtualMachine(vm.getId(),
targetOffering.getId(), new HashMap<>())) {
+ throw new CloudRuntimeException("Unable to scale cluster VM "
+ vm.getDisplayName());
+ }
+ resizeSucceeded = true;
+ UserVmVO resizedVm = userVmDao.findById(vm.getId());
+ if (resizedVm == null || !Objects.equals(targetOffering.getId(),
resizedVm.getServiceOfferingId())) {
+ throw new CloudRuntimeException("VM " + vm.getUuid() + " did
not reach target offering");
+ }
+ vm = resizedVm;
+ }
+
+ private void reconcileCapacity() {
+ if (!reconcileKubernetes) {
+ return;
+ }
+
kubernetesClusterNodeCapacityReconciler.onlineGuestResources(access);
+ kubernetesClusterNodeCapacityReconciler.verifyGuestResources(vm,
targetOffering, before, access, scaleTimeoutTime);
+ if
(!kubernetesClusterNodeCapacityReconciler.isKubernetesResourcesCurrent(before,
targetOffering)) {
+ kubernetesClusterNodeCapacityReconciler.restartKubelet(vm,
access, scaleTimeoutTime);
+
kubernetesClusterNodeCapacityReconciler.waitForKubernetesResources(kubernetesCluster,
vm, targetOffering, before, access, scaleTimeoutTime);
+ }
+
kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster,
vm, before, access, scaleTimeoutTime);
+ }
+
+ private void restoreAfterFailedResize() {
+ if (!reconcileKubernetes || !cksCordonCompleted ||
resizeSucceeded) {
+ return;
+ }
+ try {
+
kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster,
vm, before, access, scaleTimeoutTime);
+ } catch (CloudRuntimeException cleanupException) {
+ logger.warn("Unable to restore schedulability for CKS node {}
after a pre-resize failure", vm.getUuid(), cleanupException);
+ }
Review Comment:
You're right. Two changes: errors from parsing the `kubectl` and guest
output are now turned into `NodeCapacityReconciliationException` inside the
reconciler, with the phase and the VM in the message. And the per-node handler
catches `RuntimeException` again together with the three checked exceptions,
the same list as on main before this PR, so an unexpected error still goes
through the cordon cleanup and marks the scale as failed. The original
exception is kept as the cause and logged with its stack trace. I also replaced
the test that expected the old behaviour.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
