This is an automated email from the ASF dual-hosted git repository. shwstppr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit b29ec2bf12abff1ce6ea3abc098d5b799bf007c9 Merge: 68a3e9e8398 9fd410be36d Author: Abhishek Kumar <[email protected]> AuthorDate: Fri Mar 1 17:40:58 2024 +0530 Merge remote-tracking branch 'apache/4.19' .github/workflows/build.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/codecov.yml | 2 +- .github/workflows/main-sonar-check.yml | 2 +- .github/workflows/rat.yml | 2 +- .github/workflows/sonar-check.yml | 2 +- api/src/main/java/com/cloud/event/EventTypes.java | 1 + .../java/com/cloud/storage/VolumeApiService.java | 4 + .../org/apache/cloudstack/api/ApiConstants.java | 4 + .../user/volume/CheckAndRepairVolumeCmd.java | 139 ++++++++++++++ .../cloudstack/api/response/VolumeResponse.java | 25 +++ .../api/storage/CheckAndRepairVolumeAnswer.java | 57 ++++++ .../api/storage/CheckAndRepairVolumeCommand.java | 77 ++++++++ .../subsystem/api/storage/VolumeService.java | 4 + .../com/cloud/vm/VmWorkCheckAndRepairVolume.java | 42 +++++ .../engine/orchestration/VolumeOrchestrator.java | 12 ++ .../storage/volume/VolumeServiceImpl.java | 65 ++++++- .../storage/volume/VolumeServiceTest.java | 106 ++++++++++- .../LibvirtCheckAndRepairVolumeCommandWrapper.java | 192 ++++++++++++++++++++ .../org/apache/cloudstack/utils/qemu/QemuImg.java | 41 +++++ ...virtCheckAndRepairVolumeCommandWrapperTest.java | 98 ++++++++++ .../kvm/storage/LibvirtStoragePoolTest.java | 3 + .../apache/cloudstack/utils/qemu/QemuImgTest.java | 17 ++ .../hypervisor/vmware/resource/VmwareResource.java | 12 +- .../com/cloud/server/ManagementServerImpl.java | 2 + .../main/java/com/cloud/server/StatsCollector.java | 14 +- .../cloud/storage/CheckAndRepairVolumePayload.java | 41 +++++ .../com/cloud/storage/VolumeApiServiceImpl.java | 202 ++++++++++++++++++++- .../cloud/storage/VolumeApiServiceImplTest.java | 165 ++++++++++++++++- .../consoleproxy/ConsoleProxyNoVncClient.java | 24 +-- ui/public/locales/en.json | 2 +- ui/src/config/section/infra/phynetworks.js | 5 + ui/src/config/section/network.js | 2 +- ui/src/views/infra/network/IpRangesTabPublic.vue | 6 + .../src/main/java/com/cloud/utils/StringUtils.java | 22 +++ .../main/java/com/cloud/utils/script/Script.java | 132 ++++++++++++++ 36 files changed, 1488 insertions(+), 40 deletions(-) diff --cc api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 9f959db9262,18d25a0cfc3..416072f1210 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@@ -380,8 -378,8 +380,9 @@@ public class ApiConstants public static final String RECEIVED_BYTES = "receivedbytes"; public static final String RECONNECT = "reconnect"; public static final String RECOVER = "recover"; + public static final String REPAIR = "repair"; public static final String REQUIRES_HVM = "requireshvm"; + public static final String RESOURCE_COUNT = "resourcecount"; public static final String RESOURCE_NAME = "resourcename"; public static final String RESOURCE_TYPE = "resourcetype"; public static final String RESOURCE_TYPE_NAME = "resourcetypename"; diff --cc engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 9911ef26c02,75f652da379..56141d993d5 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@@ -80,9 -82,9 +82,10 @@@ import org.apache.cloudstack.storage.da import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; diff --cc engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java index f4c6df7dd40,55ff2f659af..3a7fcfb6338 --- a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java +++ b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java @@@ -19,15 -19,25 +19,12 @@@ package org.apache.cloudstack.storage.volume; - import com.cloud.storage.Storage; -import com.cloud.agent.api.storage.CheckAndRepairVolumeAnswer; -import com.cloud.agent.api.storage.CheckAndRepairVolumeCommand; -import com.cloud.agent.api.to.StorageFilerTO; -import com.cloud.exception.StorageUnavailableException; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.storage.CheckAndRepairVolumePayload; -import com.cloud.storage.StorageManager; -import com.cloud.storage.StoragePool; --import com.cloud.storage.VolumeVO; --import com.cloud.storage.dao.VolumeDao; --import com.cloud.storage.snapshot.SnapshotManager; ++ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.ExecutionException; - import junit.framework.TestCase; + -import com.cloud.utils.Pair; -import junit.framework.TestCase; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@@ -42,6 -52,6 +39,23 @@@ import org.mockito.Mockito import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; ++import com.cloud.agent.api.storage.CheckAndRepairVolumeAnswer; ++import com.cloud.agent.api.storage.CheckAndRepairVolumeCommand; ++import com.cloud.agent.api.to.StorageFilerTO; ++import com.cloud.exception.StorageUnavailableException; ++import com.cloud.host.HostVO; ++import com.cloud.host.dao.HostDao; ++import com.cloud.storage.CheckAndRepairVolumePayload; ++import com.cloud.storage.Storage; ++import com.cloud.storage.StorageManager; ++import com.cloud.storage.StoragePool; ++import com.cloud.storage.VolumeVO; ++import com.cloud.storage.dao.VolumeDao; ++import com.cloud.storage.snapshot.SnapshotManager; ++import com.cloud.utils.Pair; ++ ++import junit.framework.TestCase; ++ @RunWith(MockitoJUnitRunner.class) public class VolumeServiceTest extends TestCase{ diff --cc server/src/main/java/com/cloud/server/StatsCollector.java index 6623d8dcde8,2467416155a..f9ad0f51966 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@@ -1710,17 -1712,21 +1710,21 @@@ public class StatsCollector extends Man storagePoolStats.put(pool.getId(), (StorageStats)answer); boolean poolNeedsUpdating = false; + long capacityBytes = ((StorageStats)answer).getCapacityBytes(); + long usedBytes = ((StorageStats)answer).getByteUsed(); // Seems like we have dynamically updated the pool size since the prev. size and the current do not match - if (_storagePoolStats.get(poolId) != null && _storagePoolStats.get(poolId).getCapacityBytes() != ((StorageStats)answer).getCapacityBytes()) { - if (((StorageStats)answer).getCapacityBytes() > 0) { - pool.setCapacityBytes(((StorageStats)answer).getCapacityBytes()); + if ((_storagePoolStats.get(poolId) != null && _storagePoolStats.get(poolId).getCapacityBytes() != capacityBytes) + || pool.getCapacityBytes() != capacityBytes) { + if (capacityBytes > 0) { + pool.setCapacityBytes(capacityBytes); poolNeedsUpdating = true; } else { - LOGGER.warn("Not setting capacity bytes, received " + ((StorageStats)answer).getCapacityBytes() + " capacity for pool ID " + poolId); + logger.warn("Not setting capacity bytes, received " + ((StorageStats)answer).getCapacityBytes() + " capacity for pool ID " + poolId); } } - if (pool.getUsedBytes() != ((StorageStats)answer).getByteUsed() && (pool.getStorageProviderName().equalsIgnoreCase(DataStoreProvider.DEFAULT_PRIMARY) || _storageManager.canPoolProvideStorageStats(pool))) { - pool.setUsedBytes(((StorageStats) answer).getByteUsed()); + if (((_storagePoolStats.get(poolId) != null && _storagePoolStats.get(poolId).getByteUsed() != usedBytes) + || pool.getUsedBytes() != usedBytes) && (pool.getStorageProviderName().equalsIgnoreCase(DataStoreProvider.DEFAULT_PRIMARY) || _storageManager.canPoolProvideStorageStats(pool))) { + pool.setUsedBytes(usedBytes); poolNeedsUpdating = true; } if (poolNeedsUpdating) { diff --cc server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 887bbc91d02,2e8b36da446..2e154478d0c --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@@ -1813,10 -1819,161 +1818,161 @@@ public class VolumeApiServiceImpl exten .publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), offeringId, volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplay()); - s_logger.debug(String.format("Volume [%s] has been successfully recovered, thus a new usage event %s has been published.", volume.getUuid(), EventTypes.EVENT_VOLUME_CREATE)); + logger.debug(String.format("Volume [%s] has been successfully recovered, thus a new usage event %s has been published.", volume.getUuid(), EventTypes.EVENT_VOLUME_CREATE)); } + @Override + @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CHECK, eventDescription = "checking volume and repair if needed", async = true) + public Pair<String, String> checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException { + long volumeId = cmd.getId(); + String repair = cmd.getRepair(); + final VolumeVO volume = _volsDao.findById(volumeId); + validationsForCheckVolumeOperation(volume); + + Long vmId = volume.getInstanceId(); + if (vmId != null) { + // serialize VM operation + return handleCheckAndRepairVolumeJob(vmId, volumeId, repair); + } else { + return handleCheckAndRepairVolume(volumeId, repair); + } + } + + private Pair<String, String> handleCheckAndRepairVolume(Long volumeId, String repair) { + CheckAndRepairVolumePayload payload = new CheckAndRepairVolumePayload(repair); + VolumeInfo volumeInfo = volFactory.getVolume(volumeId); + volumeInfo.addPayload(payload); + + Pair<String, String> result = volService.checkAndRepairVolume(volumeInfo); + return result; + } + + private Pair<String, String> handleCheckAndRepairVolumeJob(Long vmId, Long volumeId, String repair) throws ResourceAllocationException { + AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); + if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { + // avoid re-entrance + VmWorkJobVO placeHolder = null; + placeHolder = createPlaceHolderWork(vmId); + try { + Pair<String, String> result = orchestrateCheckAndRepairVolume(volumeId, repair); + return result; + } finally { + _workJobDao.expunge(placeHolder.getId()); + } + } else { + Outcome<Pair> outcome = checkAndRepairVolumeThroughJobQueue(vmId, volumeId, repair); + try { + outcome.get(); + } catch (InterruptedException e) { + throw new RuntimeException("Operation is interrupted", e); + } catch (ExecutionException e) { + throw new RuntimeException("Execution exception--", e); + } + + Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); + if (jobResult != null) { + if (jobResult instanceof ConcurrentOperationException) { + throw (ConcurrentOperationException)jobResult; + } else if (jobResult instanceof ResourceAllocationException) { + throw (ResourceAllocationException)jobResult; + } else if (jobResult instanceof Throwable) { + throw new RuntimeException("Unexpected exception", (Throwable)jobResult); + } + } + + // retrieve the entity url from job result + if (jobResult != null && jobResult instanceof Pair) { + return (Pair<String, String>) jobResult; + } + + return null; + } + } + + protected void validationsForCheckVolumeOperation(VolumeVO volume) { + Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, volume); + + String volumeName = volume.getName(); + Long vmId = volume.getInstanceId(); + if (vmId != null) { + validateVMforCheckVolumeOperation(vmId, volumeName); + } + + if (volume.getState() != Volume.State.Ready) { + throw new InvalidParameterValueException(String.format("Volume: %s is not in Ready state", volumeName)); + } + + HypervisorType hypervisorType = _volsDao.getHypervisorType(volume.getId()); + if (!HypervisorType.KVM.equals(hypervisorType)) { + throw new InvalidParameterValueException(String.format("Check and Repair volumes is supported only for KVM hypervisor")); + } + + if (!Arrays.asList(ImageFormat.QCOW2, ImageFormat.VDI).contains(volume.getFormat())) { + throw new InvalidParameterValueException("Volume format is not supported for checking and repair"); + } + } + + private void validateVMforCheckVolumeOperation(Long vmId, String volumeName) { + Account caller = CallContext.current().getCallingAccount(); + UserVmVO vm = _userVmDao.findById(vmId); + if (vm == null) { + throw new InvalidParameterValueException(String.format("VM not found, please check the VM to which this volume %s is attached", volumeName)); + } + + _accountMgr.checkAccess(caller, null, true, vm); + + if (vm.getState() != State.Stopped) { + throw new InvalidParameterValueException(String.format("VM to which the volume %s is attached should be in stopped state", volumeName)); + } + } + + private Pair<String, String> orchestrateCheckAndRepairVolume(Long volumeId, String repair) { + + VolumeInfo volume = volFactory.getVolume(volumeId); + + if (volume == null) { + throw new InvalidParameterValueException("Checking volume and repairing failed due to volume:" + volumeId + " doesn't exist"); + } + + CheckAndRepairVolumePayload payload = new CheckAndRepairVolumePayload(repair); + volume.addPayload(payload); + + return volService.checkAndRepairVolume(volume); + } + + public Outcome<Pair> checkAndRepairVolumeThroughJobQueue(final Long vmId, final Long volumeId, String repair) { + + final CallContext context = CallContext.current(); + final User callingUser = context.getCallingUser(); + final Account callingAccount = context.getCallingAccount(); + + final VMInstanceVO vm = _vmInstanceDao.findById(vmId); + + VmWorkJobVO workJob = new VmWorkJobVO(context.getContextId()); + + workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); + workJob.setCmd(VmWorkCheckAndRepairVolume.class.getName()); + + workJob.setAccountId(callingAccount.getId()); + workJob.setUserId(callingUser.getId()); + workJob.setStep(VmWorkJobVO.Step.Starting); + workJob.setVmType(VirtualMachine.Type.Instance); + workJob.setVmInstanceId(vm.getId()); + workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); + + // save work context info (there are some duplications) + VmWorkCheckAndRepairVolume workInfo = new VmWorkCheckAndRepairVolume(callingUser.getId(), callingAccount.getId(), vm.getId(), + VolumeApiServiceImpl.VM_WORK_JOB_HANDLER, volumeId, repair); + workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); + + _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + + AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); + + return new VmJobCheckAndRepairVolumeOutcome(workJob); + } @Override @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CHANGE_DISK_OFFERING, eventDescription = "Changing disk offering of a volume") diff --cc services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java index 38e5a3d4104,cfa62114c3d..e89984b1749 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java @@@ -16,13 -16,12 +16,6 @@@ // under the License. package com.cloud.consoleproxy; --import org.apache.commons.lang3.StringUtils; - import org.apache.logging.log4j.Logger; - import org.apache.logging.log4j.LogManager; -import org.apache.log4j.Logger; --import org.eclipse.jetty.websocket.api.Session; --import org.eclipse.jetty.websocket.api.WebSocketException; --import org.eclipse.jetty.websocket.api.extensions.Frame; -- import java.awt.Image; import java.io.IOException; import java.net.URI; @@@ -30,6 -29,6 +23,13 @@@ import java.nio.ByteBuffer import java.nio.charset.StandardCharsets; import java.util.List; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.logging.log4j.LogManager; ++import org.apache.logging.log4j.Logger; ++import org.eclipse.jetty.websocket.api.Session; ++import org.eclipse.jetty.websocket.api.WebSocketException; ++import org.eclipse.jetty.websocket.api.extensions.Frame; ++ import com.cloud.consoleproxy.vnc.NoVncClient; public class ConsoleProxyNoVncClient implements ConsoleProxyClient { @@@ -140,10 -134,15 +135,15 @@@ connectionAlive = false; } } + try { + Thread.sleep(1); + } catch (InterruptedException e) { - s_logger.error("Error on sleep for vnc sessions", e); ++ logger.error("Error on sleep for vnc sessions", e); + } } - s_logger.info(String.format("Connection with client [%s] is dead.", clientId)); + logger.info(String.format("Connection with client [%s] is dead.", clientId)); } catch (IOException e) { - s_logger.error("Error on VNC client", e); + logger.error("Error on VNC client", e); } }
