rp- commented on a change in pull request #4994:
URL: https://github.com/apache/cloudstack/pull/4994#discussion_r629885536
##########
File path:
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,566 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiCallChecker;
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger =
Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a
better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new
StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
+
+ p.waitFor();
+ p.destroy();
+ return result;
+ } catch (IOException | InterruptedException exc) {
+ throw new CloudRuntimeException("Unable to run hostname command.");
+ }
+ }
+
+ private void handleLinstorApiAnswers(ApiCallRcList answers, String
excMessage) {
+ if (ApiCallChecker.hasError(answers)) {
+ for (ApiCallRc answer : answers) {
+ s_logger.error(answer.getMessage());
+ }
+ throw new CloudRuntimeException(excMessage);
+ }
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
+ s_logger.debug("Linstor getStoragePool: " + uuid + " -> " +
refreshInfo);
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMPhysicalDisk getPhysicalDisk(String name, KVMStoragePool pool)
+ {
+ s_logger.debug("Linstor: getPhysicalDisk for " + name);
+ if (name == null) {
+ return null;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try {
+ final String rscName = getLinstorRscName(name);
+
+ List<VolumeDefinition> volumeDefs =
api.volumeDefinitionList(rscName, null, null);
+ final long size = volumeDefs.isEmpty() ? 0 :
volumeDefs.get(0).getSizeKib() * 1024;
+
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ kvmDisk.setSize(size);
+ kvmDisk.setVirtualSize(size);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes for " + rscName);
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public KVMStoragePool createStoragePool(String name, String host, int
port, String path, String userInfo,
+ Storage.StoragePoolType type)
+ {
+ s_logger.debug(String.format(
+ "Linstor createStoragePool: name: '%s', host: '%s', path: %s,
userinfo: %s", name, host, path, userInfo));
+ LinstorStoragePool storagePool = new LinstorStoragePool(name, host,
port, userInfo, type, this);
+
+ MapStorageUuidToStoragePool.put(name, storagePool);
+
+ return storagePool;
+ }
+
+ @Override
+ public boolean deleteStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.remove(uuid) != null;
+ }
+
+ @Override
+ public boolean deleteStoragePool(KVMStoragePool pool) {
+ return deleteStoragePool(pool.getUuid());
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool
pool, QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType
provisioningType, long size)
+ {
+ LinstorStoragePool lpool = (LinstorStoragePool) pool;
+ final DevelopersApi api = getLinstorAPI(pool);
+ ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn();
+ final String rscName = getLinstorRscName(name);
+ rgSpawn.setResourceDefinitionName(rscName);
+ rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB
+ try {
+ s_logger.debug("Linstor: Spawn resource " + rscName);
+ ApiCallRcList answers =
api.resourceGroupSpawn(lpool.getResourceGroup(), rgSpawn);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to spawn
resource.");
+
+ // query linstor for the device path
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ s_logger.info("Linstor: Created drbd device: " + devPath);
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes.");
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool,
Map<String, String> details)
+ {
+ s_logger.debug(String.format("Linstor: connectPhysicalDisk %s:%s ->
%s", pool.getUuid(), volumePath, details));
+ if (volumePath == null) {
+ s_logger.warn("volumePath is null, ignoring");
+ return false;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try
+ {
+ final String rscName = getLinstorRscName(volumePath);
+ final String localNode = getHostname();
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(localNode),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+
+ if (resources.isEmpty()) {
+ s_logger.info("Linstor: resource not created yet: " + rscName);
+ ResourceCreate rscCreate = new ResourceCreate();
+ Resource rsc = new Resource();
+ rsc.setNodeName(localNode);
+ rscCreate.setResource(rsc);
+ ApiCallRcList answers = api.resourceCreate(rscName,
Collections.singletonList(rscCreate));
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Could not create resource " + rscName);
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+ // now diskful resource should be deployed
+ }
+ // keep diskless, maybe add an option, but I'm concerned about out
of space situations
+// else {
+// final ResourceWithVolumes rsc = resources.get(0);
+// if (rsc.getFlags() != null &&
rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS)) {
+// ApiCallRcList answers =
api.resourceToggleDiskful(rscName, localNode);
+// if (ApiCallChecker.hasError(answers)) {
+// s_logger.error("Unable to toggle disk on " +
rscName);
+// throw new
CloudRuntimeException(answers.get(0).getMessage());
+// }
+// }
+// }
+
+ // allow 2 primaries for live migration, should be removed by
disconnect on the other end
+ ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+ Properties props = new Properties();
+ props.put("DrbdOptions/Net/allow-two-primaries", "yes");
+ rdm.setOverrideProps(props);
+ ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Unable to set 'allow-two-primaries' on " +
rscName);
+ throw new CloudRuntimeException(answers.get(0).getMessage());
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc.getMessage());
+ }
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool
pool)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk " + pool.getUuid() +
":" + volumePath);
+ // TODO remove diskless/diskful on node
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(Map<String, String>
volumeToDisconnect)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk map");
+ // TODO remove diskless/diskful on node
+ return true;
+ }
+
+ private Optional<ResourceWithVolumes> getResourceByPath(final
List<ResourceWithVolumes> resources, String path) {
+ return resources.stream()
+ .filter(rsc -> rsc.getVolumes().stream()
+ .anyMatch(v -> v.getDevicePath().equals(path)))
+ .findFirst();
+ }
+
+ /**
+ * disconnectPhysicalDiskByPath is called after e.g. a live migration.
+ * The problem is we have no idea just from the path to which
linstor-controller
+ * this resource would belong to. But as it should be highly unlikely that
someone
+ * uses more than one linstor-controller to manage resource on the same
kvm host.
+ * We will just take the first stored storagepool.
+ */
+ @Override
+ public boolean disconnectPhysicalDiskByPath(String localPath)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDiskByPath " + localPath);
+ // get first storage pool from the map, as we don't now any better:
+ if (!MapStorageUuidToStoragePool.isEmpty())
+ {
+ String firstKey =
MapStorageUuidToStoragePool.keySet().stream().findFirst().get();
+ final KVMStoragePool pool =
MapStorageUuidToStoragePool.get(firstKey);
+
+ s_logger.debug("Linstor: Using storpool: " + pool.getUuid());
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try
+ {
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(getHostname()),
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ Optional<ResourceWithVolumes> rsc =
getResourceByPath(resources, localPath);
+
+ if (rsc.isPresent())
+ {
+ ResourceDefinitionModify rdm = new
ResourceDefinitionModify();
+
rdm.deleteProps(Collections.singletonList("DrbdOptions/Net/allow-two-primaries"));
+ ApiCallRcList answers =
api.resourceDefinitionModify(rsc.get().getName(), rdm);
+ if (ApiCallChecker.hasError(answers))
+ {
+ s_logger.error("Failed to remove 'allow-two-primaries'
on " + rsc.get().getName());
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+
+ return true;
+ }
+ s_logger.warn("Linstor: Couldn't find resource for this path:
" + localPath);
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc.getMessage());
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean deletePhysicalDisk(String name, KVMStoragePool pool,
Storage.ImageFormat format)
+ {
+ s_logger.debug("Linstor: deletePhysicalDisk " + name);
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try {
+ final String rscName = getLinstorRscName(name);
+ s_logger.debug("Linstor: delete resource definition " + rscName);
+ ApiCallRcList answers = api.resourceDefinitionDelete(rscName);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to delete
resource definition " + rscName);
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ return true;
+ }
+
+ @Override
+ public KVMPhysicalDisk createDiskFromTemplate(
+ KVMPhysicalDisk template,
+ String name,
+ QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType provisioningType,
+ long size,
+ KVMStoragePool destPool,
+ int timeout)
+ {
+ s_logger.info("Linstor: createDiskFromTemplate");
+ // if source is linstor lvm-thin snapshot we could snapshot and
restore from, but do plain copy for now
Review comment:
well it kinda is explained in the comment.
Linstor only supports snapshots from thin-lvm and ZFS backends, but with ZFS
there exists a parent-child relation between snapshots. This means it could
happen that you wan't to delete an old volume which once was your base of a
template, but you can't deleted it, because you have several snapshots from it.
(only way is to reverse the parent-child relation)
So from this experience we decided in other plugins to do rather plain
volume copies because none of those problems would arise.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
