GutoVeronezi commented on a change in pull request #4994:
URL: https://github.com/apache/cloudstack/pull/4994#discussion_r629444059
##########
File path:
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,566 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiCallChecker;
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger =
Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a
better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new
StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
Review comment:
As `result` is first used here, you could declare it here too.
##########
File path:
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,566 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiCallChecker;
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger =
Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a
better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new
StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
+
+ p.waitFor();
+ p.destroy();
+ return result;
+ } catch (IOException | InterruptedException exc) {
+ throw new CloudRuntimeException("Unable to run hostname command.");
Review comment:
You can pass the exception as parameter of `CloudRuntimeException`.
##########
File path:
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,566 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiCallChecker;
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger =
Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a
better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new
StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
+
+ p.waitFor();
+ p.destroy();
+ return result;
+ } catch (IOException | InterruptedException exc) {
+ throw new CloudRuntimeException("Unable to run hostname command.");
+ }
+ }
+
+ private void handleLinstorApiAnswers(ApiCallRcList answers, String
excMessage) {
+ if (ApiCallChecker.hasError(answers)) {
+ for (ApiCallRc answer : answers) {
+ s_logger.error(answer.getMessage());
+ }
+ throw new CloudRuntimeException(excMessage);
+ }
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
+ s_logger.debug("Linstor getStoragePool: " + uuid + " -> " +
refreshInfo);
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMPhysicalDisk getPhysicalDisk(String name, KVMStoragePool pool)
+ {
+ s_logger.debug("Linstor: getPhysicalDisk for " + name);
+ if (name == null) {
+ return null;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try {
+ final String rscName = getLinstorRscName(name);
+
+ List<VolumeDefinition> volumeDefs =
api.volumeDefinitionList(rscName, null, null);
+ final long size = volumeDefs.isEmpty() ? 0 :
volumeDefs.get(0).getSizeKib() * 1024;
+
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ kvmDisk.setSize(size);
+ kvmDisk.setVirtualSize(size);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes for " + rscName);
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public KVMStoragePool createStoragePool(String name, String host, int
port, String path, String userInfo,
+ Storage.StoragePoolType type)
+ {
+ s_logger.debug(String.format(
+ "Linstor createStoragePool: name: '%s', host: '%s', path: %s,
userinfo: %s", name, host, path, userInfo));
+ LinstorStoragePool storagePool = new LinstorStoragePool(name, host,
port, userInfo, type, this);
+
+ MapStorageUuidToStoragePool.put(name, storagePool);
+
+ return storagePool;
+ }
+
+ @Override
+ public boolean deleteStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.remove(uuid) != null;
+ }
+
+ @Override
+ public boolean deleteStoragePool(KVMStoragePool pool) {
+ return deleteStoragePool(pool.getUuid());
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool
pool, QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType
provisioningType, long size)
+ {
+ LinstorStoragePool lpool = (LinstorStoragePool) pool;
+ final DevelopersApi api = getLinstorAPI(pool);
+ ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn();
+ final String rscName = getLinstorRscName(name);
+ rgSpawn.setResourceDefinitionName(rscName);
+ rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB
+ try {
+ s_logger.debug("Linstor: Spawn resource " + rscName);
+ ApiCallRcList answers =
api.resourceGroupSpawn(lpool.getResourceGroup(), rgSpawn);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to spawn
resource.");
+
+ // query linstor for the device path
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ s_logger.info("Linstor: Created drbd device: " + devPath);
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes.");
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool,
Map<String, String> details)
+ {
+ s_logger.debug(String.format("Linstor: connectPhysicalDisk %s:%s ->
%s", pool.getUuid(), volumePath, details));
+ if (volumePath == null) {
+ s_logger.warn("volumePath is null, ignoring");
+ return false;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try
+ {
+ final String rscName = getLinstorRscName(volumePath);
+ final String localNode = getHostname();
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(localNode),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+
+ if (resources.isEmpty()) {
+ s_logger.info("Linstor: resource not created yet: " + rscName);
+ ResourceCreate rscCreate = new ResourceCreate();
+ Resource rsc = new Resource();
+ rsc.setNodeName(localNode);
+ rscCreate.setResource(rsc);
+ ApiCallRcList answers = api.resourceCreate(rscName,
Collections.singletonList(rscCreate));
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Could not create resource " + rscName);
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+ // now diskful resource should be deployed
+ }
+ // keep diskless, maybe add an option, but I'm concerned about out
of space situations
+// else {
+// final ResourceWithVolumes rsc = resources.get(0);
+// if (rsc.getFlags() != null &&
rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS)) {
+// ApiCallRcList answers =
api.resourceToggleDiskful(rscName, localNode);
+// if (ApiCallChecker.hasError(answers)) {
+// s_logger.error("Unable to toggle disk on " +
rscName);
+// throw new
CloudRuntimeException(answers.get(0).getMessage());
+// }
+// }
+// }
+
+ // allow 2 primaries for live migration, should be removed by
disconnect on the other end
+ ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+ Properties props = new Properties();
+ props.put("DrbdOptions/Net/allow-two-primaries", "yes");
+ rdm.setOverrideProps(props);
+ ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Unable to set 'allow-two-primaries' on " +
rscName);
+ throw new CloudRuntimeException(answers.get(0).getMessage());
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc.getMessage());
+ }
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool
pool)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk " + pool.getUuid() +
":" + volumePath);
+ // TODO remove diskless/diskful on node
Review comment:
Would be better to register TODO in issues instead of comments in the
code.
##########
File path:
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,566 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiCallChecker;
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger =
Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a
better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new
StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
+
+ p.waitFor();
+ p.destroy();
+ return result;
+ } catch (IOException | InterruptedException exc) {
+ throw new CloudRuntimeException("Unable to run hostname command.");
+ }
+ }
+
+ private void handleLinstorApiAnswers(ApiCallRcList answers, String
excMessage) {
+ if (ApiCallChecker.hasError(answers)) {
+ for (ApiCallRc answer : answers) {
+ s_logger.error(answer.getMessage());
+ }
+ throw new CloudRuntimeException(excMessage);
+ }
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
+ s_logger.debug("Linstor getStoragePool: " + uuid + " -> " +
refreshInfo);
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMPhysicalDisk getPhysicalDisk(String name, KVMStoragePool pool)
+ {
+ s_logger.debug("Linstor: getPhysicalDisk for " + name);
+ if (name == null) {
+ return null;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try {
+ final String rscName = getLinstorRscName(name);
+
+ List<VolumeDefinition> volumeDefs =
api.volumeDefinitionList(rscName, null, null);
+ final long size = volumeDefs.isEmpty() ? 0 :
volumeDefs.get(0).getSizeKib() * 1024;
+
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ kvmDisk.setSize(size);
+ kvmDisk.setVirtualSize(size);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes for " + rscName);
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public KVMStoragePool createStoragePool(String name, String host, int
port, String path, String userInfo,
+ Storage.StoragePoolType type)
+ {
+ s_logger.debug(String.format(
+ "Linstor createStoragePool: name: '%s', host: '%s', path: %s,
userinfo: %s", name, host, path, userInfo));
+ LinstorStoragePool storagePool = new LinstorStoragePool(name, host,
port, userInfo, type, this);
+
+ MapStorageUuidToStoragePool.put(name, storagePool);
+
+ return storagePool;
+ }
+
+ @Override
+ public boolean deleteStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.remove(uuid) != null;
+ }
+
+ @Override
+ public boolean deleteStoragePool(KVMStoragePool pool) {
+ return deleteStoragePool(pool.getUuid());
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool
pool, QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType
provisioningType, long size)
+ {
+ LinstorStoragePool lpool = (LinstorStoragePool) pool;
+ final DevelopersApi api = getLinstorAPI(pool);
+ ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn();
+ final String rscName = getLinstorRscName(name);
+ rgSpawn.setResourceDefinitionName(rscName);
+ rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB
+ try {
+ s_logger.debug("Linstor: Spawn resource " + rscName);
+ ApiCallRcList answers =
api.resourceGroupSpawn(lpool.getResourceGroup(), rgSpawn);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to spawn
resource.");
+
+ // query linstor for the device path
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ s_logger.info("Linstor: Created drbd device: " + devPath);
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes.");
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool,
Map<String, String> details)
+ {
+ s_logger.debug(String.format("Linstor: connectPhysicalDisk %s:%s ->
%s", pool.getUuid(), volumePath, details));
+ if (volumePath == null) {
+ s_logger.warn("volumePath is null, ignoring");
+ return false;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try
+ {
+ final String rscName = getLinstorRscName(volumePath);
+ final String localNode = getHostname();
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(localNode),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+
+ if (resources.isEmpty()) {
+ s_logger.info("Linstor: resource not created yet: " + rscName);
+ ResourceCreate rscCreate = new ResourceCreate();
+ Resource rsc = new Resource();
+ rsc.setNodeName(localNode);
+ rscCreate.setResource(rsc);
+ ApiCallRcList answers = api.resourceCreate(rscName,
Collections.singletonList(rscCreate));
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Could not create resource " + rscName);
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+ // now diskful resource should be deployed
+ }
+ // keep diskless, maybe add an option, but I'm concerned about out
of space situations
+// else {
+// final ResourceWithVolumes rsc = resources.get(0);
+// if (rsc.getFlags() != null &&
rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS)) {
+// ApiCallRcList answers =
api.resourceToggleDiskful(rscName, localNode);
+// if (ApiCallChecker.hasError(answers)) {
+// s_logger.error("Unable to toggle disk on " +
rscName);
+// throw new
CloudRuntimeException(answers.get(0).getMessage());
+// }
+// }
+// }
+
+ // allow 2 primaries for live migration, should be removed by
disconnect on the other end
+ ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+ Properties props = new Properties();
+ props.put("DrbdOptions/Net/allow-two-primaries", "yes");
+ rdm.setOverrideProps(props);
+ ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Unable to set 'allow-two-primaries' on " +
rscName);
+ throw new CloudRuntimeException(answers.get(0).getMessage());
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc.getMessage());
+ }
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool
pool)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk " + pool.getUuid() +
":" + volumePath);
+ // TODO remove diskless/diskful on node
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(Map<String, String>
volumeToDisconnect)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk map");
+ // TODO remove diskless/diskful on node
+ return true;
+ }
+
+ private Optional<ResourceWithVolumes> getResourceByPath(final
List<ResourceWithVolumes> resources, String path) {
+ return resources.stream()
+ .filter(rsc -> rsc.getVolumes().stream()
+ .anyMatch(v -> v.getDevicePath().equals(path)))
+ .findFirst();
+ }
+
+ /**
+ * disconnectPhysicalDiskByPath is called after e.g. a live migration.
+ * The problem is we have no idea just from the path to which
linstor-controller
+ * this resource would belong to. But as it should be highly unlikely that
someone
+ * uses more than one linstor-controller to manage resource on the same
kvm host.
+ * We will just take the first stored storagepool.
+ */
+ @Override
+ public boolean disconnectPhysicalDiskByPath(String localPath)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDiskByPath " + localPath);
+ // get first storage pool from the map, as we don't now any better:
+ if (!MapStorageUuidToStoragePool.isEmpty())
+ {
+ String firstKey =
MapStorageUuidToStoragePool.keySet().stream().findFirst().get();
+ final KVMStoragePool pool =
MapStorageUuidToStoragePool.get(firstKey);
+
+ s_logger.debug("Linstor: Using storpool: " + pool.getUuid());
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try
+ {
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(getHostname()),
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ Optional<ResourceWithVolumes> rsc =
getResourceByPath(resources, localPath);
+
+ if (rsc.isPresent())
+ {
+ ResourceDefinitionModify rdm = new
ResourceDefinitionModify();
+
rdm.deleteProps(Collections.singletonList("DrbdOptions/Net/allow-two-primaries"));
+ ApiCallRcList answers =
api.resourceDefinitionModify(rsc.get().getName(), rdm);
+ if (ApiCallChecker.hasError(answers))
+ {
+ s_logger.error("Failed to remove 'allow-two-primaries'
on " + rsc.get().getName());
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+
+ return true;
+ }
+ s_logger.warn("Linstor: Couldn't find resource for this path:
" + localPath);
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc.getMessage());
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean deletePhysicalDisk(String name, KVMStoragePool pool,
Storage.ImageFormat format)
+ {
+ s_logger.debug("Linstor: deletePhysicalDisk " + name);
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try {
+ final String rscName = getLinstorRscName(name);
+ s_logger.debug("Linstor: delete resource definition " + rscName);
+ ApiCallRcList answers = api.resourceDefinitionDelete(rscName);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to delete
resource definition " + rscName);
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ return true;
+ }
+
+ @Override
+ public KVMPhysicalDisk createDiskFromTemplate(
+ KVMPhysicalDisk template,
+ String name,
+ QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType provisioningType,
+ long size,
+ KVMStoragePool destPool,
+ int timeout)
+ {
+ s_logger.info("Linstor: createDiskFromTemplate");
+ // if source is linstor lvm-thin snapshot we could snapshot and
restore from, but do plain copy for now
+ return copyPhysicalDisk(template, name, destPool, timeout);
+ }
+
+ @Override
+ public List<KVMPhysicalDisk> listPhysicalDisks(String storagePoolUuid,
KVMStoragePool pool)
+ {
+ throw new UnsupportedOperationException("Listing disks is not
supported for this configuration.");
+ }
+
+ @Override
+ public KVMPhysicalDisk createTemplateFromDisk(
+ KVMPhysicalDisk disk,
+ String name,
+ QemuImg.PhysicalDiskFormat format,
+ long size,
+ KVMStoragePool destPool)
+ {
+ throw new UnsupportedOperationException("Copying a template from disk
is not supported in this configuration.");
+ }
+
+ @Override
+ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name,
KVMStoragePool destPools, int timeout)
+ {
+ s_logger.debug("Linstor: copyPhysicalDisk");
+ final QemuImg.PhysicalDiskFormat sourceFormat = disk.getFormat();
+ final String sourcePath = disk.getPath();
+ final QemuImg qemu = new QemuImg(timeout);
+
+ final QemuImgFile srcFile = new QemuImgFile(sourcePath, sourceFormat);
+
+ // create linstor resource
Review comment:
The method name is clear and auto explanatory, I think we do not need
this comment.
##########
File path:
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,566 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiCallChecker;
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger =
Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a
better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new
StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
+
+ p.waitFor();
+ p.destroy();
+ return result;
+ } catch (IOException | InterruptedException exc) {
+ throw new CloudRuntimeException("Unable to run hostname command.");
+ }
+ }
+
+ private void handleLinstorApiAnswers(ApiCallRcList answers, String
excMessage) {
+ if (ApiCallChecker.hasError(answers)) {
+ for (ApiCallRc answer : answers) {
+ s_logger.error(answer.getMessage());
+ }
+ throw new CloudRuntimeException(excMessage);
+ }
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
+ s_logger.debug("Linstor getStoragePool: " + uuid + " -> " +
refreshInfo);
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMPhysicalDisk getPhysicalDisk(String name, KVMStoragePool pool)
+ {
+ s_logger.debug("Linstor: getPhysicalDisk for " + name);
+ if (name == null) {
+ return null;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try {
+ final String rscName = getLinstorRscName(name);
+
+ List<VolumeDefinition> volumeDefs =
api.volumeDefinitionList(rscName, null, null);
+ final long size = volumeDefs.isEmpty() ? 0 :
volumeDefs.get(0).getSizeKib() * 1024;
+
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ kvmDisk.setSize(size);
+ kvmDisk.setVirtualSize(size);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes for " + rscName);
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public KVMStoragePool createStoragePool(String name, String host, int
port, String path, String userInfo,
+ Storage.StoragePoolType type)
+ {
+ s_logger.debug(String.format(
+ "Linstor createStoragePool: name: '%s', host: '%s', path: %s,
userinfo: %s", name, host, path, userInfo));
+ LinstorStoragePool storagePool = new LinstorStoragePool(name, host,
port, userInfo, type, this);
+
+ MapStorageUuidToStoragePool.put(name, storagePool);
+
+ return storagePool;
+ }
+
+ @Override
+ public boolean deleteStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.remove(uuid) != null;
+ }
+
+ @Override
+ public boolean deleteStoragePool(KVMStoragePool pool) {
+ return deleteStoragePool(pool.getUuid());
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool
pool, QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType
provisioningType, long size)
+ {
+ LinstorStoragePool lpool = (LinstorStoragePool) pool;
+ final DevelopersApi api = getLinstorAPI(pool);
+ ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn();
+ final String rscName = getLinstorRscName(name);
+ rgSpawn.setResourceDefinitionName(rscName);
+ rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB
+ try {
+ s_logger.debug("Linstor: Spawn resource " + rscName);
+ ApiCallRcList answers =
api.resourceGroupSpawn(lpool.getResourceGroup(), rgSpawn);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to spawn
resource.");
+
+ // query linstor for the device path
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ s_logger.info("Linstor: Created drbd device: " + devPath);
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes.");
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool,
Map<String, String> details)
+ {
+ s_logger.debug(String.format("Linstor: connectPhysicalDisk %s:%s ->
%s", pool.getUuid(), volumePath, details));
+ if (volumePath == null) {
+ s_logger.warn("volumePath is null, ignoring");
+ return false;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try
+ {
+ final String rscName = getLinstorRscName(volumePath);
+ final String localNode = getHostname();
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(localNode),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+
+ if (resources.isEmpty()) {
+ s_logger.info("Linstor: resource not created yet: " + rscName);
+ ResourceCreate rscCreate = new ResourceCreate();
+ Resource rsc = new Resource();
+ rsc.setNodeName(localNode);
+ rscCreate.setResource(rsc);
+ ApiCallRcList answers = api.resourceCreate(rscName,
Collections.singletonList(rscCreate));
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Could not create resource " + rscName);
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+ // now diskful resource should be deployed
+ }
+ // keep diskless, maybe add an option, but I'm concerned about out
of space situations
+// else {
+// final ResourceWithVolumes rsc = resources.get(0);
+// if (rsc.getFlags() != null &&
rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS)) {
+// ApiCallRcList answers =
api.resourceToggleDiskful(rscName, localNode);
+// if (ApiCallChecker.hasError(answers)) {
+// s_logger.error("Unable to toggle disk on " +
rscName);
+// throw new
CloudRuntimeException(answers.get(0).getMessage());
+// }
+// }
+// }
+
+ // allow 2 primaries for live migration, should be removed by
disconnect on the other end
+ ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+ Properties props = new Properties();
+ props.put("DrbdOptions/Net/allow-two-primaries", "yes");
+ rdm.setOverrideProps(props);
+ ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Unable to set 'allow-two-primaries' on " +
rscName);
+ throw new CloudRuntimeException(answers.get(0).getMessage());
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc.getMessage());
+ }
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool
pool)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk " + pool.getUuid() +
":" + volumePath);
+ // TODO remove diskless/diskful on node
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(Map<String, String>
volumeToDisconnect)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk map");
+ // TODO remove diskless/diskful on node
+ return true;
+ }
+
+ private Optional<ResourceWithVolumes> getResourceByPath(final
List<ResourceWithVolumes> resources, String path) {
+ return resources.stream()
+ .filter(rsc -> rsc.getVolumes().stream()
+ .anyMatch(v -> v.getDevicePath().equals(path)))
+ .findFirst();
+ }
+
+ /**
+ * disconnectPhysicalDiskByPath is called after e.g. a live migration.
+ * The problem is we have no idea just from the path to which
linstor-controller
+ * this resource would belong to. But as it should be highly unlikely that
someone
+ * uses more than one linstor-controller to manage resource on the same
kvm host.
+ * We will just take the first stored storagepool.
+ */
+ @Override
+ public boolean disconnectPhysicalDiskByPath(String localPath)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDiskByPath " + localPath);
+ // get first storage pool from the map, as we don't now any better:
+ if (!MapStorageUuidToStoragePool.isEmpty())
+ {
+ String firstKey =
MapStorageUuidToStoragePool.keySet().stream().findFirst().get();
+ final KVMStoragePool pool =
MapStorageUuidToStoragePool.get(firstKey);
+
+ s_logger.debug("Linstor: Using storpool: " + pool.getUuid());
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try
+ {
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(getHostname()),
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ Optional<ResourceWithVolumes> rsc =
getResourceByPath(resources, localPath);
+
+ if (rsc.isPresent())
+ {
+ ResourceDefinitionModify rdm = new
ResourceDefinitionModify();
+
rdm.deleteProps(Collections.singletonList("DrbdOptions/Net/allow-two-primaries"));
+ ApiCallRcList answers =
api.resourceDefinitionModify(rsc.get().getName(), rdm);
+ if (ApiCallChecker.hasError(answers))
+ {
+ s_logger.error("Failed to remove 'allow-two-primaries'
on " + rsc.get().getName());
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+
+ return true;
+ }
+ s_logger.warn("Linstor: Couldn't find resource for this path:
" + localPath);
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc.getMessage());
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean deletePhysicalDisk(String name, KVMStoragePool pool,
Storage.ImageFormat format)
+ {
+ s_logger.debug("Linstor: deletePhysicalDisk " + name);
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try {
+ final String rscName = getLinstorRscName(name);
+ s_logger.debug("Linstor: delete resource definition " + rscName);
+ ApiCallRcList answers = api.resourceDefinitionDelete(rscName);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to delete
resource definition " + rscName);
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ return true;
+ }
+
+ @Override
+ public KVMPhysicalDisk createDiskFromTemplate(
+ KVMPhysicalDisk template,
+ String name,
+ QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType provisioningType,
+ long size,
+ KVMStoragePool destPool,
+ int timeout)
+ {
+ s_logger.info("Linstor: createDiskFromTemplate");
+ // if source is linstor lvm-thin snapshot we could snapshot and
restore from, but do plain copy for now
+ return copyPhysicalDisk(template, name, destPool, timeout);
+ }
+
+ @Override
+ public List<KVMPhysicalDisk> listPhysicalDisks(String storagePoolUuid,
KVMStoragePool pool)
+ {
+ throw new UnsupportedOperationException("Listing disks is not
supported for this configuration.");
+ }
+
+ @Override
+ public KVMPhysicalDisk createTemplateFromDisk(
+ KVMPhysicalDisk disk,
+ String name,
+ QemuImg.PhysicalDiskFormat format,
+ long size,
+ KVMStoragePool destPool)
+ {
+ throw new UnsupportedOperationException("Copying a template from disk
is not supported in this configuration.");
+ }
+
+ @Override
+ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name,
KVMStoragePool destPools, int timeout)
+ {
+ s_logger.debug("Linstor: copyPhysicalDisk");
+ final QemuImg.PhysicalDiskFormat sourceFormat = disk.getFormat();
+ final String sourcePath = disk.getPath();
+ final QemuImg qemu = new QemuImg(timeout);
+
+ final QemuImgFile srcFile = new QemuImgFile(sourcePath, sourceFormat);
+
+ // create linstor resource
+ final KVMPhysicalDisk dstDisk = destPools.createPhysicalDisk(
+ name, QemuImg.PhysicalDiskFormat.RAW,
Storage.ProvisioningType.FAT, disk.getVirtualSize());
+
+ final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath());
+ destFile.setFormat(dstDisk.getFormat());
+ destFile.setSize(disk.getVirtualSize());
+
+ try {
+ qemu.convert(srcFile, destFile);
+ } catch (QemuImgException | LibvirtException e) {
+ s_logger.error("Failed to copy " + srcFile.getFileName() + " to " +
+ destFile.getFileName() + " the error was: " + e.getMessage());
Review comment:
You can pass the exception as parameter here.
##########
File path:
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,566 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiCallChecker;
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger =
Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a
better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new
StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
+
+ p.waitFor();
+ p.destroy();
+ return result;
+ } catch (IOException | InterruptedException exc) {
+ throw new CloudRuntimeException("Unable to run hostname command.");
+ }
+ }
+
+ private void handleLinstorApiAnswers(ApiCallRcList answers, String
excMessage) {
+ if (ApiCallChecker.hasError(answers)) {
+ for (ApiCallRc answer : answers) {
+ s_logger.error(answer.getMessage());
+ }
+ throw new CloudRuntimeException(excMessage);
+ }
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
+ s_logger.debug("Linstor getStoragePool: " + uuid + " -> " +
refreshInfo);
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMPhysicalDisk getPhysicalDisk(String name, KVMStoragePool pool)
+ {
+ s_logger.debug("Linstor: getPhysicalDisk for " + name);
+ if (name == null) {
+ return null;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try {
+ final String rscName = getLinstorRscName(name);
+
+ List<VolumeDefinition> volumeDefs =
api.volumeDefinitionList(rscName, null, null);
+ final long size = volumeDefs.isEmpty() ? 0 :
volumeDefs.get(0).getSizeKib() * 1024;
+
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ kvmDisk.setSize(size);
+ kvmDisk.setVirtualSize(size);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes for " + rscName);
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public KVMStoragePool createStoragePool(String name, String host, int
port, String path, String userInfo,
+ Storage.StoragePoolType type)
+ {
+ s_logger.debug(String.format(
+ "Linstor createStoragePool: name: '%s', host: '%s', path: %s,
userinfo: %s", name, host, path, userInfo));
+ LinstorStoragePool storagePool = new LinstorStoragePool(name, host,
port, userInfo, type, this);
+
+ MapStorageUuidToStoragePool.put(name, storagePool);
+
+ return storagePool;
+ }
+
+ @Override
+ public boolean deleteStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.remove(uuid) != null;
+ }
+
+ @Override
+ public boolean deleteStoragePool(KVMStoragePool pool) {
+ return deleteStoragePool(pool.getUuid());
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool
pool, QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType
provisioningType, long size)
+ {
+ LinstorStoragePool lpool = (LinstorStoragePool) pool;
+ final DevelopersApi api = getLinstorAPI(pool);
+ ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn();
+ final String rscName = getLinstorRscName(name);
+ rgSpawn.setResourceDefinitionName(rscName);
+ rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB
+ try {
+ s_logger.debug("Linstor: Spawn resource " + rscName);
+ ApiCallRcList answers =
api.resourceGroupSpawn(lpool.getResourceGroup(), rgSpawn);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to spawn
resource.");
+
+ // query linstor for the device path
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ s_logger.info("Linstor: Created drbd device: " + devPath);
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes.");
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool,
Map<String, String> details)
+ {
+ s_logger.debug(String.format("Linstor: connectPhysicalDisk %s:%s ->
%s", pool.getUuid(), volumePath, details));
+ if (volumePath == null) {
+ s_logger.warn("volumePath is null, ignoring");
+ return false;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try
+ {
+ final String rscName = getLinstorRscName(volumePath);
+ final String localNode = getHostname();
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(localNode),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+
+ if (resources.isEmpty()) {
+ s_logger.info("Linstor: resource not created yet: " + rscName);
+ ResourceCreate rscCreate = new ResourceCreate();
+ Resource rsc = new Resource();
+ rsc.setNodeName(localNode);
+ rscCreate.setResource(rsc);
+ ApiCallRcList answers = api.resourceCreate(rscName,
Collections.singletonList(rscCreate));
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Could not create resource " + rscName);
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+ // now diskful resource should be deployed
+ }
+ // keep diskless, maybe add an option, but I'm concerned about out
of space situations
+// else {
+// final ResourceWithVolumes rsc = resources.get(0);
+// if (rsc.getFlags() != null &&
rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS)) {
+// ApiCallRcList answers =
api.resourceToggleDiskful(rscName, localNode);
+// if (ApiCallChecker.hasError(answers)) {
+// s_logger.error("Unable to toggle disk on " +
rscName);
+// throw new
CloudRuntimeException(answers.get(0).getMessage());
+// }
+// }
+// }
Review comment:
Why commit a commented code?
##########
File path:
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,566 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiCallChecker;
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger =
Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a
better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new
StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
+
+ p.waitFor();
+ p.destroy();
+ return result;
+ } catch (IOException | InterruptedException exc) {
+ throw new CloudRuntimeException("Unable to run hostname command.");
+ }
+ }
+
+ private void handleLinstorApiAnswers(ApiCallRcList answers, String
excMessage) {
+ if (ApiCallChecker.hasError(answers)) {
+ for (ApiCallRc answer : answers) {
+ s_logger.error(answer.getMessage());
+ }
+ throw new CloudRuntimeException(excMessage);
+ }
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
+ s_logger.debug("Linstor getStoragePool: " + uuid + " -> " +
refreshInfo);
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMPhysicalDisk getPhysicalDisk(String name, KVMStoragePool pool)
+ {
+ s_logger.debug("Linstor: getPhysicalDisk for " + name);
+ if (name == null) {
+ return null;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try {
+ final String rscName = getLinstorRscName(name);
+
+ List<VolumeDefinition> volumeDefs =
api.volumeDefinitionList(rscName, null, null);
+ final long size = volumeDefs.isEmpty() ? 0 :
volumeDefs.get(0).getSizeKib() * 1024;
+
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ kvmDisk.setSize(size);
+ kvmDisk.setVirtualSize(size);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes for " + rscName);
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public KVMStoragePool createStoragePool(String name, String host, int
port, String path, String userInfo,
+ Storage.StoragePoolType type)
+ {
+ s_logger.debug(String.format(
+ "Linstor createStoragePool: name: '%s', host: '%s', path: %s,
userinfo: %s", name, host, path, userInfo));
+ LinstorStoragePool storagePool = new LinstorStoragePool(name, host,
port, userInfo, type, this);
+
+ MapStorageUuidToStoragePool.put(name, storagePool);
+
+ return storagePool;
+ }
+
+ @Override
+ public boolean deleteStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.remove(uuid) != null;
+ }
+
+ @Override
+ public boolean deleteStoragePool(KVMStoragePool pool) {
+ return deleteStoragePool(pool.getUuid());
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool
pool, QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType
provisioningType, long size)
+ {
+ LinstorStoragePool lpool = (LinstorStoragePool) pool;
+ final DevelopersApi api = getLinstorAPI(pool);
+ ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn();
+ final String rscName = getLinstorRscName(name);
+ rgSpawn.setResourceDefinitionName(rscName);
+ rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB
+ try {
+ s_logger.debug("Linstor: Spawn resource " + rscName);
+ ApiCallRcList answers =
api.resourceGroupSpawn(lpool.getResourceGroup(), rgSpawn);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to spawn
resource.");
+
+ // query linstor for the device path
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() &&
!resources.get(0).getVolumes().isEmpty()) {
+ final String devPath =
resources.get(0).getVolumes().get(0).getDevicePath();
+ s_logger.info("Linstor: Created drbd device: " + devPath);
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath,
name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources
or volumes.");
+ throw new CloudRuntimeException("Linstor: viewResources didn't
return resources or volumes.");
+ }
+ } catch (ApiException apiExc) {
+ throw new CloudRuntimeException(apiExc);
+ }
+ }
+
+ @Override
+ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool,
Map<String, String> details)
+ {
+ s_logger.debug(String.format("Linstor: connectPhysicalDisk %s:%s ->
%s", pool.getUuid(), volumePath, details));
+ if (volumePath == null) {
+ s_logger.warn("volumePath is null, ignoring");
+ return false;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try
+ {
+ final String rscName = getLinstorRscName(volumePath);
+ final String localNode = getHostname();
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(localNode),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+
+ if (resources.isEmpty()) {
+ s_logger.info("Linstor: resource not created yet: " + rscName);
+ ResourceCreate rscCreate = new ResourceCreate();
+ Resource rsc = new Resource();
+ rsc.setNodeName(localNode);
+ rscCreate.setResource(rsc);
+ ApiCallRcList answers = api.resourceCreate(rscName,
Collections.singletonList(rscCreate));
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Could not create resource " + rscName);
+ throw new
CloudRuntimeException(answers.get(0).getMessage());
+ }
+ // now diskful resource should be deployed
+ }
+ // keep diskless, maybe add an option, but I'm concerned about out
of space situations
+// else {
+// final ResourceWithVolumes rsc = resources.get(0);
+// if (rsc.getFlags() != null &&
rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS)) {
+// ApiCallRcList answers =
api.resourceToggleDiskful(rscName, localNode);
+// if (ApiCallChecker.hasError(answers)) {
+// s_logger.error("Unable to toggle disk on " +
rscName);
+// throw new
CloudRuntimeException(answers.get(0).getMessage());
+// }
+// }
+// }
+
+ // allow 2 primaries for live migration, should be removed by
disconnect on the other end
+ ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+ Properties props = new Properties();
+ props.put("DrbdOptions/Net/allow-two-primaries", "yes");
+ rdm.setOverrideProps(props);
+ ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+ if (ApiCallChecker.hasError(answers)) {
+ s_logger.error("Unable to set 'allow-two-primaries' on " +
rscName);
+ throw new CloudRuntimeException(answers.get(0).getMessage());
+ }
+ } catch (ApiException apiExc) {
+ s_logger.error(apiExc);
+ throw new CloudRuntimeException(apiExc.getMessage());
+ }
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool
pool)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk " + pool.getUuid() +
":" + volumePath);
+ // TODO remove diskless/diskful on node
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(Map<String, String>
volumeToDisconnect)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk map");
+ // TODO remove diskless/diskful on node
Review comment:
Would be better to register TODO in issues instead of comments in the
code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
