rg9975 commented on code in PR #7889: URL: https://github.com/apache/cloudstack/pull/7889#discussion_r1353025468
########## plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java: ########## @@ -0,0 +1,911 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.adapter.primera; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; + +import org.apache.cloudstack.storage.datastore.adapter.ProviderAdapter; +import org.apache.cloudstack.storage.datastore.adapter.ProviderAdapterContext; +import org.apache.cloudstack.storage.datastore.adapter.ProviderAdapterDataObject; +import org.apache.cloudstack.storage.datastore.adapter.ProviderAdapterDiskOffering; +import org.apache.cloudstack.storage.datastore.adapter.ProviderSnapshot; +import org.apache.cloudstack.storage.datastore.adapter.ProviderVolume; +import org.apache.cloudstack.storage.datastore.adapter.ProviderVolume.AddressType; +import org.apache.cloudstack.storage.datastore.adapter.ProviderVolumeNamer; +import org.apache.cloudstack.storage.datastore.adapter.ProviderVolumeStats; +import org.apache.cloudstack.storage.datastore.adapter.ProviderVolumeStorageStats; +import org.apache.cloudstack.storage.datastore.adapter.ProviderAdapterDiskOffering.ProvisioningType; +import org.apache.http.Header; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustAllStrategy; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.log4j.Logger; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class PrimeraAdapter implements ProviderAdapter { + + static final Logger logger = Logger.getLogger(PrimeraAdapter.class); + + public static final String HOSTSET = "hostset"; + public static final String CPG = "cpg"; + public static final String SNAP_CPG = "snapCpg"; + public static final String KEY_TTL = "keyttl"; + public static final String CONNECT_TIMEOUT_MS = "connectTimeoutMs"; + public static final String POST_COPY_WAIT_MS = "postCopyWaitMs"; + public static final String TASK_WAIT_TIMEOUT_MS = "taskWaitTimeoutMs"; + + private static final long KEY_TTL_DEFAULT = (1000 * 60 * 14); + private static final long CONNECT_TIMEOUT_MS_DEFAULT = 600000; + private static final long TASK_WAIT_TIMEOUT_MS_DEFAULT = 10 * 60 * 1000; + public static final long BYTES_IN_MiB = 1048576; + + static final ObjectMapper mapper = new ObjectMapper(); + public String cpg = null; + public String snapCpg = null; + public String hostset = null; + private String username; + private String password; + private String key; + private String url; + private long keyExpiration = -1; + private long keyTtl = KEY_TTL_DEFAULT; + private long connTimeout = CONNECT_TIMEOUT_MS_DEFAULT; + private long taskWaitTimeoutMs = TASK_WAIT_TIMEOUT_MS_DEFAULT; + private CloseableHttpClient _client = null; + private boolean skipTlsValidation; + + private Map<String, String> connectionDetails = null; + + public PrimeraAdapter(String url, Map<String, String> details) { + this.url = url; + this.connectionDetails = details; + login(); + } + + @Override + public void refresh(Map<String, String> details) { + this.connectionDetails = details; + this.refreshSession(true); + } + + @Override + public void validate() { + login(); + // check if hostgroup and pod from details really exist - we will + // require a distinct configuration object/connection object for each type + if (this.getHostset(hostset) == null) { + throw new RuntimeException("Hostgroup [" + hostset + "] not found in FlashArray at [" + url + + "], please validate configuration"); + } + + if (this.getCpg(cpg) == null) { + throw new RuntimeException( + "Pod [" + cpg + "] not found in FlashArray at [" + url + "], please validate configuration"); + } + } + + @Override + public void disconnect() { + return; + } + + @Override + public ProviderVolume create(ProviderAdapterContext context, ProviderAdapterDataObject dataIn, + ProviderAdapterDiskOffering diskOffering, long sizeInBytes) { + PrimeraVolumeRequest request = new PrimeraVolumeRequest(); + String externalName = ProviderVolumeNamer.generateObjectName(context, dataIn); + request.setName(externalName); + request.setCpg(cpg); + request.setSnapCPG(snapCpg); + if (sizeInBytes < BYTES_IN_MiB) { + request.setSizeMiB(1); + } else { + request.setSizeMiB(sizeInBytes/BYTES_IN_MiB); + } + + // determine volume type based on offering + // THIN: tpvv=true, reduce=false + // SPARSE: tpvv=true, reduce=true + // THICK: tpvv=false, tpZeroFill=true (not supported) + if (diskOffering != null) { + if (diskOffering.getType() == ProvisioningType.THIN) { + request.setTpvv(true); + request.setReduce(false); + } else if (diskOffering.getType() == ProvisioningType.SPARSE) { + request.setTpvv(false); + request.setReduce(true); + } else if (diskOffering.getType() == ProvisioningType.FAT) { + throw new RuntimeException("This storage provider does not support FAT provisioned volumes"); + } + + // sets the amount of space allowed for snapshots as a % of the volumes size + if (diskOffering.getHypervisorSnapshotReserve() != null) { + request.setSsSpcAllocLimitPct(diskOffering.getHypervisorSnapshotReserve()); + } + } else { + // default to deduplicated volume + request.setReduce(true); + request.setTpvv(false); + } + + request.setComment(ProviderVolumeNamer.generateObjectComment(context, dataIn)); + POST("/volumes", request, null); + dataIn.setExternalName(externalName); + ProviderVolume volume = getVolume(context, dataIn); + return volume; + } + + @Override + public String attach(ProviderAdapterContext context, ProviderAdapterDataObject dataIn) { + assert dataIn.getExternalName() != null : "External name not provided internally on volume attach"; + PrimeraHostset.PrimeraHostsetVLUNRequest request = new PrimeraHostset.PrimeraHostsetVLUNRequest(); + request.setHostname("set:" + hostset); + request.setVolumeName(dataIn.getExternalName()); + request.setAutoLun(true); + // auto-lun returned here: Location: /api/v1/vluns/test_vv02,252,mysystem,2:2:4 + String location = POST("/vluns", request, new TypeReference<String>() {}); + if (location == null) { + throw new RuntimeException("Attach volume failed with empty location response to vlun add command on storage provider"); + } + String[] toks = location.split(","); + if (toks.length <2) { + throw new RuntimeException("Attach volume failed with invalid location response to vlun add command on storage provider. Provided location: " + location); + } + return toks[1]; + } + + @Override + public void detach(ProviderAdapterContext context, ProviderAdapterDataObject request) { + // we expect to only be attaching one hostset to the vluns, so on detach we'll + // remove ALL vluns we find. + assert request.getExternalName() != null : "External name not provided internally on volume detach"; + removeAllVluns(request.getExternalName()); + } + + public void removeVlun(String name, Integer lunid, String hostString) { + // hostString can be a hostname OR "set:<hostsetname>". It is stored this way + // in the appliance and returned as the vlun's name/string. + DELETE("/vluns/" + name + "," + lunid + "," + hostString); + } + + /** + * Removes all vluns - this should only be done when you are sure the volume is no longer in use + * @param name + */ + public void removeAllVluns(String name) { + PrimeraVlunList list = getVolumeHostsets(name); + if (list != null && list.getMembers() != null) { + for (PrimeraVlun vlun: list.getMembers()) { + removeVlun(vlun.getVolumeName(), vlun.getLun(), vlun.getHostname()); + } + } + } + + public PrimeraVlunList getVolumeHostsets(String name) { + String query = "%22volumeName%20EQ%20" + name + "%22"; + return GET("/vluns?query=" + query, new TypeReference<PrimeraVlunList>() {}); + } + + @Override + public void delete(ProviderAdapterContext context, ProviderAdapterDataObject request) { + assert request.getExternalName() != null : "External name not provided internally on volume delete"; + + // first remove vluns (take volumes from vluns) from hostset + removeAllVluns(request.getExternalName()); + DELETE("/volumes/" + request.getExternalName()); + } + + @Override + public ProviderVolume copy(ProviderAdapterContext context, ProviderAdapterDataObject sourceVolume, + ProviderAdapterDataObject targetVolume) { + PrimeraVolumeCopyRequest request = new PrimeraVolumeCopyRequest(); + PrimeraVolumeCopyRequestParameters parms = new PrimeraVolumeCopyRequestParameters(); + + assert sourceVolume.getExternalName() != null: "External provider name not provided on copy request to Primera volume provider"; + + // if we have no external name, treat it as a new volume + if (targetVolume.getExternalName() == null) { + targetVolume.setExternalName(ProviderVolumeNamer.generateObjectName(context, targetVolume)); + } + + parms.setDestVolume(targetVolume.getExternalName()); + parms.setDestCPG(cpg); + parms.setOnline(true); + parms.setReduce(true); + parms.setSnapCPG(snapCpg); + request.setParameters(parms); + + PrimeraTaskReference taskref = POST("/volumes/" + sourceVolume.getExternalName(), request, new TypeReference<PrimeraTaskReference>() {}); + if (taskref == null) { + throw new RuntimeException("Unable to retrieve task used to copy to newly created volume"); + } + + waitForTaskToComplete(taskref.getTaskid(), "copy volume " + sourceVolume.getExternalName() + " to " + + targetVolume.getExternalName(), taskWaitTimeoutMs); + + String newVolLocation = taskref.getLocation(); + if (newVolLocation == null) { + throw new RuntimeException("Unable to retrieve link to newly created copy volume"); + } + String subpath = newVolLocation.substring(newVolLocation.lastIndexOf("/api/v1", 0)).replaceAll("/api/v1", ""); + + // update the comment for the metadata (since we did a copy the new volume doesn't have any metadata comment yet) + PrimeraVolumeUpdateRequest updateobj = new PrimeraVolumeUpdateRequest(); + updateobj.setComment(ProviderVolumeNamer.generateObjectComment(context, targetVolume)); + PUT(subpath, updateobj, null); + + // return a re-fetched volume object + return GET(subpath, new TypeReference<PrimeraVolume>() { + }); + } + + private void waitForTaskToComplete(String taskid, String taskDescription, Long timeoutMs) { + // first wait for task to complete + long taskWaitTimeout = System.currentTimeMillis() + timeoutMs; + boolean timedOut = true; + PrimeraTaskStatus status = null; + long starttime = System.currentTimeMillis(); + while (System.currentTimeMillis() <= taskWaitTimeout) { + status = this.getTaskStatus(taskid); + if (status != null && status.isFinished()) { + timedOut = false; + if (!status.isSuccess()) { + throw new RuntimeException("Task " + taskDescription + " was cancelled. TaskID: " + status.getId() + "; Final Status: " + status.getStatusName()); + } + break; + } else { + if (status != null) { + logger.info("Task " + taskDescription + " is still running. TaskID: " + status.getId() + "; Current Status: " + status.getStatusName()); + } + // ugly...to keep from hot-polling API + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + + } + } + } + + if (timedOut) { + if (status != null) { + throw new RuntimeException("Task " + taskDescription + " timed out. TaskID: " + status.getId() + ", Last Known Status: " + status.getStatusName()); + } else { + throw new RuntimeException("Task " + taskDescription + " timed out and a current status could not be retrieved from storage endpoint"); + } + } + + logger.info(taskDescription + " completed in " + ((System.currentTimeMillis() - starttime)/1000) + " seconds"); + } + + private PrimeraTaskStatus getTaskStatus(String taskid) { + return GET("/tasks/" + taskid + "?view=excludeDetail", new TypeReference<PrimeraTaskStatus>() { + }); + } + + @Override + public ProviderSnapshot snapshot(ProviderAdapterContext context, ProviderAdapterDataObject sourceVolume, + ProviderAdapterDataObject targetSnapshot) { + assert sourceVolume.getExternalName() != null : "External name not set"; + PrimeraVolumeSnapshotRequest request = new PrimeraVolumeSnapshotRequest(); + PrimeraVolumeSnapshotRequestParameters parms = new PrimeraVolumeSnapshotRequestParameters(); + parms.setName(ProviderVolumeNamer.generateObjectName(context, targetSnapshot)); + request.setParameters(parms); + POST("/volumes/" + sourceVolume.getExternalName(), request, null); + targetSnapshot.setExternalName(parms.getName()); + return getSnapshot(context, targetSnapshot); + } + + @Override + public ProviderVolume revert(ProviderAdapterContext context, ProviderAdapterDataObject dataIn) { + assert dataIn.getExternalName() != null: "External name not internally set for provided snapshot when requested storage provider to revert"; + // first get original volume + PrimeraVolume snapVol = (PrimeraVolume)getVolume(context, dataIn); + assert snapVol != null: "Storage volume associated with snapshot externally named [" + dataIn.getExternalName() + "] not found"; + assert snapVol.getParentId() != null: "Unable to determine parent volume/snapshot for snapshot named [" + dataIn.getExternalName() + "]"; + + PrimeraVolumeRevertSnapshotRequest request = new PrimeraVolumeRevertSnapshotRequest(); + request.setOnline(true); + request.setPriority(2); + PrimeraTaskReference taskref = PUT("/volumes/" + dataIn.getExternalName(), request, new TypeReference<PrimeraTaskReference>() {}); + if (taskref == null) { + throw new RuntimeException("Unable to retrieve task used to revert snapshot to base volume"); + } + + waitForTaskToComplete(taskref.getTaskid(), "revert snapshot " + dataIn.getExternalName(), taskWaitTimeoutMs); + + return getVolumeById(context, snapVol.getParentId()); + } + + @Override + public void resize(ProviderAdapterContext context, ProviderAdapterDataObject request, long totalNewSizeInBytes) { + assert request.getExternalName() != null: "External name not internally set for provided volume when requesting resize of volume"; + PrimeraVolume volume = new PrimeraVolume(); + volume.setSizeMiB((int) totalNewSizeInBytes); + // GROW_VOLUME=3; + // https://support.hpe.com/hpesc/public/docDisplay?docId=a00118636en_us&page=v25706371.html + volume.setAction(3); + PUT("/volumes/" + request.getExternalName(), volume, null); + } + + @Override + public ProviderVolume getVolume(ProviderAdapterContext context, ProviderAdapterDataObject request) { + String externalName; + + // if the external name isn't provided, look for the derived contextual name. some failure scenarios + // may result in the volume for this context being created but a subsequent failure causing the external + // name to not be persisted for later use. This is true of template-type objects being cached on primary + // storage + if (request.getExternalName() == null) { + externalName = ProviderVolumeNamer.generateObjectName(context, request); + } else { + externalName = request.getExternalName(); + } + + return GET("/volumes/" + externalName, new TypeReference<PrimeraVolume>() { + }); + } + + private ProviderVolume getVolumeById(ProviderAdapterContext context, Integer id) { + String query = "%22id%20EQ%20" + id + "%22"; + return GET("/volumes?query=" + query, new TypeReference<PrimeraVolume>() {}); + } + + @Override + public ProviderSnapshot getSnapshot(ProviderAdapterContext context, ProviderAdapterDataObject request) { + assert request.getExternalName() != null: "External name not provided internally when finding snapshot on storage provider"; + return GET("/volumes/" + request.getExternalName(), new TypeReference<PrimeraVolume>() { + }); + } + + @Override + public ProviderVolume getVolumeByAddress(ProviderAdapterContext context, AddressType addressType, String address) { + assert address != null: "External volume address not provided"; + assert AddressType.FIBERWWN.equals(addressType): "This volume provider currently does not support address type " + addressType.name(); + String query = "%22wwn%20EQ%20" + address + "%22"; + return GET("/volumes?query=" + query, new TypeReference<PrimeraVolume>() {}); + } + + @Override + public ProviderVolumeStorageStats getManagedStorageStats() { + PrimeraCpg cpgobj = getCpg(cpg); + // just in case + if (cpgobj == null || cpgobj.getTotalSpaceMiB() == 0) { + return null; + } + Long capacityBytes = cpgobj.getTotalSpaceMiB() * PrimeraAdapter.BYTES_IN_MiB; + Long usedBytes = (cpgobj.getTotalSpaceMiB() - cpgobj.getFreeSpaceMiB()) + * PrimeraAdapter.BYTES_IN_MiB; + Long virtualUsedInBytes = (cpgobj.getRawTotalSpaceMiB() - cpgobj.getRawFreeSpaceMiB()) * PrimeraAdapter.BYTES_IN_MiB; + ProviderVolumeStorageStats stats = new ProviderVolumeStorageStats(); + stats.setActualUsedInBytes(usedBytes); + stats.setCapacityInBytes(capacityBytes); + stats.setVirtualUsedInBytes(virtualUsedInBytes); + return stats; + } + + @Override + public ProviderVolumeStats getVolumeStats(ProviderAdapterContext context, ProviderAdapterDataObject request) { + PrimeraVolume vol = (PrimeraVolume)getVolume(context, request); + if (vol == null || vol.getSizeMiB() == null || vol.getSizeMiB() == 0) { + return null; + } + + Long virtualSizeInBytes = vol.getHostWriteMiB() * PrimeraAdapter.BYTES_IN_MiB; + Long allocatedSizeInBytes = vol.getSizeMiB() * PrimeraAdapter.BYTES_IN_MiB; + Long actualUsedInBytes = vol.getTotalUsedMiB() * PrimeraAdapter.BYTES_IN_MiB; + ProviderVolumeStats stats = new ProviderVolumeStats(); + stats.setActualUsedInBytes(actualUsedInBytes); + stats.setAllocatedInBytes(allocatedSizeInBytes); + stats.setVirtualUsedInBytes(virtualSizeInBytes); + return stats; + } + + @Override + public boolean canAccessHost(ProviderAdapterContext context, String hostname) { + PrimeraHostset hostset = getHostset(this.hostset); + + List<String> members = hostset.getSetmembers(); + + // check for fqdn and shortname combinations. this assumes there is at least a shortname match in both the storage array and cloudstack + // hostname configuration + String shortname; + if (hostname.indexOf('.') > 0) { + shortname = hostname.substring(0, (hostname.indexOf('.'))); + } else { + shortname = hostname; + } + for (String member: members) { + // exact match (short or long names) + if (member.equals(hostname)) { + return true; + } + + // primera has short name and cloudstack had long name + if (member.equals(shortname)) { + return true; + } + + // member has long name but cloudstack had shortname + if (member.substring(0, (member.indexOf('.'))).equals(shortname)) { + return true; + } + } + + return false; + } + + private PrimeraCpg getCpg(String name) { + return GET("/cpgs/" + name, new TypeReference<PrimeraCpg>() { + }); + } + + private PrimeraHostset getHostset(String name) { + return GET("/hostsets/" + name, new TypeReference<PrimeraHostset>() { + }); + } + + private String getSessionKey() { + refreshSession(false); + return key; + } + + private synchronized void refreshSession(boolean force) { + try { + if (force || keyExpiration < System.currentTimeMillis()) { + // close client to force connection reset on appliance -- not doing this can result in NotAuthorized error...guessing + _client.close();; + _client = null; + login(); + keyExpiration = System.currentTimeMillis() + keyTtl; + } + } catch (Exception e) { + // retry frequently but not every request to avoid DDOS on storage API + logger.warn("Failed to refresh Primera API key for " + username + "@" + url + ", will retry in 5 seconds", e); + keyExpiration = System.currentTimeMillis() + (5*1000); + } + } + /** + * Login to the array and get an access token + */ + private void login() { Review Comment: Updated in upcoming commit. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
