Repository: cloudstack Updated Branches: refs/heads/volume-upload a31e1bff3 -> 2a2d4364e
made changes to move a file to the specified localtion Signed-off-by: Rajani Karuturi <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/2a2d4364 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/2a2d4364 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/2a2d4364 Branch: refs/heads/volume-upload Commit: 2a2d4364e14903d51cdc3d3a7b4e2eb979a60445 Parents: a31e1bf Author: Bharat Kumar <[email protected]> Authored: Mon Jan 19 10:13:03 2015 +0530 Committer: Rajani Karuturi <[email protected]> Committed: Tue Jan 20 16:02:33 2015 +0530 ---------------------------------------------------------------------- .../resource/NfsSecondaryStorageResource.java | 214 +++++++++++++++++-- .../storage/template/DownloadManager.java | 2 + .../storage/template/DownloadManagerImpl.java | 7 +- .../storage/template/UploadEntity.java | 76 ++++++- 4 files changed, 274 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2a2d4364/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 246138d..6ad7df5 100755 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -45,6 +45,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.UUID; @@ -225,7 +226,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S protected String _parent = "/mnt/SecStorage"; final private String _tmpltpp = "template.properties"; protected String createTemplateFromSnapshotXenScript; - private HashMap<Long,UploadEntity> uploadEntityStateMap = new HashMap<Long,UploadEntity>(); + private HashMap<String,UploadEntity> uploadEntityStateMap = new HashMap<String,UploadEntity>(); public void setParentPath(String path) { _parent = path; @@ -1361,7 +1362,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S // Create request handler registry UriHttpAsyncRequestHandlerMapper reqistry = new UriHttpAsyncRequestHandlerMapper(); // Register the default handler for all URIs - reqistry.register("/upload*", new PostUploadRequestHandler()); + reqistry.register("/upload*", new PostUploadRequestHandler(_params,this,_dlMgr)); // Create server-side HTTP protocol handler HttpAsyncService protocolHandler = new HttpAsyncService(httpproc, reqistry) { @@ -2628,32 +2629,100 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S private class PostUploadRequestHandler implements HttpAsyncRequestHandler<HttpRequest> { //private static final Logger s_logger = Logger.getLogger(PostUploadRequestHandler.class); - public void handleuplod(long entityId, String absolutePath, String entityName, long entitySize, byte[] data) { + private String createTmpltScr; + private String createVolScr; + private String scriptsDir; + private int installTimeoutPerGig = 180 * 60 * 1000; + private Map<String,ImageFormat> formatMap = new HashMap<>(); + private DownloadManager _downloadMgr; + private SecondaryStorageResource _resource; + + public PostUploadRequestHandler(Map<String, Object> params, SecondaryStorageResource resource, DownloadManager dlmgr) { + configure(params, dlmgr); + this._resource=resource; + } + + public void configure(Map<String, Object> params, DownloadManager dlmgr) { + + _downloadMgr=dlmgr; + String scriptsDir = (String)params.get("template.scripts.dir"); + if (scriptsDir == null) { + scriptsDir = "scripts/storage/secondary"; + } + try { + createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh"); + if (createTmpltScr == null) { + throw new ConfigurationException("Unable to find createtmplt.sh"); + } + s_logger.info("createtmplt.sh found in " + createTmpltScr); + createVolScr = Script.findScript(scriptsDir, "createvolume.sh"); + if (createVolScr == null) { + throw new ConfigurationException("Unable to find createvolume.sh"); + } + s_logger.info("createvolume.sh found in " + createVolScr); + + }catch (Exception e) { + s_logger.debug("failed to configure the postupload handler "+e); + } + + formatMap.put("qcow2",ImageFormat.QCOW2); + formatMap.put("vhd",ImageFormat.VHD); + formatMap.put("iso",ImageFormat.ISO); + formatMap.put("vmdk",ImageFormat.VMDK); + formatMap.put("tar",ImageFormat.TAR); + formatMap.put("ova",ImageFormat.OVA); + + } + + public void handleuplod(long entityId, String installPathPrefix, String uuid, String localTemppath, String entityName, long entitySize, String resourceType, String imageFormat, byte[] data, boolean isHvm, String chksum, String dataStoreUrl) { UploadEntity uploadEntity=null; try { - if (uploadEntityStateMap.containsKey(entityId)) { + if (uploadEntityStateMap.containsKey(uuid)) { //the file upload entity has been created. - uploadEntity = uploadEntityStateMap.get(entityId); - uploadEntity.getFilewriter().write(data); - uploadEntity.incremetByteCount(data.length); - if (uploadEntity.getDownloadedsize() == uploadEntity.getEntitysize()) { - uploadEntity.setStatus(UploadEntity.Status.COMPLETED); - uploadEntity.getFile().renameTo(new File(uploadEntity.getAbsoluteFilePath())); - uploadEntity.getFilewriter().close(); - } else { - uploadEntity.setStatus(UploadEntity.Status.IN_PROGRESS); + uploadEntity = uploadEntityStateMap.get(uuid); + if (uploadEntity.getUploadState() == UploadEntity.Status.ERROR) { + s_logger.debug("received data write requres for a uplod entity in error state. Ignoring."); + }else if (uploadEntity.getUploadState()== UploadEntity.Status.COMPLETED) { + s_logger.debug("received a write request for entity which is alredy downloaded. Ignoring."); + }else if (uploadEntity.getUploadState()== UploadEntity.Status.IN_PROGRESS) { + uploadEntity.getFilewriter().write(data); + uploadEntity.incremetByteCount(data.length); + if (uploadEntity.getDownloadedsize() == uploadEntity.getEntitysize()) { + postUpload(uploadEntity); + } else { + uploadEntity.setStatus(UploadEntity.Status.IN_PROGRESS); + } } }else{ //this is a new upload. - uploadEntity = new UploadEntity(entitySize, UploadEntity.Status.IN_PROGRESS, entityName, absolutePath); - uploadEntityStateMap.put(entityId, uploadEntity); - File tempFile = File.createTempFile("dnld_" + entityId,absolutePath); + uploadEntity = new UploadEntity(uuid, entityId, entitySize, UploadEntity.Status.IN_PROGRESS, entityName, installPathPrefix); + uploadEntity.setResourceType("volume".equalsIgnoreCase(resourceType) ? UploadEntity.ResourceType.VOLUME : UploadEntity.ResourceType.TEMPLATE); + uploadEntity.setFormat(formatMap.get(imageFormat.toLowerCase())); + //relative path wihour ssvm mount info. + uploadEntity.setTemplatePath(installPathPrefix); + installPathPrefix = _resource.getRootDir(dataStoreUrl) + File.separator + installPathPrefix; + uploadEntity.setInstallPathPrefix(installPathPrefix); + uploadEntity.setEntitysize(entitySize); + uploadEntity.setHvm(isHvm); + uploadEntity.setFilename(entityName); + if (!_storage.exists(installPathPrefix)) { + _storage.mkdir(installPathPrefix); + } + File tempFile = File.createTempFile("dnld_", entityName,new File(installPathPrefix)); + uploadEntity.setFile(tempFile); FileOutputStream filewriter = new FileOutputStream(tempFile.getAbsolutePath()); uploadEntity.setFilewriter(filewriter); filewriter.write(data); uploadEntity.setFile(tempFile); uploadEntity.incremetByteCount(data.length); + uploadEntityStateMap.put(uuid, uploadEntity); + if (uploadEntity.getDownloadedsize() == uploadEntity.getEntitysize()) { + postUpload(uploadEntity); + + } else { + uploadEntity.setStatus(UploadEntity.Status.IN_PROGRESS); + } } } catch (Exception e) { uploadEntity.setErrorMessage(e.getMessage()); @@ -2667,6 +2736,117 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S s_logger.debug("exception occured while writing to a file " + e); } } + public String postUpload(UploadEntity uploadEntity) { + + String resourcePath = uploadEntity.getInstallPathPrefix(); // install path prefix // path with mount + // directory + String finalResourcePath = uploadEntity.getTmpltPath(); // template download + // path on secondary + // storage + UploadEntity.ResourceType resourceType = uploadEntity.getResourceType(); + + File originalTemplate = uploadEntity.getFile(); + //String checkSum = computeCheckSum(originalTemplate); + //if (checkSum == null) { + // s_logger.warn("Something wrong happened when try to calculate the checksum of downloaded template!"); + //} + //dnld.setCheckSum(checkSum); + + int imgSizeGigs = (int)Math.ceil(_storage.getSize(originalTemplate.getPath()) * 1.0d / (1024 * 1024 * 1024)); + imgSizeGigs++; // add one just in case + long timeout = (long)imgSizeGigs * installTimeoutPerGig; + Script scr = null; + String script = resourceType == UploadEntity.ResourceType.TEMPLATE ? createTmpltScr : createVolScr; + scr = new Script(script, timeout, s_logger); + scr.add("-s", Integer.toString(imgSizeGigs)); + scr.add("-S", Long.toString(UploadEntity.s_maxTemplateSize)); + //if (uploadEntity.getDescription() != null && dnld.getDescription().length() > 1) { + // scr.add("-d", dnld.getDescription()); + //} + if (uploadEntity.isHvm()) { + scr.add("-h"); + } + + // add options common to ISO and template + String extension = uploadEntity.getFormat().getFileExtension(); + String templateName = ""; + if (extension.equals("iso")) { + templateName = uploadEntity.getUuid().trim().replace(" ", "_"); + } else { + templateName = java.util.UUID.nameUUIDFromBytes((uploadEntity.getFilename() + System.currentTimeMillis()).getBytes()).toString(); + } + + // run script to mv the temporary template file to the final template + // file + String templateFilename = templateName + "." + extension; + uploadEntity.setTemplatePath(finalResourcePath + "/" + templateFilename); + scr.add("-n", templateFilename); + + scr.add("-t", resourcePath); + scr.add("-f", uploadEntity.getFile().getAbsolutePath()); // this is the temporary + // template file downloaded + if (uploadEntity.getChksum() != null && uploadEntity.getChksum().length() > 1) { + scr.add("-c", uploadEntity.getChksum()); + } + scr.add("-u"); // cleanup + String result; + result = scr.execute(); + + if (result != null) { + return result; + } + + // Set permissions for the downloaded template + File downloadedTemplate = new File(resourcePath + "/" + templateFilename); + _storage.setWorldReadableAndWriteable(downloadedTemplate); + + // Set permissions for template/volume.properties + String propertiesFile = resourcePath; + if (resourceType == UploadEntity.ResourceType.TEMPLATE) { + propertiesFile += "/template.properties"; + } else { + propertiesFile += "/volume.properties"; + } + File templateProperties = new File(propertiesFile); + _storage.setWorldReadableAndWriteable(templateProperties); + + TemplateLocation loc = new TemplateLocation(_storage, resourcePath); + try { + loc.create(uploadEntity.getEntityId(), true, uploadEntity.getFilename()); + } catch (IOException e) { + s_logger.warn("Something is wrong with template location " + resourcePath, e); + loc.purge(); + return "Unable to download due to " + e.getMessage(); + } + + Iterator<Processor> en = _downloadMgr.getProcessesors().values().iterator(); + while (en.hasNext()) { + Processor processor = en.next(); + + FormatInfo info = null; + try { + info = processor.process(resourcePath, null, templateName); + } catch (InternalErrorException e) { + s_logger.error("Template process exception ", e); + return e.toString(); + } + if (info != null) { + loc.addFormat(info); + //dnld.setTemplatesize(info.virtualSize); + //dnld.setTemplatePhysicalSize(info.size); + break; + } + } + + if (!loc.save()) { + s_logger.warn("Cleaning up because we're unable to save the formats"); + loc.purge(); + } + uploadEntity.setStatus(UploadEntity.Status.COMPLETED); + uploadEntityStateMap.put(uploadEntity.getUuid(), uploadEntity); + + return null; + } private void parsePostBody(InputStream input, OutputStream output, Map<String, String> params) throws IOException { BufferedReader reader = new BufferedReader(new InputStreamReader(input)); @@ -2745,7 +2925,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S TemplateOrVolumePostUploadCommand cmd = gson.fromJson(metadata, TemplateOrVolumePostUploadCommand.class); //call handle upload method. - handleuplod(cmd.getEntityId(), cmd.getAbsolutePath(), cmd.getName(), output.size(), output.toByteArray()); + handleuplod(cmd.getEntityId(),cmd.getAbsolutePath(), cmd.getEntityUUID(),cmd.getLocalPath(),cmd.getName(),output.size(), cmd.getType(), cmd.getImageFormat(), output.toByteArray(), false, cmd.getChecksum(), cmd.getDataTo()); s_logger.error(new String(data)); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2a2d4364/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java index d0abe2c..9003aec 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java @@ -52,6 +52,8 @@ public interface DownloadManager extends Manager { public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType); + Map getProcessesors(); + /** * Get the status of a download job * @param jobId job Id http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2a2d4364/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 9678b6b..f9b2ff1 100755 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -88,7 +88,7 @@ import com.cloud.utils.script.Script; public class DownloadManagerImpl extends ManagerBase implements DownloadManager { private String _name; StorageLayer _storage; - Map<String, Processor> _processors; + public Map<String, Processor> _processors; public class Completion implements DownloadCompleteCallback { private final String jobId; @@ -103,6 +103,11 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } } + @Override + public Map getProcessesors() { + return _processors; + } + private static class DownloadJob { private final TemplateDownloader td; private final String tmpltName; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2a2d4364/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java index 7741ca5..3066c6b 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.storage.template; +import com.cloud.storage.Storage; import java.io.File; import java.io.FileOutputStream; @@ -25,10 +26,19 @@ public class UploadEntity { private long filesize; private long downloadedsize; private String filename; - private String absoluteFilePath; + private String installPathPrefix; + private String templatePath; + private boolean isHvm; + private Storage.ImageFormat format; + private String uuid; + private long entityId; + private String chksum; + public static enum ResourceType { + VOLUME, TEMPLATE + } public static enum Status { UNKNOWN, IN_PROGRESS, COMPLETED, ERROR @@ -38,13 +48,18 @@ public class UploadEntity { private FileOutputStream filewriter = null; private String errorMessage=null; private File file; + private ResourceType resourceType; + + public static long s_maxTemplateSize = 50L * 1024L * 1024L * 1024L; - public UploadEntity(long filesize, Status status, String filename, String absoluteFilePath) { + public UploadEntity(String uuid,long entityId,long filesize, Status status, String filename, String installPathPrefix){ + this.uuid=uuid; this.filesize=filesize; this.uploadState=status; this.downloadedsize=0l; this.filename=filename; - this.absoluteFilePath=absoluteFilePath; + this.installPathPrefix = installPathPrefix; + this.entityId=entityId; } public void setEntitysize(long filesize) { @@ -102,13 +117,60 @@ public class UploadEntity { public void setFilename(String filename) { this.filename = filename; } - public String getAbsoluteFilePath() { - return absoluteFilePath; + public String getInstallPathPrefix() { + return installPathPrefix; + } + + public void setInstallPathPrefix(String absoluteFilePath) { + this.installPathPrefix = absoluteFilePath; } - public void setAbsoluteFilePath(String absoluteFilePath) { - this.absoluteFilePath = absoluteFilePath; + public String getTmpltPath() { + return templatePath; } + public void setTemplatePath(String templatePath) { + this.templatePath=templatePath; + } + + public ResourceType getResourceType() { + return resourceType; + } + + public void setResourceType(ResourceType resourceType) { + this.resourceType = resourceType; + } + + public boolean isHvm() { + return isHvm; + } + + public void setHvm(boolean isHvm) { + this.isHvm = isHvm; + } + + public Storage.ImageFormat getFormat() { + return format; + } + + public void setFormat(Storage.ImageFormat format) { + this.format = format; + } + + public String getUuid() { + return uuid; + } + + public long getEntityId() { + return entityId; + } + + public String getChksum() { + return chksum; + } + + public void setChksum(String chksum) { + this.chksum = chksum; + } }
