http://git-wip-us.apache.org/repos/asf/cloudstack/blob/342624e0/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java ---------------------------------------------------------------------- diff --cc services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index b1cebf4,0000000..dc08ba6 mode 100755,000000..100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@@ -1,1147 -1,0 +1,1164 @@@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.template; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import javax.ejb.Local; +import javax.naming.ConfigurationException; + ++import org.apache.log4j.Logger; ++ +import org.apache.cloudstack.storage.command.DownloadCommand; - import org.apache.cloudstack.storage.command.DownloadProgressCommand; +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; ++import org.apache.cloudstack.storage.command.DownloadProgressCommand; +import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; +import org.apache.cloudstack.storage.resource.SecondaryStorageResource; + - import org.apache.log4j.Logger; - +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.storage.Proxy; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.S3TO; +import com.cloud.exception.InternalErrorException; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.StorageLayer; +import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.template.HttpTemplateDownloader; +import com.cloud.storage.template.IsoProcessor; +import com.cloud.storage.template.LocalTemplateDownloader; +import com.cloud.storage.template.Processor; +import com.cloud.storage.template.Processor.FormatInfo; +import com.cloud.storage.template.QCOW2Processor; +import com.cloud.storage.template.RawImageProcessor; +import com.cloud.storage.template.S3TemplateDownloader; +import com.cloud.storage.template.ScpTemplateDownloader; +import com.cloud.storage.template.TemplateConstants; +import com.cloud.storage.template.TemplateDownloader; +import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback; +import com.cloud.storage.template.TemplateDownloader.Status; +import com.cloud.storage.template.TemplateLocation; +import com.cloud.storage.template.TemplateProp; +import com.cloud.storage.template.VhdProcessor; +import com.cloud.storage.template.VmdkProcessor; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; + +@Local(value = DownloadManager.class) +public class DownloadManagerImpl extends ManagerBase implements DownloadManager { + private String _name; + StorageLayer _storage; + Map<String, Processor> _processors; + + public class Completion implements DownloadCompleteCallback { + private final String jobId; + + public Completion(String jobId) { + this.jobId = jobId; + } + + @Override + public void downloadComplete(Status status) { + setDownloadStatus(jobId, status); + } + } + + private static class DownloadJob { + private final TemplateDownloader td; + private final String jobId; + private final String tmpltName; + private final boolean hvm; + private final ImageFormat format; + private String tmpltPath; + private final String description; + private String checksum; + private final Long accountId; + private final String installPathPrefix; + private long templatesize; + private long templatePhysicalSize; + private final long id; + private final ResourceType resourceType; + + public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, + String descr, String cksum, String installPathPrefix, ResourceType resourceType) { + super(); + this.td = td; + this.jobId = jobId; + this.tmpltName = tmpltName; + this.format = format; + this.hvm = hvm; + this.accountId = accountId; - this.description = descr; - this.checksum = cksum; ++ description = descr; ++ checksum = cksum; + this.installPathPrefix = installPathPrefix; - this.templatesize = 0; ++ templatesize = 0; + this.id = id; + this.resourceType = resourceType; + } + + public TemplateDownloader getTd() { + return td; + } + + public String getDescription() { + return description; + } + + public String getChecksum() { + return checksum; + } + + public TemplateDownloader getTemplateDownloader() { + return td; + } + + public String getJobId() { + return jobId; + } + + public String getTmpltName() { + return tmpltName; + } + + public ImageFormat getFormat() { + return format; + } + + public boolean isHvm() { + return hvm; + } + + public Long getAccountId() { + return accountId; + } + + public long getId() { + return id; + } + + public ResourceType getResourceType() { + return resourceType; + } + + public void setTmpltPath(String tmpltPath) { + this.tmpltPath = tmpltPath; + } + + public String getTmpltPath() { + return tmpltPath; + } + + public String getInstallPathPrefix() { + return installPathPrefix; + } + + public void cleanup() { + if (td != null) { + String dnldPath = td.getDownloadLocalPath(); + if (dnldPath != null) { + File f = new File(dnldPath); + File dir = f.getParentFile(); + f.delete(); + if (dir != null) { + dir.delete(); + } + } + } + + } + + public void setTemplatesize(long templatesize) { + this.templatesize = templatesize; + } + + public long getTemplatesize() { + return templatesize; + } + + public void setTemplatePhysicalSize(long templatePhysicalSize) { + this.templatePhysicalSize = templatePhysicalSize; + } + + public long getTemplatePhysicalSize() { + return templatePhysicalSize; + } + + public void setCheckSum(String checksum) { + this.checksum = checksum; + } + } + + public static final Logger s_logger = Logger.getLogger(DownloadManagerImpl.class); + private String _templateDir; + private String _volumeDir; + private String createTmpltScr; + private String createVolScr; + private List<Processor> processors; + + private ExecutorService threadPool; + + private final Map<String, DownloadJob> jobs = new ConcurrentHashMap<String, DownloadJob>(); + private String listTmpltScr; + private String listVolScr; + private int installTimeoutPerGig = 180 * 60 * 1000; + private boolean _sslCopy; + - + public void setThreadPool(ExecutorService threadPool) { + this.threadPool = threadPool; + } + - public void setStorageLayer(StorageLayer storage){ - this._storage = storage; ++ public void setStorageLayer(StorageLayer storage) { ++ _storage = storage; + } + + /** + * Get notified of change of job status. Executed in context of downloader + * thread + * + * @param jobId + * the id of the job + * @param status + * the status of the job + */ + public void setDownloadStatus(String jobId, Status status) { + DownloadJob dj = jobs.get(jobId); + if (dj == null) { + s_logger.warn("setDownloadStatus for jobId: " + jobId + ", status=" + status + " no job found"); + return; + } + TemplateDownloader td = dj.getTemplateDownloader(); + s_logger.info("Download Completion for jobId: " + jobId + ", status=" + status); + s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" + + td.getDownloadPercent()); + + switch (status) { + case ABORTED: + case NOT_STARTED: + case UNRECOVERABLE_ERROR: + // TODO + dj.cleanup(); + break; + case UNKNOWN: + return; + case IN_PROGRESS: + s_logger.info("Resuming jobId: " + jobId + ", status=" + status); + td.setResume(true); + threadPool.execute(td); + break; + case RECOVERABLE_ERROR: + threadPool.execute(td); + break; + case DOWNLOAD_FINISHED: + if (!(td instanceof S3TemplateDownloader)) { + // we currently only create template.properties for NFS by + // running some post download script + td.setDownloadError("Download success, starting install "); + String result = postDownload(jobId); + if (result != null) { + s_logger.error("Failed post download script: " + result); + td.setStatus(Status.UNRECOVERABLE_ERROR); + td.setDownloadError("Failed post download script: " + result); + } else { + td.setStatus(Status.POST_DOWNLOAD_FINISHED); + td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); + } - } else{ - // for s3 and swift, we skip post download step and just set status to trigger callback. ++ } else { ++ // for s3 and swift, we skip post download step and just set ++ // status to trigger callback. + td.setStatus(Status.POST_DOWNLOAD_FINISHED); ++ // set template size for S3 ++ if (td instanceof S3TemplateDownloader){ ++ long size = ((S3TemplateDownloader)td).totalBytes; ++ DownloadJob dnld = jobs.get(jobId); ++ dnld.setTemplatesize(size); ++ dnld.setTemplatePhysicalSize(size); ++ } + } + dj.cleanup(); + break; + default: + break; + } + } + + private String computeCheckSum(File f) { + byte[] buffer = new byte[8192]; + int read = 0; + MessageDigest digest; + String checksum = null; + InputStream is = null; + try { + digest = MessageDigest.getInstance("MD5"); + is = new FileInputStream(f); + while ((read = is.read(buffer)) > 0) { + digest.update(buffer, 0, read); + } + byte[] md5sum = digest.digest(); + BigInteger bigInt = new BigInteger(1, md5sum); + checksum = String.format("%032x", bigInt); + return checksum; + } catch (IOException e) { + return null; + } catch (NoSuchAlgorithmException e) { + return null; + } finally { + try { + if (is != null) + is.close(); + } catch (IOException e) { + return null; + } + } + } + + /** + * Post download activity (install and cleanup). Executed in context of + * downloader thread + * + * @throws IOException + */ + private String postDownload(String jobId) { + DownloadJob dnld = jobs.get(jobId); + TemplateDownloader td = dnld.getTemplateDownloader(); - String resourcePath = dnld.getInstallPathPrefix(); // path with mount directory - String finalResourcePath = dnld.getTmpltPath(); // template download path on secondary storage ++ String resourcePath = dnld.getInstallPathPrefix(); // path with mount ++ // directory ++ String finalResourcePath = dnld.getTmpltPath(); // template download ++ // path on secondary ++ // storage + ResourceType resourceType = dnld.getResourceType(); + + /* + // once template path is set, remove the parent dir so that the template + // is installed with a relative path + String finalResourcePath = ""; + if (resourceType == ResourceType.TEMPLATE) { + finalResourcePath += _templateDir + File.separator + dnld.getAccountId() + File.separator + dnld.getId() + File.separator; + resourcePath = dnld.getInstallPathPrefix() + dnld.getAccountId() + File.separator + dnld.getId() + File.separator;// dnld.getTmpltName(); + } else { + finalResourcePath += _volumeDir + File.separator + dnld.getId() + File.separator; + resourcePath = dnld.getInstallPathPrefix() + dnld.getId() + File.separator;// dnld.getTmpltName(); + } + + _storage.mkdirs(resourcePath); + dnld.setTmpltPath(finalResourcePath); + */ + + File originalTemplate = new File(td.getDownloadLocalPath()); + String checkSum = computeCheckSum(originalTemplate); + if (checkSum == null) { + s_logger.warn("Something wrong happened when try to calculate the checksum of downloaded template!"); + } + dnld.setCheckSum(checkSum); + + int imgSizeGigs = (int) Math.ceil(_storage.getSize(td.getDownloadLocalPath()) * 1.0d / (1024 * 1024 * 1024)); + imgSizeGigs++; // add one just in case + long timeout = imgSizeGigs * installTimeoutPerGig; + Script scr = null; + String script = resourceType == ResourceType.TEMPLATE ? createTmpltScr : createVolScr; + scr = new Script(script, timeout, s_logger); + scr.add("-s", Integer.toString(imgSizeGigs)); + scr.add("-S", Long.toString(td.getMaxTemplateSizeInBytes())); + if (dnld.getDescription() != null && dnld.getDescription().length() > 1) { + scr.add("-d", dnld.getDescription()); + } + if (dnld.isHvm()) { + scr.add("-h"); + } + + // add options common to ISO and template + String extension = dnld.getFormat().getFileExtension(); + String templateName = ""; + if (extension.equals("iso")) { + templateName = jobs.get(jobId).getTmpltName().trim().replace(" ", "_"); + } else { + templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName() + System.currentTimeMillis()).getBytes()).toString(); + } + - // run script to mv the temporary template file to the final template file ++ // run script to mv the temporary template file to the final template ++ // file + String templateFilename = templateName + "." + extension; + dnld.setTmpltPath(finalResourcePath + "/" + templateFilename); + scr.add("-n", templateFilename); + + scr.add("-t", resourcePath); - scr.add("-f", td.getDownloadLocalPath()); // this is the temporary template file downloaded ++ scr.add("-f", td.getDownloadLocalPath()); // this is the temporary ++ // template file downloaded + if (dnld.getChecksum() != null && dnld.getChecksum().length() > 1) { + scr.add("-c", dnld.getChecksum()); + } + scr.add("-u"); // cleanup + String result; + result = scr.execute(); + + if (result != null) { + return result; + } + + // Set permissions for the downloaded template + File downloadedTemplate = new File(resourcePath + "/" + templateFilename); + _storage.setWorldReadableAndWriteable(downloadedTemplate); + + // Set permissions for template/volume.properties + String propertiesFile = resourcePath; + if (resourceType == ResourceType.TEMPLATE) { + propertiesFile += "/template.properties"; + } else { + propertiesFile += "/volume.properties"; + } + File templateProperties = new File(propertiesFile); + _storage.setWorldReadableAndWriteable(templateProperties); + + TemplateLocation loc = new TemplateLocation(_storage, resourcePath); + try { + loc.create(dnld.getId(), true, dnld.getTmpltName()); + } catch (IOException e) { + s_logger.warn("Something is wrong with template location " + resourcePath, e); + loc.purge(); + return "Unable to download due to " + e.getMessage(); + } + + Iterator<Processor> en = _processors.values().iterator(); + while (en.hasNext()) { + Processor processor = en.next(); + + FormatInfo info = null; + try { + info = processor.process(resourcePath, null, templateName); + } catch (InternalErrorException e) { + s_logger.error("Template process exception ", e); + return e.toString(); + } + if (info != null) { + loc.addFormat(info); + dnld.setTemplatesize(info.virtualSize); + dnld.setTemplatePhysicalSize(info.size); + break; + } + } + + if (!loc.save()) { + s_logger.warn("Cleaning up because we're unable to save the formats"); + loc.purge(); + } + + return null; + } + + @Override + public Status getDownloadStatus(String jobId) { + DownloadJob job = jobs.get(jobId); + if (job != null) { + TemplateDownloader td = job.getTemplateDownloader(); + if (td != null) { + return td.getStatus(); + } + } + return Status.UNKNOWN; + } + + @Override + public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, + String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { + UUID uuid = UUID.randomUUID(); + String jobId = uuid.toString(); + + URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + url); + } + TemplateDownloader td; + if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + td = new S3TemplateDownloader(s3, url, installPathPrefix, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, + resourceType); + } else { + throw new CloudRuntimeException("Scheme is not supported " + url); + } + } else { + throw new CloudRuntimeException("Unable to download from URL: " + url); + } + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); + dj.setTmpltPath(installPathPrefix); + jobs.put(jobId, dj); + threadPool.execute(td); + + return jobId; + } + + @Override + public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, - String cksum, String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { ++ String cksum, String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ++ ResourceType resourceType) { + UUID uuid = UUID.randomUUID(); + String jobId = uuid.toString(); + String tmpDir = installPathPrefix; + + try { + + if (!_storage.mkdirs(tmpDir)) { + s_logger.warn("Unable to create " + tmpDir); + return "Unable to create " + tmpDir; + } + // TO DO - define constant for volume properties. + File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage + .getFile(tmpDir + File.separator + "volume.properties"); + if (file.exists()) { + file.delete(); + } + + if (!file.createNewFile()) { + s_logger.warn("Unable to create new file: " + file.getAbsolutePath()); + return "Unable to create new file: " + file.getAbsolutePath(); + } + + URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + url); + } + TemplateDownloader td; + if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, + resourceType); + } else if (uri.getScheme().equalsIgnoreCase("file")) { + td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); + } else if (uri.getScheme().equalsIgnoreCase("scp")) { + td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); + } else if (uri.getScheme().equalsIgnoreCase("nfs")) { + td = null; + // TODO: implement this. + throw new CloudRuntimeException("Scheme is not supported " + url); + } else { + throw new CloudRuntimeException("Scheme is not supported " + url); + } + } else { + throw new CloudRuntimeException("Unable to download from URL: " + url); + } - // NOTE the difference between installPathPrefix and templatePath here. instalPathPrefix is the absolute path for template including mount directory - // on ssvm, while templatePath is the final relative path on secondary storage. ++ // NOTE the difference between installPathPrefix and templatePath ++ // here. instalPathPrefix is the absolute path for template ++ // including mount directory ++ // on ssvm, while templatePath is the final relative path on ++ // secondary storage. + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); + dj.setTmpltPath(templatePath); + jobs.put(jobId, dj); + threadPool.execute(td); + + return jobId; + } catch (IOException e) { + s_logger.warn("Unable to download to " + tmpDir, e); + return null; + } + } + + @Override + public String getDownloadError(String jobId) { + DownloadJob dj = jobs.get(jobId); + if (dj != null) { + return dj.getTemplateDownloader().getDownloadError(); + } + return null; + } + + public long getDownloadTemplateSize(String jobId) { + DownloadJob dj = jobs.get(jobId); + if (dj != null) { + return dj.getTemplatesize(); + } + return 0; + } + + public String getDownloadCheckSum(String jobId) { + DownloadJob dj = jobs.get(jobId); + if (dj != null) { + return dj.getChecksum(); + } + return null; + } + + public long getDownloadTemplatePhysicalSize(String jobId) { + DownloadJob dj = jobs.get(jobId); + if (dj != null) { + return dj.getTemplatePhysicalSize(); + } + return 0; + } + + // @Override + public String getDownloadLocalPath(String jobId) { + DownloadJob dj = jobs.get(jobId); + if (dj != null) { + return dj.getTemplateDownloader().getDownloadLocalPath(); + } + return null; + } + + @Override + public int getDownloadPct(String jobId) { + DownloadJob dj = jobs.get(jobId); + if (dj != null) { + return dj.getTemplateDownloader().getDownloadPercent(); + } + return 0; + } + + public static VMTemplateHostVO.Status convertStatus(Status tds) { + switch (tds) { + case ABORTED: + return VMTemplateHostVO.Status.NOT_DOWNLOADED; + case DOWNLOAD_FINISHED: + return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS; + case IN_PROGRESS: + return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS; + case NOT_STARTED: + return VMTemplateHostVO.Status.NOT_DOWNLOADED; + case RECOVERABLE_ERROR: + return VMTemplateHostVO.Status.NOT_DOWNLOADED; + case UNKNOWN: + return VMTemplateHostVO.Status.UNKNOWN; + case UNRECOVERABLE_ERROR: + return VMTemplateHostVO.Status.DOWNLOAD_ERROR; + case POST_DOWNLOAD_FINISHED: + return VMTemplateHostVO.Status.DOWNLOADED; + default: + return VMTemplateHostVO.Status.UNKNOWN; + } + } + + @Override + public com.cloud.storage.VMTemplateHostVO.Status getDownloadStatus2(String jobId) { + return convertStatus(getDownloadStatus(jobId)); + } + + @Override + public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) { + ResourceType resourceType = cmd.getResourceType(); + if (cmd instanceof DownloadProgressCommand) { + return handleDownloadProgressCmd(resource, (DownloadProgressCommand) cmd); + } + + if (cmd.getUrl() == null) { + return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download", + VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + } + + if (cmd.getName() == null) { + return new DownloadAnswer("Invalid Name", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + } + + DataStoreTO dstore = cmd.getDataStore(); + String installPathPrefix = cmd.getInstallPath(); + // for NFS, we need to get mounted path + if (dstore instanceof NfsTO) { + if (ResourceType.TEMPLATE == resourceType) { + installPathPrefix = resource.getRootDir(cmd) + File.separator + installPathPrefix; + } else { + installPathPrefix = resource.getRootDir(cmd) + File.separator + installPathPrefix; + } - } else if (dstore instanceof S3TO ){ ++ } else if (dstore instanceof S3TO) { + // S3 key has template name inside to help template sync + installPathPrefix = installPathPrefix + File.separator + cmd.getName(); + } + String user = null; + String password = null; + if (cmd.getAuth() != null) { + user = cmd.getAuth().getUserName(); + password = new String(cmd.getAuth().getPassword()); + } + // TO DO - Define Volume max size as well + long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd + .getMaxDownloadSizeInBytes()); + String jobId = null; + if (dstore instanceof S3TO) { + jobId = downloadS3Template((S3TO) dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), + cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + } else { + jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), - cmd.getDescription(), cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); ++ cmd.getDescription(), cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, ++ cmd.getProxy(), resourceType); + } + sleep(); + if (jobId == null) { + return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + } + return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), + getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId)); + } + + private void sleep() { + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + // ignore + } + } + + private DownloadAnswer handleDownloadProgressCmd(SecondaryStorageResource resource, DownloadProgressCommand cmd) { + String jobId = cmd.getJobId(); + DownloadAnswer answer; + DownloadJob dj = null; + if (jobId != null) { + dj = jobs.get(jobId); + } + if (dj == null) { + if (cmd.getRequest() == RequestType.GET_OR_RESTART) { + DownloadCommand dcmd = new DownloadCommand(cmd); + return handleDownloadCommand(resource, dcmd); + } else { + return new DownloadAnswer("Cannot find job", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR.UNKNOWN); + } + } + TemplateDownloader td = dj.getTemplateDownloader(); + switch (cmd.getRequest()) { + case GET_STATUS: + break; + case ABORT: + td.stopDownload(); + sleep(); + break; + case RESTART: + td.stopDownload(); + sleep(); + threadPool.execute(td); + break; + case PURGE: + td.stopDownload(); + answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), + getDownloadLocalPath(jobId), getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), + getDownloadCheckSum(jobId)); + jobs.remove(jobId); + return answer; + default: + break; // TODO + } + return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), + getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId)); + } + + private String getInstallPath(String jobId) { + DownloadJob dj = jobs.get(jobId); + if (dj != null) { + return dj.getTmpltPath(); + } + return null; + } + + private String createTempDir(File rootDir, String name) throws IOException { + + File f = File.createTempFile(name, "", rootDir); + f.delete(); + f.mkdir(); + _storage.setWorldReadableAndWriteable(f); + return f.getAbsolutePath(); + + } + + private List<String> listVolumes(String rootdir) { + List<String> result = new ArrayList<String>(); + + Script script = new Script(listVolScr, s_logger); + script.add("-r", rootdir); + ZfsPathParser zpp = new ZfsPathParser(rootdir); + script.execute(zpp); + result.addAll(zpp.getPaths()); + s_logger.info("found " + zpp.getPaths().size() + " volumes" + zpp.getPaths()); + return result; + } + + private List<String> listTemplates(String rootdir) { + List<String> result = new ArrayList<String>(); + + Script script = new Script(listTmpltScr, s_logger); + script.add("-r", rootdir); + ZfsPathParser zpp = new ZfsPathParser(rootdir); + script.execute(zpp); + result.addAll(zpp.getPaths()); + s_logger.info("found " + zpp.getPaths().size() + " templates" + zpp.getPaths()); + return result; + } + + @Override + public Map<String, TemplateProp> gatherTemplateInfo(String rootDir) { + Map<String, TemplateProp> result = new HashMap<String, TemplateProp>(); + String templateDir = rootDir + File.separator + _templateDir; + + if (!_storage.exists(templateDir)) { + _storage.mkdirs(templateDir); + } + + List<String> publicTmplts = listTemplates(templateDir); + for (String tmplt : publicTmplts) { + String path = tmplt.substring(0, tmplt.lastIndexOf(File.separator)); + TemplateLocation loc = new TemplateLocation(_storage, path); + try { + if (!loc.load()) { + s_logger.warn("Post download installation was not completed for " + path); + // loc.purge(); + _storage.cleanup(path, templateDir); + continue; + } + } catch (IOException e) { + s_logger.warn("Unable to load template location " + path, e); + continue; + } + + TemplateProp tInfo = loc.getTemplateInfo(); + + if ((tInfo.getSize() == tInfo.getPhysicalSize()) + && (tInfo.getInstallPath().endsWith(ImageFormat.OVA.getFileExtension()))) { + try { + Processor processor = _processors.get("VMDK Processor"); + VmdkProcessor vmdkProcessor = (VmdkProcessor) processor; + long vSize = + vmdkProcessor.getTemplateVirtualSize( + path, + tInfo.getInstallPath().substring( + tInfo.getInstallPath().lastIndexOf(File.separator) + 1)); + tInfo.setSize(vSize); + loc.updateVirtualSize(vSize); + loc.save(); + } catch (Exception e) { + s_logger.error("Unable to get the virtual size of the template: " + tInfo.getInstallPath() + + " due to " + e.getMessage()); + } + } + + result.put(tInfo.getTemplateName(), tInfo); + s_logger.debug("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt); + } + /* + for (String tmplt : isoTmplts) { + String tmp[]; + tmp = tmplt.split("/"); + String tmpltName = tmp[tmp.length - 2]; + tmplt = tmplt.substring(tmplt.lastIndexOf("iso/")); + TemplateInfo tInfo = new TemplateInfo(tmpltName, tmplt, false); + s_logger.debug("Added iso template name: " + tmpltName + ", path: " + tmplt); + result.put(tmpltName, tInfo); + } + */ + return result; + } + + @Override + public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) { + Map<Long, TemplateProp> result = new HashMap<Long, TemplateProp>(); + String volumeDir = rootDir + File.separator + _volumeDir; + + if (!_storage.exists(volumeDir)) { + _storage.mkdirs(volumeDir); + } + + List<String> vols = listVolumes(volumeDir); + for (String vol : vols) { + String path = vol.substring(0, vol.lastIndexOf(File.separator)); + TemplateLocation loc = new TemplateLocation(_storage, path); + try { + if (!loc.load()) { + s_logger.warn("Post download installation was not completed for " + path); + // loc.purge(); + _storage.cleanup(path, volumeDir); + continue; + } + } catch (IOException e) { + s_logger.warn("Unable to load volume location " + path, e); + continue; + } + + TemplateProp vInfo = loc.getTemplateInfo(); + + if ((vInfo.getSize() == vInfo.getPhysicalSize()) + && (vInfo.getInstallPath().endsWith(ImageFormat.OVA.getFileExtension()))) { + try { + Processor processor = _processors.get("VMDK Processor"); + VmdkProcessor vmdkProcessor = (VmdkProcessor)processor; + long vSize = + vmdkProcessor.getTemplateVirtualSize( + path, + vInfo.getInstallPath().substring( + vInfo.getInstallPath().lastIndexOf(File.separator) + 1)); + vInfo.setSize(vSize); + loc.updateVirtualSize(vSize); + loc.save(); + } catch (Exception e) { + s_logger.error("Unable to get the virtual size of the volume: " + vInfo.getInstallPath() + + " due to " + e.getMessage()); + } + } + + result.put(vInfo.getId(), vInfo); + s_logger.debug("Added volume name: " + vInfo.getTemplateName() + ", path: " + vol); + } + return result; + } + + private int deleteDownloadDirectories(File downloadPath, int deleted) { + try { + if (downloadPath.exists()) { + File[] files = downloadPath.listFiles(); + for (int i = 0; i < files.length; i++) { + if (files[i].isDirectory()) { + deleteDownloadDirectories(files[i], deleted); + files[i].delete(); + deleted++; + } else { + files[i].delete(); + deleted++; + } + } + } + } catch (Exception ex) { + s_logger.info("Failed to clean up template downloads directory " + ex.toString()); + } + return deleted; + } + + public static class ZfsPathParser extends OutputInterpreter { + String _parent; + List<String> paths = new ArrayList<String>(); + + public ZfsPathParser(String parent) { + _parent = parent; + } + + @Override + public String interpret(BufferedReader reader) throws IOException { + String line = null; + while ((line = reader.readLine()) != null) { + paths.add(line); + } + return null; + } + + public List<String> getPaths() { + return paths; + } + + @Override + public boolean drain() { + return true; + } + } + + public DownloadManagerImpl() { + } + + @Override + @SuppressWarnings("unchecked") + public boolean configure(String name, Map<String, Object> params) throws ConfigurationException { + _name = name; + + String value = null; + + _storage = (StorageLayer) params.get(StorageLayer.InstanceConfigKey); + if (_storage == null) { + value = (String) params.get(StorageLayer.ClassConfigKey); + if (value == null) { + throw new ConfigurationException("Unable to find the storage layer"); + } + + Class<StorageLayer> clazz; + try { + clazz = (Class<StorageLayer>) Class.forName(value); + _storage = clazz.newInstance(); + } catch (ClassNotFoundException e) { + throw new ConfigurationException("Unable to instantiate " + value); + } catch (InstantiationException e) { + throw new ConfigurationException("Unable to instantiate " + value); + } catch (IllegalAccessException e) { + throw new ConfigurationException("Unable to instantiate " + value); + } + } + String useSsl = (String) params.get("sslcopy"); + if (useSsl != null) { + _sslCopy = Boolean.parseBoolean(useSsl); + + } + String inSystemVM = (String) params.get("secondary.storage.vm"); + if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) { + s_logger.info("DownloadManager: starting additional services since we are inside system vm"); + startAdditionalServices(); + blockOutgoingOnPrivate(); + } + + value = (String) params.get("install.timeout.pergig"); - this.installTimeoutPerGig = NumbersUtil.parseInt(value, 15 * 60) * 1000; ++ installTimeoutPerGig = NumbersUtil.parseInt(value, 15 * 60) * 1000; + + value = (String) params.get("install.numthreads"); + final int numInstallThreads = NumbersUtil.parseInt(value, 10); + + String scriptsDir = (String) params.get("template.scripts.dir"); + if (scriptsDir == null) { + scriptsDir = "scripts/storage/secondary"; + } + + listTmpltScr = Script.findScript(scriptsDir, "listvmtmplt.sh"); + if (listTmpltScr == null) { + throw new ConfigurationException("Unable to find the listvmtmplt.sh"); + } + s_logger.info("listvmtmplt.sh found in " + listTmpltScr); + + createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh"); + if (createTmpltScr == null) { + throw new ConfigurationException("Unable to find createtmplt.sh"); + } + s_logger.info("createtmplt.sh found in " + createTmpltScr); + + listVolScr = Script.findScript(scriptsDir, "listvolume.sh"); + if (listVolScr == null) { + throw new ConfigurationException("Unable to find the listvolume.sh"); + } + s_logger.info("listvolume.sh found in " + listVolScr); + + createVolScr = Script.findScript(scriptsDir, "createvolume.sh"); + if (createVolScr == null) { + throw new ConfigurationException("Unable to find createvolume.sh"); + } + s_logger.info("createvolume.sh found in " + createVolScr); + + _processors = new HashMap<String, Processor>(); + + Processor processor = new VhdProcessor(); + processor.configure("VHD Processor", params); + _processors.put("VHD Processor", processor); + + processor = new IsoProcessor(); + processor.configure("ISO Processor", params); + _processors.put("ISO Processor", processor); + + processor = new QCOW2Processor(); + processor.configure("QCOW2 Processor", params); + _processors.put("QCOW2 Processor", processor); + + processor = new VmdkProcessor(); + processor.configure("VMDK Processor", params); + _processors.put("VMDK Processor", processor); + + processor = new RawImageProcessor(); + processor.configure("Raw Image Processor", params); + _processors.put("Raw Image Processor", processor); + + _templateDir = (String) params.get("public.templates.root.dir"); + if (_templateDir == null) { + _templateDir = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR; + } + _templateDir += File.separator + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR; + _volumeDir = TemplateConstants.DEFAULT_VOLUME_ROOT_DIR + File.separator; + // Add more processors here. + threadPool = Executors.newFixedThreadPool(numInstallThreads); + return true; + } + + private void blockOutgoingOnPrivate() { + Script command = new Script("/bin/bash", s_logger); + String intf = "eth1"; + command.add("-c"); + command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;" + + "iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;"); + + String result = command.execute(); + if (result != null) { + s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result); + return; + } + } + + @Override + public String getName() { + return _name; + } + + @Override + public boolean start() { + return true; + } + + @Override + public boolean stop() { + return true; + } + + private void startAdditionalServices() { + + Script command = new Script("/bin/bash", s_logger); + command.add("-c"); + command.add("if [ -d /etc/apache2 ] ; then service apache2 stop; else service httpd stop; fi "); + String result = command.execute(); + if (result != null) { + s_logger.warn("Error in stopping httpd service err=" + result); + } + String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT); + String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF; + + command = new Script("/bin/bash", s_logger); + command.add("-c"); + command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" + "iptables -I INPUT -i " + + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;"); + + result = command.execute(); + if (result != null) { + s_logger.warn("Error in opening up httpd port err=" + result); + return; + } + + command = new Script("/bin/bash", s_logger); + command.add("-c"); + command.add("if [ -d /etc/apache2 ] ; then service apache2 start; else service httpd start; fi "); + result = command.execute(); + if (result != null) { + s_logger.warn("Error in starting httpd service err=" + result); + return; + } + command = new Script("mkdir", s_logger); + command.add("-p"); + command.add("/var/www/html/copy/template"); + result = command.execute(); + if (result != null) { + s_logger.warn("Error in creating directory =" + result); + return; + } + } + +}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/342624e0/setup/db/db/schema-410to420.sql ---------------------------------------------------------------------- diff --cc setup/db/db/schema-410to420.sql index 85b5c83,74f3388..eb7d78f --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@@ -1450,24 -733,6 +1451,27 @@@ CREATE VIEW `cloud`.`volume_view` A `cloud`.`async_job` ON async_job.instance_id = volumes.id and async_job.instance_type = 'Volume' and async_job.job_status = 0; +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'midonet.apiserver.address', 'http://localhost:8081', 'Specify the address at which the Midonet API server can be contacted (if using Midonet)'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'midonet.providerrouter.id', 'd7c5e6a3-e2f4-426b-b728-b7ce6a0448e5', 'Specifies the UUID of the Midonet provider router (if using Midonet)'); + +alter table cloud.vpc_gateways add column source_nat boolean default false; +alter table cloud.private_ip_address add column source_nat boolean default false; + +CREATE TABLE `cloud`.`account_vnet_map` ( + `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT, + `uuid` varchar(255) UNIQUE, + `vnet_range` varchar(255) NOT NULL COMMENT 'dedicated guest vlan range', + `account_id` bigint unsigned NOT NULL COMMENT 'account id. foreign key to account table', + `physical_network_id` bigint unsigned NOT NULL COMMENT 'physical network id. foreign key to the the physical network table', + PRIMARY KEY (`id`), + CONSTRAINT `fk_account_vnet_map__physical_network_id` FOREIGN KEY (`physical_network_id`) REFERENCES `physical_network` (`id`) ON DELETE CASCADE, + INDEX `i_account_vnet_map__physical_network_id`(`physical_network_id`), + CONSTRAINT `fk_account_vnet_map__account_id` FOREIGN KEY (`account_id`) REFERENCES `account` (`id`) ON DELETE CASCADE, + INDEX `i_account_vnet_map__account_id`(`account_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +ALTER TABLE `cloud`.`op_dc_vnet_alloc` ADD COLUMN account_vnet_map_id bigint unsigned; +ALTER TABLE `cloud`.`op_dc_vnet_alloc` ADD CONSTRAINT `fk_op_dc_vnet_alloc__account_vnet_map_id` FOREIGN KEY `fk_op_dc_vnet_alloc__account_vnet_map_id` (`account_vnet_map_id`) REFERENCES `account_vnet_map` (`id`); + + update `cloud`.`vm_template` set state='Allocated' where state is NULL; + update `cloud`.`vm_template` set update_count=0 where update_count is NULL;
