Updated Branches: refs/heads/object_store 235825dc3 -> efd249d61
CLOUDSTACK-2584: fix create template from s3 Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/827a1a22 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/827a1a22 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/827a1a22 Branch: refs/heads/object_store Commit: 827a1a229b0107fbf6d818fd64571f3c36c4da53 Parents: 235825d Author: Edison Su <[email protected]> Authored: Tue May 21 16:53:38 2013 -0700 Committer: Edison Su <[email protected]> Committed: Tue May 21 16:55:41 2013 -0700 ---------------------------------------------------------------------- .../apache/cloudstack/storage/to/ImageStoreTO.java | 1 - .../storage/motion/AncientDataMotionStrategy.java | 104 ++---------- .../resource/NfsSecondaryStorageResource.java | 124 ++++++--------- 3 files changed, 69 insertions(+), 160 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/827a1a22/engine/api/src/org/apache/cloudstack/storage/to/ImageStoreTO.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/storage/to/ImageStoreTO.java b/engine/api/src/org/apache/cloudstack/storage/to/ImageStoreTO.java index 28d5ef3..96e9479 100644 --- a/engine/api/src/org/apache/cloudstack/storage/to/ImageStoreTO.java +++ b/engine/api/src/org/apache/cloudstack/storage/to/ImageStoreTO.java @@ -74,5 +74,4 @@ public class ImageStoreTO implements DataStoreTO { public DataStoreRole getRole() { return this.role; } - } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/827a1a22/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java ---------------------------------------------------------------------- diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index cea4069..c0a3ec6 100644 --- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -137,37 +137,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { return true; } - @DB - protected Answer copyVolumeFromImage(DataObject srcData, DataObject destData) { - String value = configDao.getValue(Config.CopyVolumeWait.key()); - int _copyvolumewait = NumbersUtil.parseInt(value, - Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); - - if (needCacheStorage(srcData, destData)) { - //need to copy it to image cache store - Scope destScope = destData.getDataStore().getScope(); - if (destScope instanceof ClusterScope){ - ClusterScope clusterScope = (ClusterScope)destScope; - destScope = new ZoneScope(clusterScope.getZoneId()); - } else if (destScope instanceof HostScope){ - HostScope hostScope = (HostScope)destScope; - destScope = new ZoneScope(hostScope.getZoneId()); - } - DataObject cacheData = cacheMgr.createCacheObject(srcData, destScope); - CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _copyvolumewait); - EndPoint ep = selector.select(cacheData, destData); - Answer answer = ep.sendMessage(cmd); - return answer; - } else { - //handle copy it to/from cache store - CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _copyvolumewait); - EndPoint ep = selector.select(srcData, destData); - Answer answer = ep.sendMessage(cmd); - return answer; - } - } - - private Answer copyTemplate(DataObject srcData, DataObject destData) { + protected Answer copyObject(DataObject srcData, DataObject destData) { String value = configDao.getValue(Config.PrimaryStorageDownloadWait.toString()); int _primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue())); if (needCacheStorage(srcData, destData)) { @@ -233,7 +203,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { CopyCommand cmd = new CopyCommand(srcData.getTO(), volObj.getTO(), _createVolumeFromSnapshotWait); EndPoint ep = selector.select(snapObj, volObj); Answer answer = ep.sendMessage(cmd); - + return answer; } catch (Exception e) { s_logger.error(basicErrMsg, e); @@ -275,30 +245,24 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { Answer answer = null; String errMsg = null; try { - if (destData.getType() == DataObjectType.VOLUME - && srcData.getType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Image) { - answer = copyVolumeFromImage(srcData, destData); - } else if (destData.getType() == DataObjectType.TEMPLATE - && srcData.getType() == DataObjectType.TEMPLATE) { - answer = copyTemplate(srcData, destData); - } else if (srcData.getType() == DataObjectType.SNAPSHOT + + if (srcData.getType() == DataObjectType.SNAPSHOT && destData.getType() == DataObjectType.VOLUME) { answer = copyVolumeFromSnapshot(srcData, destData); } else if (srcData.getType() == DataObjectType.SNAPSHOT && destData.getType() == DataObjectType.TEMPLATE) { answer = createTemplateFromSnapshot(srcData, destData); - } else if (srcData.getType() == DataObjectType.VOLUME - && destData.getType() == DataObjectType.TEMPLATE) { - answer = createTemplateFromVolume(srcData, destData); } else if (srcData.getType() == DataObjectType.TEMPLATE && destData.getType() == DataObjectType.VOLUME) { answer = cloneVolume(srcData, destData); } else if (destData.getType() == DataObjectType.VOLUME - && srcData.getType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary) { + && srcData.getType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary && destData.getDataStore().getRole() == DataStoreRole.Primary) { answer = copyVolumeBetweenPools(srcData, destData); } else if (srcData.getType() == DataObjectType.SNAPSHOT && destData.getType() == DataObjectType.SNAPSHOT) { answer = copySnapshot(srcData, destData); + } else { + answer = copyObject(srcData, destData); } if (answer != null && !answer.getResult()) { @@ -337,58 +301,26 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { return answer; } + protected Answer copySnapshot(DataObject srcData, DataObject destData) { + String value = configDao.getValue(Config.BackupSnapshotWait.toString()); + int _backupsnapshotwait = NumbersUtil.parseInt(value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue())); - private Answer createTemplateFromVolume(DataObject srcData, - DataObject destData) { - - String value = configDao - .getValue(Config.CreatePrivateTemplateFromVolumeWait.toString()); - int _createprivatetemplatefromvolumewait = NumbersUtil.parseInt(value, - Integer.parseInt(Config.CreatePrivateTemplateFromVolumeWait - .getDefaultValue())); - + DataObject cacheData = null; + try { if (needCacheStorage(srcData, destData)) { - //need to copy it to image cache store - DataObject cacheData = cacheMgr.createCacheObject(srcData, destData.getDataStore().getScope()); - CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _createprivatetemplatefromvolumewait); - EndPoint ep = selector.select(cacheData, destData); + cacheData = cacheMgr.getCacheObject(srcData, destData.getDataStore().getScope()); + + CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait); + cmd.setCacheTO(cacheData.getTO()); + EndPoint ep = selector.select(srcData, destData); Answer answer = ep.sendMessage(cmd); return answer; } else { - //handle copy it to/from cache store - CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _createprivatetemplatefromvolumewait); + CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait); EndPoint ep = selector.select(srcData, destData); Answer answer = ep.sendMessage(cmd); return answer; } - } - - protected Answer copySnapshot(DataObject srcData, DataObject destData) { - String value = configDao.getValue(Config.BackupSnapshotWait.toString()); - int _backupsnapshotwait = NumbersUtil.parseInt(value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue())); - - DataObject cacheData = null; - Answer answer = null; - try { - if (needCacheStorage(srcData, destData)) { - cacheData = cacheMgr.getCacheObject(srcData, destData.getDataStore().getScope()); - - CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait); - cmd.setCacheTO(cacheData.getTO()); - EndPoint ep = selector.select(srcData, destData); - answer = ep.sendMessage(cmd); - } else { - CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait); - EndPoint ep = selector.select(srcData, destData); - answer = ep.sendMessage(cmd); - } - // clean up cache entry in case of failure - if (answer == null || !answer.getResult()) { - if (cacheData != null) { - cacheMgr.deleteCacheObject(cacheData); - } - } - return answer; } catch (Exception e) { s_logger.debug("copy snasphot failed: " + e.toString()); if (cacheData != null) { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/827a1a22/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 6fbb06b..8dca6f5 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage.resource; import static com.cloud.utils.S3Utils.getDirectory; +import static com.cloud.utils.S3Utils.putFile; import static com.cloud.utils.StringUtils.join; import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock; import static java.lang.String.format; @@ -27,6 +28,7 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; @@ -118,7 +120,9 @@ import com.cloud.storage.template.TemplateProp; import com.cloud.storage.template.VhdProcessor; import com.cloud.utils.NumbersUtil; import com.cloud.utils.S3Utils; +import com.cloud.utils.S3Utils.ClientOptions; import com.cloud.utils.S3Utils.FileNamingStrategy; +import com.cloud.utils.S3Utils.ObjectNamingStrategy; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.OutputInterpreter; @@ -214,8 +218,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return execute((DownloadTemplateFromSwiftToSecondaryStorageCommand) cmd); } else if (cmd instanceof UploadTemplateToSwiftFromSecondaryStorageCommand) { return execute((UploadTemplateToSwiftFromSecondaryStorageCommand) cmd); - } else if (cmd instanceof UploadTemplateToS3FromSecondaryStorageCommand) { - return execute((UploadTemplateToS3FromSecondaryStorageCommand) cmd); } else if (cmd instanceof CleanupSnapshotBackupCommand) { return execute((CleanupSnapshotBackupCommand) cmd); } else if (cmd instanceof CopyCommand) { @@ -424,6 +426,17 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } return new CopyCmdAnswer(""); } + + protected Answer copyFromNfsToImage(CopyCommand cmd) { + DataTO destData = cmd.getDestTO(); + DataStoreTO destDataStore = destData.getDataStore(); + + if (destDataStore instanceof S3TO) { + return copyFromNfsToS3(cmd); + } else { + return new CopyCmdAnswer("unsupported "); + } + } protected Answer execute(CopyCommand cmd) { DataTO srcData = cmd.getSrcTO(); @@ -440,6 +453,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S NfsTO destImageStore = (NfsTO)destDataStore; return this.copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore); } + + if (srcDataStore.getRole() == DataStoreRole.ImageCache && destDataStore.getRole() == DataStoreRole.Image) { + return copyFromNfsToImage(cmd); + } return Answer.createUnsupportedCommandAnswer(cmd); @@ -608,86 +625,47 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } - private Answer execute(UploadTemplateToS3FromSecondaryStorageCommand cmd) { - /* - final S3TO s3 = cmd.getS3(); - final Long accountId = cmd.getAccountId(); - final Long templateId = cmd.getTemplateId(); - - try { - final S3TO s3 = cmd.getS3(); - final Long accountId = cmd.getAccountId(); - final Long templateId = cmd.getTemplateId(); - + protected Answer copyFromNfsToS3(CopyCommand cmd) { + final DataTO srcData = cmd.getSrcTO(); + final DataTO destData = cmd.getDestTO(); + DataStoreTO srcDataStore = srcData.getDataStore(); + NfsTO srcStore = (NfsTO)srcDataStore; + DataStoreTO destDataStore = destData.getDataStore(); + + final S3TO s3 = (S3TO)destDataStore; + try { - final String templatePath = determineStorageTemplatePath( - cmd.getStoragePath(), accountId, templateId); + srcStore.getUrl(), srcData.getPath()); if (s_logger.isDebugEnabled()) { - s_logger.debug("Found template id " + templateId - + " account id " + accountId + " from directory " + s_logger.debug("Found " + srcData.getObjectType() + " from directory " + templatePath + " to upload to S3."); } - if (!_storage.isDirectory(templatePath)) { - final String errMsg = format("S3 Sync Failure: Directory %1$s" - + "for template id %2$s does not exist.", templatePath, - templateId); - s_logger.error(errMsg); - return new Answer(cmd, false, errMsg); - } - - if (!_storage.isFile(templatePath + "/template.properties")) { - final String errMsg = format("S3 Sync Failure: Template id " - + "%1$s does not exist on the file system.", - templatePath); - s_logger.error(errMsg); - return new Answer(cmd, false, errMsg); - } - - if (s_logger.isDebugEnabled()) { - s_logger.debug(format( - "Pushing template id %1$s from %2$s to S3...", - templateId, templatePath)); - } - final String bucket = s3.getBucketName(); - putDirectory(s3, bucket, _storage.getFile(templatePath), - new FilenameFilter() { - @Override - public boolean accept(final File directory, - final String fileName) { - File fileToUpload = new File(directory.getAbsolutePath() + "/" + fileName); - return !fileName.startsWith(".") && !fileToUpload.isDirectory(); - } - }, new ObjectNamingStrategy() { - @Override - public String determineKey(final File file) { - s_logger.debug(String - .format("Determining key using account id %1$s and template id %2$s", - accountId, templateId)); - return join( - asList(determineS3TemplateDirectory( - accountId, templateId), file - .getName()), S3Utils.SEPARATOR); - } - }); - - return new Answer( - cmd, - true, - format("Uploaded the contents of directory %1$s for template id %2$s to S3 bucket %3$s", - templatePath, templateId, bucket)); - + final File srcFile = _storage.getFile(templatePath); + String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName(); + putFile(s3, srcFile, bucket, key); + + DataTO retObj = null; + if (destData.getObjectType() == DataObjectType.TEMPLATE) { + TemplateObjectTO newTemplate = new TemplateObjectTO(); + newTemplate.setPath(key); + newTemplate.setSize(srcFile.length()); + retObj = newTemplate; + } else if (destData.getObjectType() == DataObjectType.VOLUME) { + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setPath(key); + newVol.setSize(srcFile.length()); + retObj = newVol; + } + + return new CopyCmdAnswer(retObj); } catch (Exception e) { - - final String errMsg = format("Failed to upload template id %1$s", - templateId); - s_logger.error(errMsg, e); - return new Answer(cmd, false, errMsg); - */ - return new Answer(cmd, false, "not supported "); + s_logger.error("failed to upload" + srcData.getPath(), e); + return new CopyCmdAnswer("failed to upload" + srcData.getPath() + e.toString()); + } } String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) {
