Forward merged 4.8 to master and fixed conflicts in #1331
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/88cd1822 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/88cd1822 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/88cd1822 Branch: refs/heads/master Commit: 88cd1822729decee1b86076d8494258a41a89522 Parents: b4ad38d 091ca56 Author: Will Stevens <[email protected]> Authored: Thu May 19 12:08:08 2016 -0400 Committer: Will Stevens <[email protected]> Committed: Thu May 19 12:08:08 2016 -0400 ---------------------------------------------------------------------- agent/bindir/libvirtqemuhook.in | 2 + .../datastore/ObjectInDataStoreManagerImpl.java | 1 + .../VirtualNetworkApplianceManagerImpl.java | 3 + server/src/com/cloud/test/TestAppender.java | 178 ++++++++++++ services/secondary-storage/server/pom.xml | 3 - .../resource/NfsSecondaryStorageResource.java | 288 +++++++++++++------ .../LocalNfsSecondaryStorageResourceTest.java | 44 ++- .../NfsSecondaryStorageResourceTest.java | 118 +++----- .../debian/config/opt/cloud/bin/cs/CsDatabag.py | 4 + .../debian/config/opt/cloud/bin/cs/CsFile.py | 1 + .../config/opt/cloud/bin/cs/CsRedundant.py | 3 + 11 files changed, 454 insertions(+), 191 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/88cd1822/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/88cd1822/services/secondary-storage/server/pom.xml ---------------------------------------------------------------------- diff --cc services/secondary-storage/server/pom.xml index f3d0176,97e68a8..50155cf --- a/services/secondary-storage/server/pom.xml +++ b/services/secondary-storage/server/pom.xml @@@ -23,12 -23,9 +23,9 @@@ <parent> <groupId>org.apache.cloudstack</groupId> <artifactId>cloudstack-service-secondary-storage</artifactId> - <version>4.8.1-SNAPSHOT</version> + <version>4.9.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <properties> - <skipTests>true</skipTests> - </properties> <dependencies> <dependency> <groupId>log4j</groupId> http://git-wip-us.apache.org/repos/asf/cloudstack/blob/88cd1822/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java ---------------------------------------------------------------------- diff --cc services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 08eab33,f512da0..e7a9076 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@@ -16,72 -16,7 +16,91 @@@ // under the License. package org.apache.cloudstack.storage.resource; - import static com.cloud.utils.StringUtils.join; +import static com.cloud.utils.storage.S3.S3Utils.putFile; ++import static com.cloud.utils.StringUtils.join; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static org.apache.commons.lang.StringUtils.substringAfterLast; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import javax.naming.ConfigurationException; + ++import com.cloud.exception.InvalidParameterValueException; ++import com.cloud.storage.Storage; ++import com.cloud.storage.template.TemplateConstants; ++import com.cloud.utils.EncryptionUtil; ++import com.google.gson.Gson; ++import com.google.gson.GsonBuilder; ++import io.netty.bootstrap.ServerBootstrap; ++import io.netty.channel.Channel; ++import io.netty.channel.ChannelInitializer; ++import io.netty.channel.ChannelPipeline; ++import io.netty.channel.EventLoopGroup; ++import io.netty.channel.nio.NioEventLoopGroup; ++import io.netty.channel.socket.SocketChannel; ++import io.netty.channel.socket.nio.NioServerSocketChannel; ++import io.netty.handler.codec.http.HttpContentCompressor; ++import io.netty.handler.codec.http.HttpRequestDecoder; ++import io.netty.handler.codec.http.HttpResponseEncoder; ++import io.netty.handler.logging.LogLevel; ++import io.netty.handler.logging.LoggingHandler; ++import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; ++import org.apache.cloudstack.storage.template.UploadEntity; ++import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; ++import org.apache.commons.codec.digest.DigestUtils; ++import org.apache.commons.io.FileUtils; ++import org.apache.commons.io.FilenameUtils; ++import org.apache.commons.lang.StringUtils; ++import org.apache.http.HttpEntity; ++import org.apache.http.HttpResponse; ++import org.apache.http.NameValuePair; ++import org.apache.http.client.HttpClient; ++import org.apache.http.client.methods.HttpGet; ++import org.apache.http.client.utils.URLEncodedUtils; ++import org.apache.http.impl.client.DefaultHttpClient; ++import org.apache.log4j.Logger; ++ + import com.amazonaws.services.s3.model.S3ObjectSummary; ++ +import org.apache.cloudstack.framework.security.keystore.KeystoreManager; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadProgressCommand; - import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.command.UploadStatusAnswer; +import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; +import org.apache.cloudstack.storage.command.UploadStatusCommand; +import org.apache.cloudstack.storage.template.DownloadManager; +import org.apache.cloudstack.storage.template.DownloadManagerImpl; +import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; - import org.apache.cloudstack.storage.template.UploadEntity; +import org.apache.cloudstack.storage.template.UploadManager; +import org.apache.cloudstack.storage.template.UploadManagerImpl; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; - import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; - import org.apache.commons.codec.digest.DigestUtils; - import org.apache.commons.io.FileUtils; - import org.apache.commons.io.FilenameUtils; - import org.apache.commons.lang.StringUtils; - import org.apache.http.HttpEntity; - import org.apache.http.HttpResponse; - import org.apache.http.NameValuePair; - import org.apache.http.client.HttpClient; - import org.apache.http.client.methods.HttpGet; - import org.apache.http.client.utils.URLEncodedUtils; - import org.apache.http.impl.client.DefaultHttpClient; - import org.apache.log4j.Logger; - import org.joda.time.DateTime; - import org.joda.time.format.ISODateTimeFormat; + - import com.amazonaws.services.s3.model.S3ObjectSummary; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckHealthAnswer; import com.cloud.agent.api.CheckHealthCommand; @@@ -116,13 -51,13 +135,11 @@@ import com.cloud.agent.api.to.NfsTO import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.exception.InternalErrorException; --import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.resource.ServerResourceBase; import com.cloud.storage.DataStoreRole; --import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@@ -132,40 -67,102 +149,24 @@@ import com.cloud.storage.template.Proce import com.cloud.storage.template.QCOW2Processor; import com.cloud.storage.template.RawImageProcessor; import com.cloud.storage.template.TARProcessor; --import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.template.TemplateLocation; import com.cloud.storage.template.TemplateProp; import com.cloud.storage.template.VhdProcessor; import com.cloud.storage.template.VmdkProcessor; --import com.cloud.utils.EncryptionUtil; import com.cloud.utils.NumbersUtil; ++import com.cloud.utils.storage.S3.S3Utils; import com.cloud.utils.SwiftUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; --import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.SecondaryStorageVm; --import com.google.gson.Gson; --import com.google.gson.GsonBuilder; - --import io.netty.bootstrap.ServerBootstrap; --import io.netty.channel.Channel; --import io.netty.channel.ChannelInitializer; --import io.netty.channel.ChannelPipeline; --import io.netty.channel.EventLoopGroup; --import io.netty.channel.nio.NioEventLoopGroup; --import io.netty.channel.socket.SocketChannel; --import io.netty.channel.socket.nio.NioServerSocketChannel; --import io.netty.handler.codec.http.HttpContentCompressor; --import io.netty.handler.codec.http.HttpRequestDecoder; --import io.netty.handler.codec.http.HttpResponseEncoder; --import io.netty.handler.logging.LogLevel; --import io.netty.handler.logging.LoggingHandler; -import org.apache.cloudstack.framework.security.keystore.KeystoreManager; -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DeleteCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.command.DownloadProgressCommand; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.command.UploadStatusAnswer; -import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; -import org.apache.cloudstack.storage.command.UploadStatusCommand; -import org.apache.cloudstack.storage.template.DownloadManager; -import org.apache.cloudstack.storage.template.DownloadManagerImpl; -import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; -import org.apache.cloudstack.storage.template.UploadEntity; -import org.apache.cloudstack.storage.template.UploadManager; -import org.apache.cloudstack.storage.template.UploadManagerImpl; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URLEncodedUtils; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.log4j.Logger; + import org.joda.time.DateTime; + import org.joda.time.format.ISODateTimeFormat; -import javax.naming.ConfigurationException; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.net.InetAddress; -import java.net.URI; -import java.net.UnknownHostException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static com.cloud.utils.StringUtils.join; -import static com.cloud.utils.storage.S3.S3Utils.putFile; -import static java.lang.String.format; -import static java.util.Arrays.asList; -import static org.apache.commons.lang.StringUtils.substringAfterLast; - public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource { - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); + public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; private static final String VOLUME_ROOT_DIR = "volumes"; @@@ -933,6 -925,118 +921,118 @@@ } } + /*** + *This method will create a file using the filenName and metaFileName. + *That file will contain the given attributes (unique name, file name, size, and virtualSize). + * + * @param metaFileName : The path of the metadata file + * @param filename :attribute: Filename of the template + * @param uniqueName :attribute: Unique name of the template + * @param size :attribute: physical size of the template + * @param virtualSize :attribute: virtual size of the template + * @return File representing the metadata file + * @throws IOException + */ + + protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String filename, long size, long virtualSize) throws IOException { + File metaFile = new File(metaFileName); + FileWriter writer = new FileWriter(metaFile); + BufferedWriter bufferWriter = new BufferedWriter(writer); + bufferWriter.write("uniquename=" + uniqueName); + bufferWriter.write("\n"); + bufferWriter.write("filename=" + filename); + bufferWriter.write("\n"); + bufferWriter.write("size=" + size); + bufferWriter.write("\n"); + bufferWriter.write("virtualsize=" + virtualSize); + bufferWriter.close(); + writer.close(); + return metaFile; + } + + /** + * Creates a template.properties for Swift with its correct unique name + * + * @param swift The swift object + * @param srcFile Source file on the staging NFS + * @param containerName Destination container + * @return true on successful write + */ + protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName) throws IOException { + + String uniqueName = FilenameUtils.getBaseName(srcFile.getName()); + + File uniqDir = _storage.createUniqDir(); + String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp; + _storage.create(uniqDir.getAbsolutePath(), _tmpltpp); + + long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())); + + File metaFile = swiftWriteMetadataFile(metaFileName, + uniqueName, + srcFile.getName(), + srcFile.length(), + virtualSize); + + SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp); + metaFile.delete(); + uniqDir.delete(); + + return true; + } + + /** + * Copies data from NFS and uploads it into a Swift container + * + * @param cmd CopyComand + * @return CopyCmdAnswer + */ + protected Answer copyFromNfsToSwift(CopyCommand cmd) { + + final DataTO srcData = cmd.getSrcTO(); + final DataTO destData = cmd.getDestTO(); + + DataStoreTO srcDataStore = srcData.getDataStore(); + NfsTO srcStore = (NfsTO)srcDataStore; + DataStoreTO destDataStore = destData.getDataStore(); - File srcFile = getFile(srcData.getPath(), srcStore.getUrl()); ++ File srcFile = getFile(srcData.getPath(), srcStore.getUrl(), _nfsVersion); + + SwiftTO swift = (SwiftTO)destDataStore; + + try { + + String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId()); + String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName()); + + + DataTO retObj = null; + if (destData.getObjectType() == DataObjectType.TEMPLATE) { + swiftUploadMetadataFile(swift, srcFile, containerName); + TemplateObjectTO newTemplate = new TemplateObjectTO(); + newTemplate.setPath(swiftPath); + newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()))); + newTemplate.setPhysicalSize(srcFile.length()); + newTemplate.setFormat(getTemplateFormat(srcFile.getName())); + retObj = newTemplate; + } else if (destData.getObjectType() == DataObjectType.VOLUME) { + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setPath(containerName); + newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()))); + retObj = newVol; + } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) { + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + newSnapshot.setPath(containerName); + retObj = newSnapshot; + } + + return new CopyCmdAnswer(retObj); + + } catch (Exception e) { + s_logger.error("failed to upload " + srcData.getPath(), e); + return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString()); + } + } + String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) { Script command = new Script("/bin/bash", s_logger); command.add("-c");
