volume upload: added max size check for temaplte/volume post upload used the existing configuration variables max.template.iso.size and storage.max.volume.upload.size for templates and volumes respectively.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/b16520bc Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/b16520bc Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/b16520bc Branch: refs/heads/master Commit: b16520bcecf5066259f9ee44c653fbb60847c18b Parents: dd1a8da Author: Rajani Karuturi <rajanikarut...@gmail.com> Authored: Tue Mar 3 17:22:40 2015 +0530 Committer: Rajani Karuturi <rajanikarut...@gmail.com> Committed: Tue Mar 3 17:27:40 2015 +0530 ---------------------------------------------------------------------- .../TemplateOrVolumePostUploadCommand.java | 10 +++++++ .../com/cloud/storage/VolumeApiServiceImpl.java | 2 ++ .../template/HypervisorTemplateAdapter.java | 3 ++ .../resource/HttpUploadServerHandler.java | 22 +++++++++++---- .../resource/NfsSecondaryStorageResource.java | 29 ++++++++++++++++---- .../storage/template/UploadEntity.java | 7 +++++ 6 files changed, 63 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java ---------------------------------------------------------------------- diff --git a/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java index 8872798..cc9df71 100644 --- a/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java +++ b/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java @@ -45,6 +45,8 @@ public class TemplateOrVolumePostUploadCommand { String remoteEndPoint; + String maxUploadSize; + public TemplateOrVolumePostUploadCommand(long entityId, String entityUUID, String absolutePath, String checksum, String type, String name, String imageFormat, String dataTo, String dataToRole) { this.entityId = entityId; @@ -156,4 +158,12 @@ public class TemplateOrVolumePostUploadCommand { public void setName(String name) { this.name = name; } + + public String getMaxUploadSize() { + return maxUploadSize; + } + + public void setMaxUploadSize(String maxUploadSize) { + this.maxUploadSize = maxUploadSize; + } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/server/src/com/cloud/storage/VolumeApiServiceImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 9840096..d2c1c69 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -337,6 +337,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic vol.getName(), vol.getFormat().toString(), dataObject.getDataStore().getUri(), dataObject.getDataStore().getRole().toString()); command.setLocalPath(volumeStore.getLocalDownloadPath()); + //using the existing max upload size configuration + command.setMaxUploadSize(_configDao.getValue(Config.MaxUploadVolumeSize.key())); Gson gson = new GsonBuilder().create(); String metadata = EncryptionUtil.encodeData(gson.toJson(command), key); response.setMetadata(metadata); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/server/src/com/cloud/template/HypervisorTemplateAdapter.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java index 6d81047..0cb48fc 100755 --- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java @@ -26,6 +26,7 @@ import java.util.concurrent.ExecutionException; import javax.ejb.Local; import javax.inject.Inject; +import com.cloud.configuration.Config; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -259,6 +260,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl.getChecksum(), tmpl .getType().toString(), template.getName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), templateOnStore.getDataStore().getRole() .toString()); + //using the existing max template size configuration + payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key())); payload.setRemoteEndPoint(ep.getPublicAddr()); payload.setRequiresHvm(template.requiresHvm()); payloads.add(payload); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java index 0c95707..7a17ef1 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java @@ -108,6 +108,8 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj String expires = null; String metadata = null; String hostname = null; + long contentLength = 0; + for (Entry<String, String> entry : request.headers()) { switch (entry.getKey()) { case HEADER_SIGNATURE: @@ -122,12 +124,16 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj case HEADER_HOST: hostname = entry.getValue(); break; + case HttpHeaders.Names.CONTENT_LENGTH: + contentLength = Long.valueOf(entry.getValue()); + break; } } logger.info("HEADER: signature=" + signature); logger.info("HEADER: metadata=" + metadata); logger.info("HEADER: expires=" + expires); logger.info("HEADER: hostname=" + hostname); + logger.info("HEADER: Content-Length=" + contentLength); QueryStringDecoder decoderQuery = new QueryStringDecoder(uri); Map<String, List<String>> uriAttributes = decoderQuery.parameters(); uuid = uriAttributes.get("uuid").get(0); @@ -136,9 +142,9 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj UploadEntity uploadEntity = null; try { // Validate the request here - storageResource.validatePostUploadRequest(signature, metadata, expires, hostname, uuid); + storageResource.validatePostUploadRequest(signature, metadata, expires, hostname, contentLength, uuid); //create an upload entity. This will fail if entity already exists. - uploadEntity = storageResource.createUploadEntity(uuid, metadata); + uploadEntity = storageResource.createUploadEntity(uuid, metadata, contentLength); } catch (InvalidParameterValueException ex) { logger.error("post request validation failed", ex); responseContent.append(ex.getMessage()); @@ -185,9 +191,15 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj return; } if (chunk instanceof LastHttpContent) { - readFileUploadData(); - writeResponse(ctx.channel(), HttpResponseStatus.OK); - reset(); + try { + readFileUploadData(); + writeResponse(ctx.channel(), HttpResponseStatus.OK); + reset(); + } catch (InvalidParameterValueException e) { + logger.error("error during the file install.", e); + responseContent.append("\n").append(e.getMessage()); + writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR); + } } } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index f80572a..236498c 100755 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -2602,7 +2602,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return Script.findScript(scriptsDir, scriptname); } - public UploadEntity createUploadEntity(String uuid, String metadata) { + public UploadEntity createUploadEntity(String uuid, String metadata, long contentLength) { TemplateOrVolumePostUploadCommand cmd = getTemplateOrVolumePostUploadCmd(metadata); UploadEntity uploadEntity = null; if(cmd == null ){ @@ -2610,10 +2610,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S throw new InvalidParameterValueException("unable to decode and deserialize metadata"); } else { uuid = cmd.getEntityUUID(); - if(uploadEntityStateMap.containsKey(uuid)) { + if (uploadEntityStateMap.containsKey(uuid)) { uploadEntity = uploadEntityStateMap.get(uuid); throw new InvalidParameterValueException("The one time post url is already used and the upload is in " + uploadEntity.getUploadState() + " state."); } + int maxSizeInGB = Integer.valueOf(cmd.getMaxUploadSize()); + int contentLengthInGB = getSizeInGB(contentLength); + if (contentLengthInGB > maxSizeInGB) { + throw new InvalidParameterValueException("Maximum file upload size exceeded. Content Length received: " + contentLengthInGB + "GB. Maximum allowed size: " + + maxSizeInGB + "GB."); + } try { String absolutePath = cmd.getAbsolutePath(); uploadEntity = new UploadEntity(uuid, cmd.getEntityId(), UploadEntity.Status.IN_PROGRESS, cmd.getName(), absolutePath); @@ -2627,6 +2633,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S uploadEntity.setInstallPathPrefix(installPathPrefix); uploadEntity.setHvm(cmd.getRequiresHvm()); uploadEntity.setChksum(cmd.getChecksum()); + uploadEntity.setMaxSizeInGB(maxSizeInGB); // create a install dir if (!_storage.exists(installPathPrefix)) { _storage.mkdir(installPathPrefix); @@ -2634,13 +2641,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S uploadEntityStateMap.put(uuid, uploadEntity); } catch (Exception e) { //upload entity will be null incase an exception occurs and the handler will not proceed. - s_logger.debug("exception occured while creating upload entity " + e); + s_logger.error("exception occurred while creating upload entity ", e); updateStateMapWithError(uuid, e.getMessage()); } } return uploadEntity; } + private int getSizeInGB(long sizeInBytes) { + return (int)Math.ceil(sizeInBytes * 1.0d / (1024 * 1024 * 1024)); + } public String postUpload(String uuid, String filename) { UploadEntity uploadEntity = uploadEntityStateMap.get(uuid); @@ -2657,7 +2667,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S //} //dnld.setCheckSum(checkSum); - int imgSizeGigs = (int)Math.ceil(_storage.getSize(fileSavedTempLocation) * 1.0d / (1024 * 1024 * 1024)); + int imgSizeGigs = getSizeInGB(_storage.getSize(fileSavedTempLocation)); + int maxSize = uploadEntity.getMaxSizeInGB(); + if(imgSizeGigs > maxSize) { + throw new InvalidParameterValueException("Maximum file upload size exceeded. Physical file size: "+imgSizeGigs+"GB. Maximum allowed size: "+maxSize+"GB."); + } imgSizeGigs++; // add one just in case long timeout = (long)imgSizeGigs * installTimeoutPerGig; Script scr = new Script(getScriptLocation(resourceType), timeout, s_logger); @@ -2771,13 +2785,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S uploadEntityStateMap.put(uuid, uploadEntity); } - public void validatePostUploadRequest(String signature, String metadata, String timeout, String hostname, String uuid) throws InvalidParameterValueException{ + public void validatePostUploadRequest(String signature, String metadata, String timeout, String hostname,long contentLength, String uuid) throws InvalidParameterValueException{ // check none of the params are empty if(StringUtils.isEmpty(signature) || StringUtils.isEmpty(metadata) || StringUtils.isEmpty(timeout)) { updateStateMapWithError(uuid,"signature, metadata and expires are compulsory fields."); throw new InvalidParameterValueException("signature, metadata and expires are compulsory fields."); } + //check that contentLength exists and is greater than zero + if (contentLength <= 0) { + throw new InvalidParameterValueException("content length is not set in the request or has invalid value."); + } + //validate signature String fullUrl = "https://"; + hostname + "/upload/" + uuid; String computedSignature = EncryptionUtil.generateSignature(metadata + fullUrl + timeout, getPostUploadPSK()); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java index 46ace7c..15a6ef2 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java @@ -31,6 +31,7 @@ public class UploadEntity { private long entityId; private String chksum; private long physicalSize; + private int maxSizeInGB; public static enum ResourceType { VOLUME, TEMPLATE @@ -172,5 +173,11 @@ public class UploadEntity { return physicalSize; } + public int getMaxSizeInGB() { + return maxSizeInGB; + } + public void setMaxSizeInGB(int maxSizeInGB) { + this.maxSizeInGB = maxSizeInGB; + } }
