Updated Branches: refs/heads/object_store b8c5c67fb -> e40a06dea
Fix extractTemplateCmd. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/e40a06de Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/e40a06de Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/e40a06de Branch: refs/heads/object_store Commit: e40a06deaea3c040255d8472ea3c9301f8fb2955 Parents: b8c5c67 Author: Min Chen <[email protected]> Authored: Mon Apr 22 21:48:57 2013 -0700 Committer: Min Chen <[email protected]> Committed: Mon Apr 22 21:48:57 2013 -0700 ---------------------------------------------------------------------- api/src/com/cloud/template/TemplateApiService.java | 5 +- .../apache/cloudstack/api/ResponseGenerator.java | 2 +- .../api/command/user/iso/ExtractIsoCmd.java | 14 +- .../command/user/template/ExtractTemplateCmd.java | 14 +- .../engine/subsystem/api/storage/EndPoint.java | 1 + .../storage/image/datastore/ImageStoreEntity.java | 37 +++ .../cloudstack/storage/LocalHostEndpoint.java | 10 +- .../cloudstack/storage/RemoteHostEndPoint.java | 17 +- .../storage/image/datastore/ImageStoreEntity.java | 37 --- server/src/com/cloud/api/ApiResponseHelper.java | 19 +- .../com/cloud/storage/upload/UploadMonitor.java | 10 +- .../cloud/storage/upload/UploadMonitorImpl.java | 194 ++++++++------- .../com/cloud/template/TemplateManagerImpl.java | 78 +++---- 13 files changed, 235 insertions(+), 203 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/api/src/com/cloud/template/TemplateApiService.java ---------------------------------------------------------------------- diff --git a/api/src/com/cloud/template/TemplateApiService.java b/api/src/com/cloud/template/TemplateApiService.java index e3906e1..d907b10 100755 --- a/api/src/com/cloud/template/TemplateApiService.java +++ b/api/src/com/cloud/template/TemplateApiService.java @@ -36,6 +36,7 @@ import com.cloud.exception.InternalErrorException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.StorageUnavailableException; import com.cloud.user.Account; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; public interface TemplateApiService { @@ -76,7 +77,7 @@ public interface TemplateApiService { * - the command specifying the mode and id of the ISO * @return extractId. */ - Long extract(ExtractIsoCmd cmd) throws InternalErrorException; + Pair<Long, String> extract(ExtractIsoCmd cmd) throws InternalErrorException; /** * Extracts a Template @@ -85,7 +86,7 @@ public interface TemplateApiService { * - the command specifying the mode and id of the template * @return extractId */ - Long extract(ExtractTemplateCmd cmd) throws InternalErrorException; + Pair<Long, String> extract(ExtractTemplateCmd cmd) throws InternalErrorException; VirtualMachineTemplate getTemplate(long templateId); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/api/src/org/apache/cloudstack/api/ResponseGenerator.java ---------------------------------------------------------------------- diff --git a/api/src/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/org/apache/cloudstack/api/ResponseGenerator.java index 77d3bea..c780bda 100644 --- a/api/src/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/org/apache/cloudstack/api/ResponseGenerator.java @@ -257,7 +257,7 @@ public interface ResponseGenerator { SecurityGroupResponse createSecurityGroupResponse(SecurityGroup group); - ExtractResponse createExtractResponse(Long uploadId, Long id, Long zoneId, Long accountId, String mode); + ExtractResponse createExtractResponse(Long uploadId, Long id, Long zoneId, Long accountId, String mode, String url); String toSerializedString(CreateCmdResponse response, String responseType); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/api/src/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java ---------------------------------------------------------------------- diff --git a/api/src/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java b/api/src/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java index 08a15ee..c405f2d 100644 --- a/api/src/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java @@ -33,6 +33,7 @@ import com.cloud.exception.InternalErrorException; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.UserContext; +import com.cloud.utils.Pair; @APICommand(name = "extractIso", description="Extracts an ISO", responseObject=ExtractResponse.class) public class ExtractIsoCmd extends BaseAsyncCmd { @@ -123,9 +124,16 @@ public class ExtractIsoCmd extends BaseAsyncCmd { public void execute(){ try { UserContext.current().setEventDetails(getEventDescription()); - Long uploadId = _templateService.extract(this); - if (uploadId != null){ - ExtractResponse response = _responseGenerator.createExtractResponse(uploadId, id, zoneId, getEntityOwnerId(), mode); + Pair<Long, String> uploadPair = _templateService.extract(this); + if (uploadPair != null){ + ExtractResponse response = null; + if (uploadPair.second() != null ) { + // region-wide image store + response = _responseGenerator.createExtractResponse(null, id, zoneId, getEntityOwnerId(), mode, uploadPair.second()); + } else { + // nfs image store + response = _responseGenerator.createExtractResponse(uploadPair.first(), id, zoneId, getEntityOwnerId(), mode, null); + } response.setResponseName(getCommandName()); response.setObjectName("iso"); this.setResponseObject(response); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/api/src/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java ---------------------------------------------------------------------- diff --git a/api/src/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java b/api/src/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java index 9a2dee3..2ec957e 100644 --- a/api/src/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java @@ -34,6 +34,7 @@ import com.cloud.exception.InternalErrorException; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.UserContext; +import com.cloud.utils.Pair; @APICommand(name = "extractTemplate", description="Extracts a template", responseObject=ExtractResponse.class) public class ExtractTemplateCmd extends BaseAsyncCmd { @@ -125,9 +126,16 @@ public class ExtractTemplateCmd extends BaseAsyncCmd { public void execute(){ try { UserContext.current().setEventDetails(getEventDescription()); - Long uploadId = _templateService.extract(this); - if (uploadId != null){ - ExtractResponse response = _responseGenerator.createExtractResponse(uploadId, id, zoneId, getEntityOwnerId(), mode); + Pair<Long, String> uploadPair = _templateService.extract(this); + if (uploadPair != null){ + ExtractResponse response = null; + if (uploadPair.second() != null ) { + // region-wide image store + response = _responseGenerator.createExtractResponse(null, id, zoneId, getEntityOwnerId(), mode, uploadPair.second()); + } else { + // nfs image store + response = _responseGenerator.createExtractResponse(uploadPair.first(), id, zoneId, getEntityOwnerId(), mode, null); + } response.setResponseName(getCommandName()); this.setResponseObject(response); } else { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPoint.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPoint.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPoint.java index eb6da70..4966419 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPoint.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPoint.java @@ -24,6 +24,7 @@ import com.cloud.agent.api.Command; public interface EndPoint { public long getId(); + public String getHostAddr(); public Answer sendMessage(Command cmd); public void sendMessageAsync(Command cmd, AsyncCompletionCallback<Answer> callback); void sendMessageAsyncWithListener(Command cmd, Listener listner); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java b/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java new file mode 100644 index 0000000..fb1b1d7 --- /dev/null +++ b/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.image.datastore; + +import java.util.Set; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; + +import com.cloud.storage.ImageStore; + +public interface ImageStoreEntity extends DataStore, ImageStore { + TemplateInfo getTemplate(long templateId); + VolumeInfo getVolume(long volumeId); + SnapshotInfo getSnapshot(long snapshotId); + boolean exists(DataObject object); + Set<TemplateInfo> listTemplates(); +} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java ---------------------------------------------------------------------- diff --git a/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java b/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java index 9fe0c55..8844b4c 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java +++ b/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java @@ -30,7 +30,13 @@ public class LocalHostEndpoint implements EndPoint { return 0; } + @Override + public String getHostAddr() { + return "127.0.0.0"; + } + + @Override public Answer sendMessage(Command cmd) { if (cmd instanceof CopyCommand) { return resource.executeRequest(cmd); @@ -52,7 +58,7 @@ public class LocalHostEndpoint implements EndPoint { callback.complete(answer); } } - + private class CmdRunner2 implements Runnable { final Command cmd; final AsyncCompletionCallback<DownloadAnswer> callback; @@ -71,7 +77,7 @@ public class LocalHostEndpoint implements EndPoint { AsyncCompletionCallback<Answer> callback) { executor.schedule(new CmdRunner(cmd, callback), 10, TimeUnit.SECONDS); } - + @Override public void sendMessageAsyncWithListener(Command cmd, Listener listner) { if (listner instanceof DownloadListener) { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/engine/storage/src/org/apache/cloudstack/storage/RemoteHostEndPoint.java ---------------------------------------------------------------------- diff --git a/engine/storage/src/org/apache/cloudstack/storage/RemoteHostEndPoint.java b/engine/storage/src/org/apache/cloudstack/storage/RemoteHostEndPoint.java index 9ce4e75..acf78f2 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/RemoteHostEndPoint.java +++ b/engine/storage/src/org/apache/cloudstack/storage/RemoteHostEndPoint.java @@ -51,22 +51,23 @@ public class RemoteHostEndPoint implements EndPoint { protected RemoteHostEndPoint() { executor = Executors.newScheduledThreadPool(10); } - + private void configure(long hostId, String hostAddress) { this.hostId = hostId; this.hostAddress = hostAddress; } - + public static RemoteHostEndPoint getHypervisorHostEndPoint(long hostId, String hostAddress) { RemoteHostEndPoint ep = ComponentContext.inject(RemoteHostEndPoint.class); ep.configure(hostId, hostAddress); return ep; } - + + @Override public String getHostAddr() { return this.hostAddress; } - + public long getId() { return this.hostId; } @@ -85,7 +86,7 @@ public class RemoteHostEndPoint implements EndPoint { } throw new CloudRuntimeException("Failed to send command, due to Agent:" + getId() + ", " + errMsg); } - + private class CmdRunner implements Runnable { final Command cmd; final AsyncCompletionCallback<Answer> callback; @@ -98,14 +99,14 @@ public class RemoteHostEndPoint implements EndPoint { Answer answer = sendMessage(cmd); callback.complete(answer); } - + } - + @Override public void sendMessageAsync(Command cmd, AsyncCompletionCallback<Answer> callback) { executor.schedule(new CmdRunner(cmd, callback), 10, TimeUnit.SECONDS); } - + @Override public void sendMessageAsyncWithListener(Command cmd, Listener listener) { try { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java ---------------------------------------------------------------------- diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java b/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java deleted file mode 100644 index fb1b1d7..0000000 --- a/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cloudstack.storage.image.datastore; - -import java.util.Set; - -import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; - -import com.cloud.storage.ImageStore; - -public interface ImageStoreEntity extends DataStore, ImageStore { - TemplateInfo getTemplate(long templateId); - VolumeInfo getVolume(long volumeId); - SnapshotInfo getSnapshot(long snapshotId); - boolean exists(DataObject object); - Set<TemplateInfo> listTemplates(); -} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/server/src/com/cloud/api/ApiResponseHelper.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index a0c9d38..faf6155 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -242,6 +242,7 @@ import com.cloud.storage.ImageStore; import com.cloud.storage.S3; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Upload; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Storage.TemplateType; @@ -1525,8 +1526,8 @@ public class ApiResponseHelper implements ResponseGenerator { } @Override - public ExtractResponse createExtractResponse(Long uploadId, Long id, Long zoneId, Long accountId, String mode) { - UploadVO uploadInfo = ApiDBUtils.findUploadById(uploadId); + public ExtractResponse createExtractResponse(Long uploadId, Long id, Long zoneId, Long accountId, String mode, String url) { + ExtractResponse response = new ExtractResponse(); response.setObjectName("template"); VMTemplateVO template = ApiDBUtils.findTemplateById(id); @@ -1538,11 +1539,19 @@ public class ApiResponseHelper implements ResponseGenerator { response.setZoneName(zone.getName()); } response.setMode(mode); - response.setUploadId(uploadInfo.getUuid()); - response.setState(uploadInfo.getUploadState().toString()); + if (uploadId == null) { + // region-wide image store + response.setUrl(url); + response.setState(Upload.Status.DOWNLOAD_URL_CREATED.toString()); + } else { + UploadVO uploadInfo = ApiDBUtils.findUploadById(uploadId); + response.setUploadId(uploadInfo.getUuid()); + response.setState(uploadInfo.getUploadState().toString()); + response.setUrl(uploadInfo.getUploadUrl()); + } Account account = ApiDBUtils.findAccountById(accountId); response.setAccountId(account.getUuid()); - response.setUrl(uploadInfo.getUploadUrl()); + return response; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/server/src/com/cloud/storage/upload/UploadMonitor.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/upload/UploadMonitor.java b/server/src/com/cloud/storage/upload/UploadMonitor.java index 1c3590e..dd50d1a 100755 --- a/server/src/com/cloud/storage/upload/UploadMonitor.java +++ b/server/src/com/cloud/storage/upload/UploadMonitor.java @@ -17,6 +17,8 @@ package com.cloud.storage.upload; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; + import com.cloud.async.AsyncJobManager; import com.cloud.host.HostVO; import com.cloud.storage.Upload.Mode; @@ -32,12 +34,12 @@ import com.cloud.utils.component.Manager; * Monitor upload progress of all entities. * */ -public interface UploadMonitor extends Manager{ - +public interface UploadMonitor extends Manager{ + public void cancelAllUploads(Long templateId); public Long extractTemplate(VMTemplateVO template, String url, - VMTemplateHostVO tmpltHostRef,Long dataCenterId, long eventId, long asyncJobId, AsyncJobManager asyncMgr); + TemplateDataStoreVO tmpltStoreRef,Long dataCenterId, long eventId, long asyncJobId, AsyncJobManager asyncMgr); boolean isTypeUploadInProgress(Long typeId, Type type); @@ -51,7 +53,7 @@ public interface UploadMonitor extends Manager{ long asyncJobId, AsyncJobManager asyncMgr); UploadVO createEntityDownloadURL(VMTemplateVO template, - VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId); + TemplateDataStoreVO vmTemplateStore, Long dataCenterId, long eventId); void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/server/src/com/cloud/storage/upload/UploadMonitorImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/upload/UploadMonitorImpl.java b/server/src/com/cloud/storage/upload/UploadMonitorImpl.java index 77f0d20..0495255 100755 --- a/server/src/com/cloud/storage/upload/UploadMonitorImpl.java +++ b/server/src/com/cloud/storage/upload/UploadMonitorImpl.java @@ -32,14 +32,22 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; +import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand; import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand; +import com.cloud.agent.api.storage.ListTemplateAnswer; import com.cloud.agent.api.storage.UploadCommand; import com.cloud.agent.api.storage.UploadProgressCommand.RequestType; import com.cloud.agent.manager.Commands; @@ -56,6 +64,7 @@ import com.cloud.storage.Upload; import com.cloud.storage.Upload.Mode; import com.cloud.storage.Upload.Status; import com.cloud.storage.Upload.Type; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateVO; @@ -82,17 +91,17 @@ import com.cloud.vm.dao.SecondaryStorageVmDao; public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { static final Logger s_logger = Logger.getLogger(UploadMonitorImpl.class); - - @Inject + + @Inject VMTemplateHostDao _vmTemplateHostDao; - @Inject + @Inject UploadDao _uploadDao; @Inject SecondaryStorageVmDao _secStorageVmDao; - + @Inject - HostDao _serverDao = null; + HostDao _serverDao = null; @Inject VMTemplateDao _templateDao = null; @Inject @@ -103,6 +112,10 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { ResourceManager _resourceMgr; @Inject SecondaryStorageVmManager _ssvmMgr; + @Inject + EndPointSelector _epSelector; + @Inject + DataStoreManager storeMgr; private String _name; private Boolean _sslCopy = new Boolean(false); @@ -114,46 +127,46 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { final Map<UploadVO, UploadListener> _listenerMap = new ConcurrentHashMap<UploadVO, UploadListener>(); - + @Override public void cancelAllUploads(Long templateId) { // TODO - } - + } + @Override public boolean isTypeUploadInProgress(Long typeId, Type type) { List<UploadVO> uploadsInProgress = _uploadDao.listByTypeUploadStatus(typeId, type, UploadVO.Status.UPLOAD_IN_PROGRESS); - + if(uploadsInProgress.size() > 0) { return true; } else if (type == Type.VOLUME && _uploadDao.listByTypeUploadStatus(typeId, type, UploadVO.Status.COPY_IN_PROGRESS).size() > 0){ return true; } return false; - + } - + @Override public UploadVO createNewUploadEntry(Long hostId, Long typeId, UploadVO.Status uploadState, Type type, String uploadUrl, Upload.Mode mode){ - - UploadVO uploadObj = new UploadVO(hostId, typeId, new Date(), + + UploadVO uploadObj = new UploadVO(hostId, typeId, new Date(), uploadState, type, uploadUrl, mode); _uploadDao.persist(uploadObj); - + return uploadObj; - + } - + @Override - public void extractVolume(UploadVO uploadVolumeObj, HostVO sserver, VolumeVO volume, String url, Long dataCenterId, String installPath, long eventId, long asyncJobId, AsyncJobManager asyncMgr){ - + public void extractVolume(UploadVO uploadVolumeObj, HostVO sserver, VolumeVO volume, String url, Long dataCenterId, String installPath, long eventId, long asyncJobId, AsyncJobManager asyncMgr){ + uploadVolumeObj.setUploadState(Upload.Status.NOT_UPLOADED); _uploadDao.update(uploadVolumeObj.getId(), uploadVolumeObj); - - start(); + + start(); UploadCommand ucmd = new UploadCommand(url, volume.getId(), volume.getSize(), installPath, Type.VOLUME); UploadListener ul = new UploadListener(sserver, _timer, _uploadDao, uploadVolumeObj, this, ucmd, volume.getAccountId(), volume.getName(), Type.VOLUME, eventId, asyncJobId, asyncMgr); _listenerMap.put(uploadVolumeObj, ul); @@ -164,26 +177,26 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { s_logger.warn("Unable to start upload of volume " + volume.getName() + " from " + sserver.getName() + " to " +url, e); ul.setDisconnected(); ul.scheduleStatusCheck(RequestType.GET_OR_RESTART); - } + } } @Override public Long extractTemplate( VMTemplateVO template, String url, - VMTemplateHostVO vmTemplateHost,Long dataCenterId, long eventId, long asyncJobId, AsyncJobManager asyncMgr){ + TemplateDataStoreVO vmTemplateHost,Long dataCenterId, long eventId, long asyncJobId, AsyncJobManager asyncMgr){ Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE ; - + List<HostVO> storageServers = _resourceMgr.listAllHostsInOneZoneByType(Host.Type.SecondaryStorage, dataCenterId); - HostVO sserver = storageServers.get(0); - - UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(), + HostVO sserver = storageServers.get(0); + + UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(), Upload.Status.NOT_UPLOADED, type, url, Mode.FTP_UPLOAD); - _uploadDao.persist(uploadTemplateObj); - + _uploadDao.persist(uploadTemplateObj); + if(vmTemplateHost != null) { start(); - UploadCommand ucmd = new UploadCommand(template, url, vmTemplateHost.getInstallPath(), vmTemplateHost.getSize()); - UploadListener ul = new UploadListener(sserver, _timer, _uploadDao, uploadTemplateObj, this, ucmd, template.getAccountId(), template.getName(), type, eventId, asyncJobId, asyncMgr); + UploadCommand ucmd = new UploadCommand(template, url, vmTemplateHost.getInstallPath(), vmTemplateHost.getSize()); + UploadListener ul = new UploadListener(sserver, _timer, _uploadDao, uploadTemplateObj, this, ucmd, template.getAccountId(), template.getName(), type, eventId, asyncJobId, asyncMgr); _listenerMap.put(uploadTemplateObj, ul); try { @@ -194,57 +207,54 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { ul.scheduleStatusCheck(RequestType.GET_OR_RESTART); } return uploadTemplateObj.getId(); - } - return null; - } - + } + return null; + } + @Override - public UploadVO createEntityDownloadURL(VMTemplateVO template, VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId) { - + public UploadVO createEntityDownloadURL(VMTemplateVO template, TemplateDataStoreVO vmTemplateHost, Long dataCenterId, long eventId) { + String errorString = ""; boolean success = false; - Host secStorage = ApiDBUtils.findHostById(vmTemplateHost.getHostId()); Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE ; - - //Check if ssvm is up - HostVO ssvm = _ssvmMgr.pickSsvmHost(ApiDBUtils.findHostById(vmTemplateHost.getHostId())); - if( ssvm == null ) { - throw new CloudRuntimeException("There is no secondary storage VM for secondary storage host " + secStorage.getId()); - } - + + //Check if it already exists. - List<UploadVO> extractURLList = _uploadDao.listByTypeUploadStatus(template.getId(), type, UploadVO.Status.DOWNLOAD_URL_CREATED); + List<UploadVO> extractURLList = _uploadDao.listByTypeUploadStatus(template.getId(), type, UploadVO.Status.DOWNLOAD_URL_CREATED); if (extractURLList.size() > 0) { return extractURLList.get(0); } - - // It doesn't exist so create a DB entry. - UploadVO uploadTemplateObj = new UploadVO(vmTemplateHost.getHostId(), template.getId(), new Date(), - Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD); - uploadTemplateObj.setInstallPath(vmTemplateHost.getInstallPath()); + + // It doesn't exist so create a DB entry. + UploadVO uploadTemplateObj = new UploadVO(vmTemplateHost.getDataStoreId(), template.getId(), new Date(), + Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD); + uploadTemplateObj.setInstallPath(vmTemplateHost.getInstallPath()); _uploadDao.persist(uploadTemplateObj); + + // find an endpoint to send command + DataStore store = this.storeMgr.getDataStore(vmTemplateHost.getDataStoreId(), DataStoreRole.Image); + EndPoint ep = _epSelector.select(store); try{ // Create Symlink at ssvm String path = vmTemplateHost.getInstallPath(); String uuid = UUID.randomUUID().toString() + "." + template.getFormat().getFileExtension(); // adding "." + vhd/ova... etc. - CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(secStorage.getParent(), path, uuid); - try { - send(ssvm.getId(), cmd, null); - } catch (AgentUnavailableException e) { - errorString = "Unable to create a link for " +type+ " id:"+template.getId() + "," + e.getMessage(); - s_logger.error(errorString, e); + CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreVO)store).getParent(), path, uuid); + Answer ans = ep.sendMessage(cmd); + if (ans == null || !ans.getResult()) { + errorString = "Unable to create a link for " +type+ " id:"+template.getId() + "," + ans.getDetails(); + s_logger.error(errorString); throw new CloudRuntimeException(errorString); } //Construct actual URL locally now that the symlink exists at SSVM - String extractURL = generateCopyUrl(ssvm.getPublicIpAddress(), uuid); + String extractURL = generateCopyUrl(ep.getHostAddr(), uuid); UploadVO vo = _uploadDao.createForUpdate(); vo.setLastUpdated(new Date()); vo.setUploadUrl(extractURL); vo.setUploadState(Status.DOWNLOAD_URL_CREATED); _uploadDao.update(uploadTemplateObj.getId(), vo); success = true; - return _uploadDao.findById(uploadTemplateObj.getId(), true); + return _uploadDao.findById(uploadTemplateObj.getId(), true); }finally{ if(!success){ UploadVO uploadJob = _uploadDao.createForUpdate(uploadTemplateObj.getId()); @@ -254,22 +264,22 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { _uploadDao.update(uploadTemplateObj.getId(), uploadJob); } } - + } - + @Override public void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId) { - + String errorString = ""; boolean success = false; try{ List<HostVO> storageServers = _resourceMgr.listAllHostsInOneZoneByType(Host.Type.SecondaryStorage, dataCenterId); if(storageServers == null ){ errorString = "No Storage Server found at the datacenter - " +dataCenterId; - throw new CloudRuntimeException(errorString); - } - - // Update DB for state = DOWNLOAD_URL_NOT_CREATED. + throw new CloudRuntimeException(errorString); + } + + // Update DB for state = DOWNLOAD_URL_NOT_CREATED. UploadVO uploadJob = _uploadDao.createForUpdate(uploadId); uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED); uploadJob.setLastUpdated(new Date()); @@ -283,7 +293,7 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { errorString = "There is no secondary storage VM for secondary storage host " + secStorage.getName(); throw new CloudRuntimeException(errorString); } - + CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(secStorage.getParent(), path, uuid); try { send(ssvm.getId(), cmd, null); @@ -323,7 +333,7 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { } } } - + private String generateCopyUrl(String ipAddress, String uuid){ String hostname = ipAddress; String scheme = "http"; @@ -332,9 +342,9 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { hostname = hostname + ".realhostip.com"; scheme = "https"; } - return scheme + "://" + hostname + "/userdata/" + uuid; + return scheme + "://" + hostname + "/userdata/" + uuid; } - + public void send(Long hostId, Command cmd, Listener listener) throws AgentUnavailableException { @@ -346,19 +356,19 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { throws ConfigurationException { final Map<String, String> configs = _configDao.getConfiguration("ManagementServer", params); _sslCopy = Boolean.parseBoolean(configs.get("secstorage.encrypt.copy")); - + String cert = configs.get("secstorage.secure.copy.cert"); if ("realhostip.com".equalsIgnoreCase(cert)) { s_logger.warn("Only realhostip.com ssl cert is supported, ignoring self-signed and other certs"); - } - + } + _agentMgr.registerForHostEvents(new UploadListener(this), true, false, false); String cleanupInterval = configs.get("extract.url.cleanup.interval"); _cleanupInterval = NumbersUtil.parseInt(cleanupInterval, 7200); - + String urlExpirationInterval = configs.get("extract.url.expiration.interval"); _urlExpirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400); - + String workers = (String)params.get("expunge.workers"); int wrks = NumbersUtil.parseInt(workers, 1); _executor = Executors.newScheduledThreadPool(wrks, new NamedThreadFactory("UploadMonitor-Scavenger")); @@ -366,19 +376,19 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { } @Override - public boolean start() { + public boolean start() { _executor.scheduleWithFixedDelay(new StorageGarbageCollector(), _cleanupInterval, _cleanupInterval, TimeUnit.SECONDS); _timer = new Timer(); return true; } @Override - public boolean stop() { + public boolean stop() { return true; } - + public void handleUploadEvent(HostVO host, Long accountId, String typeName, Type type, Long uploadId, com.cloud.storage.Upload.Status reason, long eventId) { - + if ((reason == Upload.Status.UPLOADED) || (reason==Upload.Status.ABANDONED)){ UploadVO uploadObj = new UploadVO(uploadId); UploadListener oldListener = _listenerMap.get(uploadObj); @@ -388,10 +398,10 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { } } - + @Override public void handleUploadSync(long sserverId) { - + HostVO storageHost = _serverDao.findById(sserverId); if (storageHost == null) { s_logger.warn("Huh? Agent id " + sserverId + " does not correspond to a row in hosts table?"); @@ -408,11 +418,11 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { uploadJob.setLastUpdated(new Date()); _uploadDao.update(uploadJob.getId(), uploadJob); } - + } - - } + + } protected class StorageGarbageCollector implements Runnable { @@ -440,26 +450,26 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { } } } - - + + private long getTimeDiff(Date date){ Calendar currentDateCalendar = Calendar.getInstance(); Calendar givenDateCalendar = Calendar.getInstance(); givenDateCalendar.setTime(date); - - return (currentDateCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000; + + return (currentDateCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000; } - + public void cleanupStorage() { final int EXTRACT_URL_LIFE_LIMIT_IN_SECONDS = _urlExpirationInterval; List<UploadVO> extractJobs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED); - + for (UploadVO extractJob : extractJobs){ - if( getTimeDiff(extractJob.getLastUpdated()) > EXTRACT_URL_LIFE_LIMIT_IN_SECONDS ){ + if( getTimeDiff(extractJob.getLastUpdated()) > EXTRACT_URL_LIFE_LIMIT_IN_SECONDS ){ String path = extractJob.getInstallPath(); HostVO secStorage = ApiDBUtils.findHostById(extractJob.getHostId()); - + // Would delete the symlink for the Type and if Type == VOLUME then also the volume DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path, extractJob.getType(),extractJob.getUploadUrl(), secStorage.getParent()); HostVO ssvm = _ssvmMgr.pickSsvmHost(secStorage); @@ -478,7 +488,7 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { } } } - + } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e40a06de/server/src/com/cloud/template/TemplateManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 0266995..81cf29c 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -67,7 +67,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; @@ -78,6 +77,7 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -88,8 +88,6 @@ import com.cloud.agent.api.ComputeChecksumCommand; import com.cloud.agent.api.downloadTemplateFromSwiftToSecondaryStorageCommand; import com.cloud.agent.api.uploadTemplateToSwiftFromSecondaryStorageCommand; import com.cloud.agent.api.storage.DestroyCommand; -import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer; -import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; import com.cloud.agent.api.to.SwiftTO; import com.cloud.api.ApiDBUtils; @@ -353,7 +351,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_EXTRACT, eventDescription = "extracting ISO", async = true) - public Long extract(ExtractIsoCmd cmd) { + public Pair<Long, String> extract(ExtractIsoCmd cmd) { Account account = UserContext.current().getCaller(); Long templateId = cmd.getId(); Long zoneId = cmd.getZoneId(); @@ -362,9 +360,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Long eventId = cmd.getStartEventId(); // FIXME: async job needs fixing - Long uploadId = extract(account, templateId, url, zoneId, mode, eventId, true, null, _asyncMgr); - if (uploadId != null){ - return uploadId; + Pair<Long, String> uploadPair = extract(account, templateId, url, zoneId, mode, eventId, true, null, _asyncMgr); + if (uploadPair != null){ + return uploadPair; }else { throw new CloudRuntimeException("Failed to extract the iso"); } @@ -372,7 +370,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_EXTRACT, eventDescription = "extracting template", async = true) - public Long extract(ExtractTemplateCmd cmd) { + public Pair<Long, String> extract(ExtractTemplateCmd cmd) { Account caller = UserContext.current().getCaller(); Long templateId = cmd.getId(); Long zoneId = cmd.getZoneId(); @@ -381,9 +379,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Long eventId = cmd.getStartEventId(); // FIXME: async job needs fixing - Long uploadId = extract(caller, templateId, url, zoneId, mode, eventId, false, null, _asyncMgr); - if (uploadId != null){ - return uploadId; + Pair<Long, String> uploadPair = extract(caller, templateId, url, zoneId, mode, eventId, false, null, _asyncMgr); + if (uploadPair != null){ + return uploadPair; }else { throw new CloudRuntimeException("Failed to extract the teamplate"); } @@ -402,7 +400,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return vmTemplate; } - private Long extract(Account caller, Long templateId, String url, Long zoneId, String mode, Long eventId, boolean isISO, AsyncJobVO job, AsyncJobManager mgr) { + private Pair<Long, String> extract(Account caller, Long templateId, String url, Long zoneId, String mode, Long eventId, boolean isISO, AsyncJobVO job, AsyncJobManager mgr) { String desc = Upload.Type.TEMPLATE.toString(); if (isISO) { desc = Upload.Type.ISO.toString(); @@ -434,15 +432,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } - if (zoneId == null && _swiftMgr.isSwiftEnabled()) { - zoneId = _swiftMgr.chooseZoneForTmpltExtract(templateId); - } - - if (zoneId == null && _s3Mgr.isS3Enabled()) { - zoneId = _s3Mgr.chooseZoneForTemplateExtract(template); - } - if (_dcDao.findById(zoneId) == null) { + if (zoneId != null && _dcDao.findById(zoneId) == null) { throw new IllegalArgumentException("Please specify a valid zone."); } @@ -452,39 +443,33 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, _accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template); - List<HostVO> sservers = getSecondaryStorageHosts(zoneId); + List<DataStore> ssStores = this.dataStoreMgr.getImageStoresByScope(new ZoneScope(zoneId)); - VMTemplateHostVO tmpltHostRef = null; - if (sservers != null) { - for(HostVO secondaryStorageHost: sservers){ - tmpltHostRef = _tmpltHostDao.findByHostTemplate(secondaryStorageHost.getId(), templateId); - if (tmpltHostRef != null){ - if (tmpltHostRef.getDownloadState() != com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - tmpltHostRef = null; - } - else { + TemplateDataStoreVO tmpltStoreRef = null; + ImageStoreEntity tmpltStore = null; + if (ssStores != null) { + for(DataStore store: ssStores){ + tmpltStoreRef = this._tmplStoreDao.findByStoreTemplate(store.getId(), templateId); + if (tmpltStoreRef != null){ + if (tmpltStoreRef.getDownloadState() == com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { + tmpltStore = (ImageStoreEntity)store; break; } } } } - if (tmpltHostRef == null && _swiftMgr.isSwiftEnabled()) { - SwiftTO swift = _swiftMgr.getSwiftTO(templateId); - if (swift != null && sservers != null) { - downloadTemplateFromSwiftToSecondaryStorage(zoneId, templateId); - } - } else if (tmpltHostRef == null && _s3Mgr.isS3Enabled()) { - if (sservers != null) { - _s3Mgr.downloadTemplateFromS3ToSecondaryStorage(zoneId, - templateId, _primaryStorageDownloadWait); - } + if (tmpltStoreRef == null) { + throw new InvalidParameterValueException("The " + desc + " has not been downloaded "); } - if (tmpltHostRef == null) { - throw new InvalidParameterValueException("The " + desc + " has not been downloaded "); + if ( tmpltStore.getProviderName().equalsIgnoreCase("S3") || tmpltStore.getProviderName().equalsIgnoreCase("Swift")){ + // for S3 and Swift, no need to do anything, just return template url for extract template, here we use "-1" to indicate these case + return new Pair<Long, String>(null, tmpltStoreRef.getInstallPath()); } + + // for NFS image store case, control will come here Upload.Mode extractMode; if (mode == null || (!mode.equalsIgnoreCase(Upload.Mode.FTP_UPLOAD.toString()) && !mode.equalsIgnoreCase(Upload.Mode.HTTP_DOWNLOAD.toString())) ){ throw new InvalidParameterValueException("Please specify a valid extract Mode. Supported modes: "+ Upload.Mode.FTP_UPLOAD + ", " + Upload.Mode.HTTP_DOWNLOAD); @@ -520,12 +505,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new IllegalArgumentException(template.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same"); } - return _uploadMonitor.extractTemplate(template, url, tmpltHostRef, zoneId, eventId, job.getId(), mgr); + return new Pair<Long, String>(_uploadMonitor.extractTemplate(template, url, tmpltStoreRef, zoneId, eventId, job.getId(), mgr), null); } - UploadVO vo = _uploadMonitor.createEntityDownloadURL(template, tmpltHostRef, zoneId, eventId); + UploadVO vo = _uploadMonitor.createEntityDownloadURL(template, tmpltStoreRef, zoneId, eventId); if (vo != null){ - return vo.getId(); + return new Pair<Long, String>(vo.getId(), null); }else{ return null; } @@ -537,7 +522,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if(pool.getDataCenterId() == zoneId) { s_logger.info("Schedule to preload template " + template.getId() + " into primary storage " + pool.getId()); this._preloadExecutor.execute(new Runnable() { - public void run() { + @Override + public void run() { try { reallyRun(); } catch(Throwable e) {
