This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/master by this push:
new 3e2ef19 Cloudstack 10064: Secondary storage Usage for uploadedVolume
is not collected (#2258)
3e2ef19 is described below
commit 3e2ef197dbd05db3dea8b436082e3dacb3459cb0
Author: PranaliM <[email protected]>
AuthorDate: Wed Dec 27 13:21:54 2017 +0530
Cloudstack 10064: Secondary storage Usage for uploadedVolume is not
collected (#2258)
Description: For Volumes on Secondary Storage, (Uploaded Volume) the usage
is not accounted for.
The fix is implemented as follows:
A new Usage Type is added for the Volume on secondary storage :
VOLUME_SECONDARY (id=26)
A new storage type, 'Volume' is defined.
When a volume is uploaded and the usage server executes next,entry will be
added to the usage_storage helper table for all the volumes uploaded since the
Usage server executed last.
When the uploaded volume is attached, the 'deleted' column in the
usage_storage table is set to the time-stamp when the volume was deleted
2 entries will be added to the cloud_usage table with usage_type=26 and
usage_type=6 (Volume usage on primary). One for the duration the volume was on
primary and other for the duration it was on secondary.
Entry is added to the helper table volume_usage for accounting for the
primary storage.Next execution of the usage server and on-wards, usage entry
for usage_type=6 only will be added.
---
.../org/apache/cloudstack/usage/UsageTypes.java | 1 +
test/integration/component/test_ss_volume_usage.py | 203 +++++++++++++++++++++
usage/src/com/cloud/usage/StorageTypes.java | 1 +
usage/src/com/cloud/usage/UsageManagerImpl.java | 43 ++++-
.../com/cloud/usage/parser/StorageUsageParser.java | 4 +
5 files changed, 251 insertions(+), 1 deletion(-)
diff --git a/api/src/org/apache/cloudstack/usage/UsageTypes.java
b/api/src/org/apache/cloudstack/usage/UsageTypes.java
index d9cfc13..08cd7dd 100644
--- a/api/src/org/apache/cloudstack/usage/UsageTypes.java
+++ b/api/src/org/apache/cloudstack/usage/UsageTypes.java
@@ -42,6 +42,7 @@ public class UsageTypes {
public static final int VM_DISK_BYTES_READ = 23;
public static final int VM_DISK_BYTES_WRITE = 24;
public static final int VM_SNAPSHOT = 25;
+ public static final int VOLUME_SECONDARY = 26;
public static List<UsageTypeResponse> listUsageTypes() {
List<UsageTypeResponse> responseList = new
ArrayList<UsageTypeResponse>();
diff --git a/test/integration/component/test_ss_volume_usage.py
b/test/integration/component/test_ss_volume_usage.py
new file mode 100644
index 0000000..548f90f
--- /dev/null
+++ b/test/integration/component/test_ss_volume_usage.py
@@ -0,0 +1,203 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+""" Test cases for checking that the secondary Storage usage is accounted.
This is verified by checking the usage_event table
+for a volume in 'Uploaded' state.
+ This test case does the following:
+ 1.Creates an account and uploads a volume.
+ 2.After the volume is uploaded successfully, connects to the database
+ 3.From the database verifies that an entry is added to cloud.events table
for the uploaded volume.
+ 4.Cleans up the resources.
+"""
+
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.lib.utils import *
+from marvin.lib.base import *
+from marvin.lib.common import *
+from nose.plugins.attrib import attr
+from marvin.sshClient import SshClient
+from marvin.codes import (BACKED_UP, PASS, FAIL)
+import time
+
+
+def verify_vm(self, vmid, state):
+ list_vm = list_virtual_machines(self.userapiclient,
+ account=self.account.name,
+ domainid=self.account.domainid,
+ id=vmid
+ )
+ self.assertEqual(
+ validateList(list_vm)[0],
+ PASS,
+ "Check List vm response for vmid: %s" %
+ vmid)
+ self.assertGreater(
+ len(list_vm),
+ 0,
+ "Check the list vm response for vm id: %s" %
+ vmid)
+ vm = list_vm[0]
+ self.assertEqual(
+ vm.id,
+ str(vmid),
+ "Vm deployed is different from the test")
+ self.assertEqual(vm.state, state, "VM is in %s state" %state)
+
+
+def uploadVolume(self):
+ # upload a volume
+ self.debug("Upload volume format is '%s'" %self.uploadVolumeformat)
+ self.testdata["configurableData"]["upload_volume"]["format"] =
self.uploadVolumeformat
+ self.testdata["configurableData"]["upload_volume"]["url"] =
self.uploadvolumeUrl
+ upload_volume = Volume.upload(
+ self.apiclient,
+ self.testdata["configurableData"]["upload_volume"],
+ account=self.account.name,
+ domainid=self.domain.id,
+ zoneid=self.zone.id
+ )
+ upload_volume.wait_for_upload(self.apiclient)
+ return upload_volume.id
+
+def restartUsageServer(self):
+ #Restart usage server
+
+ sshClient = SshClient(
+ self.mgtSvrDetails["mgtSvrIp"],
+ 22,
+ self.mgtSvrDetails["user"],
+ self.mgtSvrDetails["passwd"]
+ )
+ command = "service cloudstack-usage restart"
+ sshClient.execute(command)
+ return
+
+def checkUsage(self, uuid_upload_volume_id):
+ volume_id = self.dbclient.execute("SELECT id from cloud.volumes where
uuid='%s';" % uuid_upload_volume_id)
+ self.debug("Volume id of uploaded volume is= %s" %volume_id[0]);
+ qryresult_after_usageServerExecution = self.dbclient.execute(
+ "SELECT type FROM cloud.usage_event where resource_id = '%s';" %
(volume_id[0]))
+ self.debug("Usage Type is %s " %
qryresult_after_usageServerExecution[0][0])
+ self.assertEqual(qryresult_after_usageServerExecution[0][0],
'VOLUME.UPLOAD')
+
+class TestSecondaryVolumeUsage(cloudstackTestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ testClient = super(TestSecondaryVolumeUsage, cls).getClsTestClient()
+ cls.apiclient = testClient.getApiClient()
+ cls.dbclient = testClient.getDbConnection()
+ cls.testdata = testClient.getParsedTestDataConfig()
+ cls.hypervisor = cls.testClient.getHypervisorInfo()
+ cls.storagetype = 'shared'
+ # Get Zone, Domain and templates
+ cls.domain = get_domain(cls.apiclient)
+ cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
+ cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__
+ cls._cleanup = []
+
+ # Create an account
+ cls.account = Account.create(
+ cls.apiclient,
+ cls.testdata["account"],
+ domainid=cls.domain.id
+ )
+ cls._cleanup.append(cls.account)
+
+ # Create user api client of the account
+ cls.userapiclient = testClient.getUserApiClient(
+ UserName=cls.account.name,
+ DomainName=cls.account.domain
+ )
+
+ # Create Service offering
+ cls.service_offering = ServiceOffering.create(
+ cls.apiclient,
+ cls.testdata["service_offering"],
+ )
+ cls._cleanup.append(cls.service_offering)
+
+ cls.disk_offering = DiskOffering.create(
+ cls.apiclient,
+ cls.testdata["disk_offering"],
+ )
+
+ cls._cleanup.append(cls.disk_offering)
+
+ cls.skip = 0
+ hosts = list_hosts(
+ cls.apiclient,
+ type="Routing"
+ )
+
+ for hypervisorhost in hosts:
+ if hypervisorhost.hypervisor.lower() in ["xenserver"]:
+ cls.uploadVolumeformat = "VHD"
+ cls.uploadvolumeUrl =
"http://download.cloudstack.org/releases/2.0.0/systemvm.vhd.bz2";
+ break
+ elif hypervisorhost.hypervisor.lower() in ["vmware"]:
+ cls.uploadVolumeformat = "OVA"
+ cls.uploadvolumeUrl =
"http://download.cloudstack.org/releases/2.2.0/systemvm-redundant-router.ova";
+ break
+ elif hypervisorhost.hypervisor == "KVM":
+ cls.uploadVolumeformat = "QCOW2"
+ cls.uploadvolumeUrl =
"http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.qcow2.bz2";
+ break
+ elif hypervisorhost.hypervisor == "LXC":
+ cls.uploadvolumeformat = "QCOW2"
+ cls.uploadvolumeUrl =
"http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.qcow2.bz2";
+ break
+ else:
+ break
+
+ cls.template = get_template(
+ cls.apiclient,
+ cls.zone.id,
+ cls.testdata["ostype"])
+
+ try:
+ cls.vm = VirtualMachine.create(
+ cls.userapiclient,
+ cls.testdata["small"],
+ templateid=cls.template.id,
+ accountid=cls.account.name,
+ domainid=cls.account.domainid,
+ serviceofferingid=cls.service_offering.id,
+ zoneid=cls.zone.id
+ )
+
+ except Exception as e:
+ cls.tearDownClass()
+ raise e
+ return
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ cleanup_resources(cls.apiclient, cls._cleanup)
+ except Exception as e:
+ raise Exception("Warning: Exception during cleanup : %s" % e)
+
+ @attr(tags=["basic", "advanced"], required_hardware="true")
+ def test_01_SecondaryUsageUploadedVolume(self):
+ try:
+ uploaded_volume_id_uuid = uploadVolume(self)
+ checkUsage(self, uploaded_volume_id_uuid)
+ except Exception as e:
+ self.tearDown()
+ raise e
+ return
diff --git a/usage/src/com/cloud/usage/StorageTypes.java
b/usage/src/com/cloud/usage/StorageTypes.java
index a6f2131..9d4abdc 100644
--- a/usage/src/com/cloud/usage/StorageTypes.java
+++ b/usage/src/com/cloud/usage/StorageTypes.java
@@ -20,4 +20,5 @@ public class StorageTypes {
public static final int TEMPLATE = 1;
public static final int ISO = 2;
public static final int SNAPSHOT = 3;
+ public static final int VOLUME = 4;
}
diff --git a/usage/src/com/cloud/usage/UsageManagerImpl.java
b/usage/src/com/cloud/usage/UsageManagerImpl.java
index 840b02c..4a60b24 100644
--- a/usage/src/com/cloud/usage/UsageManagerImpl.java
+++ b/usage/src/com/cloud/usage/UsageManagerImpl.java
@@ -985,7 +985,7 @@ public class UsageManagerImpl extends ManagerBase
implements UsageManager, Runna
private boolean isVolumeEvent(String eventType) {
return eventType != null &&
- (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) ||
eventType.equals(EventTypes.EVENT_VOLUME_DELETE) ||
eventType.equals(EventTypes.EVENT_VOLUME_RESIZE));
+ (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) ||
eventType.equals(EventTypes.EVENT_VOLUME_DELETE) ||
eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) ||
eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD));
}
private boolean isTemplateEvent(String eventType) {
@@ -1390,6 +1390,21 @@ public class UsageManagerImpl extends ManagerBase
implements UsageManager, Runna
long volId = event.getResourceId();
+ if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType())) {
+ //For volumes which are 'attached' successfully, set the 'deleted'
column in the usage_storage table,
+ //so that the secondary storage should stop accounting and only
primary will be accounted.
+ SearchCriteria<UsageStorageVO> sc =
_usageStorageDao.createSearchCriteria();
+ sc.addAnd("id", SearchCriteria.Op.EQ, volId);
+ sc.addAnd("storageType", SearchCriteria.Op.EQ,
StorageTypes.VOLUME);
+ List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc,
null);
+ if (volumesVOs != null) {
+ if (volumesVOs.size() == 1) {
+ s_logger.debug("Setting the volume with id: " + volId + "
to 'deleted' in the usage_storage table.");
+ volumesVOs.get(0).setDeleted(event.getCreateDate());
+ _usageStorageDao.update(volumesVOs.get(0));
+ }
+ }
+ }
if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType()) ||
EventTypes.EVENT_VOLUME_RESIZE.equals(event.getType())) {
SearchCriteria<UsageVolumeVO> sc =
_usageVolumeDao.createSearchCriteria();
sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
@@ -1430,6 +1445,32 @@ public class UsageManagerImpl extends ManagerBase
implements UsageManager, Runna
volumesVO.setDeleted(event.getCreateDate()); // there really
shouldn't be more than one
_usageVolumeDao.update(volumesVO);
}
+ } else if (EventTypes.EVENT_VOLUME_UPLOAD.equals(event.getType())) {
+ //For Upload event add an entry to the usage_storage table.
+ SearchCriteria<UsageStorageVO> sc =
_usageStorageDao.createSearchCriteria();
+ sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
+ sc.addAnd("id", SearchCriteria.Op.EQ, volId);
+ sc.addAnd("deleted", SearchCriteria.Op.NULL);
+ List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc,
null);
+
+ if (volumesVOs.size() > 0) {
+ //This is a safeguard to avoid double counting of volumes.
+ s_logger.error("Found duplicate usage entry for volume: " +
volId + " assigned to account: " + event.getAccountId() + "; marking as
deleted...");
+ }
+ for (UsageStorageVO volumesVO : volumesVOs) {
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("deleting volume: " + volumesVO.getId() + "
from account: " + volumesVO.getAccountId());
+ }
+ volumesVO.setDeleted(event.getCreateDate());
+ _usageStorageDao.update(volumesVO);
+ }
+
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("create volume with id : " + volId + " for
account: " + event.getAccountId());
+ }
+ Account acct =
_accountDao.findByIdIncludingRemoved(event.getAccountId());
+ UsageStorageVO volumeVO = new UsageStorageVO(volId,
event.getZoneId(), event.getAccountId(), acct.getDomainId(),
StorageTypes.VOLUME, event.getTemplateId(), event.getSize(),
event.getCreateDate(), null);
+ _usageStorageDao.persist(volumeVO);
}
}
diff --git a/usage/src/com/cloud/usage/parser/StorageUsageParser.java
b/usage/src/com/cloud/usage/parser/StorageUsageParser.java
index 03aa97b..8231756 100644
--- a/usage/src/com/cloud/usage/parser/StorageUsageParser.java
+++ b/usage/src/com/cloud/usage/parser/StorageUsageParser.java
@@ -180,6 +180,10 @@ public class StorageUsageParser {
usage_type = UsageTypes.SNAPSHOT;
usageDesc += "Snapshot ";
break;
+ case StorageTypes.VOLUME:
+ usage_type = UsageTypes.VOLUME_SECONDARY;
+ usageDesc += "Volume ";
+ break;
}
//Create the usage record
usageDesc += "Id:" + storageId + " Size:" + size;
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
