rhtyd closed pull request #2258: Cloudstack 10064: Secondary storage Usage for
uploadedVolume is not collected
URL: https://github.com/apache/cloudstack/pull/2258
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/api/src/org/apache/cloudstack/usage/UsageTypes.java
b/api/src/org/apache/cloudstack/usage/UsageTypes.java
index d9cfc132e15..08cd7ddb721 100644
--- a/api/src/org/apache/cloudstack/usage/UsageTypes.java
+++ b/api/src/org/apache/cloudstack/usage/UsageTypes.java
@@ -42,6 +42,7 @@
public static final int VM_DISK_BYTES_READ = 23;
public static final int VM_DISK_BYTES_WRITE = 24;
public static final int VM_SNAPSHOT = 25;
+ public static final int VOLUME_SECONDARY = 26;
public static List<UsageTypeResponse> listUsageTypes() {
List<UsageTypeResponse> responseList = new
ArrayList<UsageTypeResponse>();
diff --git a/test/integration/component/test_ss_volume_usage.py
b/test/integration/component/test_ss_volume_usage.py
new file mode 100644
index 00000000000..548f90fc6f3
--- /dev/null
+++ b/test/integration/component/test_ss_volume_usage.py
@@ -0,0 +1,203 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+""" Test cases for checking that the secondary Storage usage is accounted.
This is verified by checking the usage_event table
+for a volume in 'Uploaded' state.
+ This test case does the following:
+ 1.Creates an account and uploads a volume.
+ 2.After the volume is uploaded successfully, connects to the database
+ 3.From the database verifies that an entry is added to cloud.events table
for the uploaded volume.
+ 4.Cleans up the resources.
+"""
+
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.lib.utils import *
+from marvin.lib.base import *
+from marvin.lib.common import *
+from nose.plugins.attrib import attr
+from marvin.sshClient import SshClient
+from marvin.codes import (BACKED_UP, PASS, FAIL)
+import time
+
+
+def verify_vm(self, vmid, state):
+ list_vm = list_virtual_machines(self.userapiclient,
+ account=self.account.name,
+ domainid=self.account.domainid,
+ id=vmid
+ )
+ self.assertEqual(
+ validateList(list_vm)[0],
+ PASS,
+ "Check List vm response for vmid: %s" %
+ vmid)
+ self.assertGreater(
+ len(list_vm),
+ 0,
+ "Check the list vm response for vm id: %s" %
+ vmid)
+ vm = list_vm[0]
+ self.assertEqual(
+ vm.id,
+ str(vmid),
+ "Vm deployed is different from the test")
+ self.assertEqual(vm.state, state, "VM is in %s state" %state)
+
+
+def uploadVolume(self):
+ # upload a volume
+ self.debug("Upload volume format is '%s'" %self.uploadVolumeformat)
+ self.testdata["configurableData"]["upload_volume"]["format"] =
self.uploadVolumeformat
+ self.testdata["configurableData"]["upload_volume"]["url"] =
self.uploadvolumeUrl
+ upload_volume = Volume.upload(
+ self.apiclient,
+ self.testdata["configurableData"]["upload_volume"],
+ account=self.account.name,
+ domainid=self.domain.id,
+ zoneid=self.zone.id
+ )
+ upload_volume.wait_for_upload(self.apiclient)
+ return upload_volume.id
+
+def restartUsageServer(self):
+ #Restart usage server
+
+ sshClient = SshClient(
+ self.mgtSvrDetails["mgtSvrIp"],
+ 22,
+ self.mgtSvrDetails["user"],
+ self.mgtSvrDetails["passwd"]
+ )
+ command = "service cloudstack-usage restart"
+ sshClient.execute(command)
+ return
+
+def checkUsage(self, uuid_upload_volume_id):
+ volume_id = self.dbclient.execute("SELECT id from cloud.volumes where
uuid='%s';" % uuid_upload_volume_id)
+ self.debug("Volume id of uploaded volume is= %s" %volume_id[0]);
+ qryresult_after_usageServerExecution = self.dbclient.execute(
+ "SELECT type FROM cloud.usage_event where resource_id = '%s';" %
(volume_id[0]))
+ self.debug("Usage Type is %s " %
qryresult_after_usageServerExecution[0][0])
+ self.assertEqual(qryresult_after_usageServerExecution[0][0],
'VOLUME.UPLOAD')
+
+class TestSecondaryVolumeUsage(cloudstackTestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ testClient = super(TestSecondaryVolumeUsage, cls).getClsTestClient()
+ cls.apiclient = testClient.getApiClient()
+ cls.dbclient = testClient.getDbConnection()
+ cls.testdata = testClient.getParsedTestDataConfig()
+ cls.hypervisor = cls.testClient.getHypervisorInfo()
+ cls.storagetype = 'shared'
+ # Get Zone, Domain and templates
+ cls.domain = get_domain(cls.apiclient)
+ cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
+ cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__
+ cls._cleanup = []
+
+ # Create an account
+ cls.account = Account.create(
+ cls.apiclient,
+ cls.testdata["account"],
+ domainid=cls.domain.id
+ )
+ cls._cleanup.append(cls.account)
+
+ # Create user api client of the account
+ cls.userapiclient = testClient.getUserApiClient(
+ UserName=cls.account.name,
+ DomainName=cls.account.domain
+ )
+
+ # Create Service offering
+ cls.service_offering = ServiceOffering.create(
+ cls.apiclient,
+ cls.testdata["service_offering"],
+ )
+ cls._cleanup.append(cls.service_offering)
+
+ cls.disk_offering = DiskOffering.create(
+ cls.apiclient,
+ cls.testdata["disk_offering"],
+ )
+
+ cls._cleanup.append(cls.disk_offering)
+
+ cls.skip = 0
+ hosts = list_hosts(
+ cls.apiclient,
+ type="Routing"
+ )
+
+ for hypervisorhost in hosts:
+ if hypervisorhost.hypervisor.lower() in ["xenserver"]:
+ cls.uploadVolumeformat = "VHD"
+ cls.uploadvolumeUrl =
"http://download.cloudstack.org/releases/2.0.0/systemvm.vhd.bz2";
+ break
+ elif hypervisorhost.hypervisor.lower() in ["vmware"]:
+ cls.uploadVolumeformat = "OVA"
+ cls.uploadvolumeUrl =
"http://download.cloudstack.org/releases/2.2.0/systemvm-redundant-router.ova";
+ break
+ elif hypervisorhost.hypervisor == "KVM":
+ cls.uploadVolumeformat = "QCOW2"
+ cls.uploadvolumeUrl =
"http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.qcow2.bz2";
+ break
+ elif hypervisorhost.hypervisor == "LXC":
+ cls.uploadvolumeformat = "QCOW2"
+ cls.uploadvolumeUrl =
"http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.qcow2.bz2";
+ break
+ else:
+ break
+
+ cls.template = get_template(
+ cls.apiclient,
+ cls.zone.id,
+ cls.testdata["ostype"])
+
+ try:
+ cls.vm = VirtualMachine.create(
+ cls.userapiclient,
+ cls.testdata["small"],
+ templateid=cls.template.id,
+ accountid=cls.account.name,
+ domainid=cls.account.domainid,
+ serviceofferingid=cls.service_offering.id,
+ zoneid=cls.zone.id
+ )
+
+ except Exception as e:
+ cls.tearDownClass()
+ raise e
+ return
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ cleanup_resources(cls.apiclient, cls._cleanup)
+ except Exception as e:
+ raise Exception("Warning: Exception during cleanup : %s" % e)
+
+ @attr(tags=["basic", "advanced"], required_hardware="true")
+ def test_01_SecondaryUsageUploadedVolume(self):
+ try:
+ uploaded_volume_id_uuid = uploadVolume(self)
+ checkUsage(self, uploaded_volume_id_uuid)
+ except Exception as e:
+ self.tearDown()
+ raise e
+ return
diff --git a/usage/src/com/cloud/usage/StorageTypes.java
b/usage/src/com/cloud/usage/StorageTypes.java
index a6f2131f4e4..9d4abdc3534 100644
--- a/usage/src/com/cloud/usage/StorageTypes.java
+++ b/usage/src/com/cloud/usage/StorageTypes.java
@@ -20,4 +20,5 @@
public static final int TEMPLATE = 1;
public static final int ISO = 2;
public static final int SNAPSHOT = 3;
+ public static final int VOLUME = 4;
}
diff --git a/usage/src/com/cloud/usage/UsageManagerImpl.java
b/usage/src/com/cloud/usage/UsageManagerImpl.java
index 840b02c5e40..4a60b24a0f0 100644
--- a/usage/src/com/cloud/usage/UsageManagerImpl.java
+++ b/usage/src/com/cloud/usage/UsageManagerImpl.java
@@ -985,7 +985,7 @@ private boolean isIPEvent(String eventType) {
private boolean isVolumeEvent(String eventType) {
return eventType != null &&
- (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) ||
eventType.equals(EventTypes.EVENT_VOLUME_DELETE) ||
eventType.equals(EventTypes.EVENT_VOLUME_RESIZE));
+ (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) ||
eventType.equals(EventTypes.EVENT_VOLUME_DELETE) ||
eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) ||
eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD));
}
private boolean isTemplateEvent(String eventType) {
@@ -1390,6 +1390,21 @@ private void createVolumeHelperEvent(UsageEventVO event)
{
long volId = event.getResourceId();
+ if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType())) {
+ //For volumes which are 'attached' successfully, set the 'deleted'
column in the usage_storage table,
+ //so that the secondary storage should stop accounting and only
primary will be accounted.
+ SearchCriteria<UsageStorageVO> sc =
_usageStorageDao.createSearchCriteria();
+ sc.addAnd("id", SearchCriteria.Op.EQ, volId);
+ sc.addAnd("storageType", SearchCriteria.Op.EQ,
StorageTypes.VOLUME);
+ List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc,
null);
+ if (volumesVOs != null) {
+ if (volumesVOs.size() == 1) {
+ s_logger.debug("Setting the volume with id: " + volId + "
to 'deleted' in the usage_storage table.");
+ volumesVOs.get(0).setDeleted(event.getCreateDate());
+ _usageStorageDao.update(volumesVOs.get(0));
+ }
+ }
+ }
if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType()) ||
EventTypes.EVENT_VOLUME_RESIZE.equals(event.getType())) {
SearchCriteria<UsageVolumeVO> sc =
_usageVolumeDao.createSearchCriteria();
sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
@@ -1430,6 +1445,32 @@ private void createVolumeHelperEvent(UsageEventVO event)
{
volumesVO.setDeleted(event.getCreateDate()); // there really
shouldn't be more than one
_usageVolumeDao.update(volumesVO);
}
+ } else if (EventTypes.EVENT_VOLUME_UPLOAD.equals(event.getType())) {
+ //For Upload event add an entry to the usage_storage table.
+ SearchCriteria<UsageStorageVO> sc =
_usageStorageDao.createSearchCriteria();
+ sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
+ sc.addAnd("id", SearchCriteria.Op.EQ, volId);
+ sc.addAnd("deleted", SearchCriteria.Op.NULL);
+ List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc,
null);
+
+ if (volumesVOs.size() > 0) {
+ //This is a safeguard to avoid double counting of volumes.
+ s_logger.error("Found duplicate usage entry for volume: " +
volId + " assigned to account: " + event.getAccountId() + "; marking as
deleted...");
+ }
+ for (UsageStorageVO volumesVO : volumesVOs) {
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("deleting volume: " + volumesVO.getId() + "
from account: " + volumesVO.getAccountId());
+ }
+ volumesVO.setDeleted(event.getCreateDate());
+ _usageStorageDao.update(volumesVO);
+ }
+
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("create volume with id : " + volId + " for
account: " + event.getAccountId());
+ }
+ Account acct =
_accountDao.findByIdIncludingRemoved(event.getAccountId());
+ UsageStorageVO volumeVO = new UsageStorageVO(volId,
event.getZoneId(), event.getAccountId(), acct.getDomainId(),
StorageTypes.VOLUME, event.getTemplateId(), event.getSize(),
event.getCreateDate(), null);
+ _usageStorageDao.persist(volumeVO);
}
}
diff --git a/usage/src/com/cloud/usage/parser/StorageUsageParser.java
b/usage/src/com/cloud/usage/parser/StorageUsageParser.java
index 03aa97b847d..8231756368d 100644
--- a/usage/src/com/cloud/usage/parser/StorageUsageParser.java
+++ b/usage/src/com/cloud/usage/parser/StorageUsageParser.java
@@ -180,6 +180,10 @@ private static void createUsageRecord(long zoneId, int
type, long runningTime, D
usage_type = UsageTypes.SNAPSHOT;
usageDesc += "Snapshot ";
break;
+ case StorageTypes.VOLUME:
+ usage_type = UsageTypes.VOLUME_SECONDARY;
+ usageDesc += "Volume ";
+ break;
}
//Create the usage record
usageDesc += "Id:" + storageId + " Size:" + size;
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
