Chandan Purushothama created CLOUDSTACK-8022:
------------------------------------------------
Summary: [Automation] Deletion of Domain with Cleanup set to true
fails
Key: CLOUDSTACK-8022
URL: https://issues.apache.org/jira/browse/CLOUDSTACK-8022
Project: CloudStack
Issue Type: Bug
Security Level: Public (Anyone can view this level - this is the default.)
Components: Automation
Affects Versions: 4.5.0
Reporter: Chandan Purushothama
Priority: Critical
Fix For: 4.5.0
The following test case in test_persistent_rules.py fails:
{code}
@attr(tags=["advanced"])
def test_vpc_force_delete_domain(self):
# steps
# 1. Create account and create VPC network in the account
# 2. Create two persistent networks within this VPC
# 3. Restart/delete VPC network
# Validations
# 1. In case of Restart operation, restart should be successful
# and persistent networks should be back in persistent state
# 2. In case of Delete operation, VR servicing the VPC should
# get destroyed and sourceNAT ip should get released
child_domain = Domain.create(self.apiclient,
services=self.services["domain"],
parentdomainid=self.domain.id)
try:
account_1 = Account.create(
self.apiclient, self.services["account"],
domainid=child_domain.id
)
account_2 = Account.create(
self.apiclient, self.services["account"],
domainid=child_domain.id
)
self.services["vpc"]["cidr"] = "10.1.1.1/16"
vpc_1 = VPC.create(
self.apiclient,
self.services["vpc"],
vpcofferingid=self.vpc_off.id,
zoneid=self.zone.id,
account=account_1.name,
domainid=account_1.domainid)
vpcs = VPC.list(self.apiclient, id=vpc_1.id)
self.assertEqual(
validateList(vpcs)[0],
PASS,
"VPC list validation failed, vpc list is %s" %
vpcs)
vpc_2 = VPC.create(
self.apiclient,
self.services["vpc"],
vpcofferingid=self.vpc_off.id,
zoneid=self.zone.id,
account=account_2.name,
domainid=account_2.domainid)
vpcs = VPC.list(self.apiclient, id=vpc_2.id)
self.assertEqual(
validateList(vpcs)[0],
PASS,
"VPC list validation failed, vpc list is %s" %
vpcs)
persistent_network_1 = Network.create(
self.api_client, self.services["isolated_network"],
networkofferingid=self.persistent_network_offering_NoLB.id,
accountid=account_1.name, domainid=account_1.domainid,
zoneid=self.zone.id, vpcid=vpc_1.id, gateway="10.1.1.1",
netmask="255.255.255.0")
response = verifyNetworkState(self.apiclient,
persistent_network_1.id,
"implemented"
)
exceptionOccured = response[0]
isNetworkInDesiredState = response[1]
exceptionMessage = response[2]
if (exceptionOccured or (not isNetworkInDesiredState)):
raise Exception(exceptionMessage)
self.assertIsNotNone(
persistent_network_1.vlan,
"vlan must not be null for persistent network %s" %
persistent_network_1.id)
persistent_network_2 = Network.create(
self.api_client, self.services["isolated_network"],
networkofferingid=self.persistent_network_offering_NoLB.id,
accountid=account_2.name, domainid=account_2.domainid,
zoneid=self.zone.id, vpcid=vpc_2.id, gateway="10.1.1.1",
netmask="255.255.255.0")
response = verifyNetworkState(
self.apiclient,
persistent_network_2.id,
"implemented")
exceptionOccured = response[0]
isNetworkInDesiredState = response[1]
exceptionMessage = response[2]
if (exceptionOccured or (not isNetworkInDesiredState)):
raise Exception(exceptionMessage)
self.assertIsNotNone(
persistent_network_2.vlan,
"vlan must not be null for persistent network: %s" %
persistent_network_2.id)
except Exception as e:
self.cleanup.append(account_1)
self.cleanup.append(account_2)
self.cleanup.append(child_domain)
self.fail(e)
# Force delete domain
child_domain.delete(self.apiclient, cleanup=True)
self.debug("Waiting for account.cleanup.interval" +
" to cleanup any remaining resouces")
# Sleep 3*account.gc to ensure that all resources are deleted
wait_for_cleanup(self.apiclient, ["account.cleanup.interval"] * 3)
with self.assertRaises(Exception):
Domain.list(self.apiclient, id=child_domain.id)
with self.assertRaises(Exception):
Account.list(
self.apiclient, name=account_1.name,
domainid=account_1.domainid, listall=True
)
with self.assertRaises(Exception):
Account.list(
self.apiclient, name=account_2.name,
domainid=account_2.domainid, listall=True
)
self.VerifyVpcCleanup(vpc_1.id)
self.VerifyVpcCleanup(vpc_2.id)
self.VerifyNetworkCleanup(persistent_network_1.id)
self.VerifyNetworkCleanup(persistent_network_2.id)
return
{code}
Following is the stack trace:
{noformat}
Stacktrace
File "/usr/lib/python2.7/unittest/case.py", line 332, in run
testMethod()
File
"/root/cloudstack/test/integration/component/test_persistent_networks.py", line
2780, in test_vpc_force_delete_domain
child_domain.delete(self.apiclient, cleanup=True)
File "/usr/local/lib/python2.7/dist-packages/marvin/lib/base.py", line 77, in
delete
apiclient.deleteDomain(cmd)
File
"/usr/local/lib/python2.7/dist-packages/marvin/cloudstackAPI/cloudstackAPIClient.py",
line 671, in deleteDomain
response = self.connection.marvinRequest(command, response_type=response,
method=method)
File "/usr/local/lib/python2.7/dist-packages/marvin/cloudstackConnection.py",
line 379, in marvinRequest
raise e
'Job failed: {jobprocstatus : 0, created : u\'2014-12-03T02:53:03+0000\', cmd :
u\'org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd\', userid :
u\'cb31c30a-7a84-11e4-bab5-967d6e0d04d3\', jobstatus : 2, jobid :
u\'b373c063-dfb6-46bc-b82a-da7f9b12937b\', jobresultcode : 530, jobresulttype :
u\'object\', jobresult : {errorcode : 530, errortext : u\'Failed to clean up
domain resources and sub domains, delete failed on domain domain-RS2LKV (id:
7).\'}, accountid : u\'cb31b43c-7a84-11e4-bab5-967d6e0d04d3\'}\n
{noformat}
============================================
Exception Information from the Management Server Log:
============================================
{noformat}
2014-12-03 02:54:33,733 DEBUG [c.c.n.v.RemoteAccessVpnManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063) there
are no Remote access vpns for public ip address id=12
2014-12-03 02:54:33,746 DEBUG [c.c.n.IpAddressManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Releasing ip id=12; sourceNat = true
2014-12-03 02:54:33,749 DEBUG [c.c.n.IpAddressManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Released a public ip id=12
2014-12-03 02:54:33,749 DEBUG [c.c.n.v.VpcManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Released ip addresses for vpc id=57 as a part of cleanup vpc process
2014-12-03 02:54:33,750 DEBUG [c.c.n.v.VpcManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063) Found
0 to revoke for the vpc 57
2014-12-03 02:54:33,753 DEBUG [c.c.n.v.VpcManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063) Vpc
[VPC [57-vpc_vpn-T13419] is destroyed succesfully
2014-12-03 02:54:33,753 DEBUG [c.c.u.AccountManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063) VPC
57 successfully deleted as a part of account id=105 cleanup.
2014-12-03 02:54:33,754 DEBUG [c.c.u.AccountManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Deleting site-to-site VPN customer gateways for account 105
2014-12-03 02:54:33,759 DEBUG [c.c.u.AccountManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Account specific Virtual IP ranges are successfully released as a part of
account id=105 cleanup.
2014-12-03 02:54:33,760 INFO [c.c.u.AccountManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
deleteAccount: Released 0 dedicated guest vlan ranges from account 105
2014-12-03 02:54:33,806 INFO [c.c.u.AccountManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Cleanup for account 105 is not needed.
2014-12-03 02:54:33,808 DEBUG [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Deleting networks for domain id=7
2014-12-03 02:54:33,812 ERROR [c.c.d.d.DomainDaoImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Unable to remove domain as domain 7 no longer exists
2014-12-03 02:54:33,817 ERROR [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Exception deleting domain with id 7
com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain
resources and sub domains, delete failed on domain domain-RS2LKV (id: 7).
at
com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:270)
at
com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:253)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
at
com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at $Proxy114.deleteDomain(Unknown Source)
at
org.apache.cloudstack.region.RegionManagerImpl.deleteDomain(RegionManagerImpl.java:242)
at
org.apache.cloudstack.region.RegionServiceImpl.deleteDomain(RegionServiceImpl.java:169)
at
org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd.execute(DeleteDomainCmd.java:103)
at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
at
com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:546)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:497)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
2014-12-03 02:54:33,819 DEBUG [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Changing domain id=7 state back to Active because it can't be removed due to
resources referencing to it
2014-12-03 02:54:33,850 ERROR [c.c.a.ApiAsyncJobDispatcher]
(API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Unexpected
exception while executing
org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd
com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain
resources and sub domains, delete failed on domain domain-RS2LKV (id: 7).
at
com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:270)
at
com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:253)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
at
com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at $Proxy114.deleteDomain(Unknown Source)
at
org.apache.cloudstack.region.RegionManagerImpl.deleteDomain(RegionManagerImpl.java:242)
at
org.apache.cloudstack.region.RegionServiceImpl.deleteDomain(RegionServiceImpl.java:169)
at
org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd.execute(DeleteDomainCmd.java:103)
at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
at
com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:546)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:497)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
2014-12-03 02:54:33,851 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Complete async
job-1474, jobStatus: FAILED, resultCode: 530, result:
org.apache.cloudstack.api.response.ExceptionResponse/null/{"uuidList":[],"errorcode":530,"errortext":"Failed
to clean up domain resources and sub domains, delete failed on domain
domain-RS2LKV (id: 7)."}
2014-12-03 02:54:33,852 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Publish async
job-1474 complete on message bus
2014-12-03 02:54:33,852 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Wake up jobs
related to job-1474
2014-12-03 02:54:33,853 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Update db status
for job-1474
2014-12-03 02:54:33,854 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Wake up jobs
joined with job-1474 and disjoin all subjobs created from job- 1474
2014-12-03 02:54:33,856 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Done executing
org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd for job-1474
2014-12-03 02:54:33,856 INFO [o.a.c.f.j.i.AsyncJobMonitor]
(API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Remove job-1474
from job monitoring
{noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
