[
https://issues.apache.org/jira/browse/CLOUDSTACK-8022?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Prachi Damle resolved CLOUDSTACK-8022.
--------------------------------------
Resolution: Fixed
> [Automation] Deletion of Domain with Cleanup set to "true" fails
> ----------------------------------------------------------------
>
> Key: CLOUDSTACK-8022
> URL: https://issues.apache.org/jira/browse/CLOUDSTACK-8022
> Project: CloudStack
> Issue Type: Bug
> Security Level: Public(Anyone can view this level - this is the
> default.)
> Components: Automation
> Affects Versions: 4.5.0
> Reporter: Chandan Purushothama
> Assignee: Prachi Damle
> Priority: Critical
> Fix For: 4.5.0
>
> Attachments: management-server.zip
>
>
> The following test case in test_persistent_rules.py fails:
> {code}
> @attr(tags=["advanced"])
> def test_vpc_force_delete_domain(self):
> # steps
> # 1. Create account and create VPC network in the account
> # 2. Create two persistent networks within this VPC
> # 3. Restart/delete VPC network
> # Validations
> # 1. In case of Restart operation, restart should be successful
> # and persistent networks should be back in persistent state
> # 2. In case of Delete operation, VR servicing the VPC should
> # get destroyed and sourceNAT ip should get released
> child_domain = Domain.create(self.apiclient,
> services=self.services["domain"],
> parentdomainid=self.domain.id)
> try:
> account_1 = Account.create(
> self.apiclient, self.services["account"],
> domainid=child_domain.id
> )
> account_2 = Account.create(
> self.apiclient, self.services["account"],
> domainid=child_domain.id
> )
> self.services["vpc"]["cidr"] = "10.1.1.1/16"
> vpc_1 = VPC.create(
> self.apiclient,
> self.services["vpc"],
> vpcofferingid=self.vpc_off.id,
> zoneid=self.zone.id,
> account=account_1.name,
> domainid=account_1.domainid)
> vpcs = VPC.list(self.apiclient, id=vpc_1.id)
> self.assertEqual(
> validateList(vpcs)[0],
> PASS,
> "VPC list validation failed, vpc list is %s" %
> vpcs)
> vpc_2 = VPC.create(
> self.apiclient,
> self.services["vpc"],
> vpcofferingid=self.vpc_off.id,
> zoneid=self.zone.id,
> account=account_2.name,
> domainid=account_2.domainid)
> vpcs = VPC.list(self.apiclient, id=vpc_2.id)
> self.assertEqual(
> validateList(vpcs)[0],
> PASS,
> "VPC list validation failed, vpc list is %s" %
> vpcs)
> persistent_network_1 = Network.create(
> self.api_client, self.services["isolated_network"],
> networkofferingid=self.persistent_network_offering_NoLB.id,
> accountid=account_1.name, domainid=account_1.domainid,
> zoneid=self.zone.id, vpcid=vpc_1.id, gateway="10.1.1.1",
> netmask="255.255.255.0")
> response = verifyNetworkState(self.apiclient,
> persistent_network_1.id,
> "implemented"
> )
> exceptionOccured = response[0]
> isNetworkInDesiredState = response[1]
> exceptionMessage = response[2]
> if (exceptionOccured or (not isNetworkInDesiredState)):
> raise Exception(exceptionMessage)
> self.assertIsNotNone(
> persistent_network_1.vlan,
> "vlan must not be null for persistent network %s" %
> persistent_network_1.id)
> persistent_network_2 = Network.create(
> self.api_client, self.services["isolated_network"],
> networkofferingid=self.persistent_network_offering_NoLB.id,
> accountid=account_2.name, domainid=account_2.domainid,
> zoneid=self.zone.id, vpcid=vpc_2.id, gateway="10.1.1.1",
> netmask="255.255.255.0")
> response = verifyNetworkState(
> self.apiclient,
> persistent_network_2.id,
> "implemented")
> exceptionOccured = response[0]
> isNetworkInDesiredState = response[1]
> exceptionMessage = response[2]
> if (exceptionOccured or (not isNetworkInDesiredState)):
> raise Exception(exceptionMessage)
> self.assertIsNotNone(
> persistent_network_2.vlan,
> "vlan must not be null for persistent network: %s" %
> persistent_network_2.id)
> except Exception as e:
> self.cleanup.append(account_1)
> self.cleanup.append(account_2)
> self.cleanup.append(child_domain)
> self.fail(e)
> # Force delete domain
> child_domain.delete(self.apiclient, cleanup=True)
> self.debug("Waiting for account.cleanup.interval" +
> " to cleanup any remaining resouces")
> # Sleep 3*account.gc to ensure that all resources are deleted
> wait_for_cleanup(self.apiclient, ["account.cleanup.interval"] * 3)
> with self.assertRaises(Exception):
> Domain.list(self.apiclient, id=child_domain.id)
> with self.assertRaises(Exception):
> Account.list(
> self.apiclient, name=account_1.name,
> domainid=account_1.domainid, listall=True
> )
> with self.assertRaises(Exception):
> Account.list(
> self.apiclient, name=account_2.name,
> domainid=account_2.domainid, listall=True
> )
> self.VerifyVpcCleanup(vpc_1.id)
> self.VerifyVpcCleanup(vpc_2.id)
> self.VerifyNetworkCleanup(persistent_network_1.id)
> self.VerifyNetworkCleanup(persistent_network_2.id)
> return
> {code}
> Following is the stack trace:
> {noformat}
> Stacktrace
> File "/usr/lib/python2.7/unittest/case.py", line 332, in run
> testMethod()
> File
> "/root/cloudstack/test/integration/component/test_persistent_networks.py",
> line 2780, in test_vpc_force_delete_domain
> child_domain.delete(self.apiclient, cleanup=True)
> File "/usr/local/lib/python2.7/dist-packages/marvin/lib/base.py", line 77,
> in delete
> apiclient.deleteDomain(cmd)
> File
> "/usr/local/lib/python2.7/dist-packages/marvin/cloudstackAPI/cloudstackAPIClient.py",
> line 671, in deleteDomain
> response = self.connection.marvinRequest(command, response_type=response,
> method=method)
> File
> "/usr/local/lib/python2.7/dist-packages/marvin/cloudstackConnection.py", line
> 379, in marvinRequest
> raise e
> 'Job failed: {jobprocstatus : 0, created : u\'2014-12-03T02:53:03+0000\', cmd
> : u\'org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd\', userid
> : u\'cb31c30a-7a84-11e4-bab5-967d6e0d04d3\', jobstatus : 2, jobid :
> u\'b373c063-dfb6-46bc-b82a-da7f9b12937b\', jobresultcode : 530, jobresulttype
> : u\'object\', jobresult : {errorcode : 530, errortext : u\'Failed to clean
> up domain resources and sub domains, delete failed on domain domain-RS2LKV
> (id: 7).\'}, accountid : u\'cb31b43c-7a84-11e4-bab5-967d6e0d04d3\'}\n
> {noformat}
> ============================================
> Exception Information from the Management Server Log:
> ============================================
> {noformat}
> 2014-12-03 02:54:33,733 DEBUG [c.c.n.v.RemoteAccessVpnManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> there are no Remote access vpns for public ip address id=12
> 2014-12-03 02:54:33,746 DEBUG [c.c.n.IpAddressManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Releasing ip id=12; sourceNat = true
> 2014-12-03 02:54:33,749 DEBUG [c.c.n.IpAddressManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Released a public ip id=12
> 2014-12-03 02:54:33,749 DEBUG [c.c.n.v.VpcManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Released ip addresses for vpc id=57 as a part of cleanup vpc process
> 2014-12-03 02:54:33,750 DEBUG [c.c.n.v.VpcManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Found 0 to revoke for the vpc 57
> 2014-12-03 02:54:33,753 DEBUG [c.c.n.v.VpcManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063) Vpc
> [VPC [57-vpc_vpn-T13419] is destroyed succesfully
> 2014-12-03 02:54:33,753 DEBUG [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063) VPC
> 57 successfully deleted as a part of account id=105 cleanup.
> 2014-12-03 02:54:33,754 DEBUG [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Deleting site-to-site VPN customer gateways for account 105
> 2014-12-03 02:54:33,759 DEBUG [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Account specific Virtual IP ranges are successfully released as a part of
> account id=105 cleanup.
> 2014-12-03 02:54:33,760 INFO [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> deleteAccount: Released 0 dedicated guest vlan ranges from account 105
> 2014-12-03 02:54:33,806 INFO [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Cleanup for account 105 is not needed.
> 2014-12-03 02:54:33,808 DEBUG [c.c.u.DomainManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Deleting networks for domain id=7
> 2014-12-03 02:54:33,812 ERROR [c.c.d.d.DomainDaoImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Unable to remove domain as domain 7 no longer exists
> 2014-12-03 02:54:33,817 ERROR [c.c.u.DomainManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Exception deleting domain with id 7
> com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain
> resources and sub domains, delete failed on domain domain-RS2LKV (id: 7).
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:270)
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:253)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:601)
> at
> org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> at
> org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> at
> org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
> at $Proxy114.deleteDomain(Unknown Source)
> at
> org.apache.cloudstack.region.RegionManagerImpl.deleteDomain(RegionManagerImpl.java:242)
> at
> org.apache.cloudstack.region.RegionServiceImpl.deleteDomain(RegionServiceImpl.java:169)
> at
> org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd.execute(DeleteDomainCmd.java:103)
> at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
> at
> com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:546)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:497)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:722)
> 2014-12-03 02:54:33,819 DEBUG [c.c.u.DomainManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Changing domain id=7 state back to Active because it can't be removed due to
> resources referencing to it
> 2014-12-03 02:54:33,850 ERROR [c.c.a.ApiAsyncJobDispatcher]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Unexpected
> exception while executing
> org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd
> com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain
> resources and sub domains, delete failed on domain domain-RS2LKV (id: 7).
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:270)
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:253)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:601)
> at
> org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> at
> org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> at
> org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
> at $Proxy114.deleteDomain(Unknown Source)
> at
> org.apache.cloudstack.region.RegionManagerImpl.deleteDomain(RegionManagerImpl.java:242)
> at
> org.apache.cloudstack.region.RegionServiceImpl.deleteDomain(RegionServiceImpl.java:169)
> at
> org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd.execute(DeleteDomainCmd.java:103)
> at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
> at
> com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:546)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:497)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:722)
> 2014-12-03 02:54:33,851 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Complete async
> job-1474, jobStatus: FAILED, resultCode: 530, result:
> org.apache.cloudstack.api.response.ExceptionResponse/null/{"uuidList":[],"errorcode":530,"errortext":"Failed
> to clean up domain resources and sub domains, delete failed on domain
> domain-RS2LKV (id: 7)."}
> 2014-12-03 02:54:33,852 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Publish async
> job-1474 complete on message bus
> 2014-12-03 02:54:33,852 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Wake up jobs
> related to job-1474
> 2014-12-03 02:54:33,853 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Update db status
> for job-1474
> 2014-12-03 02:54:33,854 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Wake up jobs
> joined with job-1474 and disjoin all subjobs created from job- 1474
> 2014-12-03 02:54:33,856 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Done executing
> org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd for job-1474
> 2014-12-03 02:54:33,856 INFO [o.a.c.f.j.i.AsyncJobMonitor]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Remove job-1474
> from job monitoring
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
