[
https://issues.apache.org/jira/browse/CLOUDSTACK-8022?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14244731#comment-14244731
]
Prachi Damle commented on CLOUDSTACK-8022:
------------------------------------------
This is a race condition between the deleteDomain thread and AccountChecker
thread. DeleteDomain thread marks the domain as inactive and proceeds for
cleanup, AccountChecker thread that runs at the same time cleans up any domains
marked as inactive.
2014-12-03 02:53:03,465 DEBUG [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Marking domain id=7 as Inactive before actually deleting it
2014-12-03 02:53:03,468 DEBUG [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Cleaning up domain id=7
2014-12-03 02:53:03,476 DEBUG [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Deleting account
Acct[5f032e8c-fdef-443e-b1a4-5af572bfa168-test-a-TestVPCNetworkOperations-test_vpc_force_delete_domain-1XBSTB]
as a part of domain id=7 cleanup
2014-12-03 02:53:26,888 DEBUG [c.c.u.AccountManagerImpl]
(AccountChecker-1:ctx-3e11737f) (logid:e842fa1a) Removing inactive domain id=7
2014-12-03 02:53:56,884 DEBUG [c.c.u.AccountManagerImpl]
(AccountChecker-1:ctx-38510231) (logid:8536d8ef) Removing inactive domain id=7
2014-12-03 02:54:16,762 DEBUG [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Deleting account
Acct[1e04fea4-f50b-44d3-8e20-83fe4f83cee1-test-a-TestVPCNetworkOperations-test_vpc_force_delete_domain-W7VIER]
as a part of domain id=7 cleanup
2014-12-03 02:54:26,881 DEBUG [c.c.u.AccountManagerImpl]
(AccountChecker-1:ctx-6ed07f3d) (logid:dddb67b8) Removing inactive domain id=7
2014-12-03 02:54:33,808 DEBUG [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Deleting networks for domain id=7
2014-12-03 02:54:33,812 ERROR [c.c.d.d.DomainDaoImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Unable to remove domain as domain 7 no longer exists
2014-12-03 02:54:33,817 ERROR [c.c.u.DomainManagerImpl]
(API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
Exception deleting domain with id 7
> [Automation] Deletion of Domain with Cleanup set to "true" fails
> ----------------------------------------------------------------
>
> Key: CLOUDSTACK-8022
> URL: https://issues.apache.org/jira/browse/CLOUDSTACK-8022
> Project: CloudStack
> Issue Type: Bug
> Security Level: Public(Anyone can view this level - this is the
> default.)
> Components: Automation
> Affects Versions: 4.5.0
> Reporter: Chandan Purushothama
> Assignee: Prachi Damle
> Priority: Critical
> Fix For: 4.5.0
>
> Attachments: management-server.zip
>
>
> The following test case in test_persistent_rules.py fails:
> {code}
> @attr(tags=["advanced"])
> def test_vpc_force_delete_domain(self):
> # steps
> # 1. Create account and create VPC network in the account
> # 2. Create two persistent networks within this VPC
> # 3. Restart/delete VPC network
> # Validations
> # 1. In case of Restart operation, restart should be successful
> # and persistent networks should be back in persistent state
> # 2. In case of Delete operation, VR servicing the VPC should
> # get destroyed and sourceNAT ip should get released
> child_domain = Domain.create(self.apiclient,
> services=self.services["domain"],
> parentdomainid=self.domain.id)
> try:
> account_1 = Account.create(
> self.apiclient, self.services["account"],
> domainid=child_domain.id
> )
> account_2 = Account.create(
> self.apiclient, self.services["account"],
> domainid=child_domain.id
> )
> self.services["vpc"]["cidr"] = "10.1.1.1/16"
> vpc_1 = VPC.create(
> self.apiclient,
> self.services["vpc"],
> vpcofferingid=self.vpc_off.id,
> zoneid=self.zone.id,
> account=account_1.name,
> domainid=account_1.domainid)
> vpcs = VPC.list(self.apiclient, id=vpc_1.id)
> self.assertEqual(
> validateList(vpcs)[0],
> PASS,
> "VPC list validation failed, vpc list is %s" %
> vpcs)
> vpc_2 = VPC.create(
> self.apiclient,
> self.services["vpc"],
> vpcofferingid=self.vpc_off.id,
> zoneid=self.zone.id,
> account=account_2.name,
> domainid=account_2.domainid)
> vpcs = VPC.list(self.apiclient, id=vpc_2.id)
> self.assertEqual(
> validateList(vpcs)[0],
> PASS,
> "VPC list validation failed, vpc list is %s" %
> vpcs)
> persistent_network_1 = Network.create(
> self.api_client, self.services["isolated_network"],
> networkofferingid=self.persistent_network_offering_NoLB.id,
> accountid=account_1.name, domainid=account_1.domainid,
> zoneid=self.zone.id, vpcid=vpc_1.id, gateway="10.1.1.1",
> netmask="255.255.255.0")
> response = verifyNetworkState(self.apiclient,
> persistent_network_1.id,
> "implemented"
> )
> exceptionOccured = response[0]
> isNetworkInDesiredState = response[1]
> exceptionMessage = response[2]
> if (exceptionOccured or (not isNetworkInDesiredState)):
> raise Exception(exceptionMessage)
> self.assertIsNotNone(
> persistent_network_1.vlan,
> "vlan must not be null for persistent network %s" %
> persistent_network_1.id)
> persistent_network_2 = Network.create(
> self.api_client, self.services["isolated_network"],
> networkofferingid=self.persistent_network_offering_NoLB.id,
> accountid=account_2.name, domainid=account_2.domainid,
> zoneid=self.zone.id, vpcid=vpc_2.id, gateway="10.1.1.1",
> netmask="255.255.255.0")
> response = verifyNetworkState(
> self.apiclient,
> persistent_network_2.id,
> "implemented")
> exceptionOccured = response[0]
> isNetworkInDesiredState = response[1]
> exceptionMessage = response[2]
> if (exceptionOccured or (not isNetworkInDesiredState)):
> raise Exception(exceptionMessage)
> self.assertIsNotNone(
> persistent_network_2.vlan,
> "vlan must not be null for persistent network: %s" %
> persistent_network_2.id)
> except Exception as e:
> self.cleanup.append(account_1)
> self.cleanup.append(account_2)
> self.cleanup.append(child_domain)
> self.fail(e)
> # Force delete domain
> child_domain.delete(self.apiclient, cleanup=True)
> self.debug("Waiting for account.cleanup.interval" +
> " to cleanup any remaining resouces")
> # Sleep 3*account.gc to ensure that all resources are deleted
> wait_for_cleanup(self.apiclient, ["account.cleanup.interval"] * 3)
> with self.assertRaises(Exception):
> Domain.list(self.apiclient, id=child_domain.id)
> with self.assertRaises(Exception):
> Account.list(
> self.apiclient, name=account_1.name,
> domainid=account_1.domainid, listall=True
> )
> with self.assertRaises(Exception):
> Account.list(
> self.apiclient, name=account_2.name,
> domainid=account_2.domainid, listall=True
> )
> self.VerifyVpcCleanup(vpc_1.id)
> self.VerifyVpcCleanup(vpc_2.id)
> self.VerifyNetworkCleanup(persistent_network_1.id)
> self.VerifyNetworkCleanup(persistent_network_2.id)
> return
> {code}
> Following is the stack trace:
> {noformat}
> Stacktrace
> File "/usr/lib/python2.7/unittest/case.py", line 332, in run
> testMethod()
> File
> "/root/cloudstack/test/integration/component/test_persistent_networks.py",
> line 2780, in test_vpc_force_delete_domain
> child_domain.delete(self.apiclient, cleanup=True)
> File "/usr/local/lib/python2.7/dist-packages/marvin/lib/base.py", line 77,
> in delete
> apiclient.deleteDomain(cmd)
> File
> "/usr/local/lib/python2.7/dist-packages/marvin/cloudstackAPI/cloudstackAPIClient.py",
> line 671, in deleteDomain
> response = self.connection.marvinRequest(command, response_type=response,
> method=method)
> File
> "/usr/local/lib/python2.7/dist-packages/marvin/cloudstackConnection.py", line
> 379, in marvinRequest
> raise e
> 'Job failed: {jobprocstatus : 0, created : u\'2014-12-03T02:53:03+0000\', cmd
> : u\'org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd\', userid
> : u\'cb31c30a-7a84-11e4-bab5-967d6e0d04d3\', jobstatus : 2, jobid :
> u\'b373c063-dfb6-46bc-b82a-da7f9b12937b\', jobresultcode : 530, jobresulttype
> : u\'object\', jobresult : {errorcode : 530, errortext : u\'Failed to clean
> up domain resources and sub domains, delete failed on domain domain-RS2LKV
> (id: 7).\'}, accountid : u\'cb31b43c-7a84-11e4-bab5-967d6e0d04d3\'}\n
> {noformat}
> ============================================
> Exception Information from the Management Server Log:
> ============================================
> {noformat}
> 2014-12-03 02:54:33,733 DEBUG [c.c.n.v.RemoteAccessVpnManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> there are no Remote access vpns for public ip address id=12
> 2014-12-03 02:54:33,746 DEBUG [c.c.n.IpAddressManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Releasing ip id=12; sourceNat = true
> 2014-12-03 02:54:33,749 DEBUG [c.c.n.IpAddressManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Released a public ip id=12
> 2014-12-03 02:54:33,749 DEBUG [c.c.n.v.VpcManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Released ip addresses for vpc id=57 as a part of cleanup vpc process
> 2014-12-03 02:54:33,750 DEBUG [c.c.n.v.VpcManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Found 0 to revoke for the vpc 57
> 2014-12-03 02:54:33,753 DEBUG [c.c.n.v.VpcManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063) Vpc
> [VPC [57-vpc_vpn-T13419] is destroyed succesfully
> 2014-12-03 02:54:33,753 DEBUG [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063) VPC
> 57 successfully deleted as a part of account id=105 cleanup.
> 2014-12-03 02:54:33,754 DEBUG [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Deleting site-to-site VPN customer gateways for account 105
> 2014-12-03 02:54:33,759 DEBUG [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Account specific Virtual IP ranges are successfully released as a part of
> account id=105 cleanup.
> 2014-12-03 02:54:33,760 INFO [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> deleteAccount: Released 0 dedicated guest vlan ranges from account 105
> 2014-12-03 02:54:33,806 INFO [c.c.u.AccountManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Cleanup for account 105 is not needed.
> 2014-12-03 02:54:33,808 DEBUG [c.c.u.DomainManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Deleting networks for domain id=7
> 2014-12-03 02:54:33,812 ERROR [c.c.d.d.DomainDaoImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Unable to remove domain as domain 7 no longer exists
> 2014-12-03 02:54:33,817 ERROR [c.c.u.DomainManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Exception deleting domain with id 7
> com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain
> resources and sub domains, delete failed on domain domain-RS2LKV (id: 7).
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:270)
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:253)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:601)
> at
> org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> at
> org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> at
> org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
> at $Proxy114.deleteDomain(Unknown Source)
> at
> org.apache.cloudstack.region.RegionManagerImpl.deleteDomain(RegionManagerImpl.java:242)
> at
> org.apache.cloudstack.region.RegionServiceImpl.deleteDomain(RegionServiceImpl.java:169)
> at
> org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd.execute(DeleteDomainCmd.java:103)
> at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
> at
> com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:546)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:497)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:722)
> 2014-12-03 02:54:33,819 DEBUG [c.c.u.DomainManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474 ctx-2ac8b66c) (logid:b373c063)
> Changing domain id=7 state back to Active because it can't be removed due to
> resources referencing to it
> 2014-12-03 02:54:33,850 ERROR [c.c.a.ApiAsyncJobDispatcher]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Unexpected
> exception while executing
> org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd
> com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain
> resources and sub domains, delete failed on domain domain-RS2LKV (id: 7).
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:270)
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:253)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:601)
> at
> org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> at
> org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> at
> org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
> at $Proxy114.deleteDomain(Unknown Source)
> at
> org.apache.cloudstack.region.RegionManagerImpl.deleteDomain(RegionManagerImpl.java:242)
> at
> org.apache.cloudstack.region.RegionServiceImpl.deleteDomain(RegionServiceImpl.java:169)
> at
> org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd.execute(DeleteDomainCmd.java:103)
> at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
> at
> com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:546)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:497)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:722)
> 2014-12-03 02:54:33,851 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Complete async
> job-1474, jobStatus: FAILED, resultCode: 530, result:
> org.apache.cloudstack.api.response.ExceptionResponse/null/{"uuidList":[],"errorcode":530,"errortext":"Failed
> to clean up domain resources and sub domains, delete failed on domain
> domain-RS2LKV (id: 7)."}
> 2014-12-03 02:54:33,852 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Publish async
> job-1474 complete on message bus
> 2014-12-03 02:54:33,852 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Wake up jobs
> related to job-1474
> 2014-12-03 02:54:33,853 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Update db status
> for job-1474
> 2014-12-03 02:54:33,854 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Wake up jobs
> joined with job-1474 and disjoin all subjobs created from job- 1474
> 2014-12-03 02:54:33,856 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Done executing
> org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd for job-1474
> 2014-12-03 02:54:33,856 INFO [o.a.c.f.j.i.AsyncJobMonitor]
> (API-Job-Executor-72:ctx-1a1bd808 job-1474) (logid:b373c063) Remove job-1474
> from job monitoring
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
