[
https://issues.apache.org/jira/browse/CLOUDSTACK-9764?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15959500#comment-15959500
]
ASF GitHub Bot commented on CLOUDSTACK-9764:
--------------------------------------------
Github user blueorangutan commented on the issue:
https://github.com/apache/cloudstack/pull/1935
<b>Trillian test result (tid-973)</b>
Environment: kvm-centos7 (x2), Advanced Networking with Mgmt server 7
Total time taken: 29259 seconds
Marvin logs:
https://github.com/blueorangutan/acs-prs/releases/download/trillian/pr1935-t973-kvm-centos7.zip
Intermitten failure detected: /marvin/tests/smoke/test_privategw_acl.py
Test completed. 48 look ok, 1 have error(s)
Test | Result | Time (s) | Test File
--- | --- | --- | ---
test_04_rvpc_privategw_static_routes | `Failure` | 335.65 |
test_privategw_acl.py
test_01_vpc_site2site_vpn | Success | 150.05 | test_vpc_vpn.py
test_01_vpc_remote_access_vpn | Success | 61.16 | test_vpc_vpn.py
test_01_redundant_vpc_site2site_vpn | Success | 245.99 | test_vpc_vpn.py
test_02_VPC_default_routes | Success | 264.01 | test_vpc_router_nics.py
test_01_VPC_nics_after_destroy | Success | 553.63 | test_vpc_router_nics.py
test_05_rvpc_multi_tiers | Success | 512.64 | test_vpc_redundant.py
test_04_rvpc_network_garbage_collector_nics | Success | 1410.93 |
test_vpc_redundant.py
test_03_create_redundant_VPC_1tier_2VMs_2IPs_2PF_ACL_reboot_routers |
Success | 548.06 | test_vpc_redundant.py
test_02_redundant_VPC_default_routes | Success | 745.11 |
test_vpc_redundant.py
test_01_create_redundant_VPC_2tiers_4VMs_4IPs_4PF_ACL | Success | 1306.39 |
test_vpc_redundant.py
test_09_delete_detached_volume | Success | 156.51 | test_volumes.py
test_08_resize_volume | Success | 156.45 | test_volumes.py
test_07_resize_fail | Success | 161.44 | test_volumes.py
test_06_download_detached_volume | Success | 156.43 | test_volumes.py
test_05_detach_volume | Success | 150.80 | test_volumes.py
test_04_delete_attached_volume | Success | 151.22 | test_volumes.py
test_03_download_attached_volume | Success | 156.38 | test_volumes.py
test_02_attach_volume | Success | 89.17 | test_volumes.py
test_01_create_volume | Success | 622.16 | test_volumes.py
test_03_delete_vm_snapshots | Success | 275.25 | test_vm_snapshots.py
test_02_revert_vm_snapshots | Success | 100.71 | test_vm_snapshots.py
test_01_create_vm_snapshots | Success | 168.82 | test_vm_snapshots.py
test_deploy_vm_multiple | Success | 272.66 | test_vm_life_cycle.py
test_deploy_vm | Success | 0.03 | test_vm_life_cycle.py
test_advZoneVirtualRouter | Success | 0.02 | test_vm_life_cycle.py
test_10_attachAndDetach_iso | Success | 26.55 | test_vm_life_cycle.py
test_09_expunge_vm | Success | 125.24 | test_vm_life_cycle.py
test_08_migrate_vm | Success | 30.88 | test_vm_life_cycle.py
test_07_restore_vm | Success | 0.13 | test_vm_life_cycle.py
test_06_destroy_vm | Success | 125.81 | test_vm_life_cycle.py
test_03_reboot_vm | Success | 125.84 | test_vm_life_cycle.py
test_02_start_vm | Success | 10.17 | test_vm_life_cycle.py
test_01_stop_vm | Success | 35.29 | test_vm_life_cycle.py
test_CreateTemplateWithDuplicateName | Success | 161.14 | test_templates.py
test_08_list_system_templates | Success | 0.04 | test_templates.py
test_07_list_public_templates | Success | 0.04 | test_templates.py
test_05_template_permissions | Success | 0.05 | test_templates.py
test_04_extract_template | Success | 5.17 | test_templates.py
test_03_delete_template | Success | 5.11 | test_templates.py
test_02_edit_template | Success | 90.13 | test_templates.py
test_01_create_template | Success | 35.41 | test_templates.py
test_10_destroy_cpvm | Success | 161.64 | test_ssvm.py
test_09_destroy_ssvm | Success | 133.24 | test_ssvm.py
test_08_reboot_cpvm | Success | 131.61 | test_ssvm.py
test_07_reboot_ssvm | Success | 133.65 | test_ssvm.py
test_06_stop_cpvm | Success | 131.74 | test_ssvm.py
test_05_stop_ssvm | Success | 163.77 | test_ssvm.py
test_04_cpvm_internals | Success | 1.24 | test_ssvm.py
test_03_ssvm_internals | Success | 3.37 | test_ssvm.py
test_02_list_cpvm_vm | Success | 0.12 | test_ssvm.py
test_01_list_sec_storage_vm | Success | 0.13 | test_ssvm.py
test_02_list_snapshots_with_removed_data_store | Success | 86.81 |
test_snapshots.py
test_01_snapshot_root_disk | Success | 16.33 | test_snapshots.py
test_04_change_offering_small | Success | 209.54 | test_service_offerings.py
test_03_delete_service_offering | Success | 0.04 | test_service_offerings.py
test_02_edit_service_offering | Success | 0.06 | test_service_offerings.py
test_01_create_service_offering | Success | 0.11 | test_service_offerings.py
test_02_sys_template_ready | Success | 0.14 | test_secondary_storage.py
test_01_sys_vm_start | Success | 0.18 | test_secondary_storage.py
test_09_reboot_router | Success | 35.35 | test_routers.py
test_08_start_router | Success | 30.32 | test_routers.py
test_07_stop_router | Success | 10.18 | test_routers.py
test_06_router_advanced | Success | 0.05 | test_routers.py
test_05_router_basic | Success | 0.04 | test_routers.py
test_04_restart_network_wo_cleanup | Success | 5.62 | test_routers.py
test_03_restart_network_cleanup | Success | 50.47 | test_routers.py
test_02_router_internal_adv | Success | 1.10 | test_routers.py
test_01_router_internal_basic | Success | 0.58 | test_routers.py
test_router_dns_guestipquery | Success | 73.75 | test_router_dns.py
test_router_dns_externalipquery | Success | 0.06 | test_router_dns.py
test_router_dhcphosts | Success | 271.94 | test_router_dhcphosts.py
test_router_dhcp_opts | Success | 21.49 | test_router_dhcphosts.py
test_01_updatevolumedetail | Success | 0.10 | test_resource_detail.py
test_01_reset_vm_on_reboot | Success | 130.93 | test_reset_vm_on_reboot.py
test_createRegion | Success | 0.06 | test_regions.py
test_create_pvlan_network | Success | 5.21 | test_pvlan.py
test_dedicatePublicIpRange | Success | 0.41 | test_public_ip_range.py
test_03_vpc_privategw_restart_vpc_cleanup | Success | 515.08 |
test_privategw_acl.py
test_02_vpc_privategw_static_routes | Success | 390.25 |
test_privategw_acl.py
test_01_vpc_privategw_acl | Success | 82.14 | test_privategw_acl.py
test_03_migration_options_storage_tags | Success | 66.30 |
test_primary_storage.py
test_02_edit_primary_storage_tags | Success | 0.10 | test_primary_storage.py
test_01_primary_storage_nfs | Success | 35.76 | test_primary_storage.py
test_01_deploy_vms_storage_tags | Success | 31.02 | test_primary_storage.py
test_createPortablePublicIPRange | Success | 15.19 |
test_portable_publicip.py
test_createPortablePublicIPAcquire | Success | 15.44 |
test_portable_publicip.py
test_isolate_network_password_server | Success | 89.20 |
test_password_server.py
test_UpdateStorageOverProvisioningFactor | Success | 0.14 |
test_over_provisioning.py
test_oobm_zchange_password | Success | 35.74 | test_outofbandmanagement.py
test_oobm_multiple_mgmt_server_ownership | Success | 16.34 |
test_outofbandmanagement.py
test_oobm_issue_power_status | Success | 10.23 | test_outofbandmanagement.py
test_oobm_issue_power_soft | Success | 15.47 | test_outofbandmanagement.py
test_oobm_issue_power_reset | Success | 15.39 | test_outofbandmanagement.py
test_oobm_issue_power_on | Success | 15.34 | test_outofbandmanagement.py
test_oobm_issue_power_off | Success | 15.32 | test_outofbandmanagement.py
test_oobm_issue_power_cycle | Success | 15.35 | test_outofbandmanagement.py
test_oobm_enabledisable_across_clusterzones | Success | 92.56 |
test_outofbandmanagement.py
test_oobm_enable_feature_valid | Success | 5.16 |
test_outofbandmanagement.py
test_oobm_enable_feature_invalid | Success | 0.10 |
test_outofbandmanagement.py
test_oobm_disable_feature_valid | Success | 0.17 |
test_outofbandmanagement.py
test_oobm_disable_feature_invalid | Success | 0.10 |
test_outofbandmanagement.py
test_oobm_configure_invalid_driver | Success | 0.07 |
test_outofbandmanagement.py
test_oobm_configure_default_driver | Success | 0.08 |
test_outofbandmanagement.py
test_oobm_background_powerstate_sync | Success | 23.41 |
test_outofbandmanagement.py
test_extendPhysicalNetworkVlan | Success | 15.38 |
test_non_contigiousvlan.py
test_01_nic | Success | 409.33 | test_nic.py
test_releaseIP | Success | 237.74 | test_network.py
test_reboot_router | Success | 403.56 | test_network.py
test_public_ip_user_account | Success | 10.25 | test_network.py
test_public_ip_admin_account | Success | 40.30 | test_network.py
test_network_rules_acquired_public_ip_3_Load_Balancer_Rule | Success |
67.03 | test_network.py
test_network_rules_acquired_public_ip_2_nat_rule | Success | 61.82 |
test_network.py
test_network_rules_acquired_public_ip_1_static_nat_rule | Success | 123.95
| test_network.py
test_delete_account | Success | 328.17 | test_network.py
test_02_port_fwd_on_non_src_nat | Success | 55.66 | test_network.py
test_01_port_fwd_on_src_nat | Success | 111.74 | test_network.py
test_nic_secondaryip_add_remove | Success | 207.71 |
test_multipleips_per_nic.py
login_test_saml_user | Success | 19.13 | test_login.py
test_assign_and_removal_lb | Success | 133.47 | test_loadbalance.py
test_02_create_lb_rule_non_nat | Success | 187.04 | test_loadbalance.py
test_01_create_lb_rule_src_nat | Success | 217.70 | test_loadbalance.py
test_03_list_snapshots | Success | 0.09 | test_list_ids_parameter.py
test_02_list_templates | Success | 0.04 | test_list_ids_parameter.py
test_01_list_volumes | Success | 0.03 | test_list_ids_parameter.py
test_07_list_default_iso | Success | 0.06 | test_iso.py
test_05_iso_permissions | Success | 0.06 | test_iso.py
test_04_extract_Iso | Success | 5.17 | test_iso.py
test_03_delete_iso | Success | 95.24 | test_iso.py
test_02_edit_iso | Success | 0.06 | test_iso.py
test_01_create_iso | Success | 20.99 | test_iso.py
test_04_rvpc_internallb_haproxy_stats_on_all_interfaces | Success | 198.19
| test_internal_lb.py
test_03_vpc_internallb_haproxy_stats_on_all_interfaces | Success | 148.41 |
test_internal_lb.py
test_02_internallb_roundrobin_1RVPC_3VM_HTTP_port80 | Success | 505.22 |
test_internal_lb.py
test_01_internallb_roundrobin_1VPC_3VM_HTTP_port80 | Success | 435.47 |
test_internal_lb.py
test_dedicateGuestVlanRange | Success | 10.43 | test_guest_vlan_range.py
test_UpdateConfigParamWithScope | Success | 0.14 | test_global_settings.py
test_rolepermission_lifecycle_update | Success | 6.16 | test_dynamicroles.py
test_rolepermission_lifecycle_list | Success | 5.97 | test_dynamicroles.py
test_rolepermission_lifecycle_delete | Success | 5.85 | test_dynamicroles.py
test_rolepermission_lifecycle_create | Success | 5.89 | test_dynamicroles.py
test_rolepermission_lifecycle_concurrent_updates | Success | 6.02 |
test_dynamicroles.py
test_role_lifecycle_update_role_inuse | Success | 5.91 |
test_dynamicroles.py
test_role_lifecycle_update | Success | 10.97 | test_dynamicroles.py
test_role_lifecycle_list | Success | 5.88 | test_dynamicroles.py
test_role_lifecycle_delete | Success | 11.00 | test_dynamicroles.py
test_role_lifecycle_create | Success | 5.89 | test_dynamicroles.py
test_role_inuse_deletion | Success | 5.85 | test_dynamicroles.py
test_role_account_acls_multiple_mgmt_servers | Success | 7.97 |
test_dynamicroles.py
test_role_account_acls | Success | 8.32 | test_dynamicroles.py
test_default_role_deletion | Success | 6.02 | test_dynamicroles.py
test_04_create_fat_type_disk_offering | Success | 0.07 |
test_disk_offerings.py
test_03_delete_disk_offering | Success | 0.04 | test_disk_offerings.py
test_02_edit_disk_offering | Success | 0.06 | test_disk_offerings.py
test_02_create_sparse_type_disk_offering | Success | 0.07 |
test_disk_offerings.py
test_01_create_disk_offering | Success | 0.32 | test_disk_offerings.py
test_deployvm_userdispersing | Success | 20.59 |
test_deploy_vms_with_varied_deploymentplanners.py
test_deployvm_userconcentrated | Success | 20.58 |
test_deploy_vms_with_varied_deploymentplanners.py
test_deployvm_firstfit | Success | 60.79 |
test_deploy_vms_with_varied_deploymentplanners.py
test_deployvm_userdata_post | Success | 10.52 |
test_deploy_vm_with_userdata.py
test_deployvm_userdata | Success | 55.84 | test_deploy_vm_with_userdata.py
test_02_deploy_vm_root_resize | Success | 5.99 |
test_deploy_vm_root_resize.py
test_01_deploy_vm_root_resize | Success | 5.97 |
test_deploy_vm_root_resize.py
test_00_deploy_vm_root_resize | Success | 262.74 |
test_deploy_vm_root_resize.py
test_deploy_vm_from_iso | Success | 212.41 | test_deploy_vm_iso.py
test_DeployVmAntiAffinityGroup | Success | 66.34 | test_affinity_groups.py
test_change_service_offering_for_vm_with_snapshots | Skipped | 0.00 |
test_vm_snapshots.py
test_01_test_vm_volume_snapshot | Skipped | 0.00 | test_vm_snapshots.py
test_06_copy_template | Skipped | 0.00 | test_templates.py
test_static_role_account_acls | Skipped | 0.02 | test_staticroles.py
test_11_ss_nfs_version_on_ssvm | Skipped | 0.02 | test_ssvm.py
test_01_scale_vm | Skipped | 0.00 | test_scale_vm.py
test_01_primary_storage_iscsi | Skipped | 0.04 | test_primary_storage.py
test_nested_virtualization_vmware | Skipped | 0.00 |
test_nested_virtualization.py
test_06_copy_iso | Skipped | 0.00 | test_iso.py
test_deploy_vgpu_enabled_vm | Skipped | 0.03 |
test_deploy_vgpu_enabled_vm.py
test_3d_gpu_support | Skipped | 0.03 | test_deploy_vgpu_enabled_vm.py
> Delete domain failure due to Account Cleanup task
> -------------------------------------------------
>
> Key: CLOUDSTACK-9764
> URL: https://issues.apache.org/jira/browse/CLOUDSTACK-9764
> Project: CloudStack
> Issue Type: Bug
> Security Level: Public(Anyone can view this level - this is the
> default.)
> Affects Versions: 4.10.0.0
> Reporter: Nicolas Vazquez
> Assignee: Nicolas Vazquez
> Fix For: 4.10.0.0
>
>
> It was noticed in production environments that {{deleteDomain}} task failed
> for domains with multiple accounts and resources. Examining logs it was found
> out that if Account Cleanup Task got executed after domain (and all of its
> subchilds) got marked as Inactive; and before delete domain task finishes, it
> produces a failure.
> {{AccountCleanupTask}} gets executed every {{account.cleanup.interval}}
> seconds looking for:
> * Removed accounts
> * Disabled accounts
> * Inactive domains
> As {{deleteDomain}} marks domain to delete (and its subchilds) as Inactive
> before deleting them, when {{AccountCleanupTask}} is executed, it removes
> marked domains. When there are resources to cleanup on domain accounts,
> domain is not found throwing exception:
> {{com.cloud.exception.InvalidParameterValueException: Please specify a valid
> domain ID}}
> h3. Example
> {{account.cleanup.interval}} = 100
> {noformat}
> 2017-01-26 06:07:03,621 DEBUG [cloud.api.ApiServlet]
> (catalina-exec-8:ctx-50cfa3b6 ctx-92ad5b38) ===END=== 10.39.251.17 -- GET
> command=deleteDomain&id=1910a3dc-6fa6-457b-ab3a-602b0cfb6686&cleanup=true&response=json&_=1485439623475
> ...
> // Domain and its subchilds marked as Inactive
> 2017-01-26 06:07:03,640 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Marking domain id=27
> as Inactive before actually deleting it
> 2017-01-26 06:07:03,646 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Cleaning up domain
> id=27
> 2017-01-26 06:07:03,670 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Cleaning up domain
> id=28
> 2017-01-26 06:07:03,685 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Cleaning up domain
> id=29
> ...
> // AccountCleanupTask removes Inactive domain id=29, no rollback for it
> 2017-01-26 06:07:44,285 INFO [cloud.user.AccountManagerImpl]
> (AccountChecker-1:ctx-b8a01824) Found 0 removed accounts to cleanup
> 2017-01-26 06:07:44,287 INFO [cloud.user.AccountManagerImpl]
> (AccountChecker-1:ctx-b8a01824) Found 0 disabled accounts to cleanup
> 2017-01-26 06:07:44,289 INFO [cloud.user.AccountManagerImpl]
> (AccountChecker-1:ctx-b8a01824) Found 3 inactive domains to cleanup
> 2017-01-26 06:07:44,292 DEBUG [cloud.user.AccountManagerImpl]
> (AccountChecker-1:ctx-b8a01824) Removing inactive domain id=27
> 2017-01-26 06:07:44,297 DEBUG [db.Transaction.Transaction]
> (AccountChecker-1:ctx-b8a01824) Rolling back the transaction: Time = 2 Name =
> AccountChecker-1; called by
> -TransactionLegacy.rollback:889-TransactionLegacy.removeUpTo:832-TransactionLegacy.close:656-TransactionContextInterceptor.invoke:36-ReflectiveMethodInvocation.proceed:161-ExposeInvocationInterceptor.invoke:91-ReflectiveMethodInvocation.proceed:172-JdkDynamicAopProxy.invoke:204-$Proxy63.remove:-1-DomainManagerImpl.removeDomain:248-NativeMethodAccessorImpl.invoke0:-2-NativeMethodAccessorImpl.invoke:62
> 2017-01-26 06:07:44,301 DEBUG [cloud.user.AccountManagerImpl]
> (AccountChecker-1:ctx-b8a01824) Removing inactive domain id=28
> 2017-01-26 06:07:44,304 DEBUG [db.Transaction.Transaction]
> (AccountChecker-1:ctx-b8a01824) Rolling back the transaction: Time = 2 Name =
> AccountChecker-1; called by
> -TransactionLegacy.rollback:889-TransactionLegacy.removeUpTo:832-TransactionLegacy.close:656-TransactionContextInterceptor.invoke:36-ReflectiveMethodInvocation.proceed:161-ExposeInvocationInterceptor.invoke:91-ReflectiveMethodInvocation.proceed:172-JdkDynamicAopProxy.invoke:204-$Proxy63.remove:-1-DomainManagerImpl.removeDomain:248-NativeMethodAccessorImpl.invoke0:-2-NativeMethodAccessorImpl.invoke:62
> 2017-01-26 06:07:44,307 DEBUG [cloud.user.AccountManagerImpl]
> (AccountChecker-1:ctx-b8a01824) Removing inactive domain id=29
> 2017-01-26 06:07:44,319 INFO [cloud.user.AccountManagerImpl]
> (AccountChecker-1:ctx-b8a01824) Found 0 disabled projects to cleanup
> ...
> // Failure due to domain is already removed
> 2017-01-26 06:07:46,369 WARN [cloud.user.AccountManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Failed to cleanup
> account Acct[6a6e63ad-d89b-4a53-b3ae-1c06ea3d1899-ac2] due to
> com.cloud.exception.InvalidParameterValueException: Please specify a valid
> domain ID.
> at
> com.cloud.resourcelimit.ResourceLimitManagerImpl.recalculateResourceCount(ResourceLimitManagerImpl.java:752)
> at com.cloud.vm.UserVmManagerImpl.expunge(UserVmManagerImpl.java:2053)
> ...
> 2017-01-26 06:07:46,381 INFO [cloud.user.AccountManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Cleanup for account
> 2580 is needed.
> 2017-01-26 06:07:46,390 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Deleting networks
> for domain id=29
> 2017-01-26 06:07:46,392 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Can't delete the
> domain yet because it has 1accounts that need a cleanup
> 2017-01-26 06:07:46,392 WARN [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Failed to cleanup
> domain id=29
> 2017-01-26 06:07:46,394 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Deleting networks
> for domain id=28
> 2017-01-26 06:07:46,416 WARN [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Failed to cleanup
> domain id=28
> 2017-01-26 06:07:46,418 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Deleting networks
> for domain id=27
> 2017-01-26 06:07:46,440 ERROR [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Exception deleting
> domain with id 27
> com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain
> resources and sub domains, delete failed on domain A (id: 27).
> ...
> 2017-01-26 06:07:46,441 DEBUG [cloud.user.DomainManagerImpl]
> (API-Job-Executor-29:ctx-23415942 job-7165 ctx-fe3d13d6) Changing domain
> id=27 state back to Active because it can't be removed due to resources
> referencing to it
> 2017-01-26 06:07:46,459 ERROR [cloud.api.ApiAsyncJobDispatcher]
> (API-Job-Executor-29:ctx-23415942 job-7165) Unexpected exception while
> executing org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd
> com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain
> resources and sub domains, delete failed on domain A (id: 27).
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:290)
> at
> com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:271)
> ...
> {noformat}}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
