I plan to do quite a bit of work on this front over the next couple weeks in prep for the RC next month. I am also trying to push the idea of distributed CI and I feel that stabilizing and improving the state of our tests is a big piece of that. On Apr 23, 2016 3:17 AM, "Paul Angus" <paul.an...@shapeblue.com> wrote:
> Hi Will, > > I little while back I created a matrix to attempt to keep track of the > tests and their status. > Various other projects have stopped me getting to the stage where I could > start putting anything in... > > https://cwiki.apache.org/confluence/display/CLOUDSTACK/Marvin+Test+Matrix > > > > > Kind regards, > > Paul Angus > > Regards, > > Paul Angus > > paul.an...@shapeblue.com > www.shapeblue.com > 53 Chandos Place, Covent Garden, London WC2N 4HSUK > @shapeblue > > -----Original Message----- > From: Will Stevens [mailto:williamstev...@gmail.com] > Sent: 22 April 2016 19:10 > To: dev@cloudstack.apache.org > Subject: Here goes nothing... :) > > In an effort to figure out which tests are working and which are not and > to start compiling a list of tests with issues I have written a small > python script which auto discovers and categorizes the tests in the 'smoke' > and 'component' directories. > > This will be interesting: > > # HARDWARE REQUIRED > echo "Running tests with required_hardware=true" > nosetests --with-marvin --marvin-config=${marvinCfg} -s -a > tags=advanced,required_hardware=true \ smoke/test_deploy_vgpu_enabled_vm.py > \ smoke/test_deploy_vm_iso.py \ smoke/test_deploy_vm_root_resize.py \ > smoke/test_deploy_vm_with_userdata.py \ smoke/test_internal_lb.py \ > smoke/test_iso.py \ smoke/test_loadbalance.py \ smoke/test_network.py \ > smoke/test_network_acl.py \ smoke/test_nic.py \ > smoke/test_password_server.py \ smoke/test_primary_storage.py \ > smoke/test_privategw_acl.py \ smoke/test_regions.py \ > smoke/test_router_dhcphosts.py \ smoke/test_routers.py \ > smoke/test_routers_iptables_default_policy.py \ > smoke/test_routers_network_ops.py \ smoke/test_service_offerings.py \ > smoke/test_snapshots.py \ smoke/test_ssvm.py \ smoke/test_templates.py \ > smoke/test_usage_events.py \ smoke/test_vm_life_cycle.py \ > smoke/test_vm_snapshots.py \ smoke/test_volumes.py \ > smoke/test_vpc_redundant.py \ smoke/test_vpc_router_nics.py \ > smoke/test_vpc_vpn.py \ component/test_accounts.py \ > component/test_add_remove_network.py \ component/test_blocker_bugs.py \ > component/test_browse_templates.py \ component/test_browse_volumes.py \ > component/test_concurrent_snapshots_limit.py \ > component/test_cpu_domain_limits.py \ component/test_cpu_limits.py \ > component/test_cpu_project_limits.py \ component/test_deploy_vgpu_vm.py \ > component/test_deploy_vm_userdata_reg.py \ > component/test_dynamic_compute_offering.py \ > component/test_egress_fw_rules.py \ > component/test_escalation_listTemplateDomainAdmin.py \ > component/test_escalations_instances.py \ > component/test_escalations_ipaddresses.py \ > component/test_escalations_isos.py \ component/test_escalations_networks.py > \ component/test_escalations_routers.py \ > component/test_escalations_securitygroups.py \ > component/test_escalations_snapshots.py \ > component/test_escalations_templates.py \ > component/test_escalations_vmware.py \ > component/test_escalations_volumes.py \ > component/test_escalations_vpncustomergateways.py \ > component/test_haproxy.py \ component/test_interop_xd_ccp.py \ > component/test_ip_reservation.py \ component/test_lb_secondary_ip.py \ > component/test_memory_limits.py \ component/test_mm_domain_limits.py \ > component/test_mm_project_limits.py \ component/test_netscaler_configs.py \ > component/test_netscaler_nw_off.py \ component/test_network_offering.py \ > component/test_overcommit.py \ component/test_persistent_networks.py \ > component/test_portable_ip.py \ component/test_project_limits.py \ > component/test_project_usage.py \ component/test_ps_domain_limits.py \ > component/test_ps_limits.py \ component/test_ps_resize_volume.py \ > component/test_ps_resource_limits_volume.py \ > component/test_recurring_snapshots.py \ component/test_reset_ssh_keypair.py > \ component/test_routers.py \ component/test_security_groups.py \ > component/test_shared_networks.py \ > component/test_simultaneous_volume_attach.py \ > component/test_snapshot_gc.py \ component/test_snapshot_limits.py \ > component/test_snapshots.py \ component/test_snapshots_improvement.py \ > component/test_ss_domain_limits.py \ component/test_ss_limits.py \ > component/test_ss_max_limits.py \ component/test_ss_project_limits.py \ > component/test_stopped_vm.py \ component/test_storage_motion.py \ > component/test_templates.py \ component/test_usage.py \ > component/test_VirtualRouter_alerts.py \ component/test_vm_passwdenabled.py > \ component/test_vmware_drs.py \ component/test_volumes.py \ > component/test_vpc.py \ component/test_vpc_network.py \ > component/test_vpc_network_lbrules.py \ > component/test_vpc_network_pfrules.py \ > component/test_vpc_network_staticnatrule.py \ > component/test_vpc_offerings.py \ component/test_vpc_routers.py \ > component/test_vpc_vm_life_cycle.py \ component/test_vpc_vms_deployment.py > > # HARDWARE NOT REQUIRED > echo "Running tests with required_hardware=false" > nosetests --with-marvin --marvin-config=${marvinCfg} -s -a > tags=advanced,required_hardware=false \ smoke/test_affinity_groups.py \ > smoke/test_affinity_groups_projects.py \ > smoke/test_deploy_vms_with_varied_deploymentplanners.py \ > smoke/test_disk_offerings.py \ smoke/test_global_settings.py \ > smoke/test_guest_vlan_range.py \ smoke/test_iso.py \ > smoke/test_multipleips_per_nic.py \ smoke/test_network.py \ > smoke/test_non_contigiousvlan.py \ smoke/test_over_provisioning.py \ > smoke/test_portable_publicip.py \ smoke/test_primary_storage.py \ > smoke/test_public_ip_range.py \ smoke/test_pvlan.py \ > smoke/test_reset_vm_on_reboot.py \ smoke/test_resource_detail.py \ > smoke/test_routers.py \ smoke/test_scale_vm.py \ > smoke/test_secondary_storage.py \ smoke/test_service_offerings.py \ > smoke/test_ssvm.py \ smoke/test_templates.py \ smoke/test_vm_life_cycle.py > \ smoke/test_volumes.py \ component/test_accounts.py \ > component/test_acl_isolatednetwork.py \ > component/test_acl_isolatednetwork_delete.py \ > component/test_acl_listsnapshot.py \ component/test_acl_listvm.py \ > component/test_acl_listvolume.py \ component/test_acl_sharednetwork.py \ > component/test_acl_sharednetwork_deployVM-impersonation.py \ > component/test_advancedsg_networks.py \ component/test_affinity_groups.py \ > component/test_affinity_groups_projects.py \ > component/test_allocation_states.py \ component/test_base_image_updation.py > \ component/test_blocker_bugs.py \ component/test_browse_templates.py \ > component/test_browse_templates2.py \ component/test_browse_volumes.py \ > component/test_cpu_domain_limits.py \ component/test_cpu_limits.py \ > component/test_cpu_max_limits.py \ component/test_cpu_project_limits.py \ > component/test_custom_hostname.py \ component/test_egress_fw_rules.py \ > component/test_escalations_instances.py \ component/test_haproxy.py \ > component/test_implicit_planner.py \ component/test_interop_xd_ccp.py \ > component/test_invalid_gw_nm.py \ component/test_ip_reservation.py \ > component/test_lb_secondary_ip.py \ component/test_ldap.py \ > component/test_memory_limits.py \ component/test_mm_domain_limits.py \ > component/test_mm_max_limits.py \ component/test_mm_project_limits.py \ > component/test_network_offering.py \ component/test_organization_states.py > \ component/test_overcommit.py \ component/test_persistent_networks.py \ > component/test_portable_ip.py \ component/test_project_configs.py \ > component/test_project_limits.py \ component/test_project_resources.py \ > component/test_project_usage.py \ component/test_projects.py \ > component/test_ps_domain_limits.py \ component/test_ps_limits.py \ > component/test_ps_project_limits.py \ > component/test_redundant_router_cleanups.py \ > component/test_redundant_router_services.py \ > component/test_redundant_router_upgrades.py \ component/test_regions.py \ > component/test_regions_accounts.py \ component/test_resource_limits.py \ > component/test_routers.py \ component/test_shared_networks.py \ > component/test_snapshots.py \ component/test_stopped_vm.py \ > component/test_storage_motion.py \ component/test_tags.py \ > component/test_templates.py \ component/test_update_vm.py \ > component/test_usage.py \ component/test_volumes.py \ component/test_vpc.py > \ component/test_vpc_network.py \ component/test_vpc_network_lbrules.py \ > component/test_vpc_offerings.py \ component/test_vpc_routers.py \ > component/test_vpc_vms_deployment.py \ component/test_vpn_users.py > > # HARDWARE UNKNOWN > echo "Running tests with no hardware details" > nosetests --with-marvin --marvin-config=${marvinCfg} -s -a tags=advanced \ > smoke/test_hosts.py \ smoke/test_nic_adapter_type.py \ > component/test_asa1000v_fw.py \ component/test_assign_vm.py \ > component/test_baremetal.py \ component/test_bigswitch_bcf.py \ > component/test_brocade_vcs.py \ component/test_dhcp_dns_offload.py \ > component/test_egress_rules.py \ component/test_eip_elb.py \ > component/test_multiple_ip_ranges.py \ > component/test_multiple_ips_per_nic.py \ component/test_netscaler_lb.py \ > component/test_netscaler_lb_algo.py \ component/test_netscaler_lb_sticky.py > \ component/test_non_contiguous_vlan.py \ component/test_ps_max_limits.py \ > component/test_region_vpc.py \ > component/test_vpc_distributed_routing_offering.py > > Hopefully this finishes this week. :P >
