Folks, It's been quite a journey, but the database work to modernize has come to a milestone.
For those not aware with the changes, we're dropping a lot of custom code, both database migration related and SQL DDL as well. The new way of creating and managing the autotest database is via Django and Django's South extension only. For reference, this has been initially discussed on github issue #589, but we've come a long way from the initial proposals there. New installations should not notice any hiccups, and extensive testing has been done using ./contrib/install-autotest-server.sh. For existing installations, migrations instructions will be posted to the WIKI shortly. Please test and report your feedback! Thanks! CR. Patch series: [PATCH 01/28] frontend: introduce MigrateVersion as a Django model [PATCH 02/28] database: remove the migrate step completely [PATCH 03/28] database: set tko_test_view_2 model as not managed [PATCH 04/28] database: make custom SQL compatible with MySQL [PATCH 05/28] database: only try to create functions if appropriate [PATCH 06/28] database: model fixes for AclGroup [PATCH 07/28] database: model fixes for Test [PATCH 08/28] database: model fixes for HostAttribute [PATCH 09/28] database: introduce database manager module [PATCH 10/28] autotest-database-turnkey: add option to read [PATCH 11/28] autotest-database-turnkey: use database manager [PATCH 12/28] database: allow to run unittests against other [PATCH 13/28] database: renamed database dir to database_legacy [PATCH 14/28] database: moved frontend migration files to [PATCH 15/28] database: get rid of the tko migration files [PATCH 16/28] frontend: add database migrate versioninig info to [PATCH 17/28] installation_support/autotest-database-turnkey: adapt [PATCH 18/28] database: add SQL for creating tko_test_view_2 [PATCH 19/28] database: use definition of tko_test_view_2 from [PATCH 20/28] database/south: add it to list of django apps [PATCH 21/28] database/south: added to the packages list on [PATCH 22/28] database/south: added to the packages list on Ubuntu [PATCH 23/28] database/south: add migrate step to [PATCH 24/28] database/south: provide a mapping of the custom afe [PATCH 25/28] database/south: add initial migration files [PATCH 26/28] database: add a 'run_sql' method to database managers [PATCH 27/28] database/south: work around South's failure of running [PATCH 28/28] database/south: Fixes for running unittests using Patch series diff stat: cli/autotest-rpc-query-results | 2 +- database/autotest-upgrade-db | 11 - database/common.py | 14 - database/database_connection.py | 328 ---- database/database_connection_unittest.py | 187 --- database/db_utils.py | 116 -- database/db_utils_unittest.py | 84 - database/migrate.py | 393 ----- database/migrate_unittest.py | 164 -- database/schema_068.sql | 1628 -------------------- database/setup.py | 48 - database_legacy/autotest-upgrade-db | 11 + database_legacy/common.py | 14 + database_legacy/database_connection.py | 328 ++++ database_legacy/database_connection_unittest.py | 187 +++ database_legacy/db_utils.py | 116 ++ database_legacy/db_utils_unittest.py | 84 + database_legacy/migrate.py | 388 +++++ database_legacy/migrate_unittest.py | 164 ++ database_legacy/migrations/001_initial_db.py | 22 + database_legacy/migrations/002_cleanup_fields.py | 12 + database_legacy/migrations/003_test_synch_type.py | 9 + database_legacy/migrations/004_add_indexes.py | 23 + database_legacy/migrations/005_one_more_index.py | 7 + .../migrations/006_host_label_invalid.py | 8 + .../migrations/007_indexes_on_acl_tables.py | 21 + .../migrations/008_add_profiler_table.py | 17 + .../migrations/009_add_timeout_to_jobs.py | 19 + .../migrations/010_add_protection_to_hosts.py | 26 + .../migrations/011_support_one_time_hosts.py | 41 + .../migrations/012_reset_access_levels.py | 5 + database_legacy/migrations/013_new_test_fields.py | 20 + database_legacy/migrations/014_run_verify.py | 6 + .../migrations/015_add_locked_by_and_lock_time.py | 12 + .../migrations/016_remove_run_verify.py | 6 + database_legacy/migrations/017_add_email_list.py | 8 + .../migrations/018_add_label_only_if_needed.py | 24 + database_legacy/migrations/019_add_abort_log.py | 13 + .../020_add_host_dirty_and_job_reboots.py | 18 + .../migrations/021_add_user_reboot_prefs.py | 19 + .../migrations/022_implement_sync_count.py | 50 + .../migrations/023_add_show_experimental_pref.py | 14 + .../migrations/024_make_label_name_unique.py | 14 + .../migrations/025_aclgroup_id_column_rename.py | 25 + .../migrations/026_remove_hqe_priority.py | 14 + database_legacy/migrations/027_fix_innodb.py | 11 + .../migrations/028_add_atomic_groups.py | 30 + .../migrations/029_add_atomic_group_invalid.py | 6 + .../migrations/030_update_hosts_invalid.py | 6 + .../migrations/031_add_hqe_aborted_flag.py | 11 + .../migrations/032_add_recurring_run.py | 23 + .../migrations/033_add_host_attributes.py | 21 + .../034_add_parse_failed_repair_option.py | 7 + database_legacy/migrations/035_job_max_runtime.py | 12 + .../migrations/036_add_special_tasks.py | 15 + database_legacy/migrations/037_db_constraints.py | 135 ++ ...add_hqe_id_and_foreign_keys_to_special_tasks.py | 32 + .../039_remove_special_tasks_log_file.py | 5 + database_legacy/migrations/040_add_foreign_keys.py | 75 + .../migrations/041_add_special_task_success.py | 14 + .../042_unique_index_on_hqe_job_and_host.py | 56 + .../043_add_special_task_requested_by.py | 11 + .../migrations/044_rename_afe_tables.py | 40 + .../migrations/045_test_planner_framework.py | 258 ++++ database_legacy/migrations/046_merge_databases.py | 41 + database_legacy/migrations/047_job_keyvals.py | 28 + .../migrations/048_expand_label_name_field.py | 7 + .../migrations/049_test_planner_additions.py | 36 + .../migrations/050_more_test_planner_additions.py | 13 + .../migrations/051_rename_planner_tests.py | 27 + .../migrations/052_expand_test_subdir_fields.py | 9 + database_legacy/migrations/053_nop.py | 6 + database_legacy/migrations/054_nop.py | 6 + .../055_ensure_invalidated_test_label_exists.py | 13 + .../migrations/056_planner_global_support.py | 19 + .../migrations/057_add_planner_triage_actions.py | 15 + database_legacy/migrations/058_drone_management.py | 85 + .../migrations/059_drone_sets_permissions.py | 42 + .../060_add_planner_additional_parameters.py | 47 + .../061_drone_sets_permissions_proper.py | 37 + .../migrations/062_drone_sets_unique.py | 54 + .../migrations/063_parameterized_tests.py | 144 ++ .../064_add_jobs_and_tests_time_indices.py | 10 + .../migrations/065_add_id_afe_hosts_labels.py | 7 + database_legacy/migrations/066_drop_planner.py | 443 ++++++ .../migrations/067_add_id_afe_acl_groups_hosts.py | 7 + .../migrations/068_add_id_afe_acl_groups_users.py | 7 + .../069_add_profile_afe_host_queue_entries.py | 7 + .../070_add_acl_group_replace_django_fixture.py | 8 + database_legacy/migrations/common.py | 14 + database_legacy/schema_068.sql | 1628 ++++++++++++++++++++ database_legacy/setup.py | 48 + frontend/afe/fixtures/initial_data.json | 5 +- frontend/afe/migrations/0001_initial.py | 672 ++++++++ frontend/afe/models.py | 41 +- frontend/migrations/001_initial_db.py | 22 - frontend/migrations/002_cleanup_fields.py | 12 - frontend/migrations/003_test_synch_type.py | 9 - frontend/migrations/004_add_indexes.py | 23 - frontend/migrations/005_one_more_index.py | 7 - frontend/migrations/006_host_label_invalid.py | 8 - frontend/migrations/007_indexes_on_acl_tables.py | 21 - frontend/migrations/008_add_profiler_table.py | 17 - frontend/migrations/009_add_timeout_to_jobs.py | 19 - frontend/migrations/010_add_protection_to_hosts.py | 26 - frontend/migrations/011_support_one_time_hosts.py | 41 - frontend/migrations/012_reset_access_levels.py | 5 - frontend/migrations/013_new_test_fields.py | 20 - frontend/migrations/014_run_verify.py | 6 - .../migrations/015_add_locked_by_and_lock_time.py | 12 - frontend/migrations/016_remove_run_verify.py | 6 - frontend/migrations/017_add_email_list.py | 8 - .../migrations/018_add_label_only_if_needed.py | 24 - frontend/migrations/019_add_abort_log.py | 13 - .../020_add_host_dirty_and_job_reboots.py | 18 - frontend/migrations/021_add_user_reboot_prefs.py | 19 - frontend/migrations/022_implement_sync_count.py | 50 - .../migrations/023_add_show_experimental_pref.py | 14 - frontend/migrations/024_make_label_name_unique.py | 14 - .../migrations/025_aclgroup_id_column_rename.py | 25 - frontend/migrations/026_remove_hqe_priority.py | 14 - frontend/migrations/027_fix_innodb.py | 11 - frontend/migrations/028_add_atomic_groups.py | 30 - .../migrations/029_add_atomic_group_invalid.py | 6 - frontend/migrations/030_update_hosts_invalid.py | 6 - frontend/migrations/031_add_hqe_aborted_flag.py | 11 - frontend/migrations/032_add_recurring_run.py | 23 - frontend/migrations/033_add_host_attributes.py | 21 - .../034_add_parse_failed_repair_option.py | 7 - frontend/migrations/035_job_max_runtime.py | 12 - frontend/migrations/036_add_special_tasks.py | 15 - frontend/migrations/037_db_constraints.py | 135 -- ...add_hqe_id_and_foreign_keys_to_special_tasks.py | 32 - .../039_remove_special_tasks_log_file.py | 5 - frontend/migrations/040_add_foreign_keys.py | 75 - .../migrations/041_add_special_task_success.py | 14 - .../042_unique_index_on_hqe_job_and_host.py | 56 - .../043_add_special_task_requested_by.py | 11 - frontend/migrations/044_rename_afe_tables.py | 40 - frontend/migrations/045_test_planner_framework.py | 258 ---- frontend/migrations/046_merge_databases.py | 41 - frontend/migrations/047_job_keyvals.py | 28 - frontend/migrations/048_expand_label_name_field.py | 7 - frontend/migrations/049_test_planner_additions.py | 36 - .../migrations/050_more_test_planner_additions.py | 13 - frontend/migrations/051_rename_planner_tests.py | 27 - .../migrations/052_expand_test_subdir_fields.py | 9 - frontend/migrations/053_nop.py | 6 - frontend/migrations/054_nop.py | 6 - .../055_ensure_invalidated_test_label_exists.py | 13 - frontend/migrations/056_planner_global_support.py | 19 - .../migrations/057_add_planner_triage_actions.py | 15 - frontend/migrations/058_drone_management.py | 85 - frontend/migrations/059_drone_sets_permissions.py | 42 - .../060_add_planner_additional_parameters.py | 47 - .../061_drone_sets_permissions_proper.py | 37 - frontend/migrations/062_drone_sets_unique.py | 54 - frontend/migrations/063_parameterized_tests.py | 144 -- .../064_add_jobs_and_tests_time_indices.py | 10 - frontend/migrations/065_add_id_afe_hosts_labels.py | 7 - frontend/migrations/066_drop_planner.py | 443 ------ .../migrations/067_add_id_afe_acl_groups_hosts.py | 7 - .../migrations/068_add_id_afe_acl_groups_users.py | 7 - .../069_add_profile_afe_host_queue_entries.py | 7 - .../070_add_acl_group_replace_django_fixture.py | 8 - frontend/migrations/common.py | 14 - frontend/pkgdeps.py | 2 + frontend/settings.py | 10 + frontend/setup_test_environment.py | 46 + frontend/tko/migrations/0001_initial.py | 329 ++++ frontend/tko/models.py | 1 + frontend/tko/rpc_interface_unittest.py | 81 +- frontend/tko/sql/test.sql | 31 + installation_support/autotest-database-turnkey | 209 +-- installation_support/database_manager/__init__.py | 74 + installation_support/database_manager/base.py | 100 ++ installation_support/database_manager/dummy.py | 53 + installation_support/database_manager/mysql.py | 147 ++ scheduler/monitor_db.py | 2 +- scheduler/monitor_db_cleanup_unittest.py | 2 +- scheduler/monitor_db_functional_unittest.py | 2 +- scheduler/monitor_db_unittest.py | 2 +- scheduler/scheduler_models.py | 2 +- scheduler/scheduler_models_unittest.py | 2 +- setup.py | 12 +- tko/migrations/001_initial_db.py | 24 - tko/migrations/002_add_job_timestamps.py | 142 -- tko/migrations/003_add_test_timestamps.py | 146 -- tko/migrations/004_add_test_started.py | 149 -- tko/migrations/005_add_testna_status.py | 6 - tko/migrations/006_add_table_query_history.py | 17 - tko/migrations/007_widen_reason_field.py | 5 - tko/migrations/008_add_iteration_attributes.py | 22 - tko/migrations/009_add_test_labels.py | 27 - tko/migrations/010_add_saved_queries.py | 20 - tko/migrations/011_rename_JOB_to_CLIENT_JOB.py | 11 - tko/migrations/012_add_running_status.py | 6 - tko/migrations/013_fix_perf_view.py | 79 - tko/migrations/014_add_test_view_2.py | 38 - tko/migrations/015_support_graphing_interface.py | 101 -- tko/migrations/016_modify_perf_view_2.py | 45 - tko/migrations/017_add_embedded_graph_caching.py | 33 - tko/migrations/018_add_indexes.py | 34 - tko/migrations/019_widen_test_name_field.py | 5 - tko/migrations/020_widen_hostname_field.py | 5 - tko/migrations/021_fix_innodb.py | 14 - tko/migrations/022_widen_attribute_value_field.py | 7 - tko/migrations/023_update_tests_invalid.py | 6 - .../024_index_test_labels_tests_test_id.py | 7 - tko/migrations/025_add_test_label_foreign_keys.py | 24 - tko/migrations/026_fix_saved_graphing_queries.py | 10 - tko/migrations/027_user_created_test_attributes.py | 9 - tko/migrations/028_add_tests_jobs_foreign_key.py | 19 - .../029_ensure_invalidated_label_exists.py | 15 - tko/migrations/030_add_afe_job_id_to_jobs.py | 77 - tko/migrations/031_rename_tko_tables.py | 381 ----- tko/migrations/common.py | 14 - tko/setup.py | 3 +- 218 files changed, 6924 insertions(+), 7051 deletions(-) _______________________________________________ Autotest-kernel mailing list [email protected] https://www.redhat.com/mailman/listinfo/autotest-kernel
