The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/5733
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) === Signed-off-by: Stéphane Graber <[email protected]>
From b524a268b35ea08899a3e073ad704504b225a674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Tue, 7 May 2019 16:12:00 -0400 Subject: [PATCH] DEBUG: clustering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/cluster/heartbeat.go | 16 +++++ test/main.sh | 136 +++++++++++++++++++-------------------- 2 files changed, 84 insertions(+), 68 deletions(-) diff --git a/lxd/cluster/heartbeat.go b/lxd/cluster/heartbeat.go index 980f8c7018..c84debd651 100644 --- a/lxd/cluster/heartbeat.go +++ b/lxd/cluster/heartbeat.go @@ -24,14 +24,24 @@ import ( // It will update the heartbeat timestamp column of the nodes table // accordingly, and also notify them of the current list of database nodes. func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule) { + serverName := "unknown" + cluster.Transaction(func(tx *db.ClusterTx) error { + var err error + serverName, err = tx.NodeName() + return err + }) + heartbeat := func(ctx context.Context) { + logger.Errorf("[%s] stgraber: in hearbeat", serverName) if gateway.server == nil || gateway.memoryDial != nil { // We're not a raft node or we're not clustered + logger.Errorf("[%s] stgraber: not clustered => out", serverName) return } raftNodes, err := gateway.currentRaftNodes() if err == raft.ErrNotLeader { + logger.Errorf("[%s] stgraber: not the leader => out", serverName) return } logger.Debugf("Starting heartbeat round") @@ -52,6 +62,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule) logger.Warnf("Failed to replace local raft nodes: %v", err) return } + logger.Errorf("[%s] stgraber: set raft nodes to: %+v", serverName, raftNodes) var nodes []db.NodeInfo var nodeAddress string // Address of this node @@ -71,6 +82,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule) logger.Warnf("Failed to get current cluster nodes: %v", err) return } + logger.Errorf("[%s] stgraber: local=%v nodes=%v", serverName, nodeAddress, nodes) heartbeats := make([]time.Time, len(nodes)) heartbeatsLock := sync.Mutex{} @@ -93,6 +105,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule) // Spread in time by waiting up to 3s less than the interval time.Sleep(time.Duration(rand.Intn((heartbeatInterval*1000)-3000)) * time.Millisecond) logger.Debugf("Sending heartbeat to %s", address) + logger.Errorf("[%s] stgraber: Contacting: %s", serverName, address) err := heartbeatNode(ctx, address, gateway.cert, raftNodes) if err == nil { @@ -103,9 +116,11 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule) } else { logger.Debugf("Failed heartbeat for %s: %v", address, err) } + logger.Errorf("[%s] stgraber: Got reply from: %s", serverName, address) }(i, node.Address) } heartbeatsWg.Wait() + logger.Errorf("[%s] stgraber: Got all replies", serverName) // If the context has been cancelled, return immediately. if ctx.Err() != nil { @@ -130,6 +145,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule) logger.Warnf("Failed to update heartbeat: %v", err) } logger.Debugf("Completed heartbeat round") + logger.Errorf("[%s] stgraber: done => out", serverName) } // Since the database APIs are blocking we need to wrap the core logic diff --git a/test/main.sh b/test/main.sh index 6d245c431c..f51357c75e 100755 --- a/test/main.sh +++ b/test/main.sh @@ -154,74 +154,74 @@ if [ "$#" -gt 0 ]; then exit fi -run_test test_check_deps "checking dependencies" -run_test test_static_analysis "static analysis" -run_test test_database_update "database schema updates" -run_test test_database_restore "database restore" -run_test test_sql "lxd sql" -run_test test_projects_default "default project" -run_test test_projects_crud "projects CRUD operations" -run_test test_projects_containers "containers inside projects" -run_test test_projects_snapshots "snapshots inside projects" -run_test test_projects_backups "backups inside projects" -run_test test_projects_profiles "profiles inside projects" -run_test test_projects_profiles_default "profiles from the global default project" -run_test test_projects_images "images inside projects" -run_test test_projects_images_default "images from the global default project" -run_test test_projects_storage "projects and storage pools" -run_test test_projects_network "projects and networks" -run_test test_remote_url "remote url handling" -run_test test_remote_admin "remote administration" -run_test test_remote_usage "remote usage" -run_test test_basic_usage "basic usage" -run_test test_container_devices_nic "container devices - nic" -run_test test_security "security features" -run_test test_security_protection "container protection" -run_test test_image_expiry "image expiry" -run_test test_image_list_all_aliases "image list all aliases" -run_test test_image_auto_update "image auto-update" -run_test test_image_prefer_cached "image prefer cached" -run_test test_image_import_dir "import image from directory" -run_test test_concurrent_exec "concurrent exec" -run_test test_concurrent "concurrent startup" -run_test test_snapshots "container snapshots" -run_test test_snap_restore "snapshot restores" -run_test test_snap_expiry "snapshot expiry" -run_test test_config_profiles "profiles and configuration" -run_test test_config_edit "container configuration edit" -run_test test_config_edit_container_snapshot_pool_config "container and snapshot volume configuration edit" -run_test test_container_metadata "manage container metadata and templates" -run_test test_container_snapshot_config "container snapshot configuration" -run_test test_server_config "server configuration" -run_test test_filemanip "file manipulations" -run_test test_network "network management" -run_test test_idmap "id mapping" -run_test test_template "file templating" -run_test test_pki "PKI mode" -run_test test_devlxd "/dev/lxd" -run_test test_fuidshift "fuidshift" -run_test test_migration "migration" -run_test test_fdleak "fd leak" -run_test test_storage "storage" -run_test test_storage_volume_snapshots "storage volume snapshots" -run_test test_init_auto "lxd init auto" -run_test test_init_interactive "lxd init interactive" -run_test test_init_preseed "lxd init preseed" -run_test test_storage_profiles "storage profiles" -run_test test_container_import "container import" -run_test test_storage_volume_attach "attaching storage volumes" -run_test test_storage_driver_ceph "ceph storage driver" -run_test test_resources "resources" -run_test test_kernel_limits "kernel limits" -run_test test_macaroon_auth "macaroon authentication" -run_test test_console "console" -run_test test_query "query" -run_test test_proxy_device "proxy device" -run_test test_storage_local_volume_handling "storage local volume handling" -run_test test_backup_import "backup import" -run_test test_backup_export "backup export" -run_test test_container_local_cross_pool_handling "container local cross pool handling" -run_test test_incremental_copy "incremental container copy" +#run_test test_check_deps "checking dependencies" +#run_test test_static_analysis "static analysis" +#run_test test_database_update "database schema updates" +#run_test test_database_restore "database restore" +#run_test test_sql "lxd sql" +#run_test test_projects_default "default project" +#run_test test_projects_crud "projects CRUD operations" +#run_test test_projects_containers "containers inside projects" +#run_test test_projects_snapshots "snapshots inside projects" +#run_test test_projects_backups "backups inside projects" +#run_test test_projects_profiles "profiles inside projects" +#run_test test_projects_profiles_default "profiles from the global default project" +#run_test test_projects_images "images inside projects" +#run_test test_projects_images_default "images from the global default project" +#run_test test_projects_storage "projects and storage pools" +#run_test test_projects_network "projects and networks" +#run_test test_remote_url "remote url handling" +#run_test test_remote_admin "remote administration" +#run_test test_remote_usage "remote usage" +#run_test test_basic_usage "basic usage" +#run_test test_container_devices_nic "container devices - nic" +#run_test test_security "security features" +#run_test test_security_protection "container protection" +#run_test test_image_expiry "image expiry" +#run_test test_image_list_all_aliases "image list all aliases" +#run_test test_image_auto_update "image auto-update" +#run_test test_image_prefer_cached "image prefer cached" +#run_test test_image_import_dir "import image from directory" +#run_test test_concurrent_exec "concurrent exec" +#run_test test_concurrent "concurrent startup" +#run_test test_snapshots "container snapshots" +#run_test test_snap_restore "snapshot restores" +#run_test test_snap_expiry "snapshot expiry" +#run_test test_config_profiles "profiles and configuration" +#run_test test_config_edit "container configuration edit" +#run_test test_config_edit_container_snapshot_pool_config "container and snapshot volume configuration edit" +#run_test test_container_metadata "manage container metadata and templates" +#run_test test_container_snapshot_config "container snapshot configuration" +#run_test test_server_config "server configuration" +#run_test test_filemanip "file manipulations" +#run_test test_network "network management" +#run_test test_idmap "id mapping" +#run_test test_template "file templating" +#run_test test_pki "PKI mode" +#run_test test_devlxd "/dev/lxd" +#run_test test_fuidshift "fuidshift" +#run_test test_migration "migration" +#run_test test_fdleak "fd leak" +#run_test test_storage "storage" +#run_test test_storage_volume_snapshots "storage volume snapshots" +#run_test test_init_auto "lxd init auto" +#run_test test_init_interactive "lxd init interactive" +#run_test test_init_preseed "lxd init preseed" +#run_test test_storage_profiles "storage profiles" +#run_test test_container_import "container import" +#run_test test_storage_volume_attach "attaching storage volumes" +#run_test test_storage_driver_ceph "ceph storage driver" +#run_test test_resources "resources" +#run_test test_kernel_limits "kernel limits" +#run_test test_macaroon_auth "macaroon authentication" +#run_test test_console "console" +#run_test test_query "query" +#run_test test_proxy_device "proxy device" +#run_test test_storage_local_volume_handling "storage local volume handling" +#run_test test_backup_import "backup import" +#run_test test_backup_export "backup export" +#run_test test_container_local_cross_pool_handling "container local cross pool handling" +#run_test test_incremental_copy "incremental container copy" run_test test_clustering_enable "clustering enable" run_test test_clustering_membership "clustering membership" run_test test_clustering_containers "clustering containers"
_______________________________________________ lxc-devel mailing list [email protected] http://lists.linuxcontainers.org/listinfo/lxc-devel
