From: "Fabio M. Di Nitto" <fdini...@redhat.com> Patch 76741bb2a94ae94e493c609d50f570d02e2f3029 had a not so obvious dependency on 08ae3ce147b2771c5ee6e1d364a5e48c88384427.
Backport portion of 08ae3ce147b2771c5ee6e1d364a5e48c88384427 to handle dual cman socket (admin and user) and use the correct socket (user) for send/receive data. Move cman_alive check and heartbeat (for dispatch) to ch_user. Resolves: rhbz#782900 Signed-off-by: Fabio M. Di Nitto <fdini...@redhat.com> --- cman/qdisk/disk.h | 5 ++- cman/qdisk/disk_util.c | 7 +++-- cman/qdisk/iostate.c | 8 +++--- cman/qdisk/main.c | 69 ++++++++++++++++++++++------------------------- 4 files changed, 43 insertions(+), 46 deletions(-) diff --git a/cman/qdisk/disk.h b/cman/qdisk/disk.h index d491de1..83167ea 100644 --- a/cman/qdisk/disk.h +++ b/cman/qdisk/disk.h @@ -270,7 +270,8 @@ typedef struct { int qc_master; /* Master?! */ int qc_status_sock; run_flag_t qc_flags; - cman_handle_t qc_ch; + cman_handle_t qc_ch_admin; + cman_handle_t qc_ch_user; char *qc_device; char *qc_label; char *qc_status_file; @@ -299,7 +300,7 @@ typedef struct { int qd_write_status(qd_ctx *ctx, int nid, disk_node_state_t state, disk_msg_t *msg, memb_mask_t mask, memb_mask_t master); int qd_read_print_status(target_info_t *disk, int nid); -int qd_init(qd_ctx *ctx, cman_handle_t ch, int me); +int qd_init(qd_ctx *ctx, cman_handle_t ch_admin, cman_handle_t ch_user, int me); void qd_destroy(qd_ctx *ctx); /* proc.c */ diff --git a/cman/qdisk/disk_util.c b/cman/qdisk/disk_util.c index f5539c0..25f4013 100644 --- a/cman/qdisk/disk_util.c +++ b/cman/qdisk/disk_util.c @@ -312,16 +312,17 @@ generate_token(void) Initialize a quorum disk context, given a CMAN handle and a nodeid. */ int -qd_init(qd_ctx *ctx, cman_handle_t ch, int me) +qd_init(qd_ctx *ctx, cman_handle_t ch_admin, cman_handle_t ch_user, int me) { - if (!ctx || !ch || !me) { + if (!ctx || !ch_admin || !ch_user || !me) { errno = EINVAL; return -1; } memset(ctx, 0, sizeof(*ctx)); ctx->qc_incarnation = generate_token(); - ctx->qc_ch = ch; + ctx->qc_ch_admin = ch_admin; + ctx->qc_ch_user = ch_user; ctx->qc_my_id = me; ctx->qc_status_sock = -1; diff --git a/cman/qdisk/iostate.c b/cman/qdisk/iostate.c index eb74ad2..ba7ad12 100644 --- a/cman/qdisk/iostate.c +++ b/cman/qdisk/iostate.c @@ -69,7 +69,7 @@ io_nanny_thread(void *arg) iostate_t last_main_state = 0, current_main_state = 0; int last_main_incarnation = 0, current_main_incarnation = 0; int logged_incarnation = 0; - cman_handle_t ch = (cman_handle_t)arg; + cman_handle_t ch_user = (cman_handle_t)arg; int32_t whine_state; /* Start with wherever we're at now */ @@ -105,7 +105,7 @@ io_nanny_thread(void *arg) /* Whine on CMAN api */ whine_state = (int32_t)current_main_state; swab32(whine_state); - cman_send_data(ch, &whine_state, sizeof(int32_t), 0, CLUSTER_PORT_QDISKD, 0); + cman_send_data(ch_user, &whine_state, sizeof(int32_t), 0, CLUSTER_PORT_QDISKD, 0); /* Don't log things twice */ if (logged_incarnation == current_main_incarnation) @@ -125,7 +125,7 @@ io_nanny_thread(void *arg) int -io_nanny_start(cman_handle_t ch, int timeout) +io_nanny_start(cman_handle_t ch_user, int timeout) { int ret; @@ -135,7 +135,7 @@ io_nanny_start(cman_handle_t ch, int timeout) qdisk_timeout = timeout; thread_active = 1; - ret = pthread_create(&io_nanny_tid, NULL, io_nanny_thread, ch); + ret = pthread_create(&io_nanny_tid, NULL, io_nanny_thread, ch_user); pthread_mutex_unlock(&state_mutex); return ret; diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c index 90d00ab..72a3c07 100644 --- a/cman/qdisk/main.c +++ b/cman/qdisk/main.c @@ -287,7 +287,7 @@ check_transitions(qd_ctx *ctx, node_info_t *ni, int max, memb_mask_t mask) if (ctx->qc_flags & RF_ALLOW_KILL) { clulog(LOG_DEBUG, "Telling CMAN to " "kill the node\n"); - cman_kill_node(ctx->qc_ch, + cman_kill_node(ctx->qc_ch_admin, ni[x].ni_status.ps_nodeid); } } @@ -325,7 +325,7 @@ check_transitions(qd_ctx *ctx, node_info_t *ni, int max, memb_mask_t mask) if (ctx->qc_flags & RF_ALLOW_KILL) { clulog(LOG_DEBUG, "Telling CMAN to " "kill the node\n"); - cman_kill_node(ctx->qc_ch, + cman_kill_node(ctx->qc_ch_admin, ni[x].ni_status.ps_nodeid); } continue; @@ -580,7 +580,7 @@ check_cman(qd_ctx *ctx, memb_mask_t mask, memb_mask_t master_mask) cman_node_t nodes[MAX_NODES_DISK]; int retnodes, x; - if (cman_get_nodes(ctx->qc_ch, MAX_NODES_DISK, + if (cman_get_nodes(ctx->qc_ch_admin, MAX_NODES_DISK, &retnodes, nodes) <0 ) return; @@ -894,7 +894,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max) _running = 1; while (_running) { /* perform a forceful cman dispatch */ - if (cman_alive(ctx->qc_ch) < 0) { + if (cman_alive(ctx->qc_ch_user) < 0) { clulog(LOG_ERR, "cman: %s\n", strerror(errno)); } @@ -931,11 +931,11 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max) msg.m_msg = M_NONE; ++msg.m_seq; bid_pending = 0; - if (cman_alive(ctx->qc_ch) < 0) { + if (cman_alive(ctx->qc_ch_user) < 0) { clulog(LOG_ERR, "cman: %s\n", strerror(errno)); } else { - cman_poll_quorum_device(ctx->qc_ch, 0); + cman_poll_quorum_device(ctx->qc_ch_admin, 0); } if (ctx->qc_flags & RF_REBOOT) reboot(RB_AUTOBOOT); @@ -1039,7 +1039,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max) /* We are the master. Poll the quorum device. We can't be the master unless we score high enough on our heuristics. */ - if (cman_alive(ctx->qc_ch) < 0) { + if (cman_alive(ctx->qc_ch_user) < 0) { clulog(LOG_ERR, "cman_dispatch: %s\n", strerror(errno)); clulog(LOG_ERR, @@ -1048,7 +1048,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max) } check_cman(ctx, mask, master_mask); if (!errors) - cman_poll_quorum_device(ctx->qc_ch, 1); + cman_poll_quorum_device(ctx->qc_ch_admin, 1); } else if (ctx->qc_status == S_RUN && ctx->qc_master && ctx->qc_master != ctx->qc_my_id) { @@ -1060,7 +1060,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max) ni[ctx->qc_master-1].ni_status.ps_master_mask, ctx->qc_my_id-1, sizeof(memb_mask_t))) { - if (cman_alive(ctx->qc_ch) < 0) { + if (cman_alive(ctx->qc_ch_user) < 0) { clulog(LOG_ERR, "cman_dispatch: %s\n", strerror(errno)); clulog(LOG_ERR, @@ -1069,7 +1069,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max) } if (!errors && (!(ctx->qc_flags & RF_MASTER_WINS))) - cman_poll_quorum_device(ctx->qc_ch, 1); + cman_poll_quorum_device(ctx->qc_ch_admin, 1); } } @@ -1506,12 +1506,7 @@ check_stop_cman(qd_ctx *ctx) clulog(LOG_WARNING, "Telling CMAN to leave the cluster; qdisk is not" " available\n"); -#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2) - if (cman_shutdown(ctx->qc_ch, 0) < 0) { -#else - int x = 0; - if (ioctl(cman_get_fd(ctx->qc_ch), SIOCCLUSTER_LEAVE_CLUSTER, &x) < 0) { -#endif + if (cman_shutdown(ctx->qc_ch_admin, 0) < 0) { clulog(LOG_CRIT, "Could not leave the cluster - rebooting\n"); sleep(5); if (ctx->qc_flags & RF_DEBUG) @@ -1552,7 +1547,8 @@ main(int argc, char **argv) cman_node_t me; int cfh, rv, forked = 0, nfd = -1, ret = -1; qd_ctx ctx; - cman_handle_t ch = NULL; + cman_handle_t ch_admin = NULL; + cman_handle_t ch_user = NULL; node_info_t ni[MAX_NODES_DISK]; struct h_data h[10]; char debug = 0, foreground = 0; @@ -1597,12 +1593,8 @@ main(int argc, char **argv) qp.ni = &ni[0]; qp.ni_len = MAX_NODES_DISK; -#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2) - ch = cman_admin_init(&qp); -#else - ch = cman_init(&qp); -#endif - if (!ch) { + ch_admin = cman_admin_init(&qp); + if (!ch_admin) { if (!foreground && !forked) { if (daemon_init(argv[0]) < 0) goto out; @@ -1614,22 +1606,24 @@ main(int argc, char **argv) do { sleep(5); -#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2) - ch = cman_admin_init(&qp); -#else - ch = cman_init(&qp); -#endif - } while (!ch); + ch_admin = cman_admin_init(&qp); + } while (!ch_admin); + } + + ch_user = cman_init(&qp); + if (!ch_user) { + clulog_and_print(LOG_CRIT, "Could not register with CMAN: %s\n", strerror(errno)); + goto out; } - if (cman_start_recv_data(ch, qdisk_whine, CLUSTER_PORT_QDISKD) != 0) { + if (cman_start_recv_data(ch_user, qdisk_whine, CLUSTER_PORT_QDISKD) != 0) { clulog_and_print(LOG_CRIT, "Could not register with CMAN: %s\n", strerror(errno)); goto out; } memset(&me, 0, sizeof(me)); - while (cman_get_node(ch, CMAN_NODEID_US, &me) < 0) { + while (cman_get_node(ch_admin, CMAN_NODEID_US, &me) < 0) { if (!foreground && !forked) { if (daemon_init(argv[0]) < 0) goto out; @@ -1639,7 +1633,7 @@ main(int argc, char **argv) sleep(5); } - qd_init(&ctx, ch, me.cn_nodeid); + qd_init(&ctx, ch_admin, ch_user, me.cn_nodeid); signal(SIGINT, int_handler); signal(SIGTERM, int_handler); @@ -1715,13 +1709,13 @@ main(int argc, char **argv) if (!_running) goto out; - if ((rv = cman_register_quorum_device(ctx.qc_ch, + if ((rv = cman_register_quorum_device(ctx.qc_ch_admin, (ctx.qc_flags&RF_CMAN_LABEL)? ctx.qc_cman_label: ctx.qc_device, ctx.qc_votes)) < 0) { if (errno == EBUSY) { - if ((rv = cman_update_quorum_device(ctx.qc_ch, + if ((rv = cman_update_quorum_device(ctx.qc_ch_admin, (ctx.qc_flags&RF_CMAN_LABEL)? ctx.qc_cman_label: ctx.qc_device, @@ -1740,17 +1734,18 @@ main(int argc, char **argv) } } - io_nanny_start(ch, ctx.qc_tko * ctx.qc_interval); + io_nanny_start(ch_user, ctx.qc_tko * ctx.qc_interval); if (quorum_loop(&ctx, ni, MAX_NODES_DISK) == 0) - cman_unregister_quorum_device(ctx.qc_ch); + cman_unregister_quorum_device(ctx.qc_ch_admin); io_nanny_stop(); quorum_logout(&ctx); /* free cman handle to avoid leak in cman */ out: - cman_finish(ctx.qc_ch); + cman_finish(ctx.qc_ch_admin); + cman_finish(ctx.qc_ch_user); qd_destroy(&ctx); return ret; -- 1.7.7.6