From a716a8ddd3df615009bcff3bd96dd9ae64cb5f68 Mon Sep 17 00:00:00 2001 From: Klaus Wenninger Date: Tue, 19 Mar 2019 21:36:15 +0100 Subject: [PATCH] Fix: sbd-pacemaker: make handling of cib-connection loss more robust Exit pcmk-servant on graceful pacemaker shutdown and go back to state before pacemaker was detected initially. Purge all cib-traces otherwise and try to reconnect within timeout. --- src/sbd-inquisitor.c | 24 ++++++++++++++++++++---- src/sbd-md.c | 30 +++++++++++++++--------------- src/sbd-pacemaker.c | 38 +++++++++++++++++++++++++++++--------- src/sbd.h | 11 +++++++---- 4 files changed, 71 insertions(+), 32 deletions(-) diff --git a/src/sbd-inquisitor.c b/src/sbd-inquisitor.c index 9be6c99..77c6e4f 100644 --- a/src/sbd-inquisitor.c +++ b/src/sbd-inquisitor.c @@ -490,19 +490,19 @@ void inquisitor_child(void) if (sbd_is_disk(s)) { if (WIFEXITED(status)) { switch(WEXITSTATUS(status)) { - case EXIT_MD_IO_FAIL: + case EXIT_MD_SERVANT_IO_FAIL: DBGLOG(LOG_INFO, "Servant for %s requests to be disowned", s->devname); break; - case EXIT_MD_REQUEST_RESET: + case EXIT_MD_SERVANT_REQUEST_RESET: cl_log(LOG_WARNING, "%s requested a reset", s->devname); do_reset(); break; - case EXIT_MD_REQUEST_SHUTOFF: + case EXIT_MD_SERVANT_REQUEST_SHUTOFF: cl_log(LOG_WARNING, "%s requested a shutoff", s->devname); do_off(); break; - case EXIT_MD_REQUEST_CRASHDUMP: + case EXIT_MD_SERVANT_REQUEST_CRASHDUMP: cl_log(LOG_WARNING, "%s requested a crashdump", s->devname); do_crashdump(); break; @@ -510,6 +510,22 @@ void inquisitor_child(void) break; } } + } else if (sbd_is_pcmk(s)) { + if (WIFEXITED(status)) { + switch(WEXITSTATUS(status)) { + case EXIT_PCMK_SERVANT_GRACEFUL_SHUTDOWN: + DBGLOG(LOG_INFO, "PCMK-Servant has exited gracefully"); + /* revert to state prior to pacemaker-detection */ + s->restarts = 0; + s->restart_blocked = 0; + cluster_appeared = 0; + s->outdated = 1; + s->t_last.tv_sec = 0; + break; + default: + break; + } + } } cleanup_servant_by_pid(pid); } diff --git a/src/sbd-md.c b/src/sbd-md.c index ba2c34d..c51d381 100644 --- a/src/sbd-md.c +++ b/src/sbd-md.c @@ -1061,19 +1061,19 @@ int servant_md(const char *diskname, int mode, const void* argp) st = open_device(diskname, LOG_WARNING); if (!st) { - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } s_header = header_get(st); if (!s_header) { cl_log(LOG_ERR, "Not a valid header on %s", diskname); - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } if (servant_check_timeout_inconsistent(s_header) < 0) { cl_log(LOG_ERR, "Timeouts on %s do not match first device", diskname); - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } if (s_header->minor_version > 0) { @@ -1086,14 +1086,14 @@ int servant_md(const char *diskname, int mode, const void* argp) cl_log(LOG_ERR, "No slot allocated, and automatic allocation failed for disk %s.", diskname); - rc = EXIT_MD_IO_FAIL; + rc = EXIT_MD_SERVANT_IO_FAIL; goto out; } s_node = sector_alloc(); if (slot_read(st, mbox, s_node) < 0) { cl_log(LOG_ERR, "Unable to read node entry on %s", diskname); - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } cl_log(LOG_NOTICE, "Monitoring slot %d on disk %s", mbox, diskname); @@ -1109,7 +1109,7 @@ int servant_md(const char *diskname, int mode, const void* argp) if (mode > 0) { if (mbox_read(st, mbox, s_mbox) < 0) { cl_log(LOG_ERR, "mbox read failed during start-up in servant."); - rc = EXIT_MD_IO_FAIL; + rc = EXIT_MD_SERVANT_IO_FAIL; goto out; } if (s_mbox->cmd != SBD_MSG_EXIT && @@ -1125,7 +1125,7 @@ int servant_md(const char *diskname, int mode, const void* argp) DBGLOG(LOG_INFO, "First servant start - zeroing inbox"); memset(s_mbox, 0, sizeof(*s_mbox)); if (mbox_write(st, mbox, s_mbox) < 0) { - rc = EXIT_MD_IO_FAIL; + rc = EXIT_MD_SERVANT_IO_FAIL; goto out; } } @@ -1154,28 +1154,28 @@ int servant_md(const char *diskname, int mode, const void* argp) s_header_retry = header_get(st); if (!s_header_retry) { cl_log(LOG_ERR, "No longer found a valid header on %s", diskname); - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } if (memcmp(s_header, s_header_retry, sizeof(*s_header)) != 0) { cl_log(LOG_ERR, "Header on %s changed since start-up!", diskname); - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } free(s_header_retry); s_node_retry = sector_alloc(); if (slot_read(st, mbox, s_node_retry) < 0) { cl_log(LOG_ERR, "slot read failed in servant."); - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } if (memcmp(s_node, s_node_retry, sizeof(*s_node)) != 0) { cl_log(LOG_ERR, "Node entry on %s changed since start-up!", diskname); - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } free(s_node_retry); if (mbox_read(st, mbox, s_mbox) < 0) { cl_log(LOG_ERR, "mbox read failed in servant."); - exit(EXIT_MD_IO_FAIL); + exit(EXIT_MD_SERVANT_IO_FAIL); } if (s_mbox->cmd > 0) { @@ -1190,14 +1190,14 @@ int servant_md(const char *diskname, int mode, const void* argp) sigqueue(ppid, SIG_TEST, signal_value); break; case SBD_MSG_RESET: - exit(EXIT_MD_REQUEST_RESET); + exit(EXIT_MD_SERVANT_REQUEST_RESET); case SBD_MSG_OFF: - exit(EXIT_MD_REQUEST_SHUTOFF); + exit(EXIT_MD_SERVANT_REQUEST_SHUTOFF); case SBD_MSG_EXIT: sigqueue(ppid, SIG_EXITREQ, signal_value); break; case SBD_MSG_CRASHDUMP: - exit(EXIT_MD_REQUEST_CRASHDUMP); + exit(EXIT_MD_SERVANT_REQUEST_CRASHDUMP); default: /* FIXME: An "unknown" message might result diff --git a/src/sbd-pacemaker.c b/src/sbd-pacemaker.c index aac355a..c69fc55 100644 --- a/src/sbd-pacemaker.c +++ b/src/sbd-pacemaker.c @@ -103,6 +103,9 @@ static pe_working_set_t *data_set = NULL; static long last_refresh = 0; +static int pcmk_clean_shutdown = 0; +static int pcmk_shutdown = 0; + static gboolean mon_timer_reconnect(gpointer data) { @@ -128,10 +131,26 @@ mon_cib_connection_destroy(gpointer user_data) { if (cib) { cib->cmds->signoff(cib); + /* retrigger as last one might have been skipped */ + mon_refresh_state(NULL); + if (pcmk_clean_shutdown) { + /* assume a graceful pacemaker-shutdown */ + clean_up(EXIT_PCMK_SERVANT_GRACEFUL_SHUTDOWN); + } + /* getting here we aren't sure about the pacemaker-state + so try to use the timeout to reconnect and get + everything sorted out again + */ + pcmk_shutdown = 0; set_servant_health(pcmk_health_transient, LOG_WARNING, "Disconnected from CIB"); timer_id_reconnect = g_timeout_add(reconnect_msec, mon_timer_reconnect, NULL); } cib_connected = 0; + /* no sense in looking into outdated cib, trying to apply patch, ... */ + if (current_cib) { + free_xml(current_cib); + current_cib = NULL; + } return; } @@ -171,7 +190,7 @@ static gboolean mon_timer_notify(gpointer data) { static int counter = 0; - int counter_max = timeout_watchdog / timeout_loop; + int counter_max = timeout_watchdog / timeout_loop / 2; if (timer_id_notify > 0) { g_source_remove(timer_id_notify); @@ -280,11 +299,6 @@ compute_status(pe_working_set_t * data_set) } else if (node->details->pending) { set_servant_health(pcmk_health_pending, LOG_WARNING, "Node state: pending"); -#if 0 - } else if (node->details->shutdown) { - set_servant_health(pcmk_health_shutdown, LOG_WARNING, "Node state: shutting down"); -#endif - } else if (data_set->flags & pe_flag_have_quorum) { set_servant_health(pcmk_health_online, LOG_INFO, "Node state: online"); ever_had_quorum = TRUE; @@ -315,6 +329,12 @@ compute_status(pe_working_set_t * data_set) } } + if (node->details->shutdown) { + pcmk_shutdown = 1; + } + if (pcmk_shutdown && !(node->details->running_rsc)) { + pcmk_clean_shutdown = 1; + } notify_parent(); return; } @@ -339,7 +359,7 @@ crm_diff_update(const char *event, xmlNode * msg) static mainloop_timer_t *refresh_timer = NULL; if(refresh_timer == NULL) { - refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL); + refresh_timer = mainloop_timer_add("refresh", reconnect_msec, FALSE, mon_trigger_refresh, NULL); refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_state, refresh_timer); } @@ -369,9 +389,9 @@ crm_diff_update(const char *event, xmlNode * msg) } /* Refresh - * - immediately if the last update was more than 5s ago + * - immediately if the last update was more than 1s ago * - every 10 updates - * - at most 2s after the last update + * - at most 1s after the last update */ if (updates > 10 || (now - last_refresh) > (reconnect_msec / 1000)) { mon_refresh_state(refresh_timer); diff --git a/src/sbd.h b/src/sbd.h index 6fe07f9..3b05a11 100644 --- a/src/sbd.h +++ b/src/sbd.h @@ -54,10 +54,13 @@ /* FIXME: should add dynamic check of SIG_XX >= SIGRTMAX */ /* exit status for disk-servant */ -#define EXIT_MD_IO_FAIL 20 -#define EXIT_MD_REQUEST_RESET 21 -#define EXIT_MD_REQUEST_SHUTOFF 22 -#define EXIT_MD_REQUEST_CRASHDUMP 23 +#define EXIT_MD_SERVANT_IO_FAIL 20 +#define EXIT_MD_SERVANT_REQUEST_RESET 21 +#define EXIT_MD_SERVANT_REQUEST_SHUTOFF 22 +#define EXIT_MD_SERVANT_REQUEST_CRASHDUMP 23 + +/* exit status for pcmk-servant */ +#define EXIT_PCMK_SERVANT_GRACEFUL_SHUTDOWN 30 #define HOG_CHAR 0xff #define SECTOR_NAME_MAX 63 -- 1.8.3.1