CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL4
Changes by: [EMAIL PROTECTED] 2007-09-27 20:31:20
Modified files:
cmirror-kernel/src: dm-cmirror-client.c dm-cmirror-common.h
dm-cmirror-server.c
Log message:
Bug 290821: cmirror write path appears deadlocked after recovery ...
In some device failure cases, regions must be marked 'out-of-sync' -
this was causing a following write to block because it thought the
region had not yet been recovered - when in fact, it had just been
put out-of-sync due to failing device.
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-cmirror-client.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.1.2.53&r2=1.1.2.54
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-cmirror-common.h.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.1.2.15&r2=1.1.2.16
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-cmirror-server.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.1.2.39&r2=1.1.2.40
--- cluster/cmirror-kernel/src/Attic/dm-cmirror-client.c 2007/09/26
03:15:40 1.1.2.53
+++ cluster/cmirror-kernel/src/Attic/dm-cmirror-client.c 2007/09/27
20:31:18 1.1.2.54
@@ -773,6 +773,7 @@
struct region_state *rs, *tmp_rs;
struct log_c *lc = (struct log_c *) log->context;
+ DMDEBUG("cluster_postsuspend");
spin_lock(&lc->state_lock);
if (!list_empty(&lc->mark_waiting)) {
DMERR("Mark requests remain at postsuspend!");
@@ -833,6 +834,7 @@
struct log_c *lc = (struct log_c *) log->context;
lc->sync_search = 0;
+ lc->recovery_halted = 0;
resume_server_requests();
atomic_set(&lc->suspended, 0);
@@ -861,7 +863,7 @@
{
int rtn;
struct log_c *lc = (struct log_c *) log->context;
-
+
if (atomic_read(&lc->in_sync) == 1) {
return 0;
}
@@ -1170,6 +1172,10 @@
region_t rtn;
struct log_c *lc = (struct log_c *) log->context;
+ if (atomic_read(&lc->suspended)) {
+ return (atomic_read(&lc->in_sync)) ? lc->region_count : 0;
+ }
+
/* Try to get sync count up to five times */
for (i = 0; i < 5 && consult_server(lc, 0, LRT_GET_SYNC_COUNT, &rtn);
i++);
if(i >= 5){
@@ -1226,6 +1232,7 @@
DMDEBUG(" ?sync_search : %d", lc->sync_search);
DMDEBUG(" in_sync : %s", (atomic_read(&lc->in_sync)) ?
"YES" : "NO");
DMDEBUG(" suspended : %s", (atomic_read(&lc->suspended)) ?
"YES" : "NO");
+ DMDEBUG(" recovery_halted : %s", (lc->recovery_halted) ? "YES"
: "NO");
DMDEBUG(" server_id : %u", lc->server_id);
DMDEBUG(" server_valid: %s",
((lc->server_id != 0xDEAD) && lc->server_valid) ? "YES"
: "NO");
--- cluster/cmirror-kernel/src/Attic/dm-cmirror-common.h 2007/09/26
03:15:40 1.1.2.15
+++ cluster/cmirror-kernel/src/Attic/dm-cmirror-common.h 2007/09/27
20:31:18 1.1.2.16
@@ -102,6 +102,7 @@
int sync_pass; /* number of passes attempting to resync */
int sync_search;
+ int recovery_halted; /* only useful for is_remote_recovering */
/* Resync flag */
enum sync {
--- cluster/cmirror-kernel/src/Attic/dm-cmirror-server.c 2007/09/26
03:15:40 1.1.2.39
+++ cluster/cmirror-kernel/src/Attic/dm-cmirror-server.c 2007/09/27
20:31:18 1.1.2.40
@@ -451,6 +451,14 @@
if ((lc->sync_search > lc->region_count) && !lc->sync_pass)
return 0;
+ if (lc->recovery_halted &&
+ (lc->recovering_region != lr->u.lr_region)) {
+ DMDEBUG("Recovery halted, allowing client: %Lu/%s",
+ lr->u.lr_region,
+ lc->uuid + (strlen(lc->uuid) - 8));
+ return 0;
+ }
+
/*
* If the region hasn't been recovered yet,
* we need to block the write
@@ -598,6 +606,12 @@
lr->u.lr_int_rtn = 0; /* Default to no work */
+ if (lc->recovery_halted) {
+ DMDEBUG("Recovery halted due to error on %s",
+ lc->uuid + (strlen(lc->uuid) - 8));
+ return 0;
+ }
+
if (lc->recovering_region != (uint64_t)-1) {
DMDEBUG("Someone is already recovering region %Lu/%s",
lc->recovering_region, lc->uuid + (strlen(lc->uuid) -
8));
@@ -704,11 +718,18 @@
/*
* Recovery failed or mirror is being marked out-of-sync
*
+ * We need to stop dishing out recovery work. If we don't
+ * writes happening to NOSYNC regions can't proceed and the
+ * mirror won't be able to suspend for reconfiguration - due
+ * to the return of is_remote_recovering().
+ *
* We can recieve multiple calls to mark out-of-sync
* if there were several writes to the same region that
* failed. In this case, there will not be a record for
* the region.
*/
+ lc->recovery_halted = 1;
+
ru = find_ru(lc, who, lr->u.lr_region);
if ((lr->u.lr_region == lc->recovering_region) && !ru) {
@@ -873,8 +894,14 @@
* New node joins and needs to know I am the server
* We shortcut the election here and respond directly
* to the inquirer
- */
+ *
if((lc->server_id == my_id) && !atomic_read(&lc->suspended)){
+ */
+ if (lc->server_id == my_id) {
+ if (atomic_read(&lc->suspended)) {
+ DMDEBUG("I'm suspended, but still responding as server:
%s",
+ lc->uuid + (strlen(lc->uuid) - 8));
+ }
lr->u.lr_coordinator = my_id;
if(!(saddr->sin_addr.s_addr =
nodeid_to_ipaddr(lr->u.lr_starter))){
return -1;
