Hiya,
Here's my initial patch to make things slightly saner during cachedir
rebuilds.
Its resulted in a correct-looking rebuild from a cache w/ COSS+AUFS which
crashed during rebuild. I haven't contacted the owner of the bugzilla
entry to see if this patch fixes his particular problem but I do believe
they're the same.
Comments please! I'd like to commit a fix to this issue as soon as is possible.
Adrian
? fs/.deps
Index: protos.h
===================================================================
RCS file: /server/cvs-server/squid/squid/src/protos.h,v
retrieving revision 1.502
diff -u -r1.502 protos.h
--- protos.h 28 Jun 2006 10:31:57 -0000 1.502
+++ protos.h 2 Jul 2006 07:51:11 -0000
@@ -957,6 +957,7 @@
extern void storeRead(storeIOState *, char *, size_t, squid_off_t, STRCB *,
void *);
extern void storeWrite(storeIOState *, char *, size_t, FREE *);
extern void storeUnlink(StoreEntry *);
+extern void storeRecycle(StoreEntry *);
extern squid_off_t storeOffset(storeIOState *);
/*
Index: store_io.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/store_io.c,v
retrieving revision 1.15
diff -u -r1.15 store_io.c
--- store_io.c 27 May 2006 15:50:16 -0000 1.15
+++ store_io.c 2 Jul 2006 07:51:11 -0000
@@ -119,6 +119,13 @@
SD->obj.unlink(SD, e);
}
+void
+storeRecycle(StoreEntry *e)
+{
+ SwapDir *SD = INDEXSD(e->swap_dirn);
+ SD->obj.recycle(SD, e);
+}
+
squid_off_t
storeOffset(storeIOState * sio)
{
Index: structs.h
===================================================================
RCS file: /server/cvs-server/squid/squid/src/structs.h,v
retrieving revision 1.489
diff -u -r1.489 structs.h
--- structs.h 30 Jun 2006 21:23:05 -0000 1.489
+++ structs.h 2 Jul 2006 07:51:12 -0000
@@ -1748,6 +1748,7 @@
STOBJREAD *read;
STOBJWRITE *write;
STOBJUNLINK *unlink;
+ STOBJRECYCLE *recycle;
} obj;
struct {
STLOGOPEN *open;
Index: typedefs.h
===================================================================
RCS file: /server/cvs-server/squid/squid/src/typedefs.h,v
retrieving revision 1.147
diff -u -r1.147 typedefs.h
--- typedefs.h 4 Jun 2006 02:50:05 -0000 1.147
+++ typedefs.h 2 Jul 2006 07:51:13 -0000
@@ -310,6 +310,7 @@
typedef void STOBJREAD(SwapDir *, storeIOState *, char *, size_t, squid_off_t,
STRCB *, void *);
typedef void STOBJWRITE(SwapDir *, storeIOState *, char *, size_t,
squid_off_t, FREE *);
typedef void STOBJUNLINK(SwapDir *, StoreEntry *);
+typedef void STOBJRECYCLE(SwapDir *, StoreEntry *);
typedef void STLOGOPEN(SwapDir *);
typedef void STLOGCLOSE(SwapDir *);
Index: fs/aufs/store_asyncufs.h
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/aufs/store_asyncufs.h,v
retrieving revision 1.14
diff -u -r1.14 store_asyncufs.h
--- fs/aufs/store_asyncufs.h 19 May 2006 00:16:14 -0000 1.14
+++ fs/aufs/store_asyncufs.h 2 Jul 2006 07:51:13 -0000
@@ -79,5 +79,6 @@
extern STOBJREAD storeAufsRead;
extern STOBJWRITE storeAufsWrite;
extern STOBJUNLINK storeAufsUnlink;
+extern STOBJRECYCLE storeAufsRecycle;
#endif
Index: fs/aufs/store_dir_aufs.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/aufs/store_dir_aufs.c,v
retrieving revision 1.57
diff -u -r1.57 store_dir_aufs.c
--- fs/aufs/store_dir_aufs.c 4 Jun 2006 02:01:38 -0000 1.57
+++ fs/aufs/store_dir_aufs.c 2 Jul 2006 07:51:14 -0000
@@ -595,15 +595,7 @@
* because adding to store_swap_size happens in
* the cleanup procedure.
*/
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeAufsDirReplRemove(e);
- storeAufsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.objcount--;
rb->counts.cancelcount++;
}
@@ -683,16 +675,7 @@
} else if (e) {
/* key already exists, this swapfile not being used */
/* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeAufsDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeAufsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.dupcount++;
} else {
/* URL doesnt exist, swapfile not in use */
@@ -763,15 +746,7 @@
* because adding to store_swap_size happens in
* the cleanup procedure.
*/
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeAufsDirReplRemove(e);
- storeAufsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.objcount--;
rb->counts.cancelcount++;
}
@@ -851,16 +826,7 @@
} else if (e) {
/* key already exists, this swapfile not being used */
/* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeAufsDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeAufsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.dupcount++;
} else {
/* URL doesnt exist, swapfile not in use */
@@ -1921,6 +1887,7 @@
sd->obj.read = storeAufsRead;
sd->obj.write = storeAufsWrite;
sd->obj.unlink = storeAufsUnlink;
+ sd->obj.recycle = storeAufsRecycle;
sd->log.open = storeAufsDirOpenSwapLog;
sd->log.close = storeAufsDirCloseSwapLog;
sd->log.write = storeAufsDirSwapLog;
Index: fs/aufs/store_io_aufs.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/aufs/store_io_aufs.c,v
retrieving revision 1.31
diff -u -r1.31 store_io_aufs.c
--- fs/aufs/store_io_aufs.c 7 Jun 2006 22:55:45 -0000 1.31
+++ fs/aufs/store_io_aufs.c 2 Jul 2006 07:51:14 -0000
@@ -246,6 +246,23 @@
statCounter.syscalls.disk.unlinks++;
}
+void
+storeAufsRecycle(SwapDir *SD, StoreEntry *e)
+{
+ debug(79, 3) ("storeAufsUnlink: fileno %08X\n", e->swap_filen);
+
+ /* Release the object without releasing the underlying physical object */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeAufsDirReplRemove(e);
+ storeAufsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+}
+
/* === STATIC =========================================================== */
static int
Index: fs/coss/store_coss.h
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/coss/store_coss.h,v
retrieving revision 1.9
diff -u -r1.9 store_coss.h
--- fs/coss/store_coss.h 11 Jun 2006 20:59:09 -0000 1.9
+++ fs/coss/store_coss.h 2 Jul 2006 07:51:14 -0000
@@ -189,6 +189,7 @@
extern STOBJREAD storeCossRead;
extern STOBJWRITE storeCossWrite;
extern STOBJUNLINK storeCossUnlink;
+extern STOBJRECYCLE storeCossRecycle;
extern STSYNC storeCossSync;
extern void storeCossAdd(SwapDir * sd, StoreEntry * e, int curstripe);
Index: fs/coss/store_dir_coss.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/coss/store_dir_coss.c,v
retrieving revision 1.47
diff -u -r1.47 store_dir_coss.c
--- fs/coss/store_dir_coss.c 22 Jun 2006 22:05:00 -0000 1.47
+++ fs/coss/store_dir_coss.c 2 Jul 2006 07:51:15 -0000
@@ -693,6 +693,7 @@
sd->obj.read = storeCossRead;
sd->obj.write = storeCossWrite;
sd->obj.unlink = storeCossUnlink;
+ sd->obj.recycle = storeCossRecycle;
sd->log.open = storeCossDirOpenSwapLog;
sd->log.close = storeCossDirCloseSwapLog;
@@ -1191,13 +1192,10 @@
storeCoss_DeleteStoreEntry(RebuildState * rb, const cache_key * key,
StoreEntry * e)
{
assert(rb->counts.objcount >= 0);
+ /* XXX are these counters even correct, considering e could be a different
storedir? */
rb->counts.objcount--;
assert(e->swap_dirn >= 0);
- storeCossRemove(INDEXSD(e->swap_dirn), e);
- e->swap_filen = -1;
- storeExpireNow(e);
- storeReleaseRequest(e);
- storeRelease(e);
+ storeRecycle(e);
}
/*
Index: fs/coss/store_io_coss.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/coss/store_io_coss.c,v
retrieving revision 1.22
diff -u -r1.22 store_io_coss.c
--- fs/coss/store_io_coss.c 22 May 2006 10:56:16 -0000 1.22
+++ fs/coss/store_io_coss.c 2 Jul 2006 07:51:16 -0000
@@ -176,6 +176,14 @@
storeCossRemove(SD, e);
}
+void
+storeCossRecycle(SwapDir * SD, StoreEntry * e)
+{
+ debug(79, 3) ("storeCossRecycle: %s: offset %d\n", SD->path,
e->swap_filen);
+ storeCossUnlink(SD, e);
+}
+
+
storeIOState *
storeCossCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, STIOCB *
callback, void *callback_data)
Index: fs/diskd/store_dir_diskd.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/diskd/store_dir_diskd.c,v
retrieving revision 1.78
diff -u -r1.78 store_dir_diskd.c
--- fs/diskd/store_dir_diskd.c 4 Jun 2006 02:01:39 -0000 1.78
+++ fs/diskd/store_dir_diskd.c 2 Jul 2006 07:51:17 -0000
@@ -785,15 +785,7 @@
* because adding to store_swap_size happens in
* the cleanup procedure.
*/
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeDiskdDirReplRemove(e);
- storeDiskdDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.objcount--;
rb->counts.cancelcount++;
}
@@ -873,16 +865,7 @@
} else if (e) {
/* key already exists, this swapfile not being used */
/* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeDiskdDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeDiskdDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.dupcount++;
} else {
/* URL doesnt exist, swapfile not in use */
@@ -978,15 +961,7 @@
* because adding to store_swap_size happens in
* the cleanup procedure.
*/
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeDiskdDirReplRemove(e);
- storeDiskdDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.objcount--;
rb->counts.cancelcount++;
}
@@ -1066,16 +1041,7 @@
} else if (e) {
/* key already exists, this swapfile not being used */
/* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeDiskdDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeDiskdDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.dupcount++;
} else {
/* URL doesnt exist, swapfile not in use */
@@ -2256,6 +2222,7 @@
sd->obj.read = storeDiskdRead;
sd->obj.write = storeDiskdWrite;
sd->obj.unlink = storeDiskdUnlink;
+ sd->obj.recycle = storeDiskdRecycle;
sd->log.open = storeDiskdDirOpenSwapLog;
sd->log.close = storeDiskdDirCloseSwapLog;
sd->log.write = storeDiskdDirSwapLog;
Index: fs/diskd/store_diskd.h
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/diskd/store_diskd.h,v
retrieving revision 1.12
diff -u -r1.12 store_diskd.h
--- fs/diskd/store_diskd.h 25 May 2006 03:20:38 -0000 1.12
+++ fs/diskd/store_diskd.h 2 Jul 2006 07:51:17 -0000
@@ -111,6 +111,7 @@
extern STOBJREAD storeDiskdRead;
extern STOBJWRITE storeDiskdWrite;
extern STOBJUNLINK storeDiskdUnlink;
+extern STOBJRECYCLE storeDiskdRecycle;
#define SHMBUF_BLKSZ SM_PAGE_SIZE
Index: fs/diskd/store_io_diskd.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/diskd/store_io_diskd.c,v
retrieving revision 1.31
diff -u -r1.31 store_io_diskd.c
--- fs/diskd/store_io_diskd.c 7 Jun 2006 22:55:45 -0000 1.31
+++ fs/diskd/store_io_diskd.c 2 Jul 2006 07:51:17 -0000
@@ -299,6 +299,24 @@
diskd_stats.unlink.ops++;
}
+void
+storeDiskdRecycle(SwapDir *SD, StoreEntry *e)
+{
+ debug(79, 3) ("storeDiskdUnlink: fileno %08X\n", e->swap_filen);
+
+ /* Release the object without releasing the underlying physical object */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeDiskdDirReplRemove(e);
+ storeDiskdDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+}
+
+
/* === STATIC =========================================================== */
@@ -454,7 +472,7 @@
storeDiskdIOCallback(storeIOState * sio, int errflag)
{
int valid = cbdataValid(sio->callback_data);
- debug(79, 3) ("storeUfsIOCallback: errflag=%d\n", errflag);
+ debug(79, 3) ("storeDiskdIOCallback: errflag=%d\n", errflag);
cbdataUnlock(sio->callback_data);
if (valid)
sio->callback(sio->callback_data, errflag, sio);
Index: fs/ufs/store_dir_ufs.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/ufs/store_dir_ufs.c,v
retrieving revision 1.57
diff -u -r1.57 store_dir_ufs.c
--- fs/ufs/store_dir_ufs.c 4 Jun 2006 02:01:40 -0000 1.57
+++ fs/ufs/store_dir_ufs.c 2 Jul 2006 07:51:18 -0000
@@ -605,15 +605,11 @@
* because adding to store_swap_size happens in
* the cleanup procedure.
*/
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeUfsDirReplRemove(e);
- storeUfsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
+ /*
+ * XXX considering we might've canceled an object from another
store;
+ * XXX what should happen with these stats?
+ */
rb->counts.objcount--;
rb->counts.cancelcount++;
}
@@ -693,16 +689,7 @@
} else if (e) {
/* key already exists, this swapfile not being used */
/* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeUfsDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeUfsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.dupcount++;
} else {
/* URL doesnt exist, swapfile not in use */
@@ -773,15 +760,8 @@
* because adding to store_swap_size happens in
* the cleanup procedure.
*/
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeUfsDirReplRemove(e);
- storeUfsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
+ /* XXX are these counters valid since e could be from another
swapfs? */
rb->counts.objcount--;
rb->counts.cancelcount++;
}
@@ -861,16 +841,7 @@
} else if (e) {
/* key already exists, this swapfile not being used */
/* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeUfsDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeUfsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
+ storeRecycle(e);
rb->counts.dupcount++;
} else {
/* URL doesnt exist, swapfile not in use */
@@ -1952,6 +1923,7 @@
sd->obj.read = storeUfsRead;
sd->obj.write = storeUfsWrite;
sd->obj.unlink = storeUfsUnlink;
+ sd->obj.recycle = storeUfsRecycle;
sd->log.open = storeUfsDirOpenSwapLog;
sd->log.close = storeUfsDirCloseSwapLog;
sd->log.write = storeUfsDirSwapLog;
Index: fs/ufs/store_io_ufs.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/ufs/store_io_ufs.c,v
retrieving revision 1.14
diff -u -r1.14 store_io_ufs.c
--- fs/ufs/store_io_ufs.c 7 Jun 2006 22:55:45 -0000 1.14
+++ fs/ufs/store_io_ufs.c 2 Jul 2006 07:51:18 -0000
@@ -195,6 +195,23 @@
storeUfsDirUnlinkFile(SD, e->swap_filen);
}
+void
+storeUfsRecycle(SwapDir *SD, StoreEntry *e)
+{
+ debug(79, 3) ("storeUfsUnlink: fileno %08X\n", e->swap_filen);
+
+ /* Release the object without releasing the underlying physical object */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeUfsDirReplRemove(e);
+ storeUfsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+}
+
/* === STATIC =========================================================== */
static void
Index: fs/ufs/store_ufs.h
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/ufs/store_ufs.h,v
retrieving revision 1.4
diff -u -r1.4 store_ufs.h
--- fs/ufs/store_ufs.h 17 May 2005 16:56:43 -0000 1.4
+++ fs/ufs/store_ufs.h 2 Jul 2006 07:51:18 -0000
@@ -47,5 +47,6 @@
extern STOBJREAD storeUfsRead;
extern STOBJWRITE storeUfsWrite;
extern STOBJUNLINK storeUfsUnlink;
+extern STOBJRECYCLE storeUfsRecycle;
#endif
