Ar there any objections to me committing the attached patch for COSS in
squid 2.6. I've been running this patch for the past week on one of our
caches, and it appears to be working well.
This patch adds an option to specify the maximum number of disk buffers
for COSS to use. With this new code, the COSS buffer management will work
in one of the following ways:
1 - Under normal conditions nothing changes from the old code
2 - If the 'memonly' buffers (hits) are all in use, COSS will now relocate
the object to the disk head (and a message is logged to the cache
log). This will cause a slight improvement in the hit rate, and will
avoid a swapin failure where possible.
3 - If the maximum number of in-memory disk stripes is specified, COSS
will use this as part of the load calculation. If there are only 1 or 2
stripes available, no new objects will be stored in the COSS cache_dir,
and a message is logged to the cache log. Once again, this is designed to
avoid swapin failures where possible.
4 - If all disk and memory stripes are full, no objects will be stored or
retrieved from the cache dir until one becomes available.
This keeps COSS in line with other behaviours of squid to stop clients
from being able to cause squid to use excessive amounts of memory.
regards
Steven
Index: src/cf.data.pre
===================================================================
RCS file: /server/cvs-server/squid/squid/src/cf.data.pre,v
retrieving revision 1.363
diff -u -r1.363 cf.data.pre
--- src/cf.data.pre 3 Aug 2006 02:31:11 -0000 1.363
+++ src/cf.data.pre 11 Aug 2006 04:55:21 -0000
@@ -1162,6 +1162,14 @@
number of memory-only buffers that COSS will use. The default value
is 10, which will use a maximum of 10MB of memory for buffers.
+ maxfullbufs=n defines the maximum number of stripes a COSS partition
+ will have in memory waiting to be freed (either because the disk is
+ under load and the stripe is unwritten, or because clients are still
+ transferring data from objects using the memory). In order to try
+ and maintain a good hit rate under load, COSS will reserve the last
+ 2 full stripes for object hits. (ie a COSS cache_dir will reject
+ new objects when the number of full stripes is 2 less than maxfullbufs)
+
Common options:
read-only, this cache_dir is read only.
Index: src/fs/coss/store_coss.h
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/coss/store_coss.h,v
retrieving revision 1.11
diff -u -r1.11 store_coss.h
--- src/fs/coss/store_coss.h 3 Aug 2006 02:31:12 -0000 1.11
+++ src/fs/coss/store_coss.h 11 Aug 2006 04:55:22 -0000
@@ -145,6 +145,9 @@
float minumum_overwrite_pct;
int minimum_stripe_distance;
int numstripes;
+ int maxfullstripes;
+ int hitonlyfullstripes;
+ int numfullstripes;
int sizerange_max;
int sizerange_min;
struct _cossstripe *stripes;
Index: src/fs/coss/store_dir_coss.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/coss/store_dir_coss.c,v
retrieving revision 1.52
diff -u -r1.52 store_dir_coss.c
--- src/fs/coss/store_dir_coss.c 5 Aug 2006 15:36:24 -0000 1.52
+++ src/fs/coss/store_dir_coss.c 11 Aug 2006 04:55:23 -0000
@@ -44,6 +44,7 @@
#endif
#define STORE_META_BUFSZ 4096
+#define HITONLY_BUFS 2
int n_coss_dirs = 0;
int max_coss_dir_size = 0;
@@ -97,10 +98,12 @@
static void storeCossDirParseOverwritePct(SwapDir *, const char *, const char
*, int);
static void storeCossDirParseMaxWaste(SwapDir *, const char *, const char *,
int);
static void storeCossDirParseMemOnlyBufs(SwapDir *, const char *, const char
*, int);
+static void storeCossDirParseMaxFullBufs(SwapDir *, const char *, const char
*, int);
static void storeCossDirDumpBlkSize(StoreEntry *, const char *, SwapDir *);
static void storeCossDirDumpOverwritePct(StoreEntry *, const char *, SwapDir
*);
static void storeCossDirDumpMaxWaste(StoreEntry *, const char *, SwapDir *);
static void storeCossDirDumpMemOnlyBufs(StoreEntry *, const char *, SwapDir *);
+static void storeCossDirDumpMaxFullBufs(StoreEntry *, const char *, SwapDir *);
static OBJH storeCossStats;
static void storeDirCoss_StartDiskRebuild(RebuildState * rb);
@@ -114,6 +117,7 @@
{"overwrite-percent", storeCossDirParseOverwritePct,
storeCossDirDumpOverwritePct},
{"max-stripe-waste", storeCossDirParseMaxWaste, storeCossDirDumpMaxWaste},
{"membufs", storeCossDirParseMemOnlyBufs, storeCossDirDumpMemOnlyBufs},
+ {"maxfullbufs", storeCossDirParseMaxFullBufs, storeCossDirDumpMaxFullBufs},
{NULL, NULL}
};
@@ -624,11 +628,17 @@
loadav += disk_size_weight * current_write_weight;
/* Remove the folowing check if we want to allow COSS partitions to get
- * "too busy"
+ * too busy to accept new objects
*/
if (loadav > MAX_LOAD_VALUE)
loadav = MAX_LOAD_VALUE;
+ /* Finally, we want to reject all new obects if the number of full stripes
+ * is too large
+ */
+ if (cs->numfullstripes > cs->hitonlyfullstripes)
+ loadav += MAX_LOAD_VALUE;
+
debug(47, 9) ("storeAufsDirCheckObj: load=%d\n", loadav);
return loadav;
#else
@@ -802,6 +812,15 @@
}
cs->minimum_stripe_distance = cs->numstripes * cs->minumum_overwrite_pct;
+ /* Make sure cs->maxfull has a default value */
+ if(cs->maxfullstripes == 0)
+ cs->maxfullstripes = cs->numstripes;
+
+ /* We will reject new objects (ie go into hit-only mode)
+ * if there are <= 2 stripes available
+ */
+ cs->hitonlyfullstripes = cs->maxfullstripes - HITONLY_BUFS;
+
debug(47, 0) ("COSS: number of memory-only stripes %d of %d bytes each\n",
cs->nummemstripes, COSS_MEMBUF_SZ);
cs->memstripes = xcalloc(cs->nummemstripes, sizeof(struct _cossstripe));
for (i = 0; i < cs->nummemstripes; i++) {
@@ -810,7 +829,7 @@
cs->memstripes[i].numdiskobjs = -1;
}
- /* Update the max size (used for load calculations */
+ /* Update the max size (used for load calculations) */
if (sd->max_size > max_coss_dir_size)
max_coss_dir_size = sd->max_size;
}
@@ -846,6 +865,18 @@
}
static void
+storeCossDirParseMaxFullBufs(SwapDir * sd, const char *name, const char
*value, int reconfiguring)
+{
+ CossInfo *cs = sd->fsdata;
+ int maxfull = atoi(value);
+ if (maxfull <= HITONLY_BUFS)
+ fatalf("COSS ERROR: There must be more than %d maxfullbufs\n",
HITONLY_BUFS);
+ if (maxfull > 500)
+ fatal("COSS ERROR: Squid will likely use too much memory if it ever
used 500MB worth of full buffers\n");
+ cs->maxfullstripes = maxfull;
+}
+
+static void
storeCossDirParseMemOnlyBufs(SwapDir * sd, const char *name, const char
*value, int reconfiguring)
{
CossInfo *cs = sd->fsdata;
@@ -918,6 +949,13 @@
}
static void
+storeCossDirDumpMaxFullBufs(StoreEntry * e, const char *option, SwapDir * sd)
+{
+ CossInfo *cs = sd->fsdata;
+ storeAppendPrintf(e, " maxfullbufs=%d MB", cs->maxfullstripes);
+}
+
+static void
storeCossDirDumpMemOnlyBufs(StoreEntry * e, const char *option, SwapDir * sd)
{
CossInfo *cs = sd->fsdata;
Index: src/fs/coss/store_io_coss.c
===================================================================
RCS file: /server/cvs-server/squid/squid/src/fs/coss/store_io_coss.c,v
retrieving revision 1.27
diff -u -r1.27 store_io_coss.c
--- src/fs/coss/store_io_coss.c 5 Aug 2006 15:06:29 -0000 1.27
+++ src/fs/coss/store_io_coss.c 11 Aug 2006 04:55:25 -0000
@@ -162,14 +162,29 @@
/* Since we're not supporting NOTIFY anymore, lets fail */
assert(which != COSS_ALLOC_NOTIFY);
- /* Check if we have overflowed the disk .. */
- if ((cs->current_offset + allocsize) > ((off_t) SD->max_size << 10)) {
+ /* Check to see if we need to allocate a membuf to start */
+ if (cs->current_membuf == NULL) {
+ if(cs->curstripe < cs->numstripes)
+ newmb = storeCossCreateMemBuf(SD, cs->curstripe + 1, checkf, &coll);
+ else
+ newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll);
+
+ cs->current_membuf = newmb;
+ if (newmb == NULL) {
+ cs->sizerange_max = SD->max_objsize;
+ return -1;
+ }
+ cs->current_offset = cs->current_membuf->diskstart;
+
+ /* Check if we have overflowed the disk .. */
+ } else if ((cs->current_offset + allocsize) > ((off_t) SD->max_size <<
10)) {
/*
* tried to allocate past the end of the disk, so wrap
* back to the beginning
*/
coss_stats.disk_overflows++;
cs->current_membuf->flags.full = 1;
+ cs->numfullstripes++;
cs->current_membuf->diskend = cs->current_offset;
storeCossMaybeWriteMemBuf(SD, cs->current_membuf);
/* cs->current_membuf may be invalid at this point */
@@ -178,6 +193,10 @@
newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll);
cs->current_membuf = newmb;
+ if (newmb == NULL) {
+ cs->sizerange_max = SD->max_objsize;
+ return -1;
+ }
/* Check if we have overflowed the MemBuf */
} else if ((cs->current_offset + allocsize) >=
cs->current_membuf->diskend) {
@@ -186,6 +205,7 @@
*/
coss_stats.stripe_overflows++;
cs->current_membuf->flags.full = 1;
+ cs->numfullstripes++;
cs->current_offset = cs->current_membuf->diskend;
storeCossMaybeWriteMemBuf(SD, cs->current_membuf);
/* cs->current_membuf may be invalid at this point */
@@ -194,6 +214,10 @@
assert(cs->curstripe < (cs->numstripes - 1));
newmb = storeCossCreateMemBuf(SD, cs->curstripe + 1, checkf, &coll);
cs->current_membuf = newmb;
+ if (newmb == NULL) {
+ cs->sizerange_max = SD->max_objsize;
+ return -1;
+ }
}
/* If we didn't get a collision, then update the current offset and return
it */
if (coll == 0) {
@@ -372,6 +396,10 @@
} else {
debug(79, 3) ("storeCossOpen: %s memory miss - not reallocating
(Current stripe : %d Object in stripe : %d)\n", SD->path, cs->curstripe,
storeCossFilenoToStripe(cs, sio->swap_filen));
nf = storeCossMemOnlyAllocate(SD, e);
+ if (nf == -1) {
+ debug(79, 3) ("storeCossOpen: %s memory miss - reallocating
because all membufs are in use\n", SD->path);
+ nf = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC);
+ }
}
if (nf == -1) {
/* We have to clean up neatly .. */
@@ -732,6 +760,10 @@
assert(cs->memstripes[mb->stripe].membuf == mb);
cs->memstripes[mb->stripe].membuf = NULL;
}
+ else
+ {
+ cs->numfullstripes--;
+ }
debug(79, 3) ("storeCossMaybeFreeBuf: %p: lockcount = 0, written = 1:
marking dead\n", mb);
mb->flags.dead = 1;
dlinkDelete(&mb->node, &cs->membufs);
@@ -807,6 +839,7 @@
CossInfo *cs = (CossInfo *) SD->fsdata;
off_t start;
int stripe;
+ static time_t last_warn = 0;
/* TODO: Maybe make this a simple search for a free membuf */
for (stripe = 0; stripe < cs->nummemstripes; stripe++) {
@@ -814,7 +847,10 @@
break;
}
if (stripe >= cs->nummemstripes) {
- debug(79, 1) ("storeCossCreateMemOnlyBuf: no free membufs. You may
beed to increase the value of membufs on the %s cache_dir\n", SD->path);
+ if (last_warn + 15 < squid_curtime) {
+ debug(79, 1) ("storeCossCreateMemOnlyBuf: no free membufs. You may
need to increase the value of membufs on the %s cache_dir\n", SD->path);
+ last_warn = squid_curtime;
+ }
return NULL;
}
cs->curmemstripe = stripe;
@@ -854,8 +890,17 @@
CossInfo *cs = (CossInfo *) SD->fsdata;
off_t start = (off_t) stripe * COSS_MEMBUF_SZ;
off_t o;
+ static time_t last_warn = 0;
assert(start >= 0);
+ if (cs->numfullstripes >= cs->maxfullstripes) {
+ if (last_warn + 15 < squid_curtime) {
+ debug(79, 1) ("storeCossCreateMemBuf: Maximum number of full
buffers reached on %s. You may need to increase the maxfullbuffers option for
this cache_dir\n", SD->path);
+ last_warn = squid_curtime;
+ }
+ return NULL;
+ }
+
/* No, we shouldn't ever try to create a membuf if we haven't freed the
one on
* this stripe. Grr */
assert(cs->stripes[stripe].membuf == NULL);
