Hi,
My name is Rodrigo Wanderley, I'm just another one working at a ISP ;). My
interest in squid began few weeks ago when our manager thought it would be nice
if we had a proxy-cache. I loved the program and started reading the code last
week. Liked it a lot, and would like to read more about how the development is
going and the ideas that you guys have.
I also made a tiny little patch yesterday, and wanted to discuss the
performance issues of it. The problem was that it was hard to guess the exact
size to put in maximum_object_size. We would like to cache big files from
certain sites (like windowsupdate) and let a small default for the others.
Well, what I did (or tryed to do) was mimic the reply-body-max-size option thus
adding acl to the maximum-object-size.
What I want to do now is profile it and see if putting acls into that option
is a overkill. Reading the list archives I saw two main options, gprof and a
patch in squid to enable profiling and view the results in the cachemgr. Since
the later is not at the code I wander if patching the squid for profiling is
really the best option...
Attached is the patch I did.
thanks,
RSW
Index: src/cf.data.pre
===================================================================
--- src/cf.data.pre (revision 1378)
+++ src/cf.data.pre (working copy)
@@ -603,24 +603,6 @@
numbers closer together.
DOC_END
-NAME: maximum_object_size
-COMMENT: (bytes)
-TYPE: b_size_t
-DEFAULT: 4096 KB
-LOC: Config.Store.maxObjectSize
-DOC_START
- Objects larger than this size will NOT be saved on disk. The
- value is specified in kilobytes, and the default is 4MB. If
- you wish to get a high BYTES hit ratio, you should probably
- increase this (one 32 MB object hit counts for 3200 10KB
- hits). If you wish to increase speed more than your want to
- save bandwidth you should leave this low.
-
- NOTE: if using the LFUDA replacement policy you should increase
- this value to maximize the byte hit rate improvement of LFUDA!
- See replacement_policy below for a discussion of this policy.
-DOC_END
-
NAME: minimum_object_size
COMMENT: (bytes)
TYPE: b_size_t
@@ -2411,6 +2393,18 @@
buffer-overflow or denial-of-service attacks.
DOC_END
+NAME: maximum_object_size
+COMMENT: bytes allow|deny acl acl...
+TYPE: body_size_t
+DEFAULT: none
+DEFAULT_IF_NONE: 4194304 allow all
+LOC: Config.Store.maxObjectSize
+DOC_START
+ This option specifies the maximum object size that will be cached
+ on disk. The first "allow" rule that match will be the one that
+ will be used.
+DOC_END
+
NAME: reply_body_max_size
COMMENT: bytes allow|deny acl acl...
TYPE: body_size_t
Index: src/cache_cf.c
===================================================================
--- src/cache_cf.c (revision 1378)
+++ src/cache_cf.c (working copy)
@@ -471,24 +471,6 @@
debug(22, 0) ("NOTICE: positive_dns_ttl must be larger than
negative_dns_ttl. Resetting negative_dns_ttl to match\n");
Config.positiveDnsTtl = Config.negativeDnsTtl;
}
-#if SIZEOF_SQUID_FILE_SZ <= 4
-#if SIZEOF_SQUID_OFF_T <= 4
- if (Config.Store.maxObjectSize > 0x7FFF0000) {
- debug(22, 0) ("NOTICE: maximum_object_size limited to %d KB due to
hardware limitations\n", 0x7FFF0000 / 1024);
- Config.Store.maxObjectSize = 0x7FFF0000;
- }
-#elif SIZEOF_OFF_T <= 4
- if (Config.Store.maxObjectSize > 0xFFFF0000) {
- debug(22, 0) ("NOTICE: maximum_object_size limited to %d KB due to OS
limitations\n", 0xFFFF0000 / 1024);
- Config.Store.maxObjectSize = 0xFFFF0000;
- }
-#else
- if (Config.Store.maxObjectSize > 0xFFFF0000) {
- debug(22, 0) ("NOTICE: maximum_object_size limited to %d KB to keep
compatibility with existing cache\n", 0xFFFF0000 / 1024);
- Config.Store.maxObjectSize = 0xFFFF0000;
- }
-#endif
-#endif
if (Config.Store.maxInMemObjSize > 8 * 1024 * 1024)
debug(22, 0) ("WARNING: Very large maximum_object_size_in_memory
settings can have negative impact on performance\n");
}
Index: src/store.c
===================================================================
--- src/store.c (revision 1378)
+++ src/store.c (working copy)
@@ -611,6 +611,35 @@
return 0;
}
+static int
+storeCheckTooBig(StoreEntry * e)
+{
+ body_size *bs;
+ aclCheck_t *checklist;
+ request_t *request = e->mem_obj->request;
+
+ bs = (body_size *) Config.Store.maxObjectSize.head;
+ while (bs) {
+ checklist = aclChecklistCreate(bs->access_list, request, NULL);
+
+ if (1 != aclCheckFast(bs->access_list, checklist)) {
+ /* deny - skip this entry */
+ bs = (body_size *) bs->node.next;
+ } else {
+ debug(20,3) ("storeCheckTooBig: file has: %"
PRINTF_OFF_T ", max is: %" PRINTF_OFF_T "\n",
+ e->mem_obj->inmem_hi, bs->maxsize);
+ /* Allow - use this entry */
+ return ((e->mem_obj->reply->content_length > 0 &&
+ e->mem_obj->reply->content_length >
bs->maxsize) ||
+ e->mem_obj->inmem_hi > bs->maxsize);
+ }
+ aclChecklistFree(checklist);
+ }
+ /* Did not match any allow match, do not cache it. */
+ debug(20, 1) ("storeCheckTooBig: did not match any allow acl, not
caching!\n");
+ return 1;
+}
+
int
storeCheckCachable(StoreEntry * e)
{
@@ -633,9 +662,7 @@
debug(20, 3) ("storeCheckCachable: NO: negative cached\n");
store_check_cachable_hist.no.negative_cached++;
return 0; /* avoid release call below */
- } else if ((e->mem_obj->reply->content_length > 0 &&
- e->mem_obj->reply->content_length > Config.Store.maxObjectSize)
||
- e->mem_obj->inmem_hi > Config.Store.maxObjectSize) {
+ } else if (storeCheckTooBig(e)) {
debug(20, 2) ("storeCheckCachable: NO: too big\n");
store_check_cachable_hist.no.too_big++;
} else if (storeCheckTooSmall(e)) {
Index: src/structs.h
===================================================================
--- src/structs.h (revision 1378)
+++ src/structs.h (working copy)
@@ -556,7 +556,7 @@
struct {
int objectsPerBucket;
squid_off_t avgObjectSize;
- squid_off_t maxObjectSize;
+ dlink_list maxObjectSize;
squid_off_t minObjectSize;
squid_off_t maxInMemObjSize;
} Store;
Index: src/store_swapout.c
===================================================================
--- src/store_swapout.c (revision 1378)
+++ src/store_swapout.c (working copy)
@@ -381,8 +381,6 @@
static int
storeSwapOutAble(const StoreEntry * e)
{
- if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize)
- return 0;
if (!EBIT_TEST(e->flags, ENTRY_CACHABLE))
return 0;
if (e->mem_obj->swapout.sio != NULL)
