hi adrian, i'm sorry but this does not care about correctness very well. it just handles headerlines like strings and feeding the helper-program. i'm pretty new to the squid code (and i'm still very confused by it) and i was lucky to get it actually doing what i wanted. it modifies only the request header but i want to use it for keeping track of the users going through our proxy to see at our internet-gateway where the original request came from. only collecting information with the helper for statistics (without modifying the header) would also be nice. we are running a pretty huge wireless network of enthusiasts in dresden/germany and now i got some new toy to play with :)
regards, martin Am Mittwoch, 24. Januar 2007 16:11 schrieb Adrian Chadd: > Cool, an interesting idea. I like it but it does introduce another round > trip of packing/unpacking into the HTTP parsing mix. > > What are you/others using this functionality for at the moment? > > Does this correctly handle HTTP continuations? > > > > Adrian > > On Tue, Jan 23, 2007, Martin Schoeffel wrote: > > hi henrik, > > > > here it is again. > > > > martin > > > > Am Dienstag, 23. Januar 2007 11:13 schrieb Henrik Nordstrom: > > > Martin, > > > > > > can you please send this message to [email protected] to have > > > your work properly archived and indexed. > > > > > > Regards > > > Henrik > > > > > > m?n 2007-01-22 klockan 22:07 +1300 skrev Martin Schoeffel: > > > > hi henrik, > > > > > > > > i modified the header-rewriter-patch from james beamish-white and it > > > > is now working for me with squid 2.6s7. i'm feeling a bit not so good > > > > about the hdrhandlereply-function (httpheadertools.c), but for my > > > > purpose it's okay. i attached the patch for you if you like to take a > > > > look at it. > > > > > > > > regards, martin > > > > diff --exclude=.svn -Naur squid-2.6.STABLE7/src/cache_cf.c > > squid-2.6.STABLE7_PATCHED/src/cache_cf.c --- > > squid-2.6.STABLE7/src/cache_cf.c 2007-01-09 23:24:41.000000000 +1300 +++ > > squid-2.6.STABLE7_PATCHED/src/cache_cf.c 2007-01-18 11:47:51.000000000 > > +1300 @@ -427,6 +427,14 @@ > > wordlistDestroy(&Config.Program.location_rewrite.command); > > } > > } > > + /* header_rewrite */ > > + if (Config.Program.header_rewrite.command) { > > + if (Config.Program.header_rewrite.children < 1){ > > + Config.Program.header_rewrite.children = 0; > > + wordlistDestroy(&Config.Program.header_rewrite.command); > > + } > > + } > > + /* end of change */ > > if (Config.appendDomain) > > if (*Config.appendDomain != '.') > > fatal("append_domain must begin with a '.'"); > > @@ -470,6 +478,10 @@ > > requirePathnameExists("url_rewrite_program", > > Config.Program.url_rewrite.command->key); if > > (Config.Program.location_rewrite.command) > > requirePathnameExists("location_rewrite_program", > > Config.Program.location_rewrite.command->key); + /* header_rewrite */ > > + if (Config.Program.header_rewrite.command) > > + > > requirePathnameExists("header_rewrite_program",Config.Program.header_rewr > >ite.command->key); + /* end of change */ > > requirePathnameExists("Icon Directory", Config.icons.directory); > > requirePathnameExists("Error Directory", Config.errorDirectory); > > authenticateConfigure(&Config.authConfig); > > diff --exclude=.svn -Naur squid-2.6.STABLE7/src/cf.data.pre > > squid-2.6.STABLE7_PATCHED/src/cf.data.pre --- > > squid-2.6.STABLE7/src/cf.data.pre 2007-01-14 05:06:42.000000000 +1300 +++ > > squid-2.6.STABLE7_PATCHED/src/cf.data.pre 2007-01-18 11:47:47.000000000 > > +1300 @@ -1807,6 +1807,97 @@ > > headers are sent. > > DOC_END > > > > +NAME: header_rewrite_program > > +TYPE: wordlist > > +LOC: Config.Program.header_rewrite.command > > +DEFAULT: none > > +DOC_START > > + Specify the location of the executable for the header rewriter. > > + Since they can perform almost any function there isn't one included. > > + > > + For each requested URL which matches the header_rewriter ACL, Squid > > + will send a block of header lines to the rewriter program terminated > > + by an empty line : > > + > > + header1: header1_contents > > + header2: header2_contents > > + header3: header3_contents > > + header4: header4_contents > > + \n > > + > > + And the rewriter should return modified or additional headers. Note > > + that all of the original request headers will be removed and replaced > > + by whatever the header_rewriter generates. > > + > > + By default, a header rewriter is not used. > > +DOC_END > > + > > +NAME: header_rewrite_children > > +TYPE: int > > +DEFAULT: 5 > > +LOC: Config.Program.header_rewrite.children > > +DOC_START > > + The number of header rewriting processes to spawn. If you start > > + too few Squid will have to wait for them to process a backlog of > > + headers, slowing it down. If you start too many they will use RAM > > + and other system resources. > > +DOC_END > > + > > +NAME: header_rewrite_access > > +TYPE: acl_access > > +DEFAULT: none > > +LOC: Config.accessList.header_rewrite > > +DOC_START > > + If defined, this access list specifies which requests are > > + sent to the header rewriting processes. By default all requests > > + are sent. > > +DOC_END > > + > > +NAME: header_rewrite_program > > +TYPE: wordlist > > +LOC: Config.Program.header_rewrite.command > > +DEFAULT: none > > +DOC_START > > + Specify the location of the executable for the header rewriter. > > + Since they can perform almost any function there isn't one included. > > + > > + For each requested URL which matches the header_rewriter ACL, Squid > > + will send a block of header lines to the rewriter program terminated > > + by an empty line : > > + > > + header1: header1_contents > > + header2: header2_contents > > + header3: header3_contents > > + header4: header4_contents > > + \n > > + > > + And the rewriter should return modified or additional headers. Note > > + that all of the original request headers will be removed and replaced > > + by whatever the header_rewriter generates. > > + > > + By default, a header rewriter is not used. > > +DOC_END > > + > > +NAME: header_rewrite_children > > +TYPE: int > > +DEFAULT: 5 > > +LOC: Config.Program.header_rewrite.children > > +DOC_START > > + The number of header rewriting processes to spawn. If you start > > + too few Squid will have to wait for them to process a backlog of > > + headers, slowing it down. If you start too many they will use RAM > > + and other system resources. > > +DOC_END > > + > > +NAME: header_rewrite_access > > +TYPE: acl_access > > +DEFAULT: none > > +LOC: Config.accessList.header_rewrite > > +DOC_START > > + If defined, this access list specifies which requests are > > + sent to the header rewriting processes. By default all requests > > + are sent. > > +DOC_END > > > > NAME: auth_param > > TYPE: authparam > > diff --exclude=.svn -Naur squid-2.6.STABLE7/src/client_side.c > > squid-2.6.STABLE7_PATCHED/src/client_side.c --- > > squid-2.6.STABLE7/src/client_side.c 2007-01-07 06:22:45.000000000 +1300 > > +++ squid-2.6.STABLE7_PATCHED/src/client_side.c 2007-01-18 > > 11:53:12.000000000 +1300 @@ -664,7 +664,11 @@ > > clientHttpRequest *http = data; > > http->request->flags.cachable = answer; > > http->acl_checklist = NULL; > > + /* header_rewrite > > clientProcessRequest(http); > > + */ > > + hdrRewriteStart(http,(RH *)clientProcessRequest, http ); > > + /* end of change */ > > } > > > > static void > > diff --exclude=.svn -Naur squid-2.6.STABLE7/src/helper.c > > squid-2.6.STABLE7_PATCHED/src/helper.c --- > > squid-2.6.STABLE7/src/helper.c 2006-09-09 07:41:24.000000000 +1200 +++ > > squid-2.6.STABLE7_PATCHED/src/helper.c 2007-01-18 11:53:08.000000000 > > +1300 @@ -742,7 +742,13 @@ > > srv->roffset = 0; > > srv->rbuf[0] = '\0'; > > } > > + /* header_rewrite > > while ((t = strchr(srv->rbuf, '\n'))) { > > + check if this helper has double cr_end else go on as usual > > + */ > > + while ( (!hlp->double_cr_end && (t = strchr(srv->rbuf, '\n'))) || > > + (hlp->double_cr_end && (t = strstr(srv->rbuf, "\r\n\r\n"))) > > ) { + /* end of change */ > > helper_request *r; > > char *msg = srv->rbuf; > > int i = 0; > > diff --exclude=.svn -Naur squid-2.6.STABLE7/src/HttpHeaderTools.c > > squid-2.6.STABLE7_PATCHED/src/HttpHeaderTools.c --- > > squid-2.6.STABLE7/src/HttpHeaderTools.c 2006-07-27 08:09:33.000000000 > > +1200 +++ squid-2.6.STABLE7_PATCHED/src/HttpHeaderTools.c 2007-01-18 > > 11:57:54.000000000 +1300 @@ -486,3 +486,246 @@ > > if (removed_headers) > > httpHeaderRefreshMask(l); > > } > > + > > +/* header_rewriter > > + * start of header rewriter helper code > > + */ > > + > > +/* Size of buffer to hold all header contents */ > > +#define HDR_BUFSIZE 8192 > > + > > +typedef struct { > > + void *data; > > + char *orig_url; /* copy of original URL */ > > + struct in_addr client_addr; > > + const char *client_ident; > > + RH *handler; > > + HttpHeader *req_header; /* Pointer to request headers */ > > +} hdrHelper_StateData; > > + > > +static HLPCB hdrHandleReply; > > +static helper *hdrHelpers = NULL; > > +static OBJH hdrHelperStats; > > +static int hdrHelper_n_bypassed = 0; > > +CBDATA_TYPE(hdrHelper_StateData); > > + > > +static void > > +hdrStateFree(hdrHelper_StateData * r) > > +{ > > + safe_free(r->orig_url); > > + cbdataFree(r); > > +} > > + > > +/* > > + hdrHandleReply > > + function which handles the replies from the header rewriter helpers > > + > > + read the reply from the handler, and add it to the rewritten_headers > > + in new_headers array > > + if the reply is END, then : > > + - remove the headers from the request > > + - copy the rewritten headers into the request > > + - call the handler ( ie, return ) > > +*/ > > +static void > > +hdrHandleReply(void *data, char *reply) > > +{ > > + hdrHelper_StateData *r = data; > > + int valid; > > + char *pos; > > + char *pos2; > > + char *msg_tmp; > > + > > + debug(66, 9) ("hdrHandleReply: {%s}\n", reply ? reply : "<NULL>"); > > + if (reply) { > > + debug(66, 9) ("hdrHandleReply: doing header parse on hdr > > %p\n",r->req_header); + /* delete old header */ > > + httpHeaderReset( r->req_header ); > > + /* parse the reply in separate lines and insert each line into > > header entries */ + msg_tmp = reply; > > + while(pos=strstr(msg_tmp,"\r\n")) { > > + if(pos-msg_tmp>2) { > > + /* end of line */ > > + strncpy (pos,"\0\0",2); > > + /* delimiter */ > > + pos2=strstr(msg_tmp,":"); > > + strncpy (pos2,"\0",1); > > + /* avoid duplication - HDR_HOST should not be > > overwritten by the > > header_rewriter-helper */ > > + > > if(!strstr(msg_tmp,httpHeaderNameById(HDR_HOST))) { > > + > > httpHeaderPutExt(r->req_header,msg_tmp,++pos2); > > + } > > + /* goto next line */ > > + msg_tmp=pos+2; > > + } else { > > + if(strlen(pos)>0) { > > + /* line was empty - goto next line */ > > + msg_tmp=pos+2; > > + } else { > > + /* no more to process - just give up */ > > + break; > > + } > > + } > > + } /* while */ > > + } /* if reply */ > > + /* End of helper request, call the callback function */ > > + valid = cbdataValid(r->data); > > + cbdataUnlock(r->data); > > + debug(66, 9) ("hdrHandleReply: rewrite complete, calling > > handler\n"); + if (valid) > > + r->handler(r->data, reply); > > + hdrStateFree(r); > > +} > > + > > +static void > > +hdrHelperStats(StoreEntry * sentry) > > +{ > > + storeAppendPrintf(sentry, "Header Rewriter Statistics:\n"); > > + helperStats(sentry, hdrHelpers); > > + storeAppendPrintf(sentry, "\nNumber of requests bypassed " > > + "because all rewriters were busy: %d\n", > > hdrHelper_n_bypassed); > > +} > > + > > +/* > > + public function hdrRewriteInit() > > + Startup the header rewriter helpers ( if configured ) > > +*/ > > +void > > +hdrRewriteInit(void) { > > + static int init = 0; > > + debug(66,1) ("hdrRewriteInit:\n"); > > + if (!Config.Program.header_rewrite.command){ > > + debug(66,1) ("hdrRewriteInit: quitting - no header_rewrite > > command\n"); + return; > > + } > > + if (hdrHelpers == NULL){ > > + debug(66,1) ("hdrRewriteInit: creating helpers\n"); > > + hdrHelpers = helperCreate("header_rewriter"); > > + } > > + hdrHelpers->cmdline = Config.Program.header_rewrite.command; > > + hdrHelpers->n_to_start = Config.Program.header_rewrite.children; > > + hdrHelpers->ipc_type = IPC_TCP_SOCKET; > > + hdrHelpers->double_cr_end = 1; > > + helperOpenServers(hdrHelpers); > > + if (!init) { > > + cachemgrRegister("header_rewriter", > > + "Header Rewriter Stats", > > + hdrHelperStats, 0, 1); > > + init = 1; > > + CBDATA_INIT_TYPE(hdrHelper_StateData); > > + } > > + debug(66,1) ("hdrRewriteInit: done\n"); > > +} > > + > > + > > +/* > > + public function hdrRewriteShutdown() > > + Shutdown the header rewriter helpers ( if configured ) > > +*/ > > +void > > +hdrRewriteShutdown() { > > + debug(66,1) ("hdrRewriteShutdown: starting\n"); > > + if (!hdrHelpers) > > + return; > > + helperShutdown(hdrHelpers); > > + if (!shutting_down) > > + return; > > + helperFree(hdrHelpers); > > + hdrHelpers = NULL; > > + debug(66,1) ("hdrRewriteShutdown: done\n"); > > +} > > + > > + > > +/* > > + public function hdrRewriteStart() > > + Call the header rewriter helpers > > + > > + > > + *** This needs serious fixing *** > > + > > +*/ > > +void > > +hdrRewriteStart(clientHttpRequest * http, RH * handler, void *data) > > +{ > > + ConnStateData *conn = http->conn; > > + hdrHelper_StateData *helper = NULL; > > + char buf[HDR_BUFSIZE], *cp; > > + HttpHeader *reqheaders = &http->request->header; > > + HttpHeaderEntry *e; > > + HttpHeaderPos p = HttpHeaderInitPos; > > + int nl, vl; > > + > > + assert(http); > > + assert(handler); > > + debug(66, 9) ("hdrRewriteStart: '%s'\n", http->uri); > > + > > + if (Config.Program.header_rewrite.command == NULL) { > > + /* No header rewriting command has been defined, so just return by > > calling the handler + */ > > + debug(66, 9) ("hdrRewriteStart: finishing because header_rewrite > > command not defined\n"); + handler(data, NULL); > > + return; > > + } > > + > > + if (Config.accessList.header_rewrite) { > > + aclCheck_t ch; > > + memset(&ch, '\0', sizeof(ch)); > > + ch.src_addr = http->conn->peer.sin_addr; > > + ch.my_addr = http->conn->me.sin_addr; > > + ch.my_port = ntohs(http->conn->me.sin_port); > > + ch.request = http->request; > > + if (!aclCheckFast(Config.accessList.header_rewrite, &ch)) { > > + /* denied -- bypass redirector */ > > + debug(66, 9) ("hdrRewriteStart: finishing because request not in > > ACL\n"); + handler(data, NULL); > > + return; > > + } > > + } > > + > > + if (hdrHelpers->stats.queue_size) { > > + /* Skip helper if there is one request queued */ > > + hdrHelper_n_bypassed++; > > + debug(66, 9) ("hdrRewriteStart: finishing because too many requests > > queued\n"); + handler(data, NULL); > > + return; > > + } > > + > > + /* > > + put together the callback struct > > + */ > > + helper = cbdataAlloc(hdrHelper_StateData); > > + helper->orig_url = xstrdup(http->uri); > > + helper->client_addr = conn->log_addr; > > + if (http->request->auth_user_request) > > + helper->client_ident = > > authenticateUserRequestUsername(http->request->auth_user_request); + > > else if (conn->rfc931[0]) { > > + helper->client_ident = conn->rfc931; > > + } else { > > + helper->client_ident = dash_str; > > + } > > + helper->handler = handler; > > + helper->data = data; > > + cbdataLock(helper->data); > > + helper->req_header = reqheaders; > > + > > + /* > > + Format the buffer in preparation for sending to helper > > + */ > > + cp = buf; > > + while ((e = httpHeaderGetEntry(reqheaders, &p))){ > > + debug(66, 9) ("hdrRewriteStart: sending header > > %s:%s\n",strBuf(e->name),strBuf(e->value)); + nl = > > strlen(strBuf(e->name)); > > + vl = strlen(strBuf(e->value)); > > + assert( (cp + nl + vl + 4) < (buf + HDR_BUFSIZE) ); > > + strcpy(cp,strBuf(e->name)); cp += nl; > > + strcpy(cp,": "); cp += 2; > > + strcpy(cp,strBuf(e->value)); cp += vl; > > + strcpy(cp,"\r\n"); cp += 2; > > + } > > + /* Extra \r\n to terminate (\r\n\r\n will mark the end of athe reply > > of the rewrite-helper)*/ + strcpy(cp,"\r\n"); > > + > > + debug(66, 9) ("hdrRewriteStart: calling helperSubmit with %s\n",buf); > > + helperSubmit(hdrHelpers, buf, hdrHandleReply, helper); > > +} > > + > > +/* end of change */ > > \ No newline at end of file > > diff --exclude=.svn -Naur squid-2.6.STABLE7/src/main.c > > squid-2.6.STABLE7_PATCHED/src/main.c --- > > squid-2.6.STABLE7/src/main.c 2007-01-14 05:10:14.000000000 +1300 +++ > > squid-2.6.STABLE7_PATCHED/src/main.c 2007-01-18 12:02:31.000000000 > > +1300 > > @@ -393,6 +393,9 @@ > > #endif > > redirectShutdown(); > > locationRewriteShutdown(); > > + /* header_rewrite */ > > + hdrRewriteShutdown(); > > + /* end of change */ > > authenticateShutdown(); > > externalAclShutdown(); > > storeDirCloseSwapLogs(); > > @@ -421,6 +424,9 @@ > > idnsInit(); > > #endif > > redirectInit(); > > + /* header_rewrite */ > > + hdrRewriteInit(); > > + /* end of change */ > > locationRewriteInit(); > > authenticateInit(&Config.authConfig); > > externalAclInit(); > > @@ -456,6 +462,9 @@ > > dnsShutdown(); > > #endif > > redirectShutdown(); > > + /* header_rewrite */ > > + hdrRewriteShutdown(); > > + /* end of change */ > > locationRewriteShutdown(); > > authenticateShutdown(); > > externalAclShutdown(); > > @@ -474,6 +483,9 @@ > > dnsInit(); > > #endif > > redirectInit(); > > + /* header_rewrite */ > > + hdrRewriteInit(); > > + /* end of change */ > > locationRewriteInit(); > > authenticateInit(&Config.authConfig); > > externalAclInit(); > > @@ -571,6 +583,9 @@ > > idnsInit(); > > #endif > > redirectInit(); > > + /* header_rewrite */ > > + hdrRewriteInit(); > > + /* end of change */ > > locationRewriteInit(); > > errorMapInit(); > > authenticateInit(&Config.authConfig); > > @@ -1076,6 +1091,9 @@ > > idnsShutdown(); > > #endif > > redirectShutdown(); > > + /* header_rewrite */ > > + hdrRewriteShutdown(); > > + /* end of change */ > > externalAclShutdown(); > > locationRewriteShutdown(); > > icpConnectionClose(); > > diff --exclude=.svn -Naur squid-2.6.STABLE7/src/protos.h > > squid-2.6.STABLE7_PATCHED/src/protos.h --- > > squid-2.6.STABLE7/src/protos.h 2007-01-07 06:22:45.000000000 +1300 +++ > > squid-2.6.STABLE7_PATCHED/src/protos.h 2007-01-18 12:03:12.000000000 > > +1300 @@ -413,6 +413,11 @@ > > extern void httpHeaderPutStrf(); > > #endif > > > > +/* header_rewrite */ > > +extern void hdrRewriteInit(); > > +extern void hdrRewriteShutdown(); > > +extern void hdrRewriteStart(clientHttpRequest *,RH *,void *); > > +/* end of change */ > > > > /* Http Header */ > > extern void httpHeaderInitModule(void); > > diff --exclude=.svn -Naur squid-2.6.STABLE7/src/structs.h > > squid-2.6.STABLE7_PATCHED/src/structs.h --- > > squid-2.6.STABLE7/src/structs.h 2006-11-30 04:58:52.000000000 +1300 +++ > > squid-2.6.STABLE7_PATCHED/src/structs.h 2007-01-18 12:09:18.000000000 > > +1300 @@ -128,6 +128,11 @@ > > auth_user_hash_pointer *usernamehash; > > /* cache of acl lookups on this username */ > > dlink_list proxy_match_cache; > > + /* header_rewrite */ > > + struct { > > + unsigned int credentials_ok:2; /*0=unchecked,1=ok,2=failed*/ > > + } flags; > > + /* end of changes */ > > /* what ip addresses has this user been seen at?, plus a list length > > cache */ dlink_list ip_list; > > int ipcount; > > @@ -551,6 +556,12 @@ > > int children; > > int concurrency; > > } location_rewrite; > > + /* header_rewrite */ > > + struct { > > + wordlist *command; > > + int children; > > + } header_rewrite; > > + /* end of changes */ > > #if USE_ICMP > > char *pinger; > > #endif > > @@ -700,6 +711,9 @@ > > #endif > > acl_access *url_rewrite; > > acl_access *location_rewrite; > > + /* header_rewrite */ > > + acl_access *header_rewrite; > > + /* end of changes */ > > acl_access *reply; > > acl_address *outgoing_address; > > acl_tos *outgoing_tos; > > @@ -2288,6 +2302,9 @@ > > int n_active; > > int ipc_type; > > int concurrency; > > + /* header_rewrite */ > > + int double_cr_end; > > + /* end of changes */ > > time_t last_queue_warn; > > struct { > > int requests;
