Re: header_rewriter for 2.6s7

Cool, an interesting idea. I like it but it does introduce another round trip of
packing/unpacking into the HTTP parsing mix.

What are you/others using this functionality for at the moment?

Does this correctly handle HTTP continuations?

Adrian

On Tue, Jan 23, 2007, Martin Schoeffel wrote:
> hi henrik,
>
> here it is again.
>
> martin
>
> Am Dienstag, 23. Januar 2007 11:13 schrieb Henrik Nordstrom:
> > Martin,
> >
> > can you please send this message to squid-dev@squid-cache.org to have
> > your work properly archived and indexed.
> >
> > Regards
> > Henrik
> >
> > m?n 2007-01-22 klockan 22:07 +1300 skrev Martin Schoeffel:
> > > hi henrik,
> > >
> > > i modified the header-rewriter-patch from james beamish-white and it is
> > > now working for me with squid 2.6s7. i'm feeling a bit not so good about
> > > the hdrhandlereply-function (httpheadertools.c), but for my purpose it's
> > > okay. i attached the patch for you if you like to take a look at it.
> > >
> > > regards, martin

> diff --exclude=.svn -Naur squid-2.6.STABLE7/src/cache_cf.c squid-2.6.STABLE7_PATCHED/src/cache_cf.c
> --- squid-2.6.STABLE7/src/cache_cf.c 2007-01-09 23:24:41.000000000 +1300
> +++ squid-2.6.STABLE7_PATCHED/src/cache_cf.c 2007-01-18 11:47:51.000000000 +1300
> @@ -427,6 +427,14 @@
> wordlistDestroy(&Config.Program.location_rewrite.command);
> }
> }
> + /* header_rewrite */
> + if (Config.Program.header_rewrite.command) {
> + if (Config.Program.header_rewrite.children < 1){
> + Config.Program.header_rewrite.children = 0;
> + wordlistDestroy(&Config.Program.header_rewrite.command);
> + }
> + }
> + /* end of change */
> if (Config.appendDomain)
> if (*Config.appendDomain != '.')
> fatal("append_domain must begin with a '.'");
> @@ -470,6 +478,10 @@
> requirePathnameExists("url_rewrite_program", Config.Program.url_rewrite.command->key);
> if (Config.Program.location_rewrite.command)
> requirePathnameExists("location_rewrite_program", Config.Program.location_rewrite.command->key);
> + /* header_rewrite */
> + if (Config.Program.header_rewrite.command)
> + requirePathnameExists("header_rewrite_program",Config.Program.header_rewrite.command->key);
> + /* end of change */
> requirePathnameExists("Icon Directory", Config.icons.directory);
> requirePathnameExists("Error Directory", Config.errorDirectory);
> authenticateConfigure(&Config.authConfig);
> diff --exclude=.svn -Naur squid-2.6.STABLE7/src/cf.data.pre squid-2.6.STABLE7_PATCHED/src/cf.data.pre
> --- squid-2.6.STABLE7/src/cf.data.pre 2007-01-14 05:06:42.000000000 +1300
> +++ squid-2.6.STABLE7_PATCHED/src/cf.data.pre 2007-01-18 11:47:47.000000000 +1300
> @@ -1807,6 +1807,97 @@
> headers are sent.
> DOC_END
>
> +NAME: header_rewrite_program
> +TYPE: wordlist
> +LOC: Config.Program.header_rewrite.command
> +DEFAULT: none
> +DOC_START
> + Specify the location of the executable for the header rewriter.
> + Since they can perform almost any function there isn't one included.
> +
> + For each requested URL which matches the header_rewriter ACL, Squid
> + will send a block of header lines to the rewriter program terminated
> + by an empty line :
> +
> + header1: header1_contents
> + header2: header2_contents
> + header3: header3_contents
> + header4: header4_contents
> + \n
> +
> + And the rewriter should return modified or additional headers. Note
> + that all of the original request headers will be removed and replaced
> + by whatever the header_rewriter generates.
> +
> + By default, a header rewriter is not used.
> +DOC_END
> +
> +NAME: header_rewrite_children
> +TYPE: int
> +DEFAULT: 5
> +LOC: Config.Program.header_rewrite.children
> +DOC_START
> + The number of header rewriting processes to spawn. If you start
> + too few Squid will have to wait for them to process a backlog of
> + headers, slowing it down. If you start too many they will use RAM
> + and other system resources.
> +DOC_END
> +
> +NAME: header_rewrite_access
> +TYPE: acl_access
> +DEFAULT: none
> +LOC: Config.accessList.header_rewrite
> +DOC_START
> + If defined, this access list specifies which requests are
> + sent to the header rewriting processes. By default all requests
> + are sent.
> +DOC_END
> +
> +NAME: header_rewrite_program
> +TYPE: wordlist
> +LOC: Config.Program.header_rewrite.command
> +DEFAULT: none
> +DOC_START
> + Specify the location of the executable for the header rewriter.
> + Since they can perform almost any function there isn't one included.
> +
> + For each requested URL which matches the header_rewriter ACL, Squid
> + will send a block of header lines to the rewriter program terminated
> + by an empty line :
> +
> + header1: header1_contents
> + header2: header2_contents
> + header3: header3_contents
> + header4: header4_contents
> + \n
> +
> + And the rewriter should return modified or additional headers. Note
> + that all of the original request headers will be removed and replaced
> + by whatever the header_rewriter generates.
> +
> + By default, a header rewriter is not used.
> +DOC_END
> +
> +NAME: header_rewrite_children
> +TYPE: int
> +DEFAULT: 5
> +LOC: Config.Program.header_rewrite.children
> +DOC_START
> + The number of header rewriting processes to spawn. If you start
> + too few Squid will have to wait for them to process a backlog of
> + headers, slowing it down. If you start too many they will use RAM
> + and other system resources.
> +DOC_END
> +
> +NAME: header_rewrite_access
> +TYPE: acl_access
> +DEFAULT: none
> +LOC: Config.accessList.header_rewrite
> +DOC_START
> + If defined, this access list specifies which requests are
> + sent to the header rewriting processes. By default all requests
> + are sent.
> +DOC_END
>
> NAME: auth_param
> TYPE: authparam
> diff --exclude=.svn -Naur squid-2.6.STABLE7/src/client_side.c squid-2.6.STABLE7_PATCHED/src/client_side.c
> --- squid-2.6.STABLE7/src/client_side.c 2007-01-07 06:22:45.000000000 +1300
> +++ squid-2.6.STABLE7_PATCHED/src/client_side.c 2007-01-18 11:53:12.000000000 +1300
> @@ -664,7 +664,11 @@
> clientHttpRequest *http = data;
> http->request->flags.cachable = answer;
> http->acl_checklist = NULL;
> + /* header_rewrite
> clientProcessRequest(http);
> + */
> + hdrRewriteStart(http,(RH *)clientProcessRequest, http );
> + /* end of change */
> }
>
> static void
> diff --exclude=.svn -Naur squid-2.6.STABLE7/src/helper.c squid-2.6.STABLE7_PATCHED/src/helper.c
> --- squid-2.6.STABLE7/src/helper.c 2006-09-09 07:41:24.000000000 +1200
> +++ squid-2.6.STABLE7_PATCHED/src/helper.c 2007-01-18 11:53:08.000000000 +1300
> @@ -742,7 +742,13 @@
> srv->roffset = 0;
> srv->rbuf[0] = '\0';
> }
> + /* header_rewrite
> while ((t = strchr(srv->rbuf, '\n'))) {
> + check if this helper has double cr_end else go on as usual
> + */
> + while ( (!hlp->double_cr_end && (t = strchr(srv->rbuf, '\n'))) ||
> + (hlp->double_cr_end && (t = strstr(srv->rbuf, "\r\n\r\n"))) ) {
> + /* end of change */
> helper_request *r;
> char *msg = srv->rbuf;
> int i = 0;
> diff --exclude=.svn -Naur squid-2.6.STABLE7/src/HttpHeaderTools.c squid-2.6.STABLE7_PATCHED/src/HttpHeaderTools.c
> --- squid-2.6.STABLE7/src/HttpHeaderTools.c 2006-07-27 08:09:33.000000000 +1200
> +++ squid-2.6.STABLE7_PATCHED/src/HttpHeaderTools.c 2007-01-18 11:57:54.000000000 +1300
> @@ -486,3 +486,246 @@
> if (removed_headers)
> httpHeaderRefreshMask(l);
> }
> +
> +/* header_rewriter
> + * start of header rewriter helper code
> + */
> +
> +/* Size of buffer to hold all header contents */
> +#define HDR_BUFSIZE 8192
> +
> +typedef struct {
> + void *data;
> + char *orig_url; /* copy of original URL */
> + struct in_addr client_addr;
> + const char *client_ident;
> + RH *handler;
> + HttpHeader *req_header; /* Pointer to request headers */
> +} hdrHelper_StateData;
> +
> +static HLPCB hdrHandleReply;
> +static helper *hdrHelpers = NULL;
> +static OBJH hdrHelperStats;
> +static int hdrHelper_n_bypassed = 0;
> +CBDATA_TYPE(hdrHelper_StateData);
> +
> +static void
> +hdrStateFree(hdrHelper_StateData * r)
> +{
> + safe_free(r->orig_url);
> + cbdataFree(r);
> +}
> +
> +/*
> + hdrHandleReply
> + function which handles the replies from the header rewriter helpers
> +
> + read the reply from the handler, and add it to the rewritten_headers
> + in new_headers array
> + if the reply is END, then :
> + - remove the headers from the request
> + - copy the rewritten headers into the request
> + - call the handler ( ie, return )
> +*/
> +static void
> +hdrHandleReply(void *data, char *reply)
> +{
> + hdrHelper_StateData *r = data;
> + int valid;
> + char *pos;
> + char *pos2;
> + char *msg_tmp;
> +
> + debug(66, 9) ("hdrHandleReply: {%s}\n", reply ? reply : "<NULL>");
> + if (reply) {
> + debug(66, 9) ("hdrHandleReply: doing header parse on hdr %p\n",r->req_header);
> + /* delete old header */
> + httpHeaderReset( r->req_header );
> + /* parse the reply in separate lines and insert each line into header entries */
> + msg_tmp = reply;
> + while(pos=strstr(msg_tmp,"\r\n")) {
> + if(pos-msg_tmp>2) {
> + /* end of line */
> + strncpy (pos,"\0\0",2);
> + /* delimiter */
> + pos2=strstr(msg_tmp,":");
> + strncpy (pos2,"\0",1);
> + /* avoid duplication - HDR_HOST should not be overwritten by the header_rewriter-helper */
> + if(!strstr(msg_tmp,httpHeaderNameById(HDR_HOST))) {
> + httpHeaderPutExt(r->req_header,msg_tmp,++pos2);
> + }
> + /* goto next line */
> + msg_tmp=pos+2;
> + } else {
> + if(strlen(pos)>0) {
> + /* line was empty - goto next line */
> + msg_tmp=pos+2;
> + } else {
> + /* no more to process - just give up */
> + break;
> + }
> + }
> + } /* while */
> + } /* if reply */
> + /* End of helper request, call the callback function */
> + valid = cbdataValid(r->data);
> + cbdataUnlock(r->data);
> + debug(66, 9) ("hdrHandleReply: rewrite complete, calling handler\n");
> + if (valid)
> + r->handler(r->data, reply);
> + hdrStateFree(r);
> +}
> +
> +static void
> +hdrHelperStats(StoreEntry * sentry)
> +{
> + storeAppendPrintf(sentry, "Header Rewriter Statistics:\n");
> + helperStats(sentry, hdrHelpers);
> + storeAppendPrintf(sentry, "\nNumber of requests bypassed "
> + "because all rewriters were busy: %d\n", hdrHelper_n_bypassed);
> +}
> +
> +/*
> + public function hdrRewriteInit()
> + Startup the header rewriter helpers ( if configured )
> +*/
> +void
> +hdrRewriteInit(void) {
> + static int init = 0;
> + debug(66,1) ("hdrRewriteInit:\n");
> + if (!Config.Program.header_rewrite.command){
> + debug(66,1) ("hdrRewriteInit: quitting - no header_rewrite command\n");
> + return;
> + }
> + if (hdrHelpers == NULL){
> + debug(66,1) ("hdrRewriteInit: creating helpers\n");
> + hdrHelpers = helperCreate("header_rewriter");
> + }
> + hdrHelpers->cmdline = Config.Program.header_rewrite.command;
> + hdrHelpers->n_to_start = Config.Program.header_rewrite.children;
> + hdrHelpers->ipc_type = IPC_TCP_SOCKET;
> + hdrHelpers->double_cr_end = 1;
> + helperOpenServers(hdrHelpers);
> + if (!init) {
> + cachemgrRegister("header_rewriter",
> + "Header Rewriter Stats",
> + hdrHelperStats, 0, 1);
> + init = 1;
> + CBDATA_INIT_TYPE(hdrHelper_StateData);
> + }
> + debug(66,1) ("hdrRewriteInit: done\n");
> +}
> +
> +
> +/*
> + public function hdrRewriteShutdown()
> + Shutdown the header rewriter helpers ( if configured )
> +*/
> +void
> +hdrRewriteShutdown() {
> + debug(66,1) ("hdrRewriteShutdown: starting\n");
> + if (!hdrHelpers)
> + return;
> + helperShutdown(hdrHelpers);
> + if (!shutting_down)
> + return;
> + helperFree(hdrHelpers);
> + hdrHelpers = NULL;
> + debug(66,1) ("hdrRewriteShutdown: done\n");
> +}
> +
> +
> +/*
> + public function hdrRewriteStart()
> + Call the header rewriter helpers
> +
> +
> + *** This needs serious fixing ***
> +
> +*/
> +void
> +hdrRewriteStart(clientHttpRequest * http, RH * handler, void *data)
> +{
> + ConnStateData *conn = http->conn;
> + hdrHelper_StateData *helper = NULL;
> + char buf[HDR_BUFSIZE], *cp;
> + HttpHeader *reqheaders = &http->request->header;
> + HttpHeaderEntry *e;
> + HttpHeaderPos p = HttpHeaderInitPos;
> + int nl, vl;
> +
> + assert(http);
> + assert(handler);
> + debug(66, 9) ("hdrRewriteStart: '%s'\n", http->uri);
> +
> + if (Config.Program.header_rewrite.command == NULL) {
> + /* No header rewriting command has been defined, so just return by calling the handler
> + */
> + debug(66, 9) ("hdrRewriteStart: finishing because header_rewrite command not defined\n");
> + handler(data, NULL);
> + return;
> + }
> +
> + if (Config.accessList.header_rewrite) {
> + aclCheck_t ch;
> + memset(&ch, '\0', sizeof(ch));
> + ch.src_addr = http->conn->peer.sin_addr;
> + ch.my_addr = http->conn->me.sin_addr;
> + ch.my_port = ntohs(http->conn->me.sin_port);
> + ch.request = http->request;
> + if (!aclCheckFast(Config.accessList.header_rewrite, &ch)) {
> + /* denied -- bypass redirector */
> + debug(66, 9) ("hdrRewriteStart: finishing because request not in ACL\n");
> + handler(data, NULL);
> + return;
> + }
> + }
> +
> + if (hdrHelpers->stats.queue_size) {
> + /* Skip helper if there is one request queued */
> + hdrHelper_n_bypassed++;
> + debug(66, 9) ("hdrRewriteStart: finishing because too many requests queued\n");
> + handler(data, NULL);
> + return;
> + }
> +
> + /*
> + put together the callback struct
> + */
> + helper = cbdataAlloc(hdrHelper_StateData);
> + helper->orig_url = xstrdup(http->uri);
> + helper->client_addr = conn->log_addr;
> + if (http->request->auth_user_request)
> + helper->client_ident = authenticateUserRequestUsername(http->request->auth_user_request);
> + else if (conn->rfc931[0]) {
> + helper->client_ident = conn->rfc931;
> + } else {
> + helper->client_ident = dash_str;
> + }
> + helper->handler = handler;
> + helper->data = data;
> + cbdataLock(helper->data);
> + helper->req_header = reqheaders;
> +
> + /*
> + Format the buffer in preparation for sending to helper
> + */
> + cp = buf;
> + while ((e = httpHeaderGetEntry(reqheaders, &p))){
> + debug(66, 9) ("hdrRewriteStart: sending header %s:%s\n",strBuf(e->name),strBuf(e->value));
> + nl = strlen(strBuf(e->name));
> + vl = strlen(strBuf(e->value));
> + assert( (cp + nl + vl + 4) < (buf + HDR_BUFSIZE) );
> + strcpy(cp,strBuf(e->name)); cp += nl;
> + strcpy(cp,": "); cp += 2;
> + strcpy(cp,strBuf(e->value)); cp += vl;
> + strcpy(cp,"\r\n"); cp += 2;
> + }
> + /* Extra \r\n to terminate (\r\n\r\n will mark the end of athe reply of the rewrite-helper)*/
> + strcpy(cp,"\r\n");
> +
> + debug(66, 9) ("hdrRewriteStart: calling helperSubmit with %s\n",buf);
> + helperSubmit(hdrHelpers, buf, hdrHandleReply, helper);
> +}
> +
> +/* end of change */
> \ No newline at end of file
> diff --exclude=.svn -Naur squid-2.6.STABLE7/src/main.c squid-2.6.STABLE7_PATCHED/src/main.c
> --- squid-2.6.STABLE7/src/main.c 2007-01-14 05:10:14.000000000 +1300
> +++ squid-2.6.STABLE7_PATCHED/src/main.c 2007-01-18 12:02:31.000000000 +1300
> @@ -393,6 +393,9 @@
> #endif
> redirectShutdown();
> locationRewriteShutdown();
> + /* header_rewrite */
> + hdrRewriteShutdown();
> + /* end of change */
> authenticateShutdown();
> externalAclShutdown();
> storeDirCloseSwapLogs();
> @@ -421,6 +424,9 @@
> idnsInit();
> #endif
> redirectInit();
> + /* header_rewrite */
> + hdrRewriteInit();
> + /* end of change */
> locationRewriteInit();
> authenticateInit(&Config.authConfig);
> externalAclInit();
> @@ -456,6 +462,9 @@
> dnsShutdown();
> #endif
> redirectShutdown();
> + /* header_rewrite */
> + hdrRewriteShutdown();
> + /* end of change */
> locationRewriteShutdown();
> authenticateShutdown();
> externalAclShutdown();
> @@ -474,6 +483,9 @@
> dnsInit();
> #endif
> redirectInit();
> + /* header_rewrite */
> + hdrRewriteInit();
> + /* end of change */
> locationRewriteInit();
> authenticateInit(&Config.authConfig);
> externalAclInit();
> @@ -571,6 +583,9 @@
> idnsInit();
> #endif
> redirectInit();
> + /* header_rewrite */
> + hdrRewriteInit();
> + /* end of change */
> locationRewriteInit();
> errorMapInit();
> authenticateInit(&Config.authConfig);
> @@ -1076,6 +1091,9 @@
> idnsShutdown();
> #endif
> redirectShutdown();
> + /* header_rewrite */
> + hdrRewriteShutdown();
> + /* end of change */
> externalAclShutdown();
> locationRewriteShutdown();
> icpConnectionClose();
> diff --exclude=.svn -Naur squid-2.6.STABLE7/src/protos.h squid-2.6.STABLE7_PATCHED/src/protos.h
> --- squid-2.6.STABLE7/src/protos.h 2007-01-07 06:22:45.000000000 +1300
> +++ squid-2.6.STABLE7_PATCHED/src/protos.h 2007-01-18 12:03:12.000000000 +1300
> @@ -413,6 +413,11 @@
> extern void httpHeaderPutStrf();
> #endif
>
> +/* header_rewrite */
> +extern void hdrRewriteInit();
> +extern void hdrRewriteShutdown();
> +extern void hdrRewriteStart(clientHttpRequest *,RH *,void *);
> +/* end of change */
>
> /* Http Header */
> extern void httpHeaderInitModule(void);
> diff --exclude=.svn -Naur squid-2.6.STABLE7/src/structs.h squid-2.6.STABLE7_PATCHED/src/structs.h
> --- squid-2.6.STABLE7/src/structs.h 2006-11-30 04:58:52.000000000 +1300
> +++ squid-2.6.STABLE7_PATCHED/src/structs.h 2007-01-18 12:09:18.000000000 +1300
> @@ -128,6 +128,11 @@
> auth_user_hash_pointer *usernamehash;
> /* cache of acl lookups on this username */
> dlink_list proxy_match_cache;
> + /* header_rewrite */
> + struct {
> + unsigned int credentials_ok:2; /*0=unchecked,1=ok,2=failed*/
> + } flags;
> + /* end of changes */
> /* what ip addresses has this user been seen at?, plus a list length cache */
> dlink_list ip_list;
> int ipcount;
> @@ -551,6 +556,12 @@
> int children;
> int concurrency;
> } location_rewrite;
> + /* header_rewrite */
> + struct {
> + wordlist *command;
> + int children;
> + } header_rewrite;
> + /* end of changes */
> #if USE_ICMP
> char *pinger;
> #endif
> @@ -700,6 +711,9 @@
> #endif
> acl_access *url_rewrite;
> acl_access *location_rewrite;
> + /* header_rewrite */
> + acl_access *header_rewrite;
> + /* end of changes */
> acl_access *reply;
> acl_address *outgoing_address;
> acl_tos *outgoing_tos;
> @@ -2288,6 +2302,9 @@
> int n_active;
> int ipc_type;
> int concurrency;
> + /* header_rewrite */
> + int double_cr_end;
> + /* end of changes */
> time_t last_queue_warn;
> struct {
> int requests;

-- 
- Xenion - http://www.xenion.com.au/ - VPS Hosting - Commercial Squid Support -
- $25/pm entry-level bandwidth-capped VPSes available in WA -