Figured some people will find this useful, until varnishncsa gets full
LogFormat support.
I've expanded Emil Isberg's initial path on the following items:
List of what I added/fixed:
- Fixed segfault in %s,%>s, when these where not found in shmlog
- Fixed segfault in %{VarnishR}i, when df_H, Protocol version, was not
found, http defaults to HTTP/1.0
- Added %D, ( proc_time + xmit_time ) in microseconds
- Added $T, same as above, but in seconds
- Added %V, faked as %{Host}i ( different meaning in apache, but usually
meant as vhost )
- Added %{Age}i, age of object being served. Quite useful, as you can graph
things like average age of served pages, cache hit ratios, etc etc.
I attach a diff against stock 2.0.4 varnish, I've tested this against
several multi-gigabyte varnish native logs, as well as have it run for a few
days now, with no issues.
-Javier
--- varnishncsa.c 2009-10-26 01:37:52.000000000 -0400
+++ varnishncsa.c.orig 2009-10-26 00:24:03.000000000 -0400
@@ -27,7 +27,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $Id: varnishncsa.c 4003 2009-03-27 12:13:31Z tfheen $
+ * $Id$
*
* Obtain log data from the shared memory log, order it by session ID, and
* display it in Apache / NCSA combined log format:
@@ -97,14 +97,11 @@
char *df_s; /* %s, Status */
struct tm df_t; /* %t, Date and time */
char *df_u; /* %u, Remote user */
- int df_D; /* %D, The time taken to serve the request, in microseconds */
- char *df_Age; /* %{Age}i Age of served object */
int bogus; /* bogus request */
} **ll;
static size_t nll;
static int prefer_x_forwarded_for = 0;
-static const char *L_fmt = "%h %l %u %t \"%{VarnishR}i\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"";
static int
isprefix(const char *str, const char *prefix, const char *end,
@@ -270,7 +267,6 @@
{
const char *end, *next;
long l;
- double idle_t,proc_t,xmit_t;
time_t t;
assert(spec & VSL_S_CLIENT);
@@ -310,12 +306,6 @@
lp->bogus = 1;
else
lp->df_s = trimline(ptr, end);
-
- break;
-
- case SLT_TxHeader:
- if (isprefix(ptr, "age:", end, &next))
- lp->df_Age = trimline(next, end);
break;
case SLT_RxHeader:
@@ -346,11 +336,10 @@
break;
case SLT_ReqEnd:
- if (sscanf(ptr, "%*u %*u.%*u %ld.%*u %lf %lf %lf", &l, &idle_t, &proc_t, &xmit_t) != 4) {
+ if (sscanf(ptr, "%*u %*u.%*u %ld.", &l) != 1) {
lp->bogus = 1;
} else {
t = l;
- lp->df_D= (proc_t + xmit_t) * 1000000;
localtime_r(&t, &lp->df_t);
}
/* got it all */
@@ -364,208 +353,13 @@
return (1);
}
-/*
- * print stuff according to fmt
- * try to be as much compliant with apache as possible
- * http://httpd.apache.org/docs/2.0/mod/mod_log_config.html#formats
- *
- * Note that return value is used to step a pointer
- * Must not return anything less than 1
- */
-static int
-fmt_print(FILE *fo, const char *fmt, unsigned spec, struct logline *lp)
-{
- char *q, tbuf[64];
- if (!strncmp(fmt, "%%", 2)) { /* %% */
- fputc('%', fo);
- return 2;
- }
-
- if (!strncmp(fmt, "%h", 2)) { /* %h */
- if (!lp->df_h && spec & VSL_S_BACKEND)
- fprintf(fo, "127.0.0.1");
- else if (lp->df_X_Forwarded_For && prefer_x_forwarded_for)
- fprintf(fo, "%s", lp->df_X_Forwarded_For);
- else
- fprintf(fo, "%s", lp->df_h ? lp->df_h : "-");
- return 2;
- }
-
- if (!strncmp(fmt, "%l", 2)) { /* %l */
- fprintf(fo, "-");
- return 2;
- }
-
- if (!strncmp(fmt, "%u", 2)) { /* %u: decode authorization string */
- if (lp->df_u != NULL) {
- char *rubuf;
- size_t rulen;
-
- base64_init();
- rulen = ((strlen(lp->df_u) + 3) * 4) / 3;
- rubuf = malloc(rulen);
- assert(rubuf != NULL);
- base64_decode(rubuf, rulen, lp->df_u);
- q = strchr(rubuf, ':');
- if (q != NULL)
- *q = '\0';
- fprintf(fo, "%s", rubuf);
- free(rubuf);
- } else {
- fprintf(fo, "-");
- }
- return 2;
- }
-
- if (!strncmp(fmt, "%t", 2)) { /* %t */
- strftime(tbuf, sizeof tbuf, "[%d/%b/%Y:%T %z]", &lp->df_t);
- fprintf(fo, "%s", tbuf);
- return 2;
- }
-
- if (!strncmp(fmt, "%{VarnishR}i", 12)) { /* %{VarnishR}i */
- /*
- * Fake "%r". This would be a lot easier if Varnish
- * normalized the request URL.
- */
- fprintf(fo, "%s ", lp->df_m);
- if (lp->df_Host) {
- if (strncmp(lp->df_Host, "http://";, 7) != 0)
- fprintf(fo, "http://";);
- fprintf(fo, "%s", lp->df_Host);
- }
- fprintf(fo, "%s ", lp->df_Uq);
- fprintf(fo, "%s", lp->df_H ? lp->df_H : "HTTP/1.0");
- return 12;
- }
-
- if (!strncmp(fmt, "%r", 2)) { /* %r */
- fprintf(fo, "%s %s %s", lp->df_m, lp->df_Uq, lp->df_H);
- return 2;
- }
-
- if (!strncmp(fmt, "%{Host}i", 8)) { /* %{Host}i */
- fprintf(fo, "%s", lp->df_Host ? lp->df_Host : "-");
- return 8;
- }
-
- if (!strncmp(fmt, "%V", 2)) { /* %V */
- fprintf(fo, "%s", lp->df_Host ? lp->df_Host : "-");
- return 2;
- }
-
-
- if (!strncmp(fmt, "%>s", 3)) { /* %>s should be the same as %s */
- fprintf(fo, "%s", lp->df_s ? lp->df_s : "-" );
- return 3;
- }
-
- if (!strncmp(fmt, "%s", 2)) { /* %s */
- fprintf(fo, "%s", lp->df_s ? lp->df_s : "-");
- return 2;
- }
-
- if (!strncmp(fmt, "%B", 2)) { /* %B */
- fprintf(fo, "%s", lp->df_b ? lp->df_b : "0");
- return 2;
- }
-
- if (!strncmp(fmt, "%b", 2)) { /* %b */
- fprintf(fo, "%s", lp->df_b ? lp->df_b : "-");
- return 2;
- }
-
- if (!strncmp(fmt, "%{Referer}i", 11)) { /* %{Referer}i */
- fprintf(fo, "%s",
- lp->df_Referer ? lp->df_Referer : "-");
- return 11;
- }
-
- if (!strncmp(fmt, "%{User-agent}i", 14)) { /* %{User-agent}i */
- fprintf(fo, "%s",
- lp->df_User_agent ? lp->df_User_agent : "-");
- return 14;
- }
-
- if (!strncmp(fmt, "%D", 2)) { /* %D : The time taken to serve the request, in microseconds. */
- if ( lp->df_D > 0 ) {
- fprintf(fo, "%i", lp->df_D);
- } else {
- fputc('-',fo);
- }
- return 2;
- }
-
- if (!strncmp(fmt, "%T", 2)) { /* %T : The time taken to serve the request, in seconds. */
- if ( lp->df_D > 0 ) {
- fprintf(fo, "%.2f", (float)lp->df_D/1000000);
- } else {
- fputc('-',fo);
- }
- return 2;
- }
-
-
- if (!strncmp(fmt, "%{Age}i", 7)) { /* %{Age}i: Age of object being served */
- fprintf(fo, "%s",
- lp->df_Age ? lp->df_Age : "-");
- return 7;
- }
-
- if (!strncmp(fmt, "%X", 2)) { /* %X : Connection status when response is completed: X,+,- */
- fputc('-',fo); /* TODO */
- return 2;
- }
-
- if (!strncmp(fmt, "%{Cookie}i", 10)) { /* %{Cookie}i : Cookie value as the client specifies it */
- fputc('-',fo); /* TODO */
- return 10;
- }
-
- if (!strncmp(fmt, "%{Set-Cookie}o", 14)) { /* %{Set-Cookie}o : Cookie value as we send to the cookie */
- fputc('-',fo); /* TODO */
- return 14;
- }
-
- /* fallback */
- fputc(*fmt, fo);
- return 1; /* never return anything less than 1 */
-}
-
-/*
- * Handle the backslash-escaped characters.
- * Try to confirm to basic printf syntax
- */
-static int
-bsl_print(FILE *fo, const char *fmt)
-{
- fmt++;
- switch(*fmt) {
- case '\\': fputc('\\', fo); return 2;break;
- case 'a': fputc('\a', fo); return 2;break;
- case 'b': fputc('\b', fo); return 2;break;
- case 'f': fputc('\f', fo); return 2;break;
- case 'n': fputc('\n', fo); return 2;break;
- case 'r': fputc('\r', fo); return 2;break;
- case 't': fputc('\t', fo); return 2;break;
- case 'v': fputc('\v', fo); return 2;break;
- case '"': fputc('"', fo); return 2;break;
- case '\'': fputc('\'', fo); return 2; break;
- case 'c': /* break parsing */ return strlen(fmt);break;
- }
- /* TODO: implement support for '\ddd', '\0ddd', '\xdd' */
-
- /* nothing match, output what we got */
- fputc('\\', fo);
- return 1; /* never return anything less than 1 */
-}
-
static int
h_ncsa(void *priv, enum shmlogtag tag, unsigned fd,
unsigned len, unsigned spec, const char *ptr)
{
struct logline *lp;
FILE *fo = priv;
+ char *q, tbuf[64];
if (fd >= nll) {
struct logline **newll = ll;
@@ -605,16 +399,66 @@
if (!lp->bogus) {
fo = priv;
- ptr = L_fmt;
- while (ptr && *ptr) {
- switch(*ptr) {
- printf("string = %s", ptr);
- case '%': ptr += fmt_print(fo, ptr, spec, lp);break;
- case '\\': ptr += bsl_print(fo, ptr);break;
- default: fputc(*ptr, fo);ptr++;break;
- }
+ /* %h */
+ if (!lp->df_h && spec & VSL_S_BACKEND)
+ fprintf(fo, "127.0.0.1 ");
+ else if (lp->df_X_Forwarded_For && prefer_x_forwarded_for)
+ fprintf(fo, "%s ", lp->df_X_Forwarded_For);
+ else
+ fprintf(fo, "%s ", lp->df_h ? lp->df_h : "-");
+
+ /* %l */
+ fprintf(fo, "- ");
+
+ /* %u: decode authorization string */
+ if (lp->df_u != NULL) {
+ char *rubuf;
+ size_t rulen;
+
+ base64_init();
+ rulen = ((strlen(lp->df_u) + 3) * 4) / 3;
+ rubuf = malloc(rulen);
+ assert(rubuf != NULL);
+ base64_decode(rubuf, rulen, lp->df_u);
+ q = strchr(rubuf, ':');
+ if (q != NULL)
+ *q = '\0';
+ fprintf(fo, "%s ", rubuf);
+ free(rubuf);
+ } else {
+ fprintf(fo, "- ");
+ }
+
+ /* %t */
+ strftime(tbuf, sizeof tbuf, "[%d/%b/%Y:%T %z]", &lp->df_t);
+ fprintf(fo, "%s ", tbuf);
+
+ /*
+ * Fake "%r". This would be a lot easier if Varnish
+ * normalized the request URL.
+ */
+ fprintf(fo, "\"%s ", lp->df_m);
+ if (lp->df_Host) {
+ if (strncmp(lp->df_Host, "http://";, 7) != 0)
+ fprintf(fo, "http://";);
+ fprintf(fo, "%s", lp->df_Host);
}
- fputc('\n', fo); /* end of line */
+ fprintf(fo, "%s ", lp->df_Uq);
+ fprintf(fo, "%s\" ", lp->df_H);
+
+ /* %s */
+ fprintf(fo, "%s ", lp->df_s);
+
+ /* %b */
+ fprintf(fo, "%s ", lp->df_b ? lp->df_b : "-");
+
+ /* %{Referer}i */
+ fprintf(fo, "\"%s\" ",
+ lp->df_Referer ? lp->df_Referer : "-");
+
+ /* %{User-agent}i */
+ fprintf(fo, "\"%s\"\n",
+ lp->df_User_agent ? lp->df_User_agent : "-");
/* flush the stream */
fflush(fo);
@@ -687,7 +531,7 @@
vd = VSL_New();
- while ((c = getopt(argc, argv, VSL_ARGS "aDn:P:Vw:fL:")) != -1) {
+ while ((c = getopt(argc, argv, VSL_ARGS "aDn:P:Vw:f")) != -1) {
switch (c) {
case 'a':
a_flag = 1;
@@ -710,10 +554,6 @@
case 'w':
w_arg = optarg;
break;
- case 'L':
- L_fmt = optarg;
- /* TODO: should check that the format is usable */
- break;
default:
if (VSL_Arg(vd, c, optarg) > 0)
break;
_______________________________________________
varnish-dev mailing list
[email protected]
http://projects.linpro.no/mailman/listinfo/varnish-dev
