This patch series implements global lists of cached files; and call-private pointers to the cached files. The diffs are against SVN revision 5639. -- Regards,
Sanjoy Das http://playingwithpointers.com http://playingwithpointers.com/custom/public_key.txt
Index: varnish/varnish-cache/lib/libvmod_std/vmod_std_fileread.c
===================================================================
--- varnish.orig/varnish-cache/lib/libvmod_std/vmod_std_fileread.c 2010-12-01 14:06:32.000000000 +0530
+++ varnish/varnish-cache/lib/libvmod_std/vmod_std_fileread.c 2010-12-01 14:06:56.000000000 +0530
@@ -52,7 +52,7 @@
struct cached_file_list {
unsigned magic;
#define CACHED_FILE_LIST_MAGIC 0x1e4fc24f
- VSLIST_HEAD(, cached_file) files;
+ VLIST_HEAD(, cached_file) files;
pthread_rwlock_t lock;
int update_counter;
};
@@ -71,7 +71,7 @@
char *contents;
time_t last_modification;
off_t file_sz;
- VSLIST_ENTRY(cached_file) next;
+ VLIST_ENTRY(cached_file) next;
};
static void
@@ -79,7 +79,7 @@
{
struct cached_file *iter, *tmp;
struct cached_file_list *list = file_list;
- VSLIST_FOREACH_SAFE(iter, &list->files, next, tmp) {
+ VLIST_FOREACH_SAFE(iter, &list->files, next, tmp) {
CHECK_OBJ(iter, CACHED_FILE_MAGIC);
free(iter->file_name);
free(iter->contents);
@@ -102,7 +102,7 @@
AZ(pthread_rwlock_rdlock(&list->lock));
- VSLIST_FOREACH(iter, &list->files, next) {
+ VLIST_FOREACH(iter, &list->files, next) {
CHECK_OBJ(iter, CACHED_FILE_MAGIC);
if (!strcmp(iter->file_name, file_name)) {
/* This thread was holding a read lock. */
@@ -129,7 +129,7 @@
* Small optimization: search through the linked list again
* only if something has been changed.
*/
- VSLIST_FOREACH(iter, &list->files, next) {
+ VLIST_FOREACH(iter, &list->files, next) {
CHECK_OBJ(iter, CACHED_FILE_MAGIC);
if (!strcmp(iter->file_name, file_name)) {
/* This thread was holding a write lock. */
@@ -153,7 +153,7 @@
iter->contents[iter->file_sz] = '\0';
- VSLIST_INSERT_HEAD(&list->files, iter, next);
+ VLIST_INSERT_HEAD(&list->files, iter, next);
list->update_counter++;
@@ -172,7 +172,7 @@
list->update_counter = 0;
AZ(pthread_rwlock_init(&list->lock, NULL));
- VSLIST_FIRST(&list->files) = NULL;
+ VLIST_FIRST(&list->files) = NULL;
return list;
}
Index: varnish/varnish-cache/lib/libvmod_std/vmod.vcc
===================================================================
--- varnish.orig/varnish-cache/lib/libvmod_std/vmod.vcc 2010-12-01 13:06:37.000000000 +0530
+++ varnish/varnish-cache/lib/libvmod_std/vmod.vcc 2010-12-01 13:27:16.000000000 +0530
@@ -33,4 +33,4 @@
Function REAL random(REAL, REAL)
Function VOID log(STRING_LIST)
Function VOID syslog(INT, STRING_LIST)
-Function STRING fileread(PRIV_CALL, STRING)
+Function STRING fileread(PRIV_VCL, STRING)
Index: varnish/varnish-cache/lib/libvmod_std/vmod_std_fileread.c
===================================================================
--- varnish.orig/varnish-cache/lib/libvmod_std/vmod_std_fileread.c 2010-12-01 13:10:56.000000000 +0530
+++ varnish/varnish-cache/lib/libvmod_std/vmod_std_fileread.c 2010-12-01 13:27:42.000000000 +0530
@@ -49,7 +49,20 @@
#include "vcc_if.h"
#include "vcl_priv.h"
-VSLIST_HEAD(cached_file_list, cached_file);
+struct cached_file_list {
+ unsigned magic;
+#define CACHED_FILE_LIST_MAGIC 0x1e4fc24f
+ VSLIST_HEAD(, cached_file) files;
+ pthread_rwlock_t lock;
+ int update_counter;
+};
+
+#define GET_FILE_LIST(list, ptr) do { \
+ struct std_vcl_priv *vcl_priv = (ptr); \
+ CHECK_OBJ_NOTNULL(vcl_priv, STD_VCL_PRIV_MAGIC); \
+ (list) = vcl_priv->fileread_data; \
+ CHECK_OBJ_NOTNULL((list), CACHED_FILE_LIST_MAGIC); \
+ } while (0)
struct cached_file {
unsigned magic;
@@ -66,7 +79,7 @@
{
struct cached_file *iter, *tmp;
struct cached_file_list *list = file_list;
- VSLIST_FOREACH_SAFE(iter, list, next, tmp) {
+ VSLIST_FOREACH_SAFE(iter, &list->files, next, tmp) {
CHECK_OBJ(iter, CACHED_FILE_MAGIC);
free(iter->file_name);
free(iter->contents);
@@ -75,72 +88,52 @@
free(file_list);
}
-static pthread_rwlock_t filelist_lock = PTHREAD_RWLOCK_INITIALIZER;
-static int filelist_update = 0;
-
const char *
vmod_fileread(struct sess *sp, struct vmod_priv *priv, const char *file_name)
{
struct cached_file *iter = NULL;
struct stat buf;
struct cached_file_list *list;
- int fd, my_filelist_update;
+ int fd, my_counter;
(void)sp;
- AZ(pthread_rwlock_rdlock(&filelist_lock));
+ GET_FILE_LIST(list, priv->priv);
- if (priv->free == NULL) {
- AZ(pthread_rwlock_unlock(&filelist_lock));
- /*
- * Another thread may already have initialized priv
- * here, making the repeat check necessary.
- */
- AZ(pthread_rwlock_wrlock(&filelist_lock));
- if (priv->free == NULL) {
- priv->free = free_cached_files;
- priv->priv = malloc(sizeof(struct cached_file_list));
- AN(priv->priv);
- list = priv->priv;
- VSLIST_INIT(list);
- }
- AZ(pthread_rwlock_unlock(&filelist_lock));
- AZ(pthread_rwlock_rdlock(&filelist_lock));
- } else {
- list = priv->priv;
- VSLIST_FOREACH(iter, list, next) {
- CHECK_OBJ(iter, CACHED_FILE_MAGIC);
- if (strcmp(iter->file_name, file_name) == 0) {
- /* This thread was holding a read lock. */
- AZ(pthread_rwlock_unlock(&filelist_lock));
- return iter->contents;
- }
+ AZ(pthread_rwlock_rdlock(&list->lock));
+
+ VSLIST_FOREACH(iter, &list->files, next) {
+ CHECK_OBJ(iter, CACHED_FILE_MAGIC);
+ if (!strcmp(iter->file_name, file_name)) {
+ /* This thread was holding a read lock. */
+ AZ(pthread_rwlock_unlock(&list->lock));
+ return iter->contents;
}
}
- my_filelist_update = filelist_update;
+ my_counter = list->update_counter;
/* This thread was holding a read lock. */
- AZ(pthread_rwlock_unlock(&filelist_lock));
+ AZ(pthread_rwlock_unlock(&list->lock));
if ((fd = open(file_name, O_RDONLY)) == -1)
return "";
fstat(fd, &buf);
- AZ(pthread_rwlock_wrlock(&filelist_lock));
+ AZ(pthread_rwlock_wrlock(&list->lock));
- if (my_filelist_update != filelist_update) {
+ if (my_counter != list->update_counter) {
/*
* Small optimization: search through the linked list again
* only if something has been changed.
*/
- VSLIST_FOREACH(iter, list, next) {
+ VSLIST_FOREACH(iter, &list->files, next) {
CHECK_OBJ(iter, CACHED_FILE_MAGIC);
- if (strcmp(iter->file_name, file_name) == 0) {
+ if (!strcmp(iter->file_name, file_name)) {
/* This thread was holding a write lock. */
- AZ(pthread_rwlock_unlock(&filelist_lock));
+ AZ(pthread_rwlock_unlock(&list->lock));
return iter->contents;
}
}
@@ -160,23 +153,37 @@
iter->contents[iter->file_sz] = '\0';
- VSLIST_INSERT_HEAD(list, iter, next);
+ VSLIST_INSERT_HEAD(&list->files, iter, next);
- filelist_update++;
+ list->update_counter++;
/* This thread was holding a write lock. */
- AZ(pthread_rwlock_unlock(&filelist_lock));
+ AZ(pthread_rwlock_unlock(&list->lock));
return iter->contents;
}
void *
vmod_create_fileread_priv(void)
{
- return NULL;
+ struct cached_file_list *list;
+
+ ALLOC_OBJ(list, CACHED_FILE_LIST_MAGIC);
+ AN(list);
+
+ list->update_counter = 0;
+ AZ(pthread_rwlock_init(&list->lock, NULL));
+ VSLIST_FIRST(&list->files) = NULL;
+
+ return list;
}
void
vmod_release_fileread_priv(void *data)
{
- AZ(data);
+ struct cached_file_list *list = data;
+
+ CHECK_OBJ_NOTNULL(list, CACHED_FILE_LIST_MAGIC);
+ pthread_rwlock_destroy(&list->lock);
+ free_cached_files(&list->files);
+ free(list);
}
Index: varnish/varnish-cache/lib/libvmod_std/vmod.vcc
===================================================================
--- varnish.orig/varnish-cache/lib/libvmod_std/vmod.vcc 2010-12-01 15:14:20.000000000 +0530
+++ varnish/varnish-cache/lib/libvmod_std/vmod.vcc 2010-12-01 20:04:26.000000000 +0530
@@ -33,4 +33,4 @@
Function REAL random(REAL, REAL)
Function VOID log(STRING_LIST)
Function VOID syslog(INT, STRING_LIST)
-Function STRING fileread(PRIV_VCL, STRING)
+Function STRING fileread(PRIV_VCL, PRIV_CALL, STRING)
Index: varnish/varnish-cache/lib/libvmod_std/vmod_std_fileread.c
===================================================================
--- varnish.orig/varnish-cache/lib/libvmod_std/vmod_std_fileread.c 2010-12-01 15:14:21.000000000 +0530
+++ varnish/varnish-cache/lib/libvmod_std/vmod_std_fileread.c 2010-12-01 20:02:44.000000000 +0530
@@ -49,6 +49,13 @@
#include "vcc_if.h"
#include "vcl_priv.h"
+#ifdef __GNUC__
+#define atomic_increment(a) __sync_fetch_and_add(&a, 1)
+#define atomic_decrement(a) __sync_fetch_and_sub(&a, 1)
+#else
+#error "atomic_increment and atomic_decrement defined only for GCC"
+#endif
+
struct cached_file_list {
unsigned magic;
#define CACHED_FILE_LIST_MAGIC 0x1e4fc24f
@@ -71,6 +78,7 @@
char *contents;
time_t last_modification;
off_t file_sz;
+ int references;
VLIST_ENTRY(cached_file) next;
};
@@ -89,16 +97,42 @@
}
const char *
-vmod_fileread(struct sess *sp, struct vmod_priv *priv, const char *file_name)
+vmod_fileread(struct sess *sp, struct vmod_priv *priv_vcl,
+ struct vmod_priv *priv_call, const char *file_name)
{
- struct cached_file *iter = NULL;
+ struct cached_file *iter = NULL, *prev_call = NULL;
struct stat buf;
struct cached_file_list *list;
int fd, my_counter;
(void)sp;
- GET_FILE_LIST(list, priv->priv);
+ GET_FILE_LIST(list, priv_vcl->priv);
+
+ /*
+ * First (quickly) check if the call-site just loaded
+ * the same file.
+ */
+ if (priv_call->priv != NULL) {
+ prev_call = priv_call->priv;
+ CHECK_OBJ_NOTNULL(prev_call, CACHED_FILE_MAGIC);
+ if (!strcmp(prev_call->file_name, file_name)) {
+ return prev_call->contents;
+ } else {
+ /* priv_call just lost one more reference. */
+ atomic_decrement(prev_call->references);
+ if (!prev_call->references) {
+ AZ(pthread_rwlock_wrlock(&list->lock));
+ /*
+ * Some thread may have added a reference while this thread
+ * waited for the write lock.
+ */
+ if (!prev_call->references)
+ VLIST_REMOVE(prev_call, next);
+ AZ(pthread_rwlock_unlock(&list->lock));
+ }
+ }
+ }
AZ(pthread_rwlock_rdlock(&list->lock));
@@ -106,6 +140,7 @@
CHECK_OBJ(iter, CACHED_FILE_MAGIC);
if (!strcmp(iter->file_name, file_name)) {
/* This thread was holding a read lock. */
+ atomic_increment(iter->references);
AZ(pthread_rwlock_unlock(&list->lock));
return iter->contents;
}
@@ -133,6 +168,7 @@
CHECK_OBJ(iter, CACHED_FILE_MAGIC);
if (!strcmp(iter->file_name, file_name)) {
/* This thread was holding a write lock. */
+ atomic_increment(iter->references);
AZ(pthread_rwlock_unlock(&list->lock));
return iter->contents;
}
@@ -152,6 +188,8 @@
AZ(close(fd));
iter->contents[iter->file_sz] = '\0';
+ iter->references = 1;
+ priv_call->priv = iter;
VLIST_INSERT_HEAD(&list->files, iter, next);
Index: varnish/varnish-cache/bin/varnishtest/tests/m00004.vtc
===================================================================
--- varnish.orig/varnish-cache/bin/varnishtest/tests/m00004.vtc 2010-12-01 14:53:02.000000000 +0530
+++ varnish/varnish-cache/bin/varnishtest/tests/m00004.vtc 2010-12-01 15:11:11.000000000 +0530
@@ -6,10 +6,12 @@
echo -n "File One" > "${tmpdir}/m00004_file_one"
echo -n "File Two" > "${tmpdir}/m00004_file_two"
echo -n "File Three" > "${tmpdir}/m00004_file_three"
+ echo -n "File Four" > "${tmpdir}/m00004_file_four"
+ echo -n "File Five" > "${tmpdir}/m00004_file_five"
}
server s1 {
- loop 3 {
+ loop 5 {
rxreq
txresp -hdr "foo: bar" -bodylen 4
}
@@ -26,6 +28,8 @@
set resp.http.two = std.fileread("${tmpdir}/m00004_file_two");
} else if (req.url == "/three") {
set resp.http.three = std.fileread("${tmpdir}/m00004_file_three");
+ } else {
+ set resp.http.other = std.fileread("${tmpdir}" + req.url);
}
}
} -start
@@ -58,59 +62,68 @@
}
client c2 {
- loop 5 {
- txreq -url "/two"
- rxresp
- expect resp.status == 200
- expect resp.http.content-length == "4"
- expect resp.http.foo == "bar"
- expect resp.http.two == "File Two"
- }
-
- loop 5 {
- txreq -url "/one"
- rxresp
- expect resp.status == 200
- expect resp.http.content-length == "4"
- expect resp.http.foo == "bar"
- expect resp.http.one == "File One"
-
- txreq -url "/three"
- rxresp
- expect resp.status == 200
- expect resp.http.content-length == "4"
- expect resp.http.foo == "bar"
- expect resp.http.three == "File Three"
- }
+ txreq -url "/two"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.content-length == "4"
+ expect resp.http.foo == "bar"
+ expect resp.http.two == "File Two"
+
+ txreq -url "/one"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.content-length == "4"
+ expect resp.http.foo == "bar"
+ expect resp.http.one == "File One"
+
+ txreq -url "/three"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.content-length == "4"
+ expect resp.http.foo == "bar"
+ expect resp.http.three == "File Three"
}
client c3 {
- loop 5 {
- txreq -url "/three"
- rxresp
- expect resp.status == 200
- expect resp.http.content-length == "4"
- expect resp.http.foo == "bar"
- expect resp.http.three == "File Three"
- }
-
- loop 5 {
- txreq -url "/two"
- rxresp
- expect resp.status == 200
- expect resp.http.content-length == "4"
- expect resp.http.foo == "bar"
- expect resp.http.two == "File Two"
+ txreq -url "/three"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.content-length == "4"
+ expect resp.http.foo == "bar"
+ expect resp.http.three == "File Three"
+
+ txreq -url "/two"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.content-length == "4"
+ expect resp.http.foo == "bar"
+ expect resp.http.two == "File Two"
+
+ txreq -url "/one"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.content-length == "4"
+ expect resp.http.foo == "bar"
+ expect resp.http.one == "File One"
+}
- txreq -url "/one"
- rxresp
- expect resp.status == 200
- expect resp.http.content-length == "4"
- expect resp.http.foo == "bar"
- expect resp.http.one == "File One"
- }
+client c4 {
+ txreq -url "/m00004_file_four"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.content-length == "4"
+ expect resp.http.foo == "bar"
+ expect resp.http.other == "File Four"
+
+ txreq -url "/m00004_file_five"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.content-length == "4"
+ expect resp.http.foo == "bar"
+ expect resp.http.other == "File Five"
}
client c1 -run
client c2 -run
client c3 -run
+client c4 -run
Index: varnish/varnish-cache/lib/libvmod_std/Makefile.am =================================================================== --- varnish.orig/varnish-cache/lib/libvmod_std/Makefile.am 2010-11-29 16:31:14.000000000 +0530 +++ varnish/varnish-cache/lib/libvmod_std/Makefile.am 2010-11-29 16:32:20.000000000 +0530 @@ -10,7 +10,8 @@ vcc_if.c \ vcc_if.h \ vmod_std.c \ - vmod_std_fileread.c + vmod_std_fileread.c \ + vmod_std_vcl_priv.h vcc_if.c vcc_if.h: $(top_srcdir)/lib/libvmod_std/vmod.py $(top_srcdir)/lib/libvmod_std/vmod.vcc @PYTHON@ $(top_srcdir)/lib/libvmod_std/vmod.py $(top_srcdir)/lib/libvmod_std/vmod.vcc Index: varnish/varnish-cache/lib/libvmod_std/vcl_priv.h =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ varnish/varnish-cache/lib/libvmod_std/vcl_priv.h 2010-11-29 16:32:20.000000000 +0530 @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2010 Linpro AS + * All rights reserved. + * + * Author: Sanjoy Das <[email protected]> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +struct std_vcl_priv { + unsigned magic; +#define STD_VCL_PRIV_MAGIC 0x8550f261 + void *tolower_data; + void *fileread_data; +}; + +void *vmod_create_fileread_priv(void); +void vmod_release_fileread_priv(void *); Index: varnish/varnish-cache/lib/libvmod_std/vmod_std.c =================================================================== --- varnish.orig/varnish-cache/lib/libvmod_std/vmod_std.c 2010-11-29 16:31:24.000000000 +0530 +++ varnish/varnish-cache/lib/libvmod_std/vmod_std.c 2010-11-29 18:35:07.000000000 +0530 @@ -35,6 +35,7 @@ #include "../../bin/varnishd/cache.h" #include "vcc_if.h" +#include "vcl_priv.h" void vmod_set_ip_tos(struct sess *sp, int tos) @@ -103,22 +104,44 @@ { const char *p; va_list ap; + struct std_vcl_priv *priv_main = priv->priv; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - assert(!strcmp(priv->priv, "FOO")); + CHECK_OBJ_NOTNULL(priv_main, STD_VCL_PRIV_MAGIC); + + assert(!strcmp(priv_main->tolower_data, "FOO")); va_start(ap, s); p = vmod_updown(sp, 0, s, ap); va_end(ap); return (p); } +static void +free_function(void *priv) +{ + struct std_vcl_priv *priv_main = priv; + + CHECK_OBJ_NOTNULL(priv_main, STD_VCL_PRIV_MAGIC); + + vmod_release_fileread_priv(priv_main->fileread_data); + free(priv_main->tolower_data); +} + int init_function(struct vmod_priv *priv, const struct VCL_conf *cfg) { + struct std_vcl_priv *priv_main; + (void)cfg; - priv->priv = strdup("FOO"); - priv->free = free; + ALLOC_OBJ(priv_main, STD_VCL_PRIV_MAGIC); + AN(priv_main); + + priv_main->fileread_data = vmod_create_fileread_priv(); + priv_main->tolower_data = strdup("FOO"); + + priv->priv = priv_main; + priv->free = free_function; return (0); } Index: varnish/varnish-cache/lib/libvmod_std/vmod_std_fileread.c =================================================================== --- varnish.orig/varnish-cache/lib/libvmod_std/vmod_std_fileread.c 2010-11-29 16:31:07.000000000 +0530 +++ varnish/varnish-cache/lib/libvmod_std/vmod_std_fileread.c 2010-11-29 18:34:58.000000000 +0530 @@ -47,6 +47,7 @@ #include "../../bin/varnishd/cache.h" #include "vcc_if.h" +#include "vcl_priv.h" VSLIST_HEAD(cached_file_list, cached_file); @@ -167,3 +168,15 @@ AZ(pthread_rwlock_unlock(&filelist_lock)); return iter->contents; } + +void * +vmod_create_fileread_priv(void) +{ + return NULL; +} + +void +vmod_release_fileread_priv(void *data) +{ + AZ(data); +}
signature.asc
Description: This is a digitally signed message part
_______________________________________________ varnish-dev mailing list [email protected] http://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
