Hi guys
At the latest dev meeting in Rotterdam it was brought up that it would
be a useful addition to be able to sync a varnishtest semaphore with
Varnish itself.
The current 'sema' construct in varnishtest lives within the same
process and is implemented using regular pthread mutexes and
condvars. I've hacked together a suggestion for doing IPC semaphores
in varnishtest/Varnish, backed by SysV semaphores.
The patches introduce a new varnishtest keyword 'ipcsema' and a new
function in vmod-debug, 'debug.sema_sync()'.
'ipcsema' in varnishtest follows the syntax and semantics of the usual
'sema' keyword (with an exception described below [1]),
ipcsema r1 sync 2
It can be used in both client/server and global scopes in varnishtest,
just like the regular 'sema' construct.
Further, to sync with this semaphore from VCL, we issue
sub vcl_backend_fetch {
debug.sema_sync("r1", 2);
}
How this works under the covers is that varnishtest will on each run
generate a random number and stick that in an environment variable
($VTC_IPC_RAND) prior to launching varnishd. This string, along with
the semaphore name (e.g. "r1") goes into a hash that is used as the
key argument to semget(2). After the test case finishes, the semaphore
will be removed (however, see [2] below).
There are a couple of open issues:
[1] I have yet to find a non-racy way to reset an ipcsema back to its
original value after the wait is over. This is something that is
supported by the regular sema construct in varnishtest, that allows
a sema to be reused multiple times within the same test case. (See
for example c00013.vtc).
[2] Potential for leaking a semaphore. If the varnishtest process is
killed and thus the cleanup code isn't called the semaphore is
leaked. It will then have to be cleaned up via ipcrm(1). Active
system semaphores can be shown via ipcs(1).
Please take a look - any ideas or feedback will be most welcome.
Patches attached.
Merry X-mas,
Dag
--
Dag Haavi Finstad
Software Developer | Varnish Software
Mobile: +47 476 64 134
We Make Websites Fly!
From c30b0fde5cb588f7cd06c2b2fa03a713e4473e0b Mon Sep 17 00:00:00 2001
From: Dag Haavi Finstad <[email protected]>
Date: Mon, 21 Dec 2015 14:58:32 +0100
Subject: [PATCH 1/2] Prototype varnishtest IPC semaphores.
---
bin/varnishtest/Makefile.am | 1 +
bin/varnishtest/tests/a00013.vtc | 49 ++++++++
bin/varnishtest/vtc.c | 2 +
bin/varnishtest/vtc.h | 2 +
bin/varnishtest/vtc_http.c | 1 +
bin/varnishtest/vtc_ipcsema.c | 248 +++++++++++++++++++++++++++++++++++++++
6 files changed, 303 insertions(+)
create mode 100644 bin/varnishtest/tests/a00013.vtc
create mode 100644 bin/varnishtest/vtc_ipcsema.c
diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index 2506a8a..2dd0a40 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -32,6 +32,7 @@ varnishtest_SOURCES = \
vtc_main.c \
vtc_log.c \
vtc_sema.c \
+ vtc_ipcsema.c \
vtc_server.c \
vtc_varnish.c \
vtc_logexp.c \
diff --git a/bin/varnishtest/tests/a00013.vtc b/bin/varnishtest/tests/a00013.vtc
new file mode 100644
index 0000000..36a8d8b
--- /dev/null
+++ b/bin/varnishtest/tests/a00013.vtc
@@ -0,0 +1,49 @@
+varnishtest "ipcsema operations (based on a00008.vtc)"
+
+server s1 {
+ rxreq
+ ipcsema r1 sync 4
+ delay .9
+ txresp
+} -start
+
+server s2 {
+ rxreq
+ ipcsema r1 sync 4
+ delay .6
+ txresp
+} -start
+
+server s3 {
+ rxreq
+ ipcsema r1 sync 4
+ delay .2
+ txresp
+} -start
+
+client c1 -connect ${s1_sock} {
+ delay .2
+ txreq
+ rxresp
+ ipcsema r2 sync 4
+} -start
+
+client c2 -connect ${s2_sock} {
+ delay .6
+ txreq
+ rxresp
+ ipcsema r2 sync 4
+} -start
+
+client c3 -connect ${s3_sock} {
+ delay .9
+ txreq
+ rxresp
+ ipcsema r2 sync 4
+} -start
+
+# Wait for all servers to have received requests
+ipcsema r1 sync 4
+
+# Wait for all clients to have received responses
+ipcsema r2 sync 4
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 3c6c884..6a6f86e 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -627,6 +627,7 @@ static const struct cmds cmds[] = {
{ "shell", cmd_shell },
{ "err_shell", cmd_err_shell },
{ "sema", cmd_sema },
+ { "ipcsema", cmd_ipcsema },
{ "random", cmd_random },
{ "feature", cmd_feature },
{ "logexpect", cmd_logexp },
@@ -650,6 +651,7 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
init_macro();
init_sema();
+ init_ipcsema();
init_server();
/* Apply extmacro definitions */
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 518494c..73215c2 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -62,6 +62,7 @@ cmd_f cmd_server;
cmd_f cmd_client;
cmd_f cmd_varnish;
cmd_f cmd_sema;
+cmd_f cmd_ipcsema;
cmd_f cmd_logexp;
cmd_f cmd_process;
@@ -74,6 +75,7 @@ extern int vtc_witness;
extern int feature_dns;
void init_sema(void);
+void init_ipcsema(void);
void init_server(void);
int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd);
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 0440542..7d83a5c 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1354,6 +1354,7 @@ static const struct cmds http_cmds[] = {
{ "chunkedlen", cmd_http_chunkedlen },
{ "delay", cmd_delay },
{ "sema", cmd_sema },
+ { "ipcsema", cmd_ipcsema },
{ "expect_close", cmd_http_expect_close },
{ "close", cmd_http_close },
{ "accept", cmd_http_accept },
diff --git a/bin/varnishtest/vtc_ipcsema.c b/bin/varnishtest/vtc_ipcsema.c
new file mode 100644
index 0000000..f8ef36c
--- /dev/null
+++ b/bin/varnishtest/vtc_ipcsema.c
@@ -0,0 +1,248 @@
+/*-
+ * Copyright (c) 2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dag Haavi Finstad <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/sem.h>
+
+#include "vtc.h"
+
+#include "vrnd.h"
+#include "vsha256.h"
+#include "vtim.h"
+
+struct ipcsema {
+ unsigned magic;
+#define IPCSEMA_MAGIC 0x825cee31
+ char *name;
+ int semid;
+ VTAILQ_ENTRY(ipcsema) list;
+ unsigned expected;
+};
+
+static pthread_mutex_t sema_mtx;
+static VTAILQ_HEAD(, ipcsema) ipcsemas = VTAILQ_HEAD_INITIALIZER(ipcsemas);
+
+
+/**********************************************************************
+ * Simple semop(2)/semctl(2) helper funcs
+ */
+
+static int
+sem_decr(int semid)
+{
+ struct sembuf sb;
+
+ sb.sem_num = 0;
+ sb.sem_op = -1;
+ sb.sem_flg = 0;
+
+ return(semop(semid, &sb, 1));
+}
+
+static int
+sem_wait(int semid)
+{
+ struct sembuf sb;
+
+ sb.sem_num = 0;
+ sb.sem_op = 0;
+ sb.sem_flg = 0;
+
+ return(semop(semid, &sb, 1));
+}
+
+static int
+sem_getval(int semid)
+{
+ return(semctl(semid, 0, GETVAL, 0));
+}
+
+
+/**********************************************************************
+ * Allocate and initialize an ipcsema
+ */
+
+static struct ipcsema *
+sema_new(const char *name, int semval, struct vtclog *vl)
+{
+ struct ipcsema *r;
+ int semid;
+ struct semid_ds buf;
+ SHA256_CTX sctx;
+ unsigned char digest[SHA256_LEN];
+ const char *rnd;
+ key_t key;
+
+ ALLOC_OBJ(r, IPCSEMA_MAGIC);
+ AN(r);
+ AN(name);
+ if (*name != 'r')
+ vtc_log(vl, 0, "Sema name must start with 'r' (%s)", name);
+ REPLACE(r->name, name);
+ r->expected = semval;
+ rnd = getenv("VTC_IPC_RAND");
+ AN(rnd);
+
+ SHA256_Init(&sctx);
+ SHA256_Update(&sctx, name, strlen(name));
+ SHA256_Update(&sctx, rnd, strlen(rnd));
+ SHA256_Final(digest, &sctx);
+ memcpy(&key, digest, sizeof key);
+
+ /* key_t has the one special value of IPC_PRIVATE. If we
+ * utterly lost the sha256 lottery, add 1 */
+ if (key == IPC_PRIVATE)
+ key++;
+
+ /* Initialization sync logic lifted from
+ https://beej.us/guide/bgipc/output/html/multipage/semaphores.html
+ which is again taken from W. Richard Stevens' UNIX Network
+ Programming 2nd edition, volume 2, lockvsem.c, page 295.*/
+ semid = semget(key, 1, IPC_CREAT|IPC_EXCL|0666);
+ if (semid >= 0) {
+ /* whoever won the IPC_EXCL race gets to initialize.
+ *
+ * We initialize to semval+1, and issue a semop(2)
+ * decrement in order to set the sem_otime for
+ * synchronization. */
+ assert(semctl(semid, 0, SETVAL, semval + 1) != -1);
+ AZ(sem_decr(semid));
+ } else if (errno == EEXIST) {
+ semid = semget(key, 1, 0);
+ assert(semid >= 0);
+ /* wait for changed otime */
+ while (1) {
+ semctl(semid, 0, IPC_STAT, &buf);
+ if (buf.sem_otime != 0)
+ break;
+ VTIM_sleep(0.01);
+ /* XXX: should fail eventually */
+ }
+ } else
+ vtc_log(vl, 0, "ipcsema initializaion failed (%s)",
+ strerror(errno));
+
+ r->semid = semid;
+ VTAILQ_INSERT_TAIL(&ipcsemas, r, list);
+ return (r);
+}
+
+/**********************************************************************
+ * Sync an ipcsema
+ */
+
+static void
+sema_sync(struct ipcsema *r, unsigned u, struct vtclog *vl)
+{
+ CHECK_OBJ_NOTNULL(r, IPCSEMA_MAGIC);
+ if (r->expected != u)
+ vtc_log(vl, 0,
+ "IPCSema(%s) use error: different expectations (%u vs %u)",
+ r->name, r->expected, u);
+
+ vtc_log(vl, 4, "IPCSema(%s) wait", r->name);
+ AZ(sem_decr(r->semid));
+ AZ(sem_wait(r->semid));
+ vtc_log(vl, 4, "IPCSema(%s) wake %u", r->name, r->expected);
+}
+
+/**********************************************************************
+ * Semaphore command dispatch
+ */
+
+void
+cmd_ipcsema(CMD_ARGS)
+{
+ struct ipcsema *r, *r2;
+ char *name;
+ unsigned semval;
+ int n;
+
+ (void)priv;
+ (void)cmd;
+
+ if (av == NULL) {
+ AZ(pthread_mutex_lock(&sema_mtx));
+ VTAILQ_FOREACH_SAFE(r, &ipcsemas, list, r2) {
+ VTAILQ_REMOVE(&ipcsemas, r, list);
+ n = sem_getval(r->semid);
+ assert(semctl(r->semid, 0, IPC_RMID) != -1);
+ assert(n == 0);
+ free(r->name);
+ FREE_OBJ(r);
+ }
+ AZ(pthread_mutex_unlock(&sema_mtx));
+ return;
+ }
+
+ AZ(strcmp(av[0], "ipcsema"));
+ av++;
+
+ if (*av == NULL)
+ vtc_log(vl, 0, "Missing ipcsema name");
+ name = *av;
+ av++;
+ if (*av == NULL)
+ vtc_log(vl, 0, "Missing sema argument");
+ if (strcmp(*av, "sync") != 0)
+ vtc_log(vl, 0, "Unknown ipcsema argument: %s", *av);
+ av++;
+ if (*av == NULL)
+ vtc_log(vl, 0, "Missing semval");
+ semval = strtoul(*av, NULL, 0);
+
+ AZ(pthread_mutex_lock(&sema_mtx));
+ VTAILQ_FOREACH(r, &ipcsemas, list)
+ if (!strcmp(r->name, name))
+ break;
+ if (r == NULL)
+ r = sema_new(name, semval, vl);
+ AZ(pthread_mutex_unlock(&sema_mtx));
+
+ sema_sync(r, semval, vl);
+}
+
+void
+init_ipcsema(void)
+{
+ char b[32];
+ unsigned r;
+
+ AZ(pthread_mutex_init(&sema_mtx, NULL));
+ r = (unsigned)random();
+ bprintf(b, "%u", r);
+ AZ(setenv("VTC_IPC_RAND", b, 1));
+}
--
2.6.4
From af1e3e6f0e736a788a0f9897e4545ac801d73bef Mon Sep 17 00:00:00 2001
From: Dag Haavi Finstad <[email protected]>
Date: Mon, 21 Dec 2015 15:00:55 +0100
Subject: [PATCH 2/2] Add .sema_sync() to libvmod-debug.
---
bin/varnishtest/tests/m00023.vtc | 33 ++++++++++
lib/libvmod_debug/Makefile.am | 3 +-
lib/libvmod_debug/vmod.vcc | 4 ++
lib/libvmod_debug/vmod_debug_sema.c | 122 ++++++++++++++++++++++++++++++++++++
4 files changed, 161 insertions(+), 1 deletion(-)
create mode 100644 bin/varnishtest/tests/m00023.vtc
create mode 100644 lib/libvmod_debug/vmod_debug_sema.c
diff --git a/bin/varnishtest/tests/m00023.vtc b/bin/varnishtest/tests/m00023.vtc
new file mode 100644
index 0000000..7a53429
--- /dev/null
+++ b/bin/varnishtest/tests/m00023.vtc
@@ -0,0 +1,33 @@
+varnishtest "Test debug.sema_sync"
+
+server s1 {
+ rxreq
+ txresp
+} -start
+
+varnish v1 -vcl+backend {
+ import ${vmod_debug};
+
+ sub vcl_recv {
+ debug.sema_sync("r1", 2);
+ }
+
+ sub vcl_backend_response {
+ debug.sema_sync("r2", 2);
+ }
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
+
+client c1 {
+ txreq
+ rxresp
+ expect resp.status == 200
+} -start
+
+ipcsema r1 sync 2
+delay 0.5
+ipcsema r2 sync 2
+
+client c1 -wait
+
diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am
index 9115713..244cc86 100644
--- a/lib/libvmod_debug/Makefile.am
+++ b/lib/libvmod_debug/Makefile.am
@@ -18,7 +18,8 @@ libvmod_debug_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version
libvmod_debug_la_SOURCES = \
vmod_debug.c \
vmod_debug_obj.c \
- vmod_debug_dyn.c
+ vmod_debug_dyn.c \
+ vmod_debug_sema.c
nodist_libvmod_debug_la_SOURCES = \
vcc_if.c \
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 42159fb..d6b9ac7 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -142,3 +142,7 @@ Find how much unallocated space there is left in a workspace.
$Function VOID vcl_release_delay(DURATION)
Hold a reference to the VCL when it goes cold for the given delay.
+
+$Function VOID sema_sync(STRING key, INT semval)
+
+Sync with a varnishtest ipc semaphore.
diff --git a/lib/libvmod_debug/vmod_debug_sema.c b/lib/libvmod_debug/vmod_debug_sema.c
new file mode 100644
index 0000000..b9ce88a
--- /dev/null
+++ b/lib/libvmod_debug/vmod_debug_sema.c
@@ -0,0 +1,122 @@
+/*-
+ * Copyright (c) 2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dag Haavi Finstad <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/sem.h>
+
+#include "cache/cache.h"
+
+#include "vrt.h"
+#include "vcc_if.h"
+#include "vsha256.h"
+#include "vtim.h"
+
+static int
+sem_decr(int semid)
+{
+ struct sembuf sb;
+
+ sb.sem_num = 0;
+ sb.sem_op = -1;
+ sb.sem_flg = 0;
+
+ return(semop(semid, &sb, 1));
+}
+
+static int
+sem_wait(int semid)
+{
+ struct sembuf sb;
+
+ sb.sem_num = 0;
+ sb.sem_op = 0;
+ sb.sem_flg = 0;
+
+ return(semop(semid, &sb, 1));
+}
+
+static int
+sem_get(const char *name, int semval)
+{
+ int semid;
+ struct semid_ds buf;
+ SHA256_CTX sctx;
+ unsigned char digest[SHA256_LEN];
+ const char *rnd;
+ key_t key;
+
+ rnd = getenv("VTC_IPC_RAND");
+ AN(rnd);
+
+ SHA256_Init(&sctx);
+ SHA256_Update(&sctx, name, strlen(name));
+ SHA256_Update(&sctx, rnd, strlen(rnd));
+ SHA256_Final(digest, &sctx);
+ memcpy(&key, digest, sizeof key);
+
+ if (key == IPC_PRIVATE)
+ key++;
+
+ /* See comments in bin/varnishtest/vtc_ipcsema.c:sema_new */
+ semid = semget(key, 1, IPC_CREAT|IPC_EXCL|0666);
+ if (semid >= 0) {
+ assert(semctl(semid, 0, SETVAL, semval + 1) != -1);
+ AZ(sem_decr(semid));
+ } else if (errno == EEXIST) {
+ semid = semget(key, 1, 0);
+ assert(semid >= 0);
+ while (1) {
+ semctl(semid, 0, IPC_STAT, &buf);
+ if (buf.sem_otime != 0)
+ break;
+ VTIM_sleep(0.01);
+ }
+ } else {
+ INCOMPL();
+ }
+
+ return (semid);
+}
+
+
+VCL_VOID
+vmod_sema_sync(VRT_CTX, VCL_STRING k, VCL_INT semval)
+{
+ int semid = sem_get(k, (int) semval);
+ VSL(SLT_Debug, 0, "sema_sync: wait %s", k);
+ AZ(sem_decr(semid));
+ AZ(sem_wait(semid));
+ VSL(SLT_Debug, 0, "sema_sync: wake %s", k);
+}
--
2.6.4
_______________________________________________
varnish-dev mailing list
[email protected]
https://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
