And this time an all-in-one patch with all the changes, and the removal of a GNU extension I let slip in in the previous patch.
Cheers, Dridi On Fri, Jun 5, 2015 at 12:29 PM, Dridi Boukelmoune <[email protected]> wrote: > As discussed, an update of the signal handling. > > Cheers, > Dridi
From 29f1b67cf16a2b7b457e71e4159c69723336aa13 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune <[email protected]> Date: Sat, 30 May 2015 11:38:58 +0200 Subject: [PATCH] Run background processes in varnishtest Available operations on processes are: - start - wait - kill (send a signal) - stop (kill with SIGTERM) - write (to its stdin) - close (its stdin) A process name starts with a 'p' and accepts a shell pipeline as the command line. Environment variables can be set directly in the command line. A process will create a macro with its PID, and its stdout and stderr are redirected to files in a dedicated directory. Also add a varnish "name" macro in varnishtest. --- bin/varnishtest/Makefile.am | 3 +- bin/varnishtest/tests/README | 1 + bin/varnishtest/tests/u00000.vtc | 34 ++++ bin/varnishtest/tests/u00001.vtc | 43 +++++ bin/varnishtest/vtc.c | 1 + bin/varnishtest/vtc.h | 1 + bin/varnishtest/vtc_process.c | 388 +++++++++++++++++++++++++++++++++++++++ bin/varnishtest/vtc_varnish.c | 1 + 8 files changed, 471 insertions(+), 1 deletion(-) create mode 100644 bin/varnishtest/tests/u00000.vtc create mode 100644 bin/varnishtest/tests/u00001.vtc create mode 100644 bin/varnishtest/vtc_process.c diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index fe599d4..2506a8a 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -34,7 +34,8 @@ varnishtest_SOURCES = \ vtc_sema.c \ vtc_server.c \ vtc_varnish.c \ - vtc_logexp.c + vtc_logexp.c \ + vtc_process.c varnishtest_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index 456bf75..52e5061 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -27,4 +27,5 @@ Naming scheme id ~ [r] --> Regression tests, same number as ticket id ~ [s] --> Slow tests, expiry, grace etc. id ~ [t] --> sTreaming tests + id ~ [u] --> Unusual background processes id ~ [v] --> VCL tests: execute VRT functions diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc new file mode 100644 index 0000000..16fed2c --- /dev/null +++ b/bin/varnishtest/tests/u00000.vtc @@ -0,0 +1,34 @@ +varnishtest "Simple process tests" + +# new & start +process p1 "cat" -start +process p2 "cat" -start +process p3 "cat" -start + +# write +process p1 -writeln "foo" +process p2 -writeln "bar" +process p3 -writeln "baz" + +# give enough time for the writes +delay 0.5 + +# stop +process p1 -stop +process p2 -close +process p3 -kill "SIGHUP" + +# wait +process p1 -wait +process p2 -wait +process p3 -wait + +# check stdout +shell "grep foo ${tmpdir}/p1/stdout >/dev/null 2>&1" +shell "grep bar ${tmpdir}/p2/stdout >/dev/null 2>&1" +shell "grep baz ${tmpdir}/p3/stdout >/dev/null 2>&1" + +# check stderr +shell "test -f ${tmpdir}/p1/stderr -a ! -s ${tmpdir}/p1/stderr" +shell "test -f ${tmpdir}/p2/stderr -a ! -s ${tmpdir}/p2/stderr" +shell "test -f ${tmpdir}/p3/stderr -a ! -s ${tmpdir}/p3/stderr" diff --git a/bin/varnishtest/tests/u00001.vtc b/bin/varnishtest/tests/u00001.vtc new file mode 100644 index 0000000..dc904c2 --- /dev/null +++ b/bin/varnishtest/tests/u00001.vtc @@ -0,0 +1,43 @@ +varnishtest "varnishncsa log file rotation" + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 + + rxreq + expect req.url == "/bar" + txresp -status 404 +} -start + +varnish v1 -vcl+backend "" -start + +process p1 "${varnishncsa} -n ${v1_name} -w ${tmpdir}/ncsa.log -F %s" -start + +# give enough time to varnishncsa to open the VSM +delay 1 + +client c1 { + txreq -url "/foo" + rxresp +} -run + +# give enough time to varnishncsa to write +delay 1 + +# rotate logs +shell "mv ${tmpdir}/ncsa.log ${tmpdir}/ncsa.old.log >/dev/null 2>&1" +process p1 -kill "SIGHUP" + +client c1 { + txreq -url "/bar" + rxresp +} -run + +# give enough time to varnishncsa to write +delay 1 + +process p1 -stop + +shell "grep 200 ${tmpdir}/ncsa.old.log >/dev/null" +shell "grep 404 ${tmpdir}/ncsa.log >/dev/null" diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index cbda7bf..ae38f68 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -620,6 +620,7 @@ static const struct cmds cmds[] = { { "random", cmd_random }, { "feature", cmd_feature }, { "logexpect", cmd_logexp }, + { "process", cmd_process }, { NULL, NULL } }; diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index e27c056..17b5790 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -63,6 +63,7 @@ cmd_f cmd_client; cmd_f cmd_varnish; cmd_f cmd_sema; cmd_f cmd_logexp; +cmd_f cmd_process; extern volatile sig_atomic_t vtc_error; /* Error, bail out */ extern int vtc_stop; /* Abandon current test, no error */ diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c new file mode 100644 index 0000000..07de092 --- /dev/null +++ b/bin/varnishtest/vtc_process.c @@ -0,0 +1,388 @@ +/*- + * Copyright (c) 2015 Varnish Software AS + * All rights reserved. + * + * Author: Dridi Boukelmoune <[email protected]> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include <sys/resource.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/wait.h> + +#include <errno.h> +#include <fcntl.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "vtc.h" + +#include "vss.h" + +struct process { + unsigned magic; +#define PROCESS_MAGIC 0x1617b43e + char *name; + struct vtclog *vl; + VTAILQ_ENTRY(process) list; + + char *spec; + char *workdir; + char *outdir; + char *out; + char *err; + int fds[2]; + pid_t pid; + + pthread_t tp; + unsigned running; + int status; +}; + +static VTAILQ_HEAD(, process) processes = + VTAILQ_HEAD_INITIALIZER(processes); + +/********************************************************************** + * Allocate and initialize a process + */ + +#define PROCESS_EXPAND(field, format, ...) \ + do { \ + bprintf(buf, format, __VA_ARGS__); \ + vsb = macro_expand(p->vl, buf); \ + AN(vsb); \ + p->field = strdup(VSB_data(vsb)); \ + AN(p->field); \ + VSB_delete(vsb); \ + } while (0) + +static struct process * +process_new(const char *name) +{ + struct process *p; + struct vsb *vsb; + char buf[1024]; + + AN(name); + ALLOC_OBJ(p, PROCESS_MAGIC); + AN(p); + REPLACE(p->name, name); + + p->vl = vtc_logopen(name); + AN(p->vl); + + PROCESS_EXPAND(workdir, "%s", "${pwd}"); + PROCESS_EXPAND(outdir, "${tmpdir}/%s", name); + PROCESS_EXPAND(out, "${tmpdir}/%s/stdout", name); + PROCESS_EXPAND(err, "${tmpdir}/%s/stderr", name); + + bprintf(buf, "rm -rf %s ; mkdir -p %s ; touch %s %s", + p->outdir, p->outdir, p->out, p->err); + AZ(system(buf)); + + p->fds[0] = -1; + p->fds[1] = -1; + + if (*p->name != 'p') + vtc_log(p->vl, 0, "Process name must start with 'p'"); + + VTAILQ_INSERT_TAIL(&processes, p, list); + return (p); +} + +#undef PROCESS_EXPAND + +/********************************************************************** + * Clean up process + */ + +static void +process_delete(struct process *p) +{ + + CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC); + vtc_logclose(p->vl); + free(p->name); + free(p->workdir); + free(p->outdir); + free(p->out); + free(p->err); + + /* + * We do not delete the outdir, it may contain useful stdout + * and stderr files. + */ + + /* XXX: MEMLEAK (?) */ + FREE_OBJ(p); +} + +/********************************************************************** + * Start the process thread + */ + +static void * +process_thread(void *priv) +{ + struct process *p; + struct rusage ru; + int r; + + CAST_OBJ_NOTNULL(p, priv, PROCESS_MAGIC); + r = wait4(p->pid, &p->status, 0, &ru); + macro_undef(p->vl, p->name, "pid"); + p->pid = 0; + p->running = 0; + vtc_log(p->vl, 2, "R %d Status: %04x (u %.6f s %.6f)", r, p->status, + ru.ru_utime.tv_sec + 1e-6 * ru.ru_utime.tv_usec, + ru.ru_stime.tv_sec + 1e-6 * ru.ru_stime.tv_usec + ); + + if (WIFEXITED(p->status) && WEXITSTATUS(p->status) == 0) + return (NULL); +#ifdef WCOREDUMP + vtc_log(p->vl, 2, "Bad exit code: %04x sig %x exit %x core %x", + p->status, WTERMSIG(p->status), WEXITSTATUS(p->status), + WCOREDUMP(p->status)); +#else + vtc_log(p->vl, 2, "Bad exit code: %04x sig %x exit %x", + p->status, WTERMSIG(p->status), WEXITSTATUS(p->status)); +#endif + + (void)close(p->fds[1]); + p->fds[1] = -1; + + return (NULL); +} + +static void +process_start(struct process *p) +{ + struct vsb *cl; + int i, out_fd, err_fd; + + CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC); + + vtc_log(p->vl, 4, "CMD: %s", p->spec); + + cl = macro_expand(p->vl, p->spec); + AN(cl); + AZ(pipe(p->fds)); + out_fd = open(p->out, O_WRONLY|O_APPEND); + assert(out_fd >= 0); + err_fd = open(p->err, O_WRONLY|O_APPEND); + assert(err_fd >= 0); + p->pid = fork(); + assert(p->pid >= 0); + p->running = 1; + if (p->pid == 0) { + assert(dup2(p->fds[0], 0) == 0); + assert(dup2(out_fd, 1) == 1); + assert(dup2(out_fd, 2) == 2); + for (i = 3; i <getdtablesize(); i++) + (void)close(i); + AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl), (char*)0)); + exit(1); + } + vtc_log(p->vl, 3, "PID: %ld", (long)p->pid); + macro_def(p->vl, p->name, "pid", "%ld", (long)p->pid); + AZ(close(p->fds[0])); + AZ(close(out_fd)); + AZ(close(err_fd)); + p->fds[0] = -1; + VSB_delete(cl); + AZ(pthread_create(&p->tp, NULL, process_thread, p)); +} + +/********************************************************************** + * Wait for process thread to stop + */ + +static void +process_wait(struct process *p) +{ + void *v; + + if (p->running && p->pid) + AZ(pthread_join(p->tp, &v)); +} + +/********************************************************************** + * Send a signal to a process + */ + +static void +process_kill(struct process *p, const char *sig) +{ + int s, l; + char buf[64]; + + CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC); + AN(sig); + + if (!p->running || !p->pid) { + vtc_log(p->vl, 0, "Cannot signal a non-running process"); + return; + } + + vtc_log(p->vl, 4, "CMD: /usr/bin/kill -%s %d", sig, p->pid); + + l = snprintf(buf, sizeof buf, "/usr/bin/kill -%s %d", sig, p->pid); + AN(l < sizeof buf); + s = system(buf); + if (s != 0) + vtc_log(p->vl, 0, "Failed to send signal (exit status: %d)", s); +} + +static inline void +process_stop(struct process *p) +{ + + process_kill(p, "SIGTERM"); +} + +static inline void +process_terminate(struct process *p) +{ + + process_kill(p, "SIGTERM"); + sleep(1); + process_kill(p, "SIGKILL"); +} + +/********************************************************************** + * Write to a process' stdin + */ + +static void +process_write(struct process *p, const char *text) +{ + int r, len; + + if (!p->running || !p->pid) { + vtc_log(p->vl, 0, "Cannot write to a non-running process"); + return; + } + + len = strlen(text); + vtc_log(p->vl, 4, "Writing %d bytes", len); + r = write(p->fds[1], text, len); + if (r < 0) + vtc_log(p->vl, 0, "Failed to write: %s (%d)", + strerror(errno), errno); +} + +static void +process_close(struct process *p) +{ + + if (!p->running || !p->pid) { + vtc_log(p->vl, 0, "Cannot close on a non-running process"); + return; + } + + AZ(close(p->fds[1])); + p->fds[1] = -1; +} + +/********************************************************************** + * Process command dispatch + */ + +void +cmd_process(CMD_ARGS) +{ + struct process *p, *p2; + + (void)priv; + (void)cmd; + (void)vl; + + if (av == NULL) { + /* Reset and free */ + VTAILQ_FOREACH_SAFE(p, &processes, list, p2) { + if (p->running && p->pid) + process_terminate(p); + VTAILQ_REMOVE(&processes, p, list); + process_delete(p); + } + return; + } + + AZ(strcmp(av[0], "process")); + av++; + + VTAILQ_FOREACH(p, &processes, list) + if (!strcmp(p->name, av[0])) + break; + if (p == NULL) + p = process_new(av[0]); + av++; + + for (; *av != NULL; av++) { + if (vtc_error) + break; + + if (!strcmp(*av, "-start")) { + process_start(p); + continue; + } + if (!strcmp(*av, "-wait")) { + process_wait(p); + continue; + } + if (!strcmp(*av, "-kill")) { + process_kill(p, av[1]); + av++; + continue; + } + if (!strcmp(*av, "-stop")) { + process_stop(p); + continue; + } + if (!strcmp(*av, "-write")) { + process_write(p, av[1]); + av++; + continue; + } + if (!strcmp(*av, "-writeln")) { + process_write(p, av[1]); + process_write(p, "\n"); + av++; + continue; + } + if (!strcmp(*av, "-close")) { + process_close(p); + continue; + } + if (**av == '-') + vtc_log(p->vl, 0, "Unknown process argument: %s", *av); + REPLACE(p->spec, *av); + } +} diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index a7b504f..c6cb52f 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -432,6 +432,7 @@ varnish_launch(struct varnish *v) } else { vtc_log(v->vl, 3, "PID: %ld", (long)v->pid); macro_def(v->vl, v->name, "pid", "%ld", (long)v->pid); + macro_def(v->vl, v->name, "name", "%s", v->workdir); } AZ(close(v->fds[0])); AZ(close(v->fds[3])); -- 2.1.0
_______________________________________________ varnish-dev mailing list [email protected] https://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
