Folks
Attached is a patch that implements a random substitution feature for
flood. You need a requesttemplate, one or more substitution variables
of the form ${varname) in the requesttemplate and a substitution file
formatted with one value per newline delimited line. There may be more
than one substitution variable per URL. The variables and files are
referred to as subst variables and subst files. Each time the URL is
sent the subst variable is replaced with one line randomly selected from
the subst file. Any part of the requesttemplate can be randomly
substituted, protocol, host, port, etc. The algorithm tries to make the
creation time of the substitution independent of the size of the subst
file. The randomness of the selection can be adversely impacted if the
sizes of the different entries in the subst file vary considerably.
In addition to the code changes implementing this there is also a sample
flood configuration file (subst-example.xml), an updated flood.dtd and a
sample subst file (subprojects) that the subst-example.xml references so
you can see the feature in action.
This code is reasonably well tested although this precise version is
somewhat new, I wanted to build against the most recent release of flood
I could get.
There are many possible enhancements to this but the current version is
a reasonable initial functionality. In particular this can only
randomize parts of the requesttemplate so headers can't be randomized,
at least at this time.
I am under the impression the patch is in the required format. Let me
know of any required changes. I hope this is found useful.
Thanks,
Guy
--
Guy Ferraiolo mailto:[EMAIL PROTECTED]
Performance Measurement & Analysis http://CNET.com
CNET tel: 1.908.541.3739
1200 Route 22 East fax: 1.908.575.7474
Bridgewater, NJ 08807 cel: 1.732.618.0250
diff --exclude-from=excludefile -urN flood-orig/config.h.in flood-patchbuild/config.h.in
--- flood-orig/config.h.in 2007-12-19 11:54:42.000000000 -0800
+++ flood-patchbuild/config.h.in 2007-12-19 12:32:12.000000000 -0800
@@ -53,6 +53,10 @@
#define XML_FARM_USEFARMER_COUNT "count"
#define XML_FARM_USEFARMER_DELAY "startdelay"
#define XML_FARM_USEFARMER_START "startcount"
+#define XML_SUBST_LIST "subst_list"
+#define XML_SUBST_ENTRY "subst_entry"
+#define XML_SUBST_VAR "subst_var"
+#define XML_SUBST_FILE "subst_file"
/* The delimiter (used above) between XML elements */
#define XML_ELEM_DELIM "."
diff --exclude-from=excludefile -urN flood-orig/examples/flood.dtd flood-patchbuild/examples/flood.dtd
--- flood-orig/examples/flood.dtd 2007-12-19 11:54:42.000000000 -0800
+++ flood-patchbuild/examples/flood.dtd 2007-12-19 12:32:12.000000000 -0800
@@ -1,5 +1,3 @@
-<?xml version="1.0" ?>
-
<!--
This is a DTD for flood configuration files.
@@ -12,8 +10,7 @@
is valid (in contrast to just "well-formed").
-->
-
-<!ELEMENT flood (urllist+,profile+,farmer+,farm+,seed?)>
+<!ELEMENT flood (urllist+,profile+,farmer+,farm+,seed?,subst_list?)>
<!-- urllist -->
<!ELEMENT urllist (name,description?,baseurl?,(url|sequence)+)>
@@ -46,6 +43,15 @@
<!ATTLIST sequence sequencename CDATA #REQUIRED>
<!ATTLIST sequence sequencelist CDATA #REQUIRED>
+<!-- subst_list -->
+<!--
+<!ELEMENT subst_list (subst_entry|subst_seq)>
+<!ELEMENT subst_entry (subst_file, subst_var)>
+<!ELEMENT subst_file (#PCDATA)>
+<!ELEMENT subst_var (#PCDATA)>
+<!ELEMENT subst_seq (subst_list+)>
+
+-->
<!-- profile -->
<!ENTITY % profile.events "(profile_init?,get_next_url?,create_req?,postprocess?,loop_condition?,profile_destroy?)+">
diff --exclude-from=excludefile -urN flood-orig/examples/subprojects flood-patchbuild/examples/subprojects
--- flood-orig/examples/subprojects 1969-12-31 16:00:00.000000000 -0800
+++ flood-patchbuild/examples/subprojects 2007-12-19 12:32:12.000000000 -0800
@@ -0,0 +1,2 @@
+test/flood
+apreq
diff --exclude-from=excludefile -urN flood-orig/examples/subst-example.xml flood-patchbuild/examples/subst-example.xml
--- flood-orig/examples/subst-example.xml 1969-12-31 16:00:00.000000000 -0800
+++ flood-patchbuild/examples/subst-example.xml 2007-12-19 12:32:12.000000000 -0800
@@ -0,0 +1,29 @@
+<flood configversion="1">
+ <urllist><name>Test Hosts</name><description>A bunch of hosts we want to hit</description><url method="GET" requesttemplate="http://httpd.apache.org/${subproject}";>http://httpd.apache.org/${subproject}</url></urllist>
+ <farm>
+ <name>Bingo</name>
+ <usefarmer count="1">Joe</usefarmer>
+ </farm>
+ <subst_list>
+ <subst_entry>
+ <subst_file>./subprojects</subst_file>
+ <subst_var>subproject</subst_var>
+ </subst_entry>
+ </subst_list>
+ <profile>
+ <name>RoundRobinProfile</name>
+ <profiletype>round_robin</profiletype>
+ <description>A Test Round Robin Configuration</description>
+ <verify_resp>verify_200</verify_resp>
+ <socket>generic</socket>
+ <report>easy</report>
+ <useurllist>Test Hosts</useurllist>
+ </profile>
+ <seed>1</seed>
+ <farmer>
+ <name>Joe</name>
+ <useprofile>RoundRobinProfile</useprofile>
+ <time>23</time>
+ </farmer>
+ <test_description>lab split test 2</test_description>
+</flood>
diff --exclude-from=excludefile -urN flood-orig/flood_round_robin.c flood-patchbuild/flood_round_robin.c
--- flood-orig/flood_round_robin.c 2007-12-19 11:54:42.000000000 -0800
+++ flood-patchbuild/flood_round_robin.c 2007-12-19 12:32:12.000000000 -0800
@@ -55,6 +55,7 @@
#include "config.h"
#include "flood_net.h"
#include "flood_round_robin.h"
+#include "flood_subst_file.h"
#include "flood_profile.h"
/* On FreeBSD, the return of regexec() is 0 or REG_NOMATCH, and REG_OK is undefined */
@@ -116,6 +117,9 @@
apr_hash_t *state;
+ int subst_count;
+ subst_rec_t* subst_list;
+
int current_round;
int current_url;
@@ -128,6 +132,16 @@
int size, matchsize;
regex_t re;
regmatch_t match[2];
+ subst_rec_t* subst_rec_p;
+ char* lookup_val;
+ char subst_buf[8096];
+ apr_pool_t *local_pool;
+
+ if (apr_pool_create(&local_pool, NULL) != APR_SUCCESS) {
+ apr_file_printf(local_stderr, "Failed apr_pool_create!\n");
+ exit(-1);
+ }
+
prev = template;
returnValue = NULL;
@@ -171,6 +185,26 @@
data = apr_hash_get(rp->state, cur+match[1].rm_so, matchsize);
}
+ /* if there is no data, maybe it's a random string subst */
+ /* try to do the substition */
+ if (!data) {
+ matchsize = match[1].rm_eo - match[1].rm_so;
+ lookup_val = apr_pstrndup(local_pool, cur+match[1].rm_so, matchsize);
+
+ memset(subst_buf, 0, sizeof(subst_buf));
+ subst_rec_p = subst_file_get(lookup_val, rp->subst_list);
+ subst_file_entry_get(&subst_rec_p->subst_file,
+ &subst_rec_p->fsize, subst_buf,
+ sizeof(subst_buf));
+
+ if (!strlen(subst_buf)) {
+ apr_file_printf(local_stderr,
+ "substitution didn't return data!\n");
+ exit(-1);
+ }
+ data = apr_pstrdup(rp->pool, subst_buf);
+ }
+
/* If there is no data, place the original string back. */
if (!data) {
data = apr_psprintf(rp->pool, "${%s}",
@@ -204,7 +238,10 @@
else
returnValue = apr_pstrcat(rp->pool, returnValue, cur, NULL);
+ subst_file_entry_unescape(returnValue, sizeof(returnValue));
+
regfree(&re);
+ apr_pool_destroy(local_pool);
return returnValue;
}
@@ -715,9 +752,13 @@
int i;
struct apr_xml_elem *root_elem, *profile_elem,
*urllist_elem, *count_elem, *useurllist_elem, *baseurl_elem,
+ *subst_list_elem, *subst_entry_elem, *subst_entry_child,
*proxyurl_elem, *e;
round_robin_profile_t *p;
char *xml_profile, *xml_urllist, *urllist_name;
+ char *xml_subst_list, *subst_list_name;
+ subst_rec_t* subst_rec_p;
+ int valid_substs = 0;
p = apr_pcalloc(pool, sizeof(round_robin_profile_t));
p->pool = pool;
@@ -833,6 +874,79 @@
/* Reset this back to 0. */
p->current_url = 0;
+ /* now initialize the subst_list for random text substitution */
+ /* get the subst_list from the config file */
+ /* the subst_list has pairs or substitution variables and files */
+ /* later on, in handle_param_string(), when a substitution variable */
+ /* is found, it will be substituted with a randomly chosen line from */
+ /* the subsitution file */
+ /* there can be an arbitrary number of such pairs */
+ /* the pairs scope is the entire configuration file */
+ /* they are not specific to a profile, url or urllist */
+
+ xml_subst_list = apr_pstrdup(pool, XML_SUBST_LIST);
+
+ if ((rv = retrieve_xml_elem_child(
+ &subst_list_elem, root_elem, XML_SUBST_LIST)) == APR_SUCCESS) {
+ /* count the subst_entries for this config file and allocate space */
+ p->subst_count = 0;
+ p->subst_count += count_xml_elem_child(subst_list_elem, XML_SUBST_ENTRY);
+ p->subst_list = apr_pcalloc(p->pool, sizeof(subst_rec_t) * (p->subst_count + 1));
+
+ /* get the subst_list info and populate the data structures */
+ subst_rec_p = p->subst_list;
+ for (e = subst_list_elem->first_child; e; e = e->next) {
+ if (strncasecmp(e->name, XML_SUBST_ENTRY, FLOOD_STRLEN_MAX) == 0) {
+ subst_entry_elem = e;
+ for (subst_entry_child = subst_entry_elem->first_child;
+ subst_entry_child;
+ subst_entry_child = subst_entry_child->next) {
+ if (strncasecmp(subst_entry_child->name,
+ XML_SUBST_VAR, FLOOD_STRLEN_MAX) == 0) {
+ if (subst_entry_child->first_cdata.first
+ && subst_entry_child->first_cdata.first->text) {
+ (subst_rec_t*)subst_rec_p->subst_var =
+ apr_pstrdup(pool,
+ subst_entry_child->first_cdata.first->text);
+ }
+ }
+
+ if (strncasecmp(subst_entry_child->name,
+ XML_SUBST_FILE, FLOOD_STRLEN_MAX) == 0) {
+ if (subst_entry_child->first_cdata.first
+ && subst_entry_child->first_cdata.first->text) {
+ (subst_rec_t*)subst_rec_p->subst_file_name =
+ apr_pstrdup(pool,
+ subst_entry_child->first_cdata.first->text);
+ }
+ }
+ }
+ }
+ /* this is the end of each subst_entry fetch */
+ /* at this point we should have the subst_var and subst_file_name */
+ if (subst_rec_p->subst_var && subst_rec_p->subst_file_name) {
+ subst_rec_p->valid = 1;
+ }
+ /* we should have the same number of valid substs as */
+ /* the subst_count above */
+ valid_substs++;
+ subst_rec_p++;
+ }
+ if (valid_substs != p->subst_count) {
+ apr_file_printf(local_stderr,
+ "Profile '%s' valid substs: %d inconsistent with subst count %d.\n",
+ profile_name, valid_substs, p->subst_count);
+ return APR_EGENERAL;
+ }
+ /* now open all the substitution files */
+ subst_rec_p = p->subst_list;
+ while(subst_rec_p->valid) {
+ subst_file_open(&(subst_rec_p->subst_file),
+ subst_rec_p->subst_file_name,
+ &(subst_rec_p->fsize), pool);
+ subst_rec_p++;
+ }
+ }
*profile = p;
diff --exclude-from=excludefile -urN flood-orig/flood_subst_file.c flood-patchbuild/flood_subst_file.c
--- flood-orig/flood_subst_file.c 1969-12-31 16:00:00.000000000 -0800
+++ flood-patchbuild/flood_subst_file.c 2007-12-19 12:32:12.000000000 -0800
@@ -0,0 +1,245 @@
+#include <assert.h>
+#include <apr_general.h>
+#include <apr_file_io.h>
+#include "flood_subst_file.h"
+#define IS_NUM(c) (('0' <= (c)) && ('9' >= (c)))
+
+apr_file_t* subst_file = NULL;
+extern apr_file_t *local_stdout;
+extern apr_file_t *local_stderr;
+apr_off_t fsize;
+char * pp = "thisis\nmytest";
+
+void subst_list_init(subst_rec_t *subst_list, int subst_list_size) {
+ int i;
+
+ for (i = 0; i < SUBST_FILE_ARR_MAX; i++) {
+ subst_list[i].subst_var = NULL;
+ subst_list[i].subst_file_name = NULL;
+ subst_list[i].subst_mode = 0;
+ subst_list[i].fsize = (apr_off_t)0;
+ subst_list[i].valid = 0;
+ subst_list[i].subst_file = NULL;
+ }
+}
+
+void subst_list_make(subst_rec_t *subst_list) {
+ int i2 = 0;
+
+ subst_list[0].subst_var = "name";
+ subst_list[0].subst_file_name = "/pmalab1/temphome/guyf/work/replace_mc5/flood_stuff/build/flood-0.4/test";
+ subst_list[0].subst_mode = 0;
+ subst_list[0].valid = 1;
+ i2++;
+
+ subst_list[i2].subst_var = "foot";
+ subst_list[i2].subst_file_name = "/pmalab1/temphome/guyf/work/replace_mc5/flood_stuff/build/flood-0.4/blort";
+ subst_list[i2].subst_mode = 0;
+ subst_list[i2].valid = 1;
+
+ i2++;
+ subst_list[i2].subst_var = "nerve";
+ subst_list[i2].subst_file_name = "/pmalab1/temphome/guyf/work/replace_mc5/flood_stuff/build/flood-0.4/cavort";
+ subst_list[i2].subst_mode = 0;
+ subst_list[i2].valid = 1;
+}
+
+subst_rec_t* subst_file_get(const char* varname, subst_rec_t* subst_list) {
+ int i;
+
+ for (i = 0; i < SUBST_FILE_ARR_MAX; i++) {
+ if ((strcmp(subst_list[i].subst_var, varname) == 0)
+ && subst_list[i].valid) {
+ return &(subst_list[i]);
+ }
+ }
+ return NULL;
+}
+
+
+void subst_file_err(const char* msgtext, const char* vartext, apr_status_t errcode) {
+ char errtext[SUBST_FILE_ERROR_BUF];
+ apr_file_t* local_stderr;
+ apr_pool_t *err_pool;
+
+ if (apr_pool_create(&err_pool, NULL) != APR_SUCCESS) {
+ printf("Failed apr_pool_create\n");
+ exit(-1);
+ }
+
+ apr_strerror(errcode, (char *) &errtext, SUBST_FILE_ERROR_BUF);
+ apr_file_open_stderr(&local_stderr, err_pool);
+
+ apr_file_printf(local_stderr, "%s %s %s\n", msgtext, vartext, errtext);
+ apr_file_close(local_stderr);
+
+ apr_pool_destroy(err_pool);
+}
+
+
+int subst_file_open(apr_file_t** subst_file, const char* fname, apr_off_t* fsize, apr_pool_t* pool) {
+ apr_finfo_t finfo;
+ apr_status_t rc = 0;
+ apr_int32_t wanted = APR_FINFO_SIZE;
+
+ rc = apr_file_open(subst_file, fname, APR_READ, APR_OS_DEFAULT, pool);
+
+ if (rc) {
+ subst_file_err("Couldn't open file", fname, rc);
+ exit(-1 );
+ }
+
+ rc = 0;
+ if (rc = apr_stat(&finfo, fname, wanted, pool)) {
+ subst_file_err("stat failed on file ", fname, rc);
+ apr_file_close(*subst_file);
+ exit(-1);
+ }
+ *fsize = finfo.size;
+
+ return 0;
+}
+
+
+int close_subst_file(apr_file_t* subst_file) {
+ apr_status_t rc = 0;
+
+ if (subst_file) {
+ rc = apr_file_close(subst_file);
+ }
+ return rc;
+}
+
+
+char* subst_file_entry_get(apr_file_t** subst_file, apr_off_t *fsize, char* line, int line_size) {
+ apr_off_t seek_val;
+ apr_off_t zero = 0;
+ apr_status_t rc = 0;
+
+ if (!subst_file ) {
+ subst_file_err("subst_file not open ", "", rc);
+ exit(-1);
+ }
+ assert (line_size > 0);
+ assert (*fsize > 0);
+ seek_val = random() % *fsize;
+
+ if (apr_file_seek(*subst_file, APR_SET, &seek_val) != 0 ) {
+ subst_file_err("error in seeking for file", "no name available", rc);
+ exit(-1 );
+ }
+
+ apr_file_gets(line, line_size, *subst_file);
+ memset(line, 0, line_size);
+ if (apr_file_gets(line, line_size, *subst_file) != (apr_status_t)0 ) {
+ if (apr_file_seek(*subst_file, APR_SET, &zero) != (apr_off_t)0 ) {
+ subst_file_err("error in seeking for file", "no name available", rc);
+ exit(-1 );
+ }
+ apr_file_gets(line, line_size, *subst_file);
+ }
+ line[strlen(line)-1] = '\0';
+ return line;
+}
+
+
+/* a substitution file entry (nomimally a single line) can now contain */
+/* escaped characters such as \n, \t, \012 so that the entry can be */
+/* a multi-line POST payload */
+char* subst_file_entry_unescape(char* line, int line_size)
+{
+ int num;
+ char *from, *to;
+ char changed_buf[SUBST_FILE_MAX_URL_SIZE];
+
+ if (line == NULL) {
+ return NULL;
+ }
+
+ from = to = line;
+ while (*from) {
+ if (*from == '\\') {
+ ++from;
+ if (IS_NUM(*from)) {
+ num = *from++ - '0';
+ if (IS_NUM(*from))
+ num = num*10 + (*from++ - '0');
+ if (IS_NUM(*from))
+ num = num*10 + (*from++ - '0');
+ if (num != 0) {
+ *to++ = num;
+ } else {
+ *to++ = '\\';
+ *to++ = '0';
+ }
+ } else {
+ switch (*from) {
+ case '\0': continue;
+ case 'n': *to++ = '\n'; break;
+ case 'r': *to++ = '\r'; break;
+ case 't': *to++ = '\t'; break;
+ default: *to++ = *from; break;
+ }
+ ++from;
+ }
+ } else {
+ *to++ = *from++;
+ }
+ }
+ *to = '\0';
+
+ return line;
+}
+
+#ifdef SUBST_MAIN
+int main(int argc, char** argv) {
+ char line[SUBST_FILE_MAX_URL_SIZE]; # why mess around, therefore static
+ int i = 20;
+ int list = 0;
+ char* the_name;
+ subst_rec_t *the_rec;
+
+ apr_initialize();
+ atexit(apr_terminate);
+
+ if (apr_pool_create(&pool, NULL) != APR_SUCCESS) {
+ printf("Failed apr_pool_create\n");
+ exit(-1);
+ }
+
+ srandom(time(NULL));
+
+ subst_list_init(subst_list, SUBST_FILE_ARR_MAX);
+ subst_list_make(subst_list);
+
+ while(subst_list[list].valid && list < 10) {
+ subst_file_open(&(subst_list[list].subst_file), subst_list[list].subst_file_name, &(subst_list[list].fsize));
+ list++;
+ }
+
+/* list = 0; */
+/* while (i--) { */
+/* while(subst_list[list].valid && list < 10) { */
+/* subst_file_entry_get(&(subst_list[list].subst_file), &(subst_list[list].fsize), line, sizeof(line)); */
+/* printf("%s\n", line); */
+/* memset(line, 0, sizeof(line)); */
+/* list++; */
+/* } */
+/* list = 0; */
+/* } */
+
+ the_name = "test";
+ the_rec = subst_file_get(the_name, subst_list);
+ subst_file_entry_get(&(the_rec->subst_file), &(the_rec->fsize), line, sizeof(line));
+ printf("%s\n", line);
+
+ list = 0;
+ while(subst_list[list].valid && list < 10) {
+ close_subst_file(subst_list[list].subst_file);
+ list++;
+ }
+
+ exit(0);
+}
+
+#endif
diff --exclude-from=excludefile -urN flood-orig/flood_subst_file.h flood-patchbuild/flood_subst_file.h
--- flood-orig/flood_subst_file.h 1969-12-31 16:00:00.000000000 -0800
+++ flood-patchbuild/flood_subst_file.h 2007-12-19 12:32:12.000000000 -0800
@@ -0,0 +1,25 @@
+#ifndef _SUBST_FILE_H
+#define _SUBST_FILE_H 1
+
+#define SUBST_FILE_MAX_URL_SIZE 8096
+#define SUBST_FILE_ERROR_BUF 256
+#define SUBST_FILE_ARR_MAX 10
+struct subst_rec_t {
+ char* subst_var;
+ char* subst_file_name;
+ apr_file_t* subst_file;
+ int subst_mode;
+ apr_off_t fsize;
+ int valid;
+};
+typedef struct subst_rec_t subst_rec_t;
+subst_rec_t subst_list[SUBST_FILE_ARR_MAX];
+//int subst_list_size = sizeof(subst_list)/sizeof(subst_rec_t);
+
+void subst_file_err(const char*, const char*, apr_status_t);
+int subst_file_open(apr_file_t**, const char*, apr_off_t*, apr_pool_t*);
+int close_subst_file(apr_file_t*);
+char* subst_file_entry_get(apr_file_t**, apr_off_t*, char*, int);
+subst_rec_t* subst_file_get(const char*, subst_rec_t*);
+
+#endif
diff --exclude-from=excludefile -urN flood-orig/Makefile.in flood-patchbuild/Makefile.in
--- flood-orig/Makefile.in 2007-12-19 11:54:42.000000000 -0800
+++ flood-patchbuild/Makefile.in 2007-12-19 12:32:12.000000000 -0800
@@ -26,7 +26,7 @@
flood_farmer.lo flood_simple_reports.lo flood_easy_reports.lo \
flood_farm.lo \
flood_socket_generic.lo flood_socket_keepalive.lo \
- flood_report_relative_times.lo
+ flood_report_relative_times.lo flood_subst_file.lo
flood_OBJECTS = flood.lo $(FLOOD_OBJS)
flood: $(flood_OBJECTS) $(PROGRAM_DEPENDENCIES)
