Hello,
Apologies in advance for the verbosity of this message.
A set of patches (from 0001 to 0006) were made to make the DeviceAtlas
module handling multiple HTTP headers, converting the convertor to the
fetch function type and in the end introducing a new optional keyword entry.
In addition, in case it might be helpful,an independant patchset (made from
the master branch) to fix some modest memory leaks.
Please cc as well ttr...@deviceatlas.com for any response.
Kindest regards.
David Carlier.
From 41104700179c8e5d2c4ccaa6004aff0eabe9edf1 Mon Sep 17 00:00:00 2001
From: David CARLIER <devne...@gmail.com>
Date: Wed, 23 Sep 2015 20:08:27 +0100
Subject: [PATCH 1/6] Added new DeviceAtlas configuration keyword
deviceatlas-cookie-name.
---
doc/configuration.txt | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/doc/configuration.txt b/doc/configuration.txt
index aa00628..44ec896 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -653,6 +653,11 @@ deviceatlas-separator <char>
Sets the character separator for the API properties results. This directive
is optional and set to | by default if not set.
+devicatlas-properties-cookie <name>
+ Sets the client cookie's name used for the detection if the DeviceAtlas Client-side
+ component was used during the request. This directive is optional and set to
+ DAPROPS by default if not set.
+
external-check
Allows the use of an external agent to perform health checks.
This is disabled by default as a security precaution.
@@ -11394,18 +11399,17 @@ crc32([<avalanche>])
also "djb2", "sdbm", "wt6" and the "hash-type" directive.
da-csv(<prop>[,<prop>*])
- Asks the DeviceAtlas converter to identify the User Agent string passed on
- input, and to emit a string made of the concatenation of the properties
- enumerated in argument, delimited by the separator defined by the global
- keyword "deviceatlas-property-separator", or by default the pipe character
- ('|'). There's a limit of 5 different properties imposed by the haproxy
+ Asks the DeviceAtlas converter to emit a string made of the concatenation of
+ the properties enumerated in argument, delimited by the separator defined by
+ the global keyword "deviceatlas-property-separator", or by default the pipe
+ character ('|'). There's a limit of 5 different properties imposed by the haproxy
configuration language.
Example:
frontend www
bind *:8881
default_backend servers
- http-request set-header X-DeviceAtlas-Data %[req.fhdr(User-Agent),da-csv(primaryHardwareType,osName,osVersion,browserName,browserVersion)]
+ http-request set-header X-DeviceAtlas-Data %[da-csv(primaryHardwareType,osName,osVersion,browserName,browserVersion)]
debug
This converter is used as debug tool. It dumps on screen the content and the
--
1.9.1
From 39b8e5bcb9c39bdd7a7242ec7b0ad370fe2bfd74 Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 20:10:10 +0100
Subject: [PATCH 2/6] Extracts two internal functions to iterate through the
list of HTTP headers and to get if possible the specific client's cookie
pushed by the DeviceAtlas Client-side component.
---
include/proto/proto_http.h | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/include/proto/proto_http.h b/include/proto/proto_http.h
index 8385dc6..39717ed 100644
--- a/include/proto/proto_http.h
+++ b/include/proto/proto_http.h
@@ -94,7 +94,11 @@ int http_find_full_header2(const char *name, int len,
int http_find_header2(const char *name, int len,
char *sol, struct hdr_idx *idx,
struct hdr_ctx *ctx);
+int http_find_next_header(char *sol, struct hdr_idx *idx,
+ struct hdr_ctx *ctx);
char *find_hdr_value_end(char *s, const char *e);
+char *extract_cookie_value(char *hdr, const char *hdr_end, char *cookie_name,
+ size_t cookie_name_l, int list, char **value, int *value_l);
int http_header_match2(const char *hdr, const char *end, const char *name, int len);
int http_remove_header2(struct http_msg *msg, struct hdr_idx *idx, struct hdr_ctx *ctx);
int http_header_add_tail2(struct http_msg *msg, struct hdr_idx *hdr_idx, const char *text, int len);
--
1.9.1
From d7cb8acbb07e4474f4d6e1077e4b52bdd7f55ebc Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 20:11:35 +0100
Subject: [PATCH 3/6] As the DeviceAtlas fetch function handles all HTTP
headers, no need anymore to store an user agent identifier, a simple
bitfield value suffices.
---
include/types/global.h | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/include/types/global.h b/include/types/global.h
index cb04074..9842bce 100644
--- a/include/types/global.h
+++ b/include/types/global.h
@@ -180,10 +180,12 @@ struct global {
struct {
void *atlasimgptr;
char *jsonpath;
+ char *cookiename;
+ size_t cookienamelen;
da_atlas_t atlas;
- da_evidence_id_t useragentid;
da_severity_t loglevel;
char separator;
+ unsigned char daset:1;
} deviceatlas;
#endif
#ifdef USE_51DEGREES
--
1.9.1
From f0d0b172227813d74227af31559a3e2bc9d3d45e Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 20:12:48 +0100
Subject: [PATCH 4/6] The main changes occurs here :
- Adding new keyword configuration to possibly update the cookie name to look at.
- Converting the convertor function to the fetch type. Ensuring all request data
are ready to process and are in good type for.
- The DeviceAtlas API filters which headers it needs to use,
with a special care for the cookie value.
---
src/da.c | 171 ++++++++++++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 131 insertions(+), 40 deletions(-)
diff --git a/src/da.c b/src/da.c
index 7f507ea..69b4aae 100644
--- a/src/da.c
+++ b/src/da.c
@@ -3,6 +3,7 @@
#include <common/cfgparse.h>
#include <proto/arg.h>
#include <proto/log.h>
+#include <proto/proto_http.h>
#include <proto/sample.h>
#include <import/da.h>
@@ -50,6 +51,20 @@ static int da_property_separator(char **args, int section_type, struct proxy *cu
return 0;
}
+static int da_cookie_name(char **args, int section_type, struct proxy *curpx,
+ struct proxy *defpx, const char *file, int line,
+ char **err)
+{
+ if (*(args[1]) == 0) {
+ memprintf(err, "deviceatlas cookie name : expects a string argument.\n");
+ return -1;
+ } else {
+ global.deviceatlas.cookiename = strdup(args[1]);
+ }
+ global.deviceatlas.cookienamelen = strlen(global.deviceatlas.cookiename);
+ return 0;
+}
+
static size_t da_haproxy_read(void *ctx, size_t len, char *buf)
{
return fread(buf, 1, len, ctx);
@@ -70,6 +85,8 @@ static void da_haproxy_log(da_severity_t severity, da_status_t status,
}
}
+#define DA_COOKIENAME_DEFAULT "DAPROPS"
+
int init_deviceatlas(void)
{
da_status_t status = DA_SYS;
@@ -105,8 +122,12 @@ int init_deviceatlas(void)
goto out;
}
- global.deviceatlas.useragentid = da_atlas_header_evidence_id(&global.deviceatlas.atlas,
- "user-agent");
+ if (global.deviceatlas.cookiename == 0) {
+ global.deviceatlas.cookiename = strdup(DA_COOKIENAME_DEFAULT);
+ global.deviceatlas.cookienamelen = strlen(global.deviceatlas.cookiename);
+ }
+
+ global.deviceatlas.daset = 1;
fprintf(stdout, "Deviceatlas module loaded.\n");
}
@@ -121,7 +142,8 @@ void deinit_deviceatlas(void)
free(global.deviceatlas.jsonpath);
}
- if (global.deviceatlas.useragentid > 0) {
+ if (global.deviceatlas.daset == 1) {
+ free(global.deviceatlas.cookiename);
da_atlas_close(&global.deviceatlas.atlas);
free(global.deviceatlas.atlasimgptr);
}
@@ -129,34 +151,99 @@ void deinit_deviceatlas(void)
da_fini();
}
-static int da_haproxy(const struct arg *args, struct sample *smp, void *private)
+#define DA_MAX_HEADERS 24
+
+static int da_haproxy(const struct arg *args, struct sample *smp, const char *kw, void *private)
{
struct chunk *tmp;
+ struct hdr_idx *hidx;
+ struct hdr_ctx hctx;
+ const struct http_msg *hmsg;
+ da_evidence_t ev[DA_MAX_HEADERS];
da_deviceinfo_t devinfo;
da_propid_t prop, *pprop;
da_type_t proptype;
da_status_t status;
- const char *useragent, *propname;
- char useragentbuf[1024];
- int i;
+ const char *propname;
+ int i, nbh = 0;
- if (global.deviceatlas.useragentid == 0) {
+ if (global.deviceatlas.daset == 0) {
return 1;
}
+ CHECK_HTTP_MESSAGE_FIRST();
+ smp->data.type = SMP_T_STR;
+
+ /**
+ * Here we go through the whole list of headers from start
+ * they will be filtered via the DeviceAtlas API itself
+ */
+ hctx.idx = 0;
+ hidx = &smp->strm->txn->hdr_idx;
+ hmsg = &smp->strm->txn->req;
+
+ while(http_find_next_header(hmsg->chn->buf->p, hidx, &hctx) == 1 &&
+ nbh < DA_MAX_HEADERS) {
+ char *pval;
+ size_t vlen;
+ da_evidence_id_t evid = -1;
+ char hbuf[24] = { 0 };
+
+ /* The HTTP headers used by the DeviceAtlas API are not longer */
+ if (hctx.del >= sizeof(hbuf)) {
+ continue;
+ }
+
+ vlen = hctx.vlen;
+ memcpy(hbuf, hctx.line, hctx.del);
+ hbuf[hctx.del] = 0;
+ pval = (hctx.line + hctx.val);
+
+ if (strcmp(hbuf, "Accept-Language") == 0) {
+ evid = da_atlas_accept_language_evidence_id(&global.deviceatlas.
+ atlas);
+ } else if (strcmp(hbuf, "Cookie") == 0) {
+ char *p, *eval;
+ int pl;
+
+ eval = pval + hctx.vlen;
+ /**
+ * The cookie value, if it exists, is located between the current header's
+ * value position and the next one
+ */
+ if (extract_cookie_value(pval, eval, global.deviceatlas.cookiename,
+ global.deviceatlas.cookienamelen, 1, &p, &pl) == NULL) {
+
+ continue;
+ }
+
+ vlen = (size_t)pl;
+ pval = p;
+ evid = da_atlas_clientprop_evidence_id(&global.deviceatlas.atlas);
+ } else {
+ evid = da_atlas_header_evidence_id(&global.deviceatlas.atlas,
+ hbuf);
+ }
+
+ if (evid == -1) {
+ continue;
+ }
+
+ ev[nbh].key = evid;
+ ev[nbh].value = malloc(vlen + 1);
+ strncpy(ev[nbh].value, pval, vlen);
+ ev[nbh].value[vlen] = 0;
+ ++ nbh;
+ }
+
tmp = get_trash_chunk();
chunk_reset(tmp);
- i = smp->data.u.str.len > sizeof(useragentbuf) ? sizeof(useragentbuf) : smp->data.u.str.len;
- memcpy(useragentbuf, smp->data.u.str.str, i - 1);
- useragentbuf[i - 1] = 0;
-
- useragent = (const char *)useragentbuf;
propname = (const char *)args[0].data.str.str;
i = 0;
- status = da_search(&global.deviceatlas.atlas, &devinfo,
- global.deviceatlas.useragentid, useragent, 0);
+ status = da_searchv(&global.deviceatlas.atlas, &devinfo,
+ ev, nbh);
if (status != DA_OK) {
return 0;
}
@@ -173,38 +260,41 @@ static int da_haproxy(const struct arg *args, struct sample *smp, void *private)
switch (proptype) {
case DA_TYPE_BOOLEAN: {
- bool val;
- status = da_getpropboolean(&devinfo, *pprop, &val);
- if (status == DA_OK) {
- chunk_appendf(tmp, "%d", val);
- }
- break;
+ bool val;
+ status = da_getpropboolean(&devinfo, *pprop, &val);
+ if (status == DA_OK) {
+ chunk_appendf(tmp, "%d", val);
+ }
+ break;
}
case DA_TYPE_INTEGER:
case DA_TYPE_NUMBER: {
- long val;
- status = da_getpropinteger(&devinfo, *pprop, &val);
- if (status == DA_OK) {
- chunk_appendf(tmp, "%ld", val);
- }
- break;
+ long val;
+ status = da_getpropinteger(&devinfo, *pprop, &val);
+ if (status == DA_OK) {
+ chunk_appendf(tmp, "%ld", val);
+ }
+ break;
}
case DA_TYPE_STRING: {
- const char *val;
- status = da_getpropstring(&devinfo, *pprop, &val);
- if (status == DA_OK) {
- chunk_appendf(tmp, "%s", val);
- }
- break;
- }
- default:
- break;
+ const char *val;
+ status = da_getpropstring(&devinfo, *pprop, &val);
+ if (status == DA_OK) {
+ chunk_appendf(tmp, "%s", val);
+ }
+ break;
+ }
+ default:
+ break;
}
chunk_appendf(tmp, "%c", global.deviceatlas.separator);
}
da_close(&devinfo);
+ for (i = 0; i < nbh; i ++) {
+ free(ev[i].value);
+ }
if (tmp->len) {
--tmp->len;
@@ -212,7 +302,7 @@ static int da_haproxy(const struct arg *args, struct sample *smp, void *private)
}
smp->data.u.str.str = tmp->str;
- smp->data.u.str.len = strlen(tmp->str);
+ smp->data.u.str.len = tmp->len;
return 1;
}
@@ -221,12 +311,13 @@ static struct cfg_kw_list dacfg_kws = {{ }, {
{ CFG_GLOBAL, "deviceatlas-json-file", da_json_file },
{ CFG_GLOBAL, "deviceatlas-log-level", da_log_level },
{ CFG_GLOBAL, "deviceatlas-property-separator", da_property_separator },
+ { CFG_GLOBAL, "deviceatlas-cookie-name", da_cookie_name },
{ 0, NULL, NULL },
}};
/* Note: must not be declared <const> as its list will be overwritten */
-static struct sample_conv_kw_list conv_kws = {ILH, {
- { "da-csv", da_haproxy, ARG5(1,STR,STR,STR,STR,STR), NULL, SMP_T_STR, SMP_T_STR },
+static struct sample_fetch_kw_list conv_kws = {ILH, {
+ { "da-csv", da_haproxy, ARG5(1,STR,STR,STR,STR,STR), NULL, SMP_T_STR, SMP_USE_HRQHV },
{ NULL, NULL, 0, 0, 0 },
}};
@@ -234,6 +325,6 @@ __attribute__((constructor))
static void __da_init(void)
{
/* register sample fetch and format conversion keywords */
- sample_register_convs(&conv_kws);
+ sample_register_fetches(&conv_kws);
cfg_register_keywords(&dacfg_kws);
}
--
1.9.1
From 4617fd5fb689e7edd1b3e9d2eb522a1c7eb3a3b1 Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 20:14:37 +0100
Subject: [PATCH 5/6] Exposing the previously internal functions.
---
src/proto_http.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/proto_http.c b/src/proto_http.c
index 9acf0a2..793871c 100644
--- a/src/proto_http.c
+++ b/src/proto_http.c
@@ -10971,7 +10971,7 @@ smp_fetch_http_auth_grp(const struct arg *args, struct sample *smp, const char *
* of values (cookie headers). This makes it faster to abort parsing when no
* list is expected.
*/
-static char *
+char *
extract_cookie_value(char *hdr, const char *hdr_end,
char *cookie_name, size_t cookie_name_l, int list,
char **value, int *value_l)
--
1.9.1
From 4e6b2855de553c9f787196c021d7151809bb2cf3 Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 20:15:20 +0100
Subject: [PATCH 6/6] Initialization of the new global DeviceAtlas fields.
---
src/haproxy.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/haproxy.c b/src/haproxy.c
index 465bb6a..013a8ee 100644
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -181,8 +181,10 @@ struct global global = {
#ifdef USE_DEVICEATLAS
.deviceatlas = {
.loglevel = DA_SEV_INFO,
- .useragentid = 0,
.jsonpath = 0,
+ .cookiename = 0,
+ .cookienamelen = 0,
+ .daset = 0,
.separator = '|',
},
#endif
--
1.9.1
From e4f395a8f225b701b1714172c0abd593f30ccc26 Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 21:54:16 +0100
Subject: [PATCH 07/11] Additional function to free config sections.
---
include/common/cfgparse.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/include/common/cfgparse.h b/include/common/cfgparse.h
index 86a0035..d785327 100644
--- a/include/common/cfgparse.h
+++ b/include/common/cfgparse.h
@@ -73,6 +73,7 @@ int check_config_validity();
int str2listener(char *str, struct proxy *curproxy, struct bind_conf *bind_conf, const char *file, int line, char **err);
int cfg_register_section(char *section_name,
int (*section_parser)(const char *, int, char **, int));
+void cfg_unregister_sections(void);
int warnif_misplaced_tcp_conn(struct proxy *proxy, const char *file, int line, const char *arg);
int warnif_misplaced_tcp_cont(struct proxy *proxy, const char *file, int line, const char *arg);
--
1.9.1
From cb2a20abce0c222e3d95135a5bc53c52898175e2 Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 21:54:55 +0100
Subject: [PATCH 08/11] New function to free the two internal trash buffers.
---
include/common/chunk.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/include/common/chunk.h b/include/common/chunk.h
index 18f41af..8225d96 100644
--- a/include/common/chunk.h
+++ b/include/common/chunk.h
@@ -48,6 +48,7 @@ int chunk_asciiencode(struct chunk *dst, struct chunk *src, char qc);
int chunk_strcmp(const struct chunk *chk, const char *str);
int chunk_strcasecmp(const struct chunk *chk, const char *str);
int alloc_trash_buffers(int bufsize);
+void free_trash_buffers(void);
struct chunk *get_trash_chunk(void);
static inline void chunk_reset(struct chunk *chk)
--
1.9.1
From d52f26412c792c816519358a6da500cc33f02fec Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 21:55:49 +0100
Subject: [PATCH 09/11] Clear the list of config sections.
---
src/cfgparse.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/src/cfgparse.c b/src/cfgparse.c
index 3b7b390..aefc225 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -8840,6 +8840,19 @@ int cfg_register_section(char *section_name,
return 1;
}
+/*
+ * free all config section entries
+ */
+void cfg_unregister_sections(void)
+{
+ struct cfg_section *cs, *ics;
+
+ list_for_each_entry_safe(cs, ics, §ions, list) {
+ LIST_DEL(&cs->list);
+ free(cs);
+ }
+}
+
/*
* Local variables:
* c-indent-level: 8
--
1.9.1
From cbbe42163e6e91bd9aae613e45e4d4a9508cbc76 Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 21:56:42 +0100
Subject: [PATCH 10/11] Function to clear the two trash buffers.
---
src/chunk.c | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/src/chunk.c b/src/chunk.c
index 3c1cfdf..2098812 100644
--- a/src/chunk.c
+++ b/src/chunk.c
@@ -24,7 +24,7 @@ static struct chunk trash_chunk1;
static struct chunk trash_chunk2;
/* trash buffers used for various conversions */
-static int trash_size;
+static int trash_size = 0;
static char *trash_buf1;
static char *trash_buf2;
@@ -66,6 +66,17 @@ int alloc_trash_buffers(int bufsize)
}
/*
+ * free the trash buffers
+ */
+void free_trash_buffers(void)
+{
+ if (trash_size > 0) {
+ free(trash_buf2);
+ free(trash_buf1);
+ }
+}
+
+/*
* Does an snprintf() at the beginning of chunk <chk>, respecting the limit of
* at most chk->size chars. If the chk->len is over, nothing is added. Returns
* the new chunk size, or < 0 in case of failure.
--
1.9.1
From f2a7ef2ce3a21ba41aae22fdaf72910ca47dfd80 Mon Sep 17 00:00:00 2001
From: David CARLIER <dcarl...@afilias.info>
Date: Wed, 23 Sep 2015 21:57:41 +0100
Subject: [PATCH 11/11] free a bunch of static variables during the deinit
phase. free the config sections, the trash buffers and clear the internal
trash's chunk.
---
src/haproxy.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/src/haproxy.c b/src/haproxy.c
index 465bb6a..18cefe0 100644
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -1445,6 +1445,11 @@ void deinit(void)
userlist_free(userlist);
+ cfg_unregister_sections();
+
+ free_trash_buffers();
+ chunk_destroy(&trash);
+
protocol_unbind_all();
#if defined(USE_DEVICEATLAS)
@@ -1464,6 +1469,9 @@ void deinit(void)
free(fdinfo); fdinfo = NULL;
free(fdtab); fdtab = NULL;
free(oldpids); oldpids = NULL;
+ free(static_table_key); static_table_key = NULL;
+ free(get_http_auth_buff); get_http_auth_buff = NULL;
+ free(swap_buffer); swap_buffer = NULL;
free(global_listener_queue_task); global_listener_queue_task = NULL;
list_for_each_entry_safe(log, logb, &global.logsrvs, list) {
--
1.9.1
