[PATCH 3/3]: deviceatlas update doc and build changes.
Last patch of the patchset concerning the doc and the build. Thanks in advance. From e3f520ebd4f23fe42a1068ee2d9b42fd529241e8 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 21 Jan 2022 20:57:27 + Subject: [PATCH 3/3] MEDIUM: deviceatlas doc and build update. Mentions of the new service and update of the build. --- Makefile | 5 - doc/DeviceAtlas-device-detection.txt | 14 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index af0f5cc87..e6f2e04ad 100644 --- a/Makefile +++ b/Makefile @@ -653,11 +653,14 @@ endif ifneq ($(USE_PCRE2),) OPTIONS_CFLAGS += -DDA_REGEX_HDR=\"dac_pcre2.c\" -DDA_REGEX_TAG=2 endif +OPTIONS_OBJS += $(DEVICEATLAS_LIB)/Os/daunix.o +OPTIONS_OBJS += $(DEVICEATLAS_LIB)/dadwcom.o +OPTIONS_OBJS += $(DEVICEATLAS_LIB)/dasch.o OPTIONS_OBJS += $(DEVICEATLAS_LIB)/json.o OPTIONS_OBJS += $(DEVICEATLAS_LIB)/dac.o endif OPTIONS_OBJS += addons/deviceatlas/da.o -OPTIONS_CFLAGS += $(if $(DEVICEATLAS_INC),-I$(DEVICEATLAS_INC)) +OPTIONS_CFLAGS += $(if $(DEVICEATLAS_INC),-I$(DEVICEATLAS_INC)) endif ifneq ($(USE_51DEGREES),) diff --git a/doc/DeviceAtlas-device-detection.txt b/doc/DeviceAtlas-device-detection.txt index 9a1c9b594..0045448f5 100644 --- a/doc/DeviceAtlas-device-detection.txt +++ b/doc/DeviceAtlas-device-detection.txt @@ -9,7 +9,8 @@ The build supports the USE_PCRE and USE_PCRE2 options. Once extracted : Optionally DEVICEATLAS_INC and DEVICEATLAS_LIB may be set to override the path to the include files and libraries respectively if they're not in the source -directory. +directory. However, if the API had been installed beforehand, DEVICEATLAS_SRC +can be omitted. Note that the DeviceAtlas C API version supported is the 2.4.0 at minimum. For HAProxy developers who need to verify that their changes didn't accidentally break the DeviceAtlas code, it is possible to build a dummy library provided in @@ -62,5 +63,16 @@ Single HTTP header acl device_type_tablet req.fhdr(User-Agent),da-csv-conv(primaryHardwareType) "Tablet" +Optionally a JSON download scheduler is provided to allow a data file being fetched automatically in a daily basis without +restarting HAProxy : + +$ cd addons/deviceatlas && make [DEVICEATLAS_SRC=] + +Similarly, if the DeviceAtlas API is installed, DEVICEATLAS_SRC can be omitted. + +$ ./dadwsch -u JSON data file URL e.g. "https://deviceatlas.com/getJSON?licencekey==zip=my=web" [-p download directory path /tmp by default] [-d scheduled hour of download, hour when the service is launched by default] + +Noted it needs to be started before HAProxy. + Please find more information about DeviceAtlas and the detection methods at https://deviceatlas.com/resources . -- 2.30.2
[PATCH 2/3]: deviceatlas addon update
Second patch of the patchset concerning the update of the addon itself.
From d541d8dfd8aa0290625b3824bd6b44d2861e997f Mon Sep 17 00:00:00 2001
From: David Carlier
Date: Fri, 21 Jan 2022 20:51:20 +
Subject: [PATCH 2/3] MEDIUM: deviceatlas update of the main module.
The DeviceAtlas addon can optionally interacts with the new service
without change of configuration in the HAProxy part.
Note that however it requires the DeviceAtlas Identification C API
2.4.0 minimum from this point.
Signed-off-by: David Carlier
---
addons/deviceatlas/da.c | 77 +
1 file changed, 77 insertions(+)
diff --git a/addons/deviceatlas/da.c b/addons/deviceatlas/da.c
index 418909226..724209936 100644
--- a/addons/deviceatlas/da.c
+++ b/addons/deviceatlas/da.c
@@ -1,4 +1,7 @@
#include
+#include
+#include
+#include
#include
#include
@@ -14,11 +17,16 @@
#include
#include
+#define ATLASTOKSZ PATH_MAX
+#define ATLASMAPNM "/hapdeviceatlas"
+
static struct {
void *atlasimgptr;
+ void *atlasmap;
char *jsonpath;
char *cookiename;
size_t cookienamelen;
+ int atlasfd;
da_atlas_t atlas;
da_evidence_id_t useragentid;
da_severity_t loglevel;
@@ -29,11 +37,17 @@ static struct {
.jsonpath = 0,
.cookiename = 0,
.cookienamelen = 0,
+ .atlasmap = NULL,
+ .atlasfd = -1,
.useragentid = 0,
.daset = 0,
.separator = '|',
};
+#ifdef USE_THREAD
+__decl_spinlock(dadwsch_lock);
+#endif
+
static int da_json_file(char **args, int section_type, struct proxy *curpx,
const struct proxy *defpx, const char *file, int line,
char **err)
@@ -163,6 +177,16 @@ static int init_deviceatlas(void)
global_deviceatlas.useragentid = da_atlas_header_evidence_id(_deviceatlas.atlas,
"user-agent");
+ if ((global_deviceatlas.atlasfd = shm_open(ATLASMAPNM, O_RDWR, 0660)) != -1) {
+ global_deviceatlas.atlasmap = mmap(NULL, ATLASTOKSZ, PROT_READ | PROT_WRITE, MAP_SHARED, global_deviceatlas.atlasfd, 0);
+ if (global_deviceatlas.atlasmap == MAP_FAILED) {
+close(global_deviceatlas.atlasfd);
+global_deviceatlas.atlasfd = -1;
+global_deviceatlas.atlasmap = NULL;
+ } else {
+fprintf(stdout, "deviceatlas : scheduling support.\n");
+ }
+ }
global_deviceatlas.daset = 1;
fprintf(stdout, "Deviceatlas module loaded.\n");
@@ -184,9 +208,58 @@ static void deinit_deviceatlas(void)
free(global_deviceatlas.atlasimgptr);
}
+ if (global_deviceatlas.atlasfd != -1) {
+ munmap(global_deviceatlas.atlasmap, ATLASTOKSZ);
+ close(global_deviceatlas.atlasfd);
+ shm_unlink(ATLASMAPNM);
+ }
+
da_fini();
}
+static void da_haproxy_checkinst(void)
+{
+ if (global_deviceatlas.atlasmap != 0) {
+ char *base;
+ base = (char *)global_deviceatlas.atlasmap;
+
+ if (base[0] != 0) {
+ void *cnew;
+ size_t atlassz;
+ char atlasp[ATLASTOKSZ] = {0};
+ da_atlas_t inst;
+ da_property_decl_t extraprops[1] = {{NULL, 0}};
+#ifdef USE_THREAD
+ HA_SPIN_LOCK(OTHER_LOCK, _lock);
+#endif
+ strlcpy2(atlasp, base, sizeof(atlasp));
+ if (da_atlas_read_mapped(atlasp, NULL, , ) == DA_OK) {
+if (da_atlas_open(, extraprops, cnew, atlassz) == DA_OK) {
+ char jsonbuf[26];
+ time_t jsond;
+
+ da_atlas_close(_deviceatlas.atlas);
+ free(global_deviceatlas.atlasimgptr);
+ global_deviceatlas.atlasimgptr = cnew;
+ global_deviceatlas.atlas = inst;
+ memset(base, 0, ATLASTOKSZ);
+ jsond = da_getdatacreation(_deviceatlas.atlas);
+ ctime_r(, jsonbuf);
+ jsonbuf[24] = 0;
+ printf("deviceatlas: new instance, data file date `%s`.\n", jsonbuf);
+} else {
+ ha_warning("deviceatlas: instance update failed.\n");
+ memset(base, 0, ATLASTOKSZ);
+ free(cnew);
+}
+ }
+#ifdef USE_THREAD
+ HA_SPIN_UNLOCK(OTHER_LOCK, _lock);
+#endif
+ }
+ }
+}
+
static int da_haproxy(const struct arg *args, struct sample *smp, da_deviceinfo_t *devinfo)
{
struct buffer *tmp;
@@ -272,6 +345,8 @@ static int da_haproxy_conv(const struct arg *args, struct sample *smp, void *pri
return 1;
}
+ da_haproxy_checkinst();
+
i = smp->data.u.str.data > sizeof(useragentbuf) ? sizeof(useragentbuf) : smp->data.u.str.data;
memcpy(useragentbuf, smp->data.u.str.area, i - 1);
useragentbuf[i - 1] = 0;
@@ -301,6 +376,8 @@ static int da_haproxy_fetch(const struct arg *args, struct sample *smp, const ch
return 0;
}
+ da_haproxy_checkinst();
+
chn = (smp->strm ? >strm->req : NULL);
htx = smp_prefetch_htx(smp, chn, NULL, 1);
if (!htx)
--
2.30.2
[PATCH 1/3] deviceatlas update new service addition
Hi here the first patch of the patchset for the DeviceAtlas addon
concerning the
new download/scheduler service.
Thanks in advance.
From eae56a44d3bf51f8e92f66baef25f2a417f837ce Mon Sep 17 00:00:00 2001
From: David Carlier
Date: Fri, 21 Jan 2022 20:46:40 +
Subject: [PATCH 1/3] MEDIUM: deviceatlas new optional data file download
scheduler service.
New specialized service to daily handle the update of download file without
interruption of service and to be preemptively started before HAProxy.
Signed-off-by: David Carlier
---
addons/deviceatlas/Makefile | 49 +
addons/deviceatlas/dadwsch.c | 195 +++
2 files changed, 244 insertions(+)
create mode 100644 addons/deviceatlas/Makefile
create mode 100644 addons/deviceatlas/dadwsch.c
diff --git a/addons/deviceatlas/Makefile b/addons/deviceatlas/Makefile
new file mode 100644
index 0..33dd9434e
--- /dev/null
+++ b/addons/deviceatlas/Makefile
@@ -0,0 +1,49 @@
+# DEVICEATLAS_SRC : DeviceAtlas API source root path
+
+
+OS := $(shell uname -s)
+OBJS:= dadwsch.o
+CFLAGS := -g -O2
+LDFLAGS :=
+
+CURL_CONFIG := curl-config
+CURLDIR := $(shell $(CURL_CONFIG) --prefix 2>/dev/null || echo /usr/local)
+CURL_INC:= $(CURLDIR)/include
+CURL_LIB:= $(CURLDIR)/lib
+CURL_LDFLAGS:= $(shell $(CURL_CONFIG) --libs 2>/dev/null || echo -L /usr/local/lib -lcurl)
+
+PCRE2_CONFIG:= pcre2-config
+PCRE2DIR:= $(shell $(PCRE2_CONFIG) --prefix 2>/dev/null || echo /usr/local)
+PCRE2_INC := $(PCRE2DIR)/include
+PCRE2_LIB := $(PCRE2DIR)/lib
+PCRE2_LDFLAGS := $(shell $(PCRE2_CONFIG) --libs8 2>/dev/null || echo /usr/local)
+
+ifeq ($(DEVICEATLAS_SRC),)
+dadwsch:dadwsch.c
+ $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
+
+LDFLAGS += -lda
+else
+DEVICEATLAS_INC = $(DEVICEATLAS_SRC)
+DEVICEATLAS_LIB = $(DEVICEATLAS_SRC)
+CFLAGS += -DDA_REGEX_HDR=\"dac_pcre2.c\" -DDA_REGEX_TAG=2
+CFLAGS += -DMOBI_CURL -DMOBI_CURLSSET -DMOBI_GZ -DMOBI_ZIP
+CFLAGS += -I$(DEVICEATLAS_INC) -I$(CURL_INC) -I$(PCRE2DIR)
+LDFLAGS += $(CURL_LDFLAGS) $(PCRE2_LDFLAGS) -lz -lzip -lpthread
+
+dadwsch:dadwsch.c $(DEVICEATLAS_SRC)/dac.c $(DEVICEATLAS_SRC)/dasch.c $(DEVICEATLAS_SRC)/dadwarc.c $(DEVICEATLAS_SRC)/dadwcom.c $(DEVICEATLAS_SRC)/dadwcurl.c $(DEVICEATLAS_SRC)/json.c $(DEVICEATLAS_SRC)/Os/daunix.c
+ $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
+endif
+
+ifeq ($(OS), Linux)
+LDFLAGS += -lrt
+endif
+ifeq ($(OS), SunOS)
+LDFLAGS += -lrt
+endif
+
+clean:
+ rm -f *.o
+ rm -f $(DEVICEATLAS_LIB)*.o
+ rm -f dadwsch
+
diff --git a/addons/deviceatlas/dadwsch.c b/addons/deviceatlas/dadwsch.c
new file mode 100644
index 0..e35566a3f
--- /dev/null
+++ b/addons/deviceatlas/dadwsch.c
@@ -0,0 +1,195 @@
+#define _GNU_SOURCE
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#define ATLASTOKSZ PATH_MAX
+#define ATLASMAPNM "/hapdeviceatlas"
+
+const char *__pgname;
+
+static struct {
+ da_dwatlas_t o;
+ int ofd;
+ void* atlasmap;
+} global_deviceatlassch = {
+ .ofd = -1,
+ .atlasmap = NULL
+};
+
+
+void usage(void)
+{
+ fprintf(stderr, "%s -u download URL [-d hour (in H:M:S format) current hour by default] [-p path for the downloaded file, /tmp by default]\n", __pgname);
+ exit(EXIT_FAILURE);
+}
+
+static size_t jsonread(void *ctx, size_t count, char *buf)
+{
+ return fread(buf, 1, count, ctx);
+}
+
+static da_status_t jsonseek(void *ctx, off_t pos)
+{
+ return fseek(ctx, pos, SEEK_SET) != -1 ? DA_OK : DA_SYS;
+}
+
+static void dadwlog(dw_config_t cfg, const char* msg)
+{
+ time_t now = time(NULL);
+ char buf[26] = {0};
+ ctime_r(, buf);
+ buf[24] = 0;
+ fprintf(stderr, "%s: %s\n", buf, msg);
+}
+
+static dw_status_t dadwnot(void *a, dw_config_t *cfg)
+{
+ da_dwatlas_t *o = (da_dwatlas_t *)a;
+ if (!o)
+ return DW_ERR;
+ char *e;
+ char jsondbuf[26] = {0}, buf[26] = {0}, atlasp[ATLASTOKSZ] = {0};
+ time_t now = time(NULL);
+ time_t jsond;
+ int fd = -1;
+ (void)a;
+ jsond = da_getdatacreation(>atlas);
+ dwgetfinalp(o->dcfg.info, atlasp, sizeof(atlasp));
+ ctime_r(, jsondbuf);
+ ctime_r(, buf);
+ jsondbuf[24] = 0;
+ buf[24] = 0;
+
+ printf("%s: data file generated on `%s`\n", buf, jsondbuf);
+ int val = 1;
+ unsigned char *ptr = (unsigned char *)global_deviceatlassch.atlasmap;
+ memset(ptr, 0, sizeof(atlasp));
+ strcpy(ptr, atlasp);
+ return DW_OK;
+}
+
+static da_status_t dadwinit(void)
+{
+ if ((global_deviceatlassch.ofd = shm_open(ATLASMAPNM, O_RDWR | O_CREAT, 0660)) == -1) {
+ fprintf(stderr, "%s\n", strerror(errno));
+ return DA_SYS;
+ }
+
+ if (ftruncate(global_deviceatlassch.ofd, ATLASTOKSZ) == -1) {
+ close(global_deviceatlassch.ofd);
+ return DA_SYS;
+ }
+ lseek(global_deviceatlassch.ofd, 0, SEEK_SET);
+ global_deviceatlassch.atlasmap = mmap(0, ATLASTOKSZ, PROT_READ | PROT_WRITE, MAP_SHARED, global_deviceatlassch.ofd, 0);
+ if (global_deviceatlassch.atlasmap ==
[PATCH] CI: introduce caching for ssl libs (except BoringSSL, QUICTLS)
Hello,
this patch introduces github actions cache for SSL libs.
hope it will save couple of minutes.
cheers,
Ilya
From 5c62945e56f3bd36432483b01cba4e734dd44979 Mon Sep 17 00:00:00 2001
From: Ilya Shipitsin
Date: Sat, 22 Jan 2022 00:00:44 +0500
Subject: [PATCH] CI: github actions: use cache for SSL libs
we have two kinds of SSL libs built - git based and version based.
this commit introduces caching for version based SSL libs.
---
.github/workflows/vtest.yml | 18 +-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/vtest.yml b/.github/workflows/vtest.yml
index d748ee28f..ac516054b 100644
--- a/.github/workflows/vtest.yml
+++ b/.github/workflows/vtest.yml
@@ -47,6 +47,22 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 100
+#
+# Github Action cache key cannot contain comma, so we calculate it based on job name
+#
+- name: Generate cache key
+ id: generate-cache-key
+ run: |
+echo "::set-output name=key::$(echo ${{ matrix.name }} | sha256sum | awk '{print $1}')"
+
+- name: Cache SSL libs
+ if: ${{ matrix.ssl && matrix.ssl != 'stock' && matrix.ssl != 'BORINGSSL=yes' && matrix.ssl != 'QUICTLS=yes' }}
+ id: cache_ssl
+ uses: actions/cache@v2
+ with:
+path: '~/opt/'
+key: ssl-${{ steps.generate-cache-key.outputs.key }}
+
- name: Cache OpenTracing
if: ${{ contains(matrix.FLAGS, 'USE_OT=1') }}
id: cache_ot
@@ -73,7 +89,7 @@ jobs:
run: |
scripts/build-vtest.sh
- name: Install SSL ${{ matrix.ssl }}
- if: ${{ matrix.ssl && matrix.ssl != 'stock' }}
+ if: ${{ matrix.ssl && matrix.ssl != 'stock' && steps.cache_ssl.outputs.cache-hit != 'true' }}
run: env ${{ matrix.ssl }} scripts/build-ssl.sh
- name: Install OpenTracing libs
if: ${{ contains(matrix.FLAGS, 'USE_OT=1') && steps.cache_ot.outputs.cache-hit != 'true' }}
--
2.34.1
