Hi Tim, Updated (see attachments). Other patches did not change. Regards,
Marcin Deranek On Thu, Jul 15, 2021 at 10:20 AM Tim Düsterhus <[email protected]> wrote: > Marcin, > > On 7/14/21 2:01 PM, Marcin Deranek wrote: > > Thank you for all comments I have received regarding JA3 Fingerprinting > > patches. Here is the new set of patches which incorporated all your > > suggestions. > > Sorry I gave a little outdated advice regarding the reg-tests. For any > new tests please use: > > feature cmd "$HAPROXY_PROGRAM -cc 'version_atleast(2.5-dev0)'" > > instead of > > #REQUIRE_VERSION=2.5 > > Other than that the tests LGTM from a glance. I didn't look at your C > and I also didn't (yet) compare the tests against the documentation you > have written. > > Best regards > Tim Düsterhus > -- Marcin Deranek Senior Site Reliability Engineer [image: Booking.com] <https://www.booking.com/> Making it easier for everyone to experience the world.
From 87dcccaa9d3fbd4474b256bf6623323621c3144b Mon Sep 17 00:00:00 2001 From: Marcin Deranek <[email protected]> Date: Tue, 13 Jul 2021 14:05:24 +0200 Subject: [PATCH 3/5] MINOR: sample: Add be2dec converter Add be2dec converter which allows to build JA3 compatible TLS fingerprints by converting big-endian binary data into string separated unsigned integers eg. http-request set-header X-SSL-JA3 %[ssl_fc_protocol_hello_id],\ %[ssl_fc_cipherlist_bin(1),be2dec(-,2)],\ %[ssl_fc_extlist_bin(1),be2dec(-,2)],\ %[ssl_fc_eclist_bin(1),be2dec(-,2)],\ %[ssl_fc_ecformats_bin,be2dec(-,1)] --- doc/configuration.txt | 12 +++++++ reg-tests/converter/be2dec.vtc | 56 +++++++++++++++++++++++++++++++++ src/sample.c | 57 ++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 reg-tests/converter/be2dec.vtc diff --git a/doc/configuration.txt b/doc/configuration.txt index ecbbcdd04..d39e90752 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -16064,6 +16064,18 @@ base64 an SSL ID can be copied in a header). For base64url("URL and Filename Safe Alphabet" (RFC 4648)) variant see "ub64enc". +be2dec(<separator>,<chunk_size>,[<truncate>]) + Converts a binary input sample to a string containing an unsigned integer + number per <chunk_size> input bytes. <separator> is put every <chunk_size> + binary input bytes if specified. <truncate> flag indicates whatever binary + input is truncated at <chunk_size> boundaries. <chunk_size> maximum value is + limited by the size of long long int (8 bytes). + + Example: + bin(01020304050607),be2dec(:,2) # 258:772:1286:7 + bin(01020304050607),be2dec(-,2,1) # 258-772-1286 + bin(01020304050607),be2dec(,2,1) # 2587721286 + bool Returns a boolean TRUE if the input value of type signed integer is non-null, otherwise returns FALSE. Used in conjunction with and(), it can be diff --git a/reg-tests/converter/be2dec.vtc b/reg-tests/converter/be2dec.vtc new file mode 100644 index 000000000..bdb903523 --- /dev/null +++ b/reg-tests/converter/be2dec.vtc @@ -0,0 +1,56 @@ +varnishtest "be2dec converter Test" + +feature cmd "$HAPROXY_PROGRAM -cc 'version_atleast(2.5-dev0)'" +feature ignore_unknown_macro + +server s1 { + rxreq + txresp +} -repeat 3 -start + +haproxy h1 -conf { + defaults + mode http + timeout connect 1s + timeout client 1s + timeout server 1s + + frontend fe + bind "fd@${fe}" + + #### requests + http-request set-var(txn.input) req.hdr(input) + + http-response set-header be2dec-1 "%[var(txn.input),be2dec(:,1)]" + http-response set-header be2dec-2 "%[var(txn.input),be2dec(-,3)]" + http-response set-header be2dec-3 "%[var(txn.input),be2dec(::,3,1)]" + + default_backend be + + backend be + server s1 ${s1_addr}:${s1_port} +} -start + +client c1 -connect ${h1_fe_sock} { + txreq -url "/" \ + -hdr "input:" + rxresp + expect resp.status == 200 + expect resp.http.be2dec-1 == "" + expect resp.http.be2dec-2 == "" + expect resp.http.be2dec-3 == "" + txreq -url "/" \ + -hdr "input: 0123456789" + rxresp + expect resp.status == 200 + expect resp.http.be2dec-1 == "48:49:50:51:52:53:54:55:56:57" + expect resp.http.be2dec-2 == "3158322-3355701-3553080-57" + expect resp.http.be2dec-3 == "3158322::3355701::3553080" + txreq -url "/" \ + -hdr "input: abcdefghijklmnopqrstuvwxyz" + rxresp + expect resp.status == 200 + expect resp.http.be2dec-1 == "97:98:99:100:101:102:103:104:105:106:107:108:109:110:111:112:113:114:115:116:117:118:119:120:121:122" + expect resp.http.be2dec-2 == "6382179-6579558-6776937-6974316-7171695-7369074-7566453-7763832-31098" + expect resp.http.be2dec-3 == "6382179::6579558::6776937::6974316::7171695::7369074::7566453::7763832" +} -run diff --git a/src/sample.c b/src/sample.c index d02034cf0..5b7ad8b34 100644 --- a/src/sample.c +++ b/src/sample.c @@ -2057,6 +2057,62 @@ static int sample_conv_crypto_hmac(const struct arg *args, struct sample *smp, v #endif /* USE_OPENSSL */ +static int sample_conv_be2dec_check(struct arg *args, struct sample_conv *conv, + const char *file, int line, char **err) +{ + if (args[1].data.sint <= 0 || args[1].data.sint > sizeof(unsigned long long)) { + memprintf(err, "chunk_size out of [1..%ld] range (%lld)", sizeof(unsigned long long), args[1].data.sint); + return 0; + } + + if (args[2].data.sint != 0 && args[2].data.sint != 1) { + memprintf(err, "Unsupported truncate value (%lld)", args[2].data.sint); + return 0; + } + + return 1; +} + +static int sample_conv_be2dec(const struct arg *args, struct sample *smp, void *private) +{ + struct buffer *trash = get_trash_chunk(); + const int last = args[2].data.sint ? smp->data.u.str.data - args[1].data.sint + 1 : smp->data.u.str.data; + int max_size = trash->size - 2; + int i; + int start; + int ptr = 0; + unsigned long long number; + char *pos; + + trash->data = 0; + + while (ptr < last && trash->data <= max_size) { + start = trash->data; + if (ptr) { + /* Separator */ + memcpy(trash->area + trash->data, args[0].data.str.area, args[0].data.str.data); + trash->data += args[0].data.str.data; + } + else + max_size -= args[0].data.str.data; + /* Integer */ + for (number = 0, i = 0; i < args[1].data.sint && ptr < smp->data.u.str.data; i++) + number = (number << 8) + (unsigned char)smp->data.u.str.area[ptr++]; + pos = ulltoa(number, trash->area + trash->data, trash->size - trash->data); + if (pos) + trash->data = pos - trash->area; + else { + trash->data = start; + break; + } + } + + smp->data.u.str = *trash; + smp->data.type = SMP_T_STR; + smp->flags &= ~SMP_F_CONST; + return 1; +} + static int sample_conv_bin2hex(const struct arg *arg_p, struct sample *smp, void *private) { struct buffer *trash = get_trash_chunk(); @@ -4251,6 +4307,7 @@ static struct sample_conv_kw_list sample_conv_kws = {ILH, { { "upper", sample_conv_str2upper, 0, NULL, SMP_T_STR, SMP_T_STR }, { "lower", sample_conv_str2lower, 0, NULL, SMP_T_STR, SMP_T_STR }, { "length", sample_conv_length, 0, NULL, SMP_T_STR, SMP_T_SINT }, + { "be2dec", sample_conv_be2dec, ARG3(1,STR,SINT,SINT), sample_conv_be2dec_check, SMP_T_BIN, SMP_T_STR }, { "hex", sample_conv_bin2hex, 0, NULL, SMP_T_BIN, SMP_T_STR }, { "hex2i", sample_conv_hex2int, 0, NULL, SMP_T_STR, SMP_T_SINT }, { "ipmask", sample_conv_ipmask, ARG2(1,MSK4,MSK6), NULL, SMP_T_ADDR, SMP_T_IPV4 }, -- 2.32.0
From 771366db734d94df4c885f78e04033f79cada502 Mon Sep 17 00:00:00 2001 From: Marcin Deranek <[email protected]> Date: Tue, 13 Jul 2021 14:08:56 +0200 Subject: [PATCH 4/5] MINOR: sample: Add be2hex converter Add be2hex converter to convert big-endian binary data into hex string with optional string separators. --- doc/configuration.txt | 14 ++++++++ reg-tests/converter/be2hex.vtc | 60 ++++++++++++++++++++++++++++++++++ src/sample.c | 54 ++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 reg-tests/converter/be2hex.vtc diff --git a/doc/configuration.txt b/doc/configuration.txt index d39e90752..d2958fc3a 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -16076,6 +16076,20 @@ be2dec(<separator>,<chunk_size>,[<truncate>]) bin(01020304050607),be2dec(-,2,1) # 258-772-1286 bin(01020304050607),be2dec(,2,1) # 2587721286 +be2hex([<separator>],[<chunk_size>],[<truncate>]) + Converts big-endian binary input sample to a hex string containing two hex + digits per input byte. It is used to log or transfer hex dumps of some + binary input data in a way that can be reliably transferred (e.g. an SSL ID + can be copied in a header). <separator> is put every <chunk_size> binary + input bytes if specified. <truncate> flag indicates whatever binary input is + truncated at <chunk_size> boundaries. + + Example: + bin(01020304050607),be2hex # 01020304050607 + bin(01020304050607),be2hex(:,2) # 0102:0304:0506:07 + bin(01020304050607),be2hex(--,2,1) # 0102--0304--0506 + bin(0102030405060708),be2hex(,3,1) # 010203040506 + bool Returns a boolean TRUE if the input value of type signed integer is non-null, otherwise returns FALSE. Used in conjunction with and(), it can be diff --git a/reg-tests/converter/be2hex.vtc b/reg-tests/converter/be2hex.vtc new file mode 100644 index 000000000..86e504292 --- /dev/null +++ b/reg-tests/converter/be2hex.vtc @@ -0,0 +1,60 @@ +varnishtest "be2hex converter Test" + +feature cmd "$HAPROXY_PROGRAM -cc 'version_atleast(2.5-dev0)'" +feature ignore_unknown_macro + +server s1 { + rxreq + txresp +} -repeat 3 -start + +haproxy h1 -conf { + defaults + mode http + timeout connect 1s + timeout client 1s + timeout server 1s + + frontend fe + bind "fd@${fe}" + + #### requests + http-request set-var(txn.input) req.hdr(input) + + http-response set-header be2hex "%[var(txn.input),be2hex,lower]" + http-response set-header be2hex-1 "%[var(txn.input),be2hex(:,1),lower]" + http-response set-header be2hex-2 "%[var(txn.input),be2hex(--,3),lower]" + http-response set-header be2hex-3 "%[var(txn.input),be2hex(.,3,1),lower]" + + default_backend be + + backend be + server s1 ${s1_addr}:${s1_port} +} -start + +client c1 -connect ${h1_fe_sock} { + txreq -url "/" \ + -hdr "input:" + rxresp + expect resp.status == 200 + expect resp.http.be2hex == "" + expect resp.http.be2hex-1 == "" + expect resp.http.be2hex-2 == "" + expect resp.http.be2hex-3 == "" + txreq -url "/" \ + -hdr "input: 0123456789" + rxresp + expect resp.status == 200 + expect resp.http.be2hex == "30313233343536373839" + expect resp.http.be2hex-1 == "30:31:32:33:34:35:36:37:38:39" + expect resp.http.be2hex-2 == "303132--333435--363738--39" + expect resp.http.be2hex-3 == "303132.333435.363738" + txreq -url "/" \ + -hdr "input: abcdefghijklmnopqrstuvwxyz" + rxresp + expect resp.status == 200 + expect resp.http.be2hex == "6162636465666768696a6b6c6d6e6f707172737475767778797a" + expect resp.http.be2hex-1 == "61:62:63:64:65:66:67:68:69:6a:6b:6c:6d:6e:6f:70:71:72:73:74:75:76:77:78:79:7a" + expect resp.http.be2hex-2 == "616263--646566--676869--6a6b6c--6d6e6f--707172--737475--767778--797a" + expect resp.http.be2hex-3 == "616263.646566.676869.6a6b6c.6d6e6f.707172.737475.767778" +} -run diff --git a/src/sample.c b/src/sample.c index 5b7ad8b34..00f296b6c 100644 --- a/src/sample.c +++ b/src/sample.c @@ -2113,6 +2113,59 @@ static int sample_conv_be2dec(const struct arg *args, struct sample *smp, void * return 1; } +static int sample_conv_be2hex_check(struct arg *args, struct sample_conv *conv, + const char *file, int line, char **err) +{ + if (args[1].data.sint <= 0 && (args[0].data.str.data > 0 || args[2].data.sint != 0)) { + memprintf(err, "chunk_size needs to be positive (%lld)", args[1].data.sint); + return 0; + } + + if (args[2].data.sint != 0 && args[2].data.sint != 1) { + memprintf(err, "Unsupported truncate value (%lld)", args[2].data.sint); + return 0; + } + + return 1; +} + +static int sample_conv_be2hex(const struct arg *args, struct sample *smp, void *private) +{ + struct buffer *trash = get_trash_chunk(); + int chunk_size = args[1].data.sint; + const int last = args[2].data.sint ? smp->data.u.str.data - chunk_size + 1 : smp->data.u.str.data; + int i; + int max_size; + int ptr = 0; + unsigned char c; + + trash->data = 0; + if (args[0].data.str.data == 0 && args[2].data.sint == 0) + chunk_size = smp->data.u.str.data; + max_size = trash->size - 2 * chunk_size; + + while (ptr < last && trash->data <= max_size) { + if (ptr) { + /* Separator */ + memcpy(trash->area + trash->data, args[0].data.str.area, args[0].data.str.data); + trash->data += args[0].data.str.data; + } + else + max_size -= args[0].data.str.data; + /* Hex */ + for (i = 0; i < chunk_size && ptr < smp->data.u.str.data; i++) { + c = smp->data.u.str.area[ptr++]; + trash->area[trash->data++] = hextab[(c >> 4) & 0xF]; + trash->area[trash->data++] = hextab[c & 0xF]; + } + } + + smp->data.u.str = *trash; + smp->data.type = SMP_T_STR; + smp->flags &= ~SMP_F_CONST; + return 1; +} + static int sample_conv_bin2hex(const struct arg *arg_p, struct sample *smp, void *private) { struct buffer *trash = get_trash_chunk(); @@ -4308,6 +4361,7 @@ static struct sample_conv_kw_list sample_conv_kws = {ILH, { { "lower", sample_conv_str2lower, 0, NULL, SMP_T_STR, SMP_T_STR }, { "length", sample_conv_length, 0, NULL, SMP_T_STR, SMP_T_SINT }, { "be2dec", sample_conv_be2dec, ARG3(1,STR,SINT,SINT), sample_conv_be2dec_check, SMP_T_BIN, SMP_T_STR }, + { "be2hex", sample_conv_be2hex, ARG3(1,STR,SINT,SINT), sample_conv_be2hex_check, SMP_T_BIN, SMP_T_STR }, { "hex", sample_conv_bin2hex, 0, NULL, SMP_T_BIN, SMP_T_STR }, { "hex2i", sample_conv_hex2int, 0, NULL, SMP_T_STR, SMP_T_SINT }, { "ipmask", sample_conv_ipmask, ARG2(1,MSK4,MSK6), NULL, SMP_T_ADDR, SMP_T_IPV4 }, -- 2.32.0
