This is an automated email from the ASF dual-hosted git repository.
bneradt pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 8057371ea9 ja3_fingerprint: add modify-incoming option (#10886)
8057371ea9 is described below
commit 8057371ea9f40e0890ed62b76fb951a44fec8d54
Author: Brian Neradt <[email protected]>
AuthorDate: Fri Dec 1 16:09:17 2023 -0600
ja3_fingerprint: add modify-incoming option (#10886)
This adds the optional modify-incoming argument to the
ja3_fingerprint.so plugin. This argument alters the header modification
behavior to add the X-JA3-* headers to the incoming client request
rather than to the outgoing proxy request. This allows other plugins
later in the plugin.config file to view the added headers as if they
were added by the client, which can be helpful to some plugins.
---
plugins/ja3_fingerprint/README | 10 +++
plugins/ja3_fingerprint/ja3_fingerprint.cc | 85 ++++++++++++----------
.../ja3_fingerprint/ja3_fingerprint.test.py | 46 +++++++++++-
.../ja3_fingerprint/modify-incoming-client.gold | 10 +++
.../ja3_fingerprint/modify-incoming-proxy.gold | 11 +++
.../ja3_fingerprint/modify-sent-client.gold | 10 +++
.../ja3_fingerprint/modify-sent-proxy.gold | 15 ++++
7 files changed, 145 insertions(+), 42 deletions(-)
diff --git a/plugins/ja3_fingerprint/README b/plugins/ja3_fingerprint/README
index 491a68bdff..7a9e5d9448 100644
--- a/plugins/ja3_fingerprint/README
+++ b/plugins/ja3_fingerprint/README
@@ -17,6 +17,16 @@ The following optional arguments can be used to configure
the plugin's behavior:
Add flag --ja3raw if `X-JA3-Raw` is desired in addition to `X-JA3-Sig`.
Add flag --ja3log if local logging in standard logging directory is desired.
+Add flag --modify-incoming if it is desired that the plugin modify the incoming
+ client request headers rather than the sent (proxy) request headers.
+ Regardless, the origin will receive the configured `X-JA3-*` headers. Using
+ this option allows other plugins that are configured later than
+ `ja3_fingerprint.so` in the `plugin.config` file to see the `X-JA3-*`
+ headers as if they were sent by the client. This option is only applicable
+ for ja3_fingerprint configured as a global plugin (i.e., a `plugin.config`
+ plugin) not as a remap plugin. This is because remap plugins by definition
+ are enaged upon remap completion and by that point it is too late to
+ meaningfully modify the client request headers.
3. plugin.config
In plugin.config, supply name of the plugin and any desired options. For
example:
diff --git a/plugins/ja3_fingerprint/ja3_fingerprint.cc
b/plugins/ja3_fingerprint/ja3_fingerprint.cc
index 82f081b64b..90e2c2b784 100644
--- a/plugins/ja3_fingerprint/ja3_fingerprint.cc
+++ b/plugins/ja3_fingerprint/ja3_fingerprint.cc
@@ -16,25 +16,19 @@
limitations under the License.
*/
-#include <cinttypes>
#include <cstdlib>
#include <cstdio>
#include <cstring>
-#include <cerrno>
#include <cstdlib>
#include <getopt.h>
#include <netinet/in.h>
#include <arpa/inet.h>
-#include <algorithm>
-#include <fstream>
-#include <sstream>
#include <string>
#include <unordered_set>
-#include <unordered_map>
#include <memory>
-#include <regex>
+#include "ts/apidefs.h"
#include "ts/ts.h"
#include "ts/remap.h"
@@ -51,10 +45,11 @@
const char *PLUGIN_NAME = "ja3_fingerprint";
static DbgCtl dbg_ctl{PLUGIN_NAME};
-static TSTextLogObject pluginlog;
-static int ja3_idx = -1;
-static int enable_raw = 0;
-static int enable_log = 0;
+static TSTextLogObject pluginlog = nullptr;
+static int ja3_idx = -1;
+static int global_raw_enabled = 0;
+static int global_log_enabled = 0;
+static int global_modify_incoming_enabled = 0;
// GREASE table as in ja3
static const std::unordered_set<uint16_t> GREASE_table = {0x0a0a, 0x1a1a,
0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a,
@@ -67,9 +62,9 @@ struct ja3_data {
};
struct ja3_remap_info {
- int raw = false;
- int log = false;
- TSCont handler = nullptr;
+ int raw_enabled = false;
+ int log_enabled = false;
+ TSCont handler = nullptr;
~ja3_remap_info()
{
@@ -259,6 +254,12 @@ client_hello_ja3_handler(TSCont contp, TSEvent event, void
*edata)
static int
req_hdr_ja3_handler(TSCont contp, TSEvent event, void *edata)
{
+ TSEvent expected_event = global_modify_incoming_enabled ?
TS_EVENT_HTTP_READ_REQUEST_HDR : TS_EVENT_HTTP_SEND_REQUEST_HDR;
+ if (event != expected_event) {
+ TSError("[%s] req_hdr_ja3_handler(): Unexpected event, got %d, expected
%d", PLUGIN_NAME, event, expected_event);
+ TSAssert(event == expected_event);
+ }
+
TSHttpTxn txnp = nullptr;
TSHttpSsn ssnp = nullptr;
TSVConn vconn = nullptr;
@@ -273,15 +274,19 @@ req_hdr_ja3_handler(TSCont contp, TSEvent event, void
*edata)
ja3_data *data = static_cast<ja3_data *>(TSUserArgGet(vconn, ja3_idx));
if (data) {
// Decide global or remap
- ja3_remap_info *info = static_cast<ja3_remap_info *>(TSContDataGet(contp));
- bool raw_flag = info ? info->raw : enable_raw;
- bool log_flag = info ? info->log : enable_log;
+ ja3_remap_info *remap_info = static_cast<ja3_remap_info
*>(TSContDataGet(contp));
+ bool raw_flag = remap_info ? remap_info->raw_enabled :
global_raw_enabled;
+ bool log_flag = remap_info ? remap_info->log_enabled :
global_log_enabled;
Dbg(dbg_ctl, "req_hdr_ja3_handler(): Found ja3 string.");
// Get handle to headers
TSMBuffer bufp;
TSMLoc hdr_loc;
- TSAssert(TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &bufp, &hdr_loc));
+ if (global_modify_incoming_enabled) {
+ TSAssert(TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &bufp, &hdr_loc));
+ } else {
+ TSAssert(TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &bufp, &hdr_loc));
+ }
// Add JA3 md5 fingerprints
append_to_field(bufp, hdr_loc, "X-JA3-Sig", 9, data->md5_string, 32);
@@ -305,12 +310,13 @@ req_hdr_ja3_handler(TSCont contp, TSEvent event, void
*edata)
}
static bool
-read_config_option(int argc, const char *argv[], int &raw, int &log)
+read_config_option(int argc, const char *argv[], int &raw, int &log, int
&modify_incoming)
{
const struct option longopts[] = {
- {"ja3raw", no_argument, &raw, 1},
- {"ja3log", no_argument, &log, 1},
- {nullptr, 0, nullptr, 0}
+ {"ja3raw", no_argument, &raw, 1},
+ {"ja3log", no_argument, &log, 1},
+ {"modify-incoming", no_argument, &modify_incoming, 1},
+ {nullptr, 0, nullptr, 0}
};
int opt = 0;
@@ -329,6 +335,7 @@ read_config_option(int argc, const char *argv[], int &raw,
int &log)
Dbg(dbg_ctl, "read_config_option(): ja3 raw is %s", (raw == 1) ? "enabled" :
"disabled");
Dbg(dbg_ctl, "read_config_option(): ja3 logging is %s", (log == 1) ?
"enabled" : "disabled");
+ Dbg(dbg_ctl, "read_config_option(): ja3 modify-incoming is %s",
(modify_incoming == 1) ? "enabled" : "disabled");
return true;
}
@@ -344,14 +351,14 @@ TSPluginInit(int argc, const char *argv[])
info.support_email = "[email protected]";
// Options
- if (!read_config_option(argc, argv, enable_raw, enable_log)) {
+ if (!read_config_option(argc, argv, global_raw_enabled, global_log_enabled,
global_modify_incoming_enabled)) {
return;
}
if (TSPluginRegister(&info) != TS_SUCCESS) {
TSError("[%s] Unable to initialize plugin. Failed to register.",
PLUGIN_NAME);
} else {
- if (enable_log && !pluginlog) {
+ if (global_log_enabled && !pluginlog) {
TSAssert(TS_SUCCESS == TSTextLogObjectCreate(PLUGIN_NAME,
TS_LOG_MODE_ADD_TIMESTAMP, &pluginlog));
Dbg(dbg_ctl, "log object created successfully");
}
@@ -360,7 +367,9 @@ TSPluginInit(int argc, const char *argv[])
TSUserArgIndexReserve(TS_USER_ARGS_VCONN, PLUGIN_NAME, "used to pass ja3",
&ja3_idx);
TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, ja3_cont);
TSHttpHookAdd(TS_VCONN_CLOSE_HOOK, ja3_cont);
- TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK,
TSContCreate(req_hdr_ja3_handler, nullptr));
+
+ TSHttpHookID const hook = global_modify_incoming_enabled ?
TS_HTTP_READ_REQUEST_HDR_HOOK : TS_HTTP_SEND_REQUEST_HDR_HOOK;
+ TSHttpHookAdd(hook, TSContCreate(req_hdr_ja3_handler, nullptr));
}
return;
@@ -391,38 +400,40 @@ TSReturnCode
TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf
ATS_UNUSED */, int /* errbuf_size ATS_UNUSED */)
{
Dbg(dbg_ctl, "New instance for client matching %s to %s", argv[0], argv[1]);
- std::unique_ptr<ja3_remap_info> pri{new ja3_remap_info};
+ std::unique_ptr<ja3_remap_info> remap_info{new ja3_remap_info};
// Parse parameters
- if (!read_config_option(argc - 1, const_cast<const char **>(argv + 1),
pri->raw, pri->log)) {
+ int discard_modify_incoming = -1; // Not used for remap.
+ if (!read_config_option(argc - 1, const_cast<const char **>(argv + 1),
remap_info->raw_enabled, remap_info->log_enabled,
+ discard_modify_incoming)) {
Dbg(dbg_ctl, "TSRemapNewInstance(): Bad arguments");
return TS_ERROR;
}
- if (pri->log && !pluginlog) {
+ if (remap_info->log_enabled && !pluginlog) {
TSAssert(TS_SUCCESS == TSTextLogObjectCreate(PLUGIN_NAME,
TS_LOG_MODE_ADD_TIMESTAMP, &pluginlog));
Dbg(dbg_ctl, "log object created successfully");
}
// Create continuation
- pri->handler = TSContCreate(req_hdr_ja3_handler, nullptr);
- TSContDataSet(pri->handler, pri.get());
+ remap_info->handler = TSContCreate(req_hdr_ja3_handler, nullptr);
+ TSContDataSet(remap_info->handler, remap_info.get());
// Pass to other remap plugin functions
- *ih = static_cast<void *>(pri.release());
+ *ih = static_cast<void *>(remap_info.release());
return TS_SUCCESS;
}
TSRemapStatus
TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri)
{
- auto pri = static_cast<ja3_remap_info *>(ih);
+ auto remap_info = static_cast<ja3_remap_info *>(ih);
// On remap, set up handler at send req hook to send JA3 data as header
- if (!pri || !rri || !(pri->handler)) {
+ if (!remap_info || !rri || !(remap_info->handler)) {
TSError("[%s] TSRemapDoRemap(): Invalid private data or RRI or handler.",
PLUGIN_NAME);
} else {
- TSHttpTxnHookAdd(rh, TS_HTTP_SEND_REQUEST_HDR_HOOK, pri->handler);
+ TSHttpTxnHookAdd(rh, TS_HTTP_SEND_REQUEST_HDR_HOOK, remap_info->handler);
}
return TSREMAP_NO_REMAP;
@@ -431,9 +442,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo
*rri)
void
TSRemapDeleteInstance(void *ih)
{
- auto pri = static_cast<ja3_remap_info *>(ih);
- if (pri) {
- delete pri;
- }
+ auto remap_info = static_cast<ja3_remap_info *>(ih);
+ delete remap_info;
ih = nullptr;
}
diff --git
a/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint.test.py
b/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint.test.py
index 3e0421319c..64067078a3 100644
--- a/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint.test.py
+++ b/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint.test.py
@@ -31,24 +31,32 @@ class JA3FingerprintTest:
_ts_counter: int = 0
_client_counter: int = 0
- def __init__(self, test_remap: bool) -> None:
+ def __init__(self, test_remap: bool, modify_incoming: bool) -> None:
"""Configure the test processes in preparation for the TestRun.
:param test_remap: Whether to configure the plugin as a remap plugin
instead of as a global plugin.
+ :param modify_incoming: Whether ja3_fingerprint should be configured to
+ modify the client request rather than the proxy request.
"""
+ if test_remap and modify_incoming:
+ raise ValueError('modify-incoming is only allowed as a global
plugin.')
+
self._test_remap = test_remap
if test_remap:
self._replay_file = 'ja3_fingerprint_remap.replay.yaml'
else:
self._replay_file = 'ja3_fingerprint_global.replay.yaml'
+ self._modify_incoming = modify_incoming
+
tr = Test.AddTestRun('Testing ja3_fingerprint plugin.')
self._configure_dns(tr)
self._configure_server(tr)
self._configure_trafficserver()
self._configure_client(tr)
self._await_ja3log()
+ self._verify_internal_headers()
def _configure_dns(self, tr: 'TestRun') -> None:
"""Configure a nameserver for the test.
@@ -86,7 +94,10 @@ class JA3FingerprintTest:
f'map https://http2.server.com
https://http2.backend.com:{server_port} '
'@plugin=ja3_fingerprint.so @pparam=--ja3log')
else:
- self._ts.Disk.plugin_config.AddLine('ja3_fingerprint.so --ja3log
--ja3raw')
+ arguments = '--ja3log --ja3raw'
+ if self._modify_incoming:
+ arguments += ' --modify-incoming'
+ self._ts.Disk.plugin_config.AddLine(f'ja3_fingerprint.so
{arguments}')
self._ts.Disk.remap_config.AddLine(f'map https://http2.server.com
https://http2.backend.com:{server_port}')
self._ts.Disk.records_config.update(
@@ -143,6 +154,33 @@ class JA3FingerprintTest:
p.Command = f'echo await {ja3_path} creation'
p.StartBefore(waiter)
+ def _verify_internal_headers(self) -> None:
+ """Verify that the correct headers were modified."""
+
+ # We use the grep command to get the small snippet of output we want
+ # from the traffic.out file. Gold file matching against long files,
like
+ # traffic.out, is exceedingly slow.
+
+ tr = Test.AddTestRun('Verify the internal client request headers.')
+ traffic_out = self._ts.Disk.traffic_out.AbsPath
+ p = tr.Processes.Default
+ p.Command = f'grep --after-context=20 "Incoming Request" {traffic_out}'
+
+ if self._modify_incoming:
+ p.Streams.All += "modify-incoming-client.gold"
+ else:
+ p.Streams.All += "modify-sent-client.gold"
+
+ tr = Test.AddTestRun('Verify the internal proxy request headers.')
+ p = tr.Processes.Default
+ p.Command = f'grep --after-context=20 "Proxy\'s Request after hooks"
{traffic_out}'
+
+ if self._modify_incoming:
+ p.Streams.All += "modify-incoming-proxy.gold"
+ else:
+ p.Streams.All += "modify-sent-proxy.gold"
+
-JA3FingerprintTest(test_remap=False)
-JA3FingerprintTest(test_remap=True)
+JA3FingerprintTest(test_remap=False, modify_incoming=False)
+JA3FingerprintTest(test_remap=True, modify_incoming=False)
+JA3FingerprintTest(test_remap=False, modify_incoming=True)
diff --git
a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-client.gold
b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-client.gold
new file mode 100644
index 0000000000..22560ff722
--- /dev/null
+++ b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-client.gold
@@ -0,0 +1,10 @@
++++++++++ Incoming Request +++++++++
+-- State Machine Id``
+GET ``
+host: ``
+content-length: ``
+x-request: ``
+uuid: ``
+X-JA3-Sig: ``
+x-JA3-RAW: ``
+``
diff --git
a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-proxy.gold
b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-proxy.gold
new file mode 100644
index 0000000000..0e7a2379cc
--- /dev/null
+++ b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-proxy.gold
@@ -0,0 +1,11 @@
++++++++++ Proxy's Request after hooks +++++++++
+``
++++++++++ Proxy's Request after hooks +++++++++
+-- State Machine Id``
+POST ``
+Host: ``
+Content-Type: ``
+uuid: ``
+x-request: ``
+X-JA3-Sig: ``
+``
diff --git
a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-client.gold
b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-client.gold
new file mode 100644
index 0000000000..f50c2dea7c
--- /dev/null
+++ b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-client.gold
@@ -0,0 +1,10 @@
++++++++++ Incoming Request +++++++++
+``
++++++++++ Incoming Request +++++++++
+-- State Machine Id``
+POST ``
+Host: ``
+Content-Type: ``
+uuid: ``
+x-request: ``
+``
diff --git a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy.gold
b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy.gold
new file mode 100644
index 0000000000..785faae93d
--- /dev/null
+++ b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy.gold
@@ -0,0 +1,15 @@
++++++++++ Proxy's Request after hooks +++++++++
+``
++++++++++ Proxy's Request after hooks +++++++++
+-- State Machine Id``
+POST ``
+Host: ``
+Content-Type: ``
+uuid: ``
+x-request: ``
+Client-ip: ``
+X-Forwarded-For: ``
+Via: ``
+Transfer-Encoding: ``
+X-JA3-Sig: ``
+``
