This is an automated email from the ASF dual-hosted git repository.
wkaras pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 2956553c24 ESI plugin: make maximum document size configurable.
(#11076)
2956553c24 is described below
commit 2956553c2486f88eccbc999fda4479ae6102f157
Author: Walt Karas <[email protected]>
AuthorDate: Fri Feb 16 12:14:50 2024 -0500
ESI plugin: make maximum document size configurable. (#11076)
Adds the --max-doc-size <number-of-bytes> command line option.
---
doc/admin-guide/plugins/esi.en.rst | 4 ++
plugins/esi/esi.cc | 74 +++++++++++++++++++----------
plugins/esi/lib/EsiParser.cc | 10 ++--
plugins/esi/lib/EsiParser.h | 6 +--
plugins/esi/lib/EsiProcessor.cc | 5 +-
plugins/esi/lib/EsiProcessor.h | 3 +-
plugins/esi/test/docnode_test.cc | 4 +-
plugins/esi/test/parser_test.cc | 2 +-
plugins/esi/test/processor_test.cc | 10 ++--
tests/gold_tests/pluginTest/esi/esi.test.py | 23 +++++++++
10 files changed, 97 insertions(+), 44 deletions(-)
diff --git a/doc/admin-guide/plugins/esi.en.rst
b/doc/admin-guide/plugins/esi.en.rst
index 1759131292..d83360ed57 100644
--- a/doc/admin-guide/plugins/esi.en.rst
+++ b/doc/admin-guide/plugins/esi.en.rst
@@ -86,6 +86,10 @@ Enabling ESI
- ``--first-byte-flush`` will enable the first byte flush feature, which will
flush content to users as soon as the entire
ESI document is received and parsed without all ESI includes fetched. The
flushing will stop at the ESI include markup
till that include is fetched.
+- ``--max-doc-size <number-of-bytes>`` gives the maximum size of the document,
in bytes. The number of bytes must be
+ be must an unsigned decimal integer, and can be followed (with no white
space) by a K, to indicate the given number is
+ multiplied by 1024, or by M, to indicate the given number is multiplied by
1024 * 1024. Example values: 500,
+ 5K, 2M. If this option is omitted, the maximum document size defaults to 1M.
3. ``HTTP_COOKIE`` variable support is turned off by default. It can be turned
on with ``-f <handler_config>`` or
``-handler <handler_config>``. For example:
diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc
index 999a25a196..0e03cd70c4 100644
--- a/plugins/esi/esi.cc
+++ b/plugins/esi/esi.cc
@@ -29,6 +29,8 @@
#include <cstring>
#include <string>
#include <list>
+#include <new>
+#include <limits>
#include <arpa/inet.h>
#include <getopt.h>
@@ -51,10 +53,11 @@ using namespace EsiLib;
using namespace Stats;
struct OptionInfo {
- bool packed_node_support;
- bool private_response;
- bool disable_gzip_output;
- bool first_byte_flush;
+ bool packed_node_support{false};
+ bool private_response{false};
+ bool disable_gzip_output{false};
+ bool first_byte_flush{false};
+ unsigned max_doc_size{1024 * 1024};
};
static HandlerManager *gHandlerManager = nullptr;
@@ -100,7 +103,7 @@ struct ContData {
EsiGunzip *esi_gunzip;
TSCont contp;
TSHttpTxn txnp;
- const struct OptionInfo *option_info = nullptr;
+ const OptionInfo *const option_info;
char *request_url;
sockaddr const *client_addr;
DataType input_type;
@@ -116,7 +119,7 @@ struct ContData {
bool os_response_cacheable;
list<string> post_headers;
- ContData(TSCont contptr, TSHttpTxn tx)
+ ContData(TSCont contptr, TSHttpTxn tx, const OptionInfo *opt_info)
: curr_state(READING_ESI_DOC),
input_vio(nullptr),
output_vio(nullptr),
@@ -129,6 +132,7 @@ struct ContData {
esi_gunzip(nullptr),
contp(contptr),
txnp(tx),
+ option_info(opt_info),
request_url(nullptr),
input_type(DATA_TYPE_RAW_ESI),
packed_node_list(""),
@@ -238,7 +242,7 @@ ContData::init()
esi_vars = new Variables(contp, gAllowlistCookies);
}
- esi_proc = new EsiProcessor(contp, *data_fetcher, *esi_vars,
*gHandlerManager);
+ esi_proc = new EsiProcessor(contp, *data_fetcher, *esi_vars,
*gHandlerManager, option_info->max_doc_size);
esi_gzip = new EsiGzip();
esi_gunzip = new EsiGunzip();
@@ -999,7 +1003,7 @@ struct RespHdrModData {
bool cache_txn;
bool gzip_encoding;
bool head_only;
- const struct OptionInfo *option_info;
+ const OptionInfo *option_info;
};
static void
@@ -1407,7 +1411,7 @@ addSendResponseHeaderHook(TSHttpTxn txnp, const ContData
*src_cont_data)
static bool
addTransform(TSHttpTxn txnp, const bool processing_os_response, const bool
intercept_header, const bool head_only,
- const struct OptionInfo *pOptionInfo)
+ const OptionInfo *pOptionInfo)
{
TSCont contp = nullptr;
ContData *cont_data = nullptr;
@@ -1418,10 +1422,9 @@ addTransform(TSHttpTxn txnp, const bool
processing_os_response, const bool inter
goto lFail;
}
- cont_data = new ContData(contp, txnp);
+ cont_data = new ContData(contp, txnp, pOptionInfo);
TSContDataSet(contp, cont_data);
- cont_data->option_info = pOptionInfo;
cont_data->cache_txn = !processing_os_response;
cont_data->intercept_header = intercept_header;
cont_data->head_only = head_only;
@@ -1474,7 +1477,7 @@ globalHookHandler(TSCont contp, TSEvent event, void
*edata)
bool intercept_header = false;
bool head_only = false;
bool intercept_req = isInterceptRequest(txnp);
- struct OptionInfo *pOptionInfo = static_cast<struct OptionInfo
*>(TSContDataGet(contp));
+ struct OptionInfo *pOptionInfo = static_cast<OptionInfo
*>(TSContDataGet(contp));
switch (event) {
case TS_EVENT_HTTP_READ_REQUEST_HDR:
@@ -1548,7 +1551,7 @@ loadHandlerConf(const char *file_name, Utils::KeyValueMap
&handler_conf)
}
static int
-esiPluginInit(int argc, const char *argv[], struct OptionInfo *pOptionInfo)
+esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo)
{
static TSStatSystem *statSystem = nullptr;
@@ -1561,7 +1564,7 @@ esiPluginInit(int argc, const char *argv[], struct
OptionInfo *pOptionInfo)
gHandlerManager = new HandlerManager();
}
- memset(pOptionInfo, 0, sizeof(struct OptionInfo));
+ new (pOptionInfo) OptionInfo;
if (argc > 1) {
int c;
@@ -1571,11 +1574,12 @@ esiPluginInit(int argc, const char *argv[], struct
OptionInfo *pOptionInfo)
{const_cast<char *>("disable-gzip-output"), no_argument, nullptr,
'z'},
{const_cast<char *>("first-byte-flush"), no_argument, nullptr,
'b'},
{const_cast<char *>("handler-filename"), required_argument, nullptr,
'f'},
+ {const_cast<char *>("max-doc-size"), required_argument, nullptr,
'd'},
{nullptr, 0, nullptr,
0 },
};
int longindex = 0;
- while ((c = getopt_long(argc, const_cast<char *const *>(argv), "npzbf:",
longopts, &longindex)) != -1) {
+ while ((c = getopt_long(argc, const_cast<char *const *>(argv), "npzbf:d:",
longopts, &longindex)) != -1) {
switch (c) {
case 'n':
pOptionInfo->packed_node_support = true;
@@ -1595,18 +1599,40 @@ esiPluginInit(int argc, const char *argv[], struct
OptionInfo *pOptionInfo)
gHandlerManager->loadObjects(handler_conf);
break;
}
- default:
+ case 'd': {
+ unsigned max, coeff{1};
+ char multiplier, crap;
+ auto num_assigned = std::sscanf(optarg, "%u%c%c", &max, &multiplier,
&crap);
+ if (2 == num_assigned) {
+ if ('K' == multiplier) {
+ coeff = 1024;
+ num_assigned = 1;
+ } else if ('M' == multiplier) {
+ coeff = 1024 * 1024;
+ num_assigned = 1;
+ }
+ }
+ if (num_assigned != 1) {
+ TSEmergency("[esi][%s] value for maximum document size (%s) has bad
format", __FUNCTION__, optarg);
+ }
+ if ((coeff != 1) && (max > (std::numeric_limits<unsigned>::max() /
coeff))) {
+ TSEmergency("[esi][%s] specified maximum document size (%u%c) too
large", __FUNCTION__, max, multiplier);
+ }
+ pOptionInfo->max_doc_size = max * coeff;
break;
}
+ default:
+ TSEmergency("[esi][%s] bad option", __FUNCTION__);
+ return -1;
+ }
}
}
Dbg(dbg_ctl_local,
"[%s] Plugin started, "
- "packed-node-support: %d, private-response: %d, "
- "disable-gzip-output: %d, first-byte-flush: %d ",
+ "packed-node-support: %d, private-response: %d, disable-gzip-output: %d,
first-byte-flush: %d, max-doc-size %u ",
__FUNCTION__, pOptionInfo->packed_node_support,
pOptionInfo->private_response, pOptionInfo->disable_gzip_output,
- pOptionInfo->first_byte_flush);
+ pOptionInfo->first_byte_flush, pOptionInfo->max_doc_size);
return 0;
}
@@ -1624,9 +1650,9 @@ TSPluginInit(int argc, const char *argv[])
return;
}
- struct OptionInfo *pOptionInfo = static_cast<struct OptionInfo
*>(TSmalloc(sizeof(struct OptionInfo)));
+ auto pOptionInfo = TSRalloc<OptionInfo>();
if (pOptionInfo == nullptr) {
- TSError("[esi][%s] malloc %d bytes fail", __FUNCTION__,
static_cast<int>(sizeof(struct OptionInfo)));
+ TSError("[esi][%s] malloc %d bytes fail", __FUNCTION__,
static_cast<int>(sizeof(OptionInfo)));
return;
}
if (esiPluginInit(argc, argv, pOptionInfo) != 0) {
@@ -1690,10 +1716,10 @@ TSRemapNewInstance(int argc, char *argv[], void **ih,
char *errbuf, int errbuf_s
}
new_argv[index] = nullptr;
- struct OptionInfo *pOptionInfo = static_cast<struct OptionInfo
*>(TSmalloc(sizeof(struct OptionInfo)));
+ OptionInfo *pOptionInfo = TSRalloc<OptionInfo>();
if (pOptionInfo == nullptr) {
- snprintf(errbuf, errbuf_size, "malloc %d bytes fail",
static_cast<int>(sizeof(struct OptionInfo)));
- TSError("[esi][%s] malloc %d bytes fail", __FUNCTION__,
static_cast<int>(sizeof(struct OptionInfo)));
+ snprintf(errbuf, errbuf_size, "malloc %d bytes fail",
static_cast<int>(sizeof(OptionInfo)));
+ TSError("[esi][%s] malloc %d bytes fail", __FUNCTION__,
static_cast<int>(sizeof(OptionInfo)));
return TS_ERROR;
}
if (esiPluginInit(index, new_argv, pOptionInfo) != 0) {
diff --git a/plugins/esi/lib/EsiParser.cc b/plugins/esi/lib/EsiParser.cc
index 6b897d944f..12c23aa1e2 100644
--- a/plugins/esi/lib/EsiParser.cc
+++ b/plugins/esi/lib/EsiParser.cc
@@ -43,8 +43,6 @@ const string EsiParser::SRC_ATTR_STR("src");
const string EsiParser::TEST_ATTR_STR("test");
const string EsiParser::HANDLER_ATTR_STR("handler");
-const unsigned int EsiParser::MAX_DOC_SIZE = 1024 * 1024;
-
const EsiParser::EsiNodeInfo EsiParser::ESI_NODES[] = {
EsiNodeInfo(DocNode::TYPE_INCLUDE, "include", 7, "/>", 2),
EsiNodeInfo(DocNode::TYPE_REMOVE, "remove>", 7, "</esi:remove>", 13),
@@ -62,11 +60,11 @@ const EsiParser::EsiNodeInfo EsiParser::ESI_NODES[] = {
const EsiParser::EsiNodeInfo
EsiParser::HTML_COMMENT_NODE_INFO(DocNode::TYPE_HTML_COMMENT, "<!--esi", 7,
"-->", 3);
-EsiParser::EsiParser() : _parse_start_pos(-1)
+EsiParser::EsiParser(unsigned max_doc_size) : _max_doc_size(max_doc_size),
_parse_start_pos(-1)
{
// do this so that object doesn't move around in memory;
// (because we return pointers into this object)
- _data.reserve(MAX_DOC_SIZE);
+ _data.reserve(_max_doc_size);
}
bool
@@ -80,9 +78,9 @@ EsiParser::_setup(string &data, int &parse_start_pos, size_t
&orig_output_list_s
if (data_len == -1) {
data_len = strlen(data_ptr);
}
- if ((data.size() + data_len) > MAX_DOC_SIZE) {
+ if ((data.size() + data_len) > _max_doc_size) {
TSError("[%s] Cannot allow attempted doc of size %d; Max allowed size is
%d", __FUNCTION__, int(data.size() + data_len),
- MAX_DOC_SIZE);
+ _max_doc_size);
retval = false;
} else {
data.append(data_ptr, data_len);
diff --git a/plugins/esi/lib/EsiParser.h b/plugins/esi/lib/EsiParser.h
index fdedde7f17..b674a7ac8e 100644
--- a/plugins/esi/lib/EsiParser.h
+++ b/plugins/esi/lib/EsiParser.h
@@ -30,7 +30,7 @@
class EsiParser
{
public:
- EsiParser();
+ EsiParser(unsigned max_doc_size);
/** clears state */
void clear();
@@ -90,6 +90,8 @@ private:
: type(t), tag_suffix(s), tag_suffix_len(s_len), closing_tag(ct),
closing_tag_len(ct_len){};
};
+ const unsigned _max_doc_size;
+
std::string _data;
int _parse_start_pos;
size_t _orig_output_list_size = 0;
@@ -104,8 +106,6 @@ private:
static const std::string TEST_ATTR_STR;
static const std::string HANDLER_ATTR_STR;
- static const unsigned int MAX_DOC_SIZE;
-
enum MATCH_TYPE {
NO_MATCH,
COMPLETE_MATCH,
diff --git a/plugins/esi/lib/EsiProcessor.cc b/plugins/esi/lib/EsiProcessor.cc
index 43cd7b601f..d7cc0e3b82 100644
--- a/plugins/esi/lib/EsiProcessor.cc
+++ b/plugins/esi/lib/EsiProcessor.cc
@@ -42,9 +42,10 @@ DbgCtl dbg_ctl{"plugin_esi_procesor"};
//
#define DBG(FMT, ...) Dbg(dbg_ctl, FMT " contp=%p", ##__VA_ARGS__, _cont_addr)
-EsiProcessor::EsiProcessor(void *cont_addr, HttpDataFetcher &fetcher,
Variables &variables, const HandlerManager &handler_mgr)
+EsiProcessor::EsiProcessor(void *cont_addr, HttpDataFetcher &fetcher,
Variables &variables, const HandlerManager &handler_mgr,
+ unsigned max_doc_size)
: _curr_state(STOPPED),
- _parser(),
+ _parser(max_doc_size),
_n_prescanned_nodes(0),
_n_processed_nodes(0),
_n_processed_try_nodes(0),
diff --git a/plugins/esi/lib/EsiProcessor.h b/plugins/esi/lib/EsiProcessor.h
index 1b32a639c6..acb4e17183 100644
--- a/plugins/esi/lib/EsiProcessor.h
+++ b/plugins/esi/lib/EsiProcessor.h
@@ -44,7 +44,8 @@ public:
PROCESS_FAILURE,
};
- EsiProcessor(void *cont_addr, HttpDataFetcher &fetcher, EsiLib::Variables
&variables, const EsiLib::HandlerManager &handler_mgr);
+ EsiProcessor(void *cont_addr, HttpDataFetcher &fetcher, EsiLib::Variables
&variables, const EsiLib::HandlerManager &handler_mgr,
+ unsigned max_doc_size);
/** Initializes the processor with the context of the request to be
processed */
bool start();
diff --git a/plugins/esi/test/docnode_test.cc b/plugins/esi/test/docnode_test.cc
index efd691f3e3..57fd6bdda8 100644
--- a/plugins/esi/test/docnode_test.cc
+++ b/plugins/esi/test/docnode_test.cc
@@ -149,7 +149,7 @@ TEST_CASE("esi docnode test")
{
SECTION("Test 1")
{
- EsiParser parser;
+ EsiParser parser{1024 * 1024};
string input_data = "foo <esi:include src=blah /> bar";
DocNodeList node_list;
@@ -180,7 +180,7 @@ TEST_CASE("esi docnode test")
SECTION("Test 2")
{
- EsiParser parser;
+ EsiParser parser{1024 * 1024};
string input_data("<esi:choose>"
"<esi:when test=c1>"
"<esi:try>"
diff --git a/plugins/esi/test/parser_test.cc b/plugins/esi/test/parser_test.cc
index df5e60c7c2..d404ae77a1 100644
--- a/plugins/esi/test/parser_test.cc
+++ b/plugins/esi/test/parser_test.cc
@@ -45,7 +45,7 @@ check_node_attr(const Attribute &attr, const char *name,
const char *value)
TEST_CASE("esi parser test")
{
- EsiParser parser;
+ EsiParser parser{1024 * 1024};
SECTION("No src attr")
{
diff --git a/plugins/esi/test/processor_test.cc
b/plugins/esi/test/processor_test.cc
index 69414152df..d407179575 100644
--- a/plugins/esi/test/processor_test.cc
+++ b/plugins/esi/test/processor_test.cc
@@ -43,7 +43,7 @@ TEST_CASE("esi processor test")
Variables esi_vars(&dummy, allowlistCookies);
HandlerManager handler_mgr;
TestHttpDataFetcher data_fetcher;
- EsiProcessor esi_proc(&dummy, data_fetcher, esi_vars, handler_mgr);
+ EsiProcessor esi_proc(&dummy, data_fetcher, esi_vars, handler_mgr, 1024 *
1024);
SECTION("call sequence")
{
@@ -847,7 +847,7 @@ TEST_CASE("esi processor test")
SECTION("using packed node list 1")
{
- EsiParser parser;
+ EsiParser parser{1024 * 1024};
DocNodeList node_list;
string input_data("<esi:try>"
"<esi:attempt>"
@@ -896,7 +896,7 @@ TEST_CASE("esi processor test")
{
string input_data("<esi:comment text=\"bleh\"/>");
- EsiParser parser;
+ EsiParser parser{1024 * 1024};
DocNodeList node_list;
string input_data2("<esi:try>"
"<esi:attempt>"
@@ -951,7 +951,7 @@ TEST_CASE("esi processor test")
SECTION("using packed node list 3")
{
- EsiParser parser;
+ EsiParser parser{1024 * 1024};
DocNodeList node_list;
string input_data("<esi:try>"
"<esi:attempt>"
@@ -970,7 +970,7 @@ TEST_CASE("esi processor test")
SECTION("using packed node list 4")
{
- EsiParser parser;
+ EsiParser parser{1024 * 1024};
DocNodeList node_list;
string input_data("<esi:try>"
"<esi:attempt>"
diff --git a/tests/gold_tests/pluginTest/esi/esi.test.py
b/tests/gold_tests/pluginTest/esi/esi.test.py
index bb42fdfed6..be9e7f8a74 100644
--- a/tests/gold_tests/pluginTest/esi/esi.test.py
+++ b/tests/gold_tests/pluginTest/esi/esi.test.py
@@ -247,6 +247,19 @@ echo date('l jS \of F Y h:i:s A');
unzipped_disk_file = tr.Disk.File(empty_body_file)
unzipped_disk_file.Size = 0
+ def run_case_max_doc_size_too_small(self):
+ tr = Test.AddTestRun("Max doc size too smal")
+ tr.Processes.Default.Command = \
+ ('curl http://127.0.0.1:{0}/esi.php -H"Host: www.example.com" '
+ '-H"Accept: */*" --verbose'.format(
+ self._ts.Variables.port))
+ tr.Processes.Default.ReturnCode = 0
+ self._ts.Disk.diags_log.Content = Testers.ContainsExpression(
+ r"ERROR: \[_setup\] Cannot allow attempted doc of size 121; Max
allowed size is 100",
+ "max doc size test should have doc size error log")
+ tr.StillRunningAfter = self._server
+ tr.StillRunningAfter = self._ts
+
def run_cases_expecting_no_gzip(self):
# Test 1: Run an ESI test where the client does not accept gzip.
tr = Test.AddTestRun("First request for esi.php: gzip not accepted.")
@@ -302,3 +315,13 @@ first_byte_flush_test.run_cases_expecting_gzip()
# --disable-gzip-output is set.
gzip_disabled_test = EsiTest(plugin_config='esi.so --disable-gzip-output')
gzip_disabled_test.run_cases_expecting_no_gzip()
+
+# Run the tests with too small max doc size.
+max_doc_100_test = EsiTest(plugin_config='esi.so --max-doc-size 100')
+max_doc_100_test.run_case_max_doc_size_too_small()
+
+# Run the tests with no default, but sufficient, max doc size.
+max_doc_2K_test = EsiTest(plugin_config='esi.so --max-doc-size 2K')
+max_doc_2K_test.run_cases_expecting_gzip()
+max_doc_20M_test = EsiTest(plugin_config='esi.so --max-doc-size 20M')
+max_doc_20M_test.run_cases_expecting_gzip()
