This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/9.0.x by this push:
new d94e6c3 For combo_handler plugin, add an optional whitelist of
allowed values for Content-Type.
d94e6c3 is described below
commit d94e6c369d308769671b4e922e1698b7701fed6b
Author: Walter Karas <[email protected]>
AuthorDate: Thu Nov 21 13:30:11 2019 -0600
For combo_handler plugin, add an optional whitelist of allowed values for
Content-Type.
The whitelist is in a text file, whose name is given on the plugin line in
plugin.config.
(cherry picked from commit d474cf757b7401c06c6b4441aa049ed0bd7588b8)
---
doc/admin-guide/plugins/combo_handler.en.rst | 10 ++
plugins/esi/README.combo | 2 +
plugins/esi/combo_handler.cc | 142 ++++++++++++++++++---
.../pluginTest/combo_handler/combo_handler.test.py | 133 +++++++++++++++++++
.../combo_handler/combo_handler_files/tr1.gold | 9 ++
.../combo_handler/combo_handler_files/tr2.gold | 18 +++
6 files changed, 297 insertions(+), 17 deletions(-)
diff --git a/doc/admin-guide/plugins/combo_handler.en.rst
b/doc/admin-guide/plugins/combo_handler.en.rst
index 528704d..926c518 100644
--- a/doc/admin-guide/plugins/combo_handler.en.rst
+++ b/doc/admin-guide/plugins/combo_handler.en.rst
@@ -47,6 +47,16 @@ The arguments in the :file:`plugin.config` line in order
represent
2. The name of the key used for signature verification (disabled by
default)
+3. A colon separated list of headers which, if present on at least one
response, will be
+ added to the combo response.
+
+4. The path of a config file with allowed content types of objects to be
combined, one per
+ line, without parameters. (Blank lines and comments starting with "#" are
ignored.)
+ Parameters in the Content-Type field value will be ignored when
+ checking if they appear in the allowed types. If the path does not start
with "/", the
+ config file must be located in the ATS config directory. By default, all
content types
+ are allowed, but if this file is specified, it must contain at least one
content type.
+
A "-" can be supplied as a value for any of these arguments to request
default value be applied.
diff --git a/plugins/esi/README.combo b/plugins/esi/README.combo
index e2042e1..211e8e0 100644
--- a/plugins/esi/README.combo
+++ b/plugins/esi/README.combo
@@ -1,6 +1,8 @@
Combohandler
--------------------
+NOTE: THIS FILE IS OBSOLETE, SEE THE TRAFFICSERVER ADMIN GUIDE.
+
This plugin provides that functionality (and more) with the same
interface but with these differences in configuration:
diff --git a/plugins/esi/combo_handler.cc b/plugins/esi/combo_handler.cc
index 72d27fa..7cc317f 100644
--- a/plugins/esi/combo_handler.cc
+++ b/plugins/esi/combo_handler.cc
@@ -23,10 +23,12 @@
#include <list>
#include <string>
+#include <string_view>
#include <sstream>
#include <vector>
#include <algorithm>
#include <ctime>
+#include <fstream>
#include <pthread.h>
#include <arpa/inet.h>
#include <limits>
@@ -36,6 +38,7 @@
#include "ts/experimental.h"
#include "ts/remap.h"
#include "tscore/ink_defs.h"
+#include "tscpp/util/TextView.h"
#include "HttpDataFetcherImpl.h"
#include "gzip.h"
@@ -168,6 +171,32 @@ struct CacheControlHeader {
bool _immutable = true;
};
+class ContentTypeHandler
+{
+public:
+ ContentTypeHandler(std::string &resp_header_fields) :
_resp_header_fields(resp_header_fields) {}
+
+ // Returns false if _content_type_whitelist is not empty, and content-type
field is either not present or not in the
+ // whitelist. Adds first Content-type field it encounters in the headers
passed to this function.
+ //
+ bool nextObjectHeader(TSMBuffer bufp, TSMLoc hdr_loc);
+
+ // Load whitelist from config file.
+ //
+ static void loadWhiteList(std::string const &file_spec);
+
+private:
+ // Add Content-Type field to these.
+ //
+ std::string &_resp_header_fields;
+
+ bool _added_content_type{false};
+
+ static vector<std::string> _content_type_whitelist;
+};
+
+vector<std::string> ContentTypeHandler::_content_type_whitelist;
+
bool
InterceptData::init(TSVConn vconn)
{
@@ -309,7 +338,6 @@ static bool writeResponse(InterceptData &int_data);
static bool writeErrorResponse(InterceptData &int_data, int &n_bytes_written);
static bool writeStandardHeaderFields(InterceptData &int_data, int
&n_bytes_written);
static void prepareResponse(InterceptData &int_data, ByteBlockList
&body_blocks, string &resp_header_fields);
-static bool getContentType(TSMBuffer bufp, TSMLoc hdr_loc, string
&resp_header_fields);
static bool getDefaultBucket(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc hdr_obj,
ClientRequest &creq);
// libesi TLS key.
@@ -392,6 +420,20 @@ TSPluginInit(int argc, const char *argv[])
LOG_DEBUG("WhiteList: %s", HEADER_WHITELIST[i].c_str());
}
+ std::string content_type_whitelist_filespec = (argc > optind &&
(argv[optind][0] != '-' || argv[optind][1])) ? argv[optind] : "";
+ if (content_type_whitelist_filespec.empty()) {
+ LOG_DEBUG("No Content-Type whitelist file specified (all content types
allowed)");
+ } else {
+ // If we have a path and it's not an absolute path, make it relative to the
+ // configuration directory.
+ if (content_type_whitelist_filespec[0] != '/') {
+ content_type_whitelist_filespec = std::string(TSConfigDirGet()) + '/' +
content_type_whitelist_filespec;
+ }
+ LOG_DEBUG("Content-Type whitelist file: %s",
content_type_whitelist_filespec.c_str());
+ ContentTypeHandler::loadWhiteList(content_type_whitelist_filespec);
+ }
+ ++optind;
+
TSReleaseAssert(pthread_key_create(&threadKey, nullptr) == 0);
TSCont rrh_contp = TSContCreate(handleReadRequestHeader, nullptr);
@@ -926,8 +968,6 @@ writeResponse(InterceptData &int_data)
static void
prepareResponse(InterceptData &int_data, ByteBlockList &body_blocks, string
&resp_header_fields)
{
- bool got_content_type = false;
-
if (int_data.creq.status == TS_HTTP_STATUS_OK) {
HttpDataFetcherImpl::ResponseData resp_data;
TSMLoc field_loc;
@@ -941,12 +981,16 @@ prepareResponse(InterceptData &int_data, ByteBlockList
&body_blocks, string &res
flags_list[i] = 0;
}
+ ContentTypeHandler cth(resp_header_fields);
+
for (StringList::iterator iter = int_data.creq.file_urls.begin(); iter !=
int_data.creq.file_urls.end(); ++iter) {
if (int_data.fetcher->getData(*iter, resp_data) && resp_data.status ==
TS_HTTP_STATUS_OK) {
body_blocks.push_back(ByteBlock(resp_data.content,
resp_data.content_len));
if (find(HEADER_WHITELIST.begin(), HEADER_WHITELIST.end(),
TS_MIME_FIELD_CONTENT_TYPE) == HEADER_WHITELIST.end()) {
- if (!got_content_type) {
- got_content_type = getContentType(resp_data.bufp,
resp_data.hdr_loc, resp_header_fields);
+ if (!cth.nextObjectHeader(resp_data.bufp, resp_data.hdr_loc)) {
+ LOG_ERROR("Content type missing or forbidden for requested URL
[%s]", iter->c_str());
+ int_data.creq.status = TS_HTTP_STATUS_FORBIDDEN;
+ break;
}
}
@@ -1040,10 +1084,9 @@ prepareResponse(InterceptData &int_data, ByteBlockList
&body_blocks, string &res
}
}
-static bool
-getContentType(TSMBuffer bufp, TSMLoc hdr_loc, string &resp_header_fields)
+bool
+ContentTypeHandler::nextObjectHeader(TSMBuffer bufp, TSMLoc hdr_loc)
{
- bool retval = false;
TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc,
TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE);
if (field_loc != TS_NULL_MLOC) {
bool values_added = false;
@@ -1052,21 +1095,86 @@ getContentType(TSMBuffer bufp, TSMLoc hdr_loc, string
&resp_header_fields)
int n_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc);
for (int i = 0; i < n_values; ++i) {
value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, i,
&value_len);
- if (!values_added) {
- resp_header_fields.append("Content-Type: ");
- values_added = true;
- } else {
- resp_header_fields.append(", ");
+ ts::TextView tv{value, value_len};
+ tv = tv.prefix(';').rtrim(std::string_view(" \t"));
+ if (_content_type_whitelist.empty()) {
+ ;
+ } else if (std::find_if(_content_type_whitelist.begin(),
_content_type_whitelist.end(), [tv](ts::TextView tv2) -> bool {
+ return strcasecmp(tv, tv2) == 0;
+ }) == _content_type_whitelist.end()) {
+ return false;
+ } else if (tv.empty()) {
+ // Whitelist is bad, contains an empty string.
+ return false;
+ }
+ if (!_added_content_type) {
+ if (!values_added) {
+ _resp_header_fields.append("Content-Type: ");
+ values_added = true;
+ } else {
+ _resp_header_fields.append(", ");
+ }
+ _resp_header_fields.append(value, value_len);
}
- resp_header_fields.append(value, value_len);
}
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
if (values_added) {
- resp_header_fields.append("\r\n");
- retval = true;
+ _resp_header_fields.append("\r\n");
+
+ // Assume that the Content-type field from the first header covers all
the responses being combined.
+ _added_content_type = true;
}
+ return true;
}
- return retval;
+ // No content type header field so doesn't pass whitelist if there is one.
+ return _content_type_whitelist.empty();
+}
+
+void
+ContentTypeHandler::loadWhiteList(std::string const &file_spec)
+{
+ std::fstream fs;
+ char line_buffer[256];
+ bool extra_junk_on_line{false};
+ int line_num = 0;
+
+ fs.open(file_spec);
+ if (fs.good()) {
+ for (;;) {
+ ++line_num;
+ fs.getline(line_buffer, sizeof(line_buffer));
+ if (!fs.good()) {
+ break;
+ }
+ constexpr std::string_view bs{" \t"sv};
+ ts::TextView line{line_buffer, std::size_t(fs.gcount() - 1)};
+ line.ltrim(bs);
+ if (line.empty() || line[0] == '#') {
+ // Empty/comment line.
+ continue;
+ }
+ ts::TextView content_type{line.take_prefix_at(bs)};
+ line.trim(bs);
+ if (line.size() && (line[0] != '#')) {
+ extra_junk_on_line = true;
+ break;
+ }
+ _content_type_whitelist.emplace_back(content_type);
+ }
+ }
+ if (fs.fail() && !(fs.eof() && (fs.gcount() == 0))) {
+ LOG_ERROR("Error reading Content-Type whitelist config file %s, line %d",
file_spec.c_str(), line_num);
+ } else if (extra_junk_on_line) {
+ LOG_ERROR("More than one type on line %d in Content-Type whitelist config
file %s", line_num, file_spec.c_str());
+ } else if (_content_type_whitelist.empty()) {
+ LOG_ERROR("Content-type whitelist config file %s must have at least one
entry", file_spec.c_str());
+ } else {
+ // End of file.
+ return;
+ }
+ _content_type_whitelist.clear();
+ // An empty string marks object as bad.
+ _content_type_whitelist.emplace_back("");
}
static const char INVARIANT_FIELD_LINES[] = {"Vary: Accept-Encoding\r\n"};
diff --git a/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py
b/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py
new file mode 100644
index 0000000..6c0820d
--- /dev/null
+++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py
@@ -0,0 +1,133 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = '''
+Test combo_handler plugin
+'''
+
+# Skip if plugin not present.
+#
+Test.SkipUnless(
+ Condition.PluginExists('combo_handler.so'),
+)
+
+# Function to generate a unique data file path (in the top level of the test's
run directory), put data (in string 'data') into
+# the file, and return the file name.
+#
+_data_file__file_count = 0
+def data_file(data):
+ global _data_file__file_count
+ file_path = Test.RunDirectory +
"/tcp_client_in_{}".format(_data_file__file_count)
+ _data_file__file_count += 1
+ with open(file_path, "x") as f:
+ f.write(data)
+ return file_path
+
+# Function to return command (string) to run tcp_client.py tool. 'host'
'port', and 'file_path' are the parameters to tcp_client.
+#
+def tcp_client_cmd(host, port, file_path):
+ return "python {}/tcp_client.py {} {}
{}".format(Test.Variables.AtsTestToolsDir, host, port, file_path)
+
+# Function to return command (string) to run tcp_client.py tool. 'host' and
'port' are the first two parameters to tcp_client.
+# 'data' is the data to put in the data file input to tcp_client.
+#
+def tcp_client(host, port, data):
+ return tcp_client_cmd(host, port, data_file(data))
+
+server = Test.MakeOriginServer("server")
+
+def add_server_obj(content_type, path):
+ request_header = {
+ "headers": "GET " + path + " HTTP/1.1\r\n" +
+ "Host: just.any.thing\r\n\r\n",
+ "timestamp": "1469733493.993",
+ "body": ""
+ }
+ response_header = {
+ "headers": "HTTP/1.1 200 OK\r\n" +
+ "Connection: close\r\n" +
+ 'Etag: "359670651"\r\n' +
+ "Cache-Control: public, max-age=31536000\r\n" +
+ "Accept-Ranges: bytes\r\n" +
+ "Content-Type: " + content_type + "\r\n" +
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "Content for " + path + "\n"
+ }
+ server.addResponse("sessionfile.log", request_header, response_header)
+
+add_server_obj("text/css ; charset=utf-8", "/obj1")
+add_server_obj("text/javascript", "/sub/obj2")
+add_server_obj("text/argh", "/obj3")
+add_server_obj("application/javascript", "/obj4")
+
+ts = Test.MakeATSProcess("ts")
+
+ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'http|combo_handler',
+})
+
+ts.Disk.plugin_config.AddLine("combo_handler.so - - - ctwl.txt")
+
+ts.Disk.remap_config.AddLine(
+ 'map http://xyz/ http://127.0.0.1/ @plugin=combo_handler.so'
+)
+ts.Disk.remap_config.AddLine(
+ 'map http://localhost/127.0.0.1/
http://127.0.0.1:{}/'.format(server.Variables.Port)
+)
+ts.Disk.remap_config.AddLine(
+ 'map http://localhost/sub/
http://127.0.0.1:{}/sub/'.format(server.Variables.Port)
+)
+
+ts.Disk.File(ts.Variables.CONFIGDIR + "/ctwl.txt", id="ctwl_cfg",
typename="ats:config")
+ts.Disk.ctwl_cfg.AddLine("# test ")
+ts.Disk.ctwl_cfg.AddLine("")
+ts.Disk.ctwl_cfg.AddLine(" text/javascript # test side comment")
+ts.Disk.ctwl_cfg.AddLine(" application/javascript")
+ts.Disk.ctwl_cfg.AddLine("text/css")
+
+tr = Test.AddTestRun()
+tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port))
+tr.Processes.Default.StartBefore(server,
ready=When.PortOpen(server.Variables.Port))
+tr.Processes.Default.Command = "echo start stuff"
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = tcp_client("127.0.0.1", ts.Variables.port,
+ "GET /admin/v1/combo?obj1&sub:obj2&obj3 HTTP/1.1\n" +
+ "Host: xyz\n" +
+ "Connection: close\n" +
+ "\n"
+)
+tr.Processes.Default.ReturnCode = 0
+f = tr.Disk.File("_output/1-tr-Default/stream.all.txt")
+f.Content = "combo_handler_files/tr1.gold"
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = tcp_client("127.0.0.1", ts.Variables.port,
+ "GET /admin/v1/combo?obj1&sub:obj2&obj4 HTTP/1.1\n" +
+ "Host: xyz\n" +
+ "Connection: close\n" +
+ "\n"
+)
+tr.Processes.Default.ReturnCode = 0
+f = tr.Disk.File("_output/2-tr-Default/stream.all.txt")
+f.Content = "combo_handler_files/tr2.gold"
+
+ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", "Some tests
are failure tests")
diff --git
a/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr1.gold
b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr1.gold
new file mode 100644
index 0000000..133179b
--- /dev/null
+++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr1.gold
@@ -0,0 +1,9 @@
+HTTP/1.1 403 Forbidden
+Date: ``
+Age: ``
+Transfer-Encoding: chunked
+Connection: close
+Server: ATS/``
+
+0
+
diff --git
a/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr2.gold
b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr2.gold
new file mode 100644
index 0000000..4d775f6
--- /dev/null
+++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr2.gold
@@ -0,0 +1,18 @@
+HTTP/1.1 200 OK
+Vary: Accept-Encoding
+Last-Modified: ``
+Content-Type: text/css ; charset=utf-8
+Cache-Control: max-age=31536000, Public
+Date: ``
+Age: ``
+Transfer-Encoding: chunked
+Connection: close
+Server: ATS/``
+
+3a
+Content for /obj1
+Content for /sub/obj2
+Content for /obj4
+
+0
+
