This is an automated email from the ASF dual-hosted git repository.
bnolsen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 211753f261 slice/cache_range_requests plugin: avoid subsequent slice
IMS requests (#12092)
211753f261 is described below
commit 211753f261ce669f8dc27b37ed4c7c3d941bf5b9
Author: Brian Olsen <[email protected]>
AuthorDate: Mon Mar 31 13:15:53 2025 -0600
slice/cache_range_requests plugin: avoid subsequent slice IMS requests
(#12092)
* crr plugin with support for 304 bypass
* add crr custom header tests
* add slice plugin autest for attaching identifier header
* add documentation for new feature
* add autest for slice+crr in combined use
* for slice_selfhealing only enable consider-ident for the primary test
---
.../plugins/cache_range_requests.en.rst | 30 ++-
doc/admin-guide/plugins/slice.en.rst | 8 +
.../cache_range_requests/cache_range_requests.cc | 118 +++++++++--
plugins/header_rewrite/parser.cc | 1 +
plugins/slice/Config.cc | 12 +-
plugins/slice/Config.h | 1 +
plugins/slice/slice.cc | 5 +-
plugins/slice/util.cc | 20 ++
.../cache_range_requests_ident.test.py | 195 ++++++++++++++++++
.../pluginTest/slice/gold/slice_crr_ident.gold | 13 ++
.../pluginTest/slice/gold/slice_ident.gold | 15 ++
.../pluginTest/slice/slice_crr_ident.test.py | 222 +++++++++++++++++++++
.../pluginTest/slice/slice_ident.test.py | 188 +++++++++++++++++
.../pluginTest/slice/slice_selfhealing.test.py | 73 ++++++-
14 files changed, 877 insertions(+), 24 deletions(-)
diff --git a/doc/admin-guide/plugins/cache_range_requests.en.rst
b/doc/admin-guide/plugins/cache_range_requests.en.rst
index 59f62f9413..e7fd928437 100644
--- a/doc/admin-guide/plugins/cache_range_requests.en.rst
+++ b/doc/admin-guide/plugins/cache_range_requests.en.rst
@@ -112,7 +112,6 @@ which includes information about the partial content range.
In this mode,
all requests (include partial content) will use consistent hashing method
for parent selection.
-
X-Crr-Ims header support
------------------------
@@ -144,6 +143,35 @@ option must have the same value (or not be defined) in
order to work.
Presence of the `--ims-header` automatically sets the `--consider-ims` option.
+X-Crr-Ident header support
+--------------------------
+
+.. option:: --consider-ident
+.. option:: -d
+.. option:: --ident-header=[header name] (default: X-Crr-Ident)
+.. option:: -j
+
+This supports the slice plugin which makes multiple adjacent range
+requests. The slice plugin will record the identifier of the first range
+request (Etag, or Last-Modified in that order) and will add the value
+to this header.
+
+.. code::
+
+ X-Crr-Ident: Etag: "foo"
+ X-Crr-Ident: Last-Modified: Tue, 19 Nov 2019 13:26:45 GMT
+
+During the cache lookup hook if a range request is considered STALE
+the identifier from this header will be compared to the stale cache
+identifier. If the values match the response will be changed to FRESH,
+preventing the transaction from contacting a parent.
+
+When used with the :program:`slice` plugin its `--crr-ident-header`
+option must have the same value (or not be defined) in order to work.
+
+Presence of the `--ident-header` automatically sets the `--consider-ident`
+option.
+
Don't modify the Cache Key
--------------------------
diff --git a/doc/admin-guide/plugins/slice.en.rst
b/doc/admin-guide/plugins/slice.en.rst
index 9215d4cc84..d0dce11b20 100644
--- a/doc/admin-guide/plugins/slice.en.rst
+++ b/doc/admin-guide/plugins/slice.en.rst
@@ -140,6 +140,14 @@ The slice plugin supports the following options::
`cache_range_requests` plugin.
-i for short
+ --crr-ident-header=<header name> (default: X-Crr-Ident)
+ Header name used by the slice plugin to tell the
+ `cache_range_requests` plugin the identifier of the
+ first/reference slice fetched. First Etag is preferred
+ followed by Last-Modified. The `cache_range_requests`
+ can use this identifier to flip a STALE asset back to
+ FRESH in order to limit unnecessary IMS requests.
+
--prefetch-count=<int> (optional)
Default is 0
Prefetches successive 'n' slice block requests in the background
diff --git a/plugins/cache_range_requests/cache_range_requests.cc
b/plugins/cache_range_requests/cache_range_requests.cc
index 9ba9417fb8..c2c149ee1b 100644
--- a/plugins/cache_range_requests/cache_range_requests.cc
+++ b/plugins/cache_range_requests/cache_range_requests.cc
@@ -50,28 +50,36 @@ enum parent_select_mode_t {
PS_CACHEKEY_URL, // Set parent selection url to cache_key url
};
-constexpr std::string_view DefaultImsHeader = {"X-Crr-Ims"};
-constexpr std::string_view SLICE_CRR_HEADER = {"Slice-Crr-Status"};
-constexpr std::string_view SLICE_CRR_VAL = "1";
-constexpr std::string_view SKIP_CRR_HDR_NAME = {"X-Skip-Crr"};
+constexpr std::string_view DefaultImsHeader = {"X-Crr-Ims"};
+constexpr std::string_view DefaultIdentHeader = {"X-Crr-Ident"};
+constexpr std::string_view SLICE_CRR_HEADER = {"Slice-Crr-Status"};
+constexpr std::string_view SLICE_CRR_VAL = "1";
+constexpr std::string_view SKIP_CRR_HDR_NAME = {"X-Skip-Crr"};
+
+std::string_view const Etag(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
+std::string_view const LastModified(TS_MIME_FIELD_LAST_MODIFIED,
TS_MIME_LEN_LAST_MODIFIED);
struct pluginconfig {
parent_select_mode_t ps_mode{PS_DEFAULT};
bool consider_ims_header{false};
+ bool consider_ident_header{false};
bool modify_cache_key{true};
bool verify_cacheability{false};
bool cache_complete_responses{false};
std::string ims_header;
+ std::string ident_header;
};
struct txndata {
- std::string range_value;
- TSHttpStatus origin_status{TS_HTTP_STATUS_NONE};
- time_t ims_time{0};
- bool verify_cacheability{false};
- bool cache_complete_responses{false};
- bool slice_response{false};
- bool slice_request{false};
+ pluginconfig const *config{nullptr};
+ std::string range_value;
+ TSHttpStatus origin_status{TS_HTTP_STATUS_NONE};
+ time_t ims_time{0};
+ bool ident_check{false};
+ bool verify_cacheability{false};
+ bool cache_complete_responses{false};
+ bool slice_response{false};
+ bool slice_request{false};
};
// pluginconfig struct (global plugin only)
@@ -111,7 +119,9 @@ create_pluginconfig(int argc, char *const argv[])
static const struct option longopts[] = {
{const_cast<char *>("consider-ims"), no_argument,
nullptr, 'c'},
+ {const_cast<char *>("consider-ident"), no_argument,
nullptr, 'd'},
{const_cast<char *>("ims-header"), required_argument,
nullptr, 'i'},
+ {const_cast<char *>("ident-header"), required_argument,
nullptr, 'j'},
{const_cast<char *>("no-modify-cachekey"), no_argument,
nullptr, 'n'},
{const_cast<char *>("ps-cachekey"), no_argument,
nullptr, 'p'},
{const_cast<char *>("verify-cacheability"), no_argument,
nullptr, 'v'},
@@ -134,11 +144,20 @@ create_pluginconfig(int argc, char *const argv[])
DEBUG_LOG("Plugin considers the ims header");
pc->consider_ims_header = true;
} break;
+ case 'd': {
+ DEBUG_LOG("Plugin considers the ident header");
+ pc->consider_ident_header = true;
+ } break;
case 'i': {
DEBUG_LOG("Plugin uses custom ims header: %s", optarg);
pc->ims_header.assign(optarg);
pc->consider_ims_header = true;
} break;
+ case 'j': {
+ DEBUG_LOG("Plugin uses custom ident header: %s", optarg);
+ pc->ident_header.assign(optarg);
+ pc->consider_ident_header = true;
+ } break;
case 'n': {
DEBUG_LOG("Plugin doesn't modify cache key");
pc->modify_cache_key = false;
@@ -171,6 +190,11 @@ create_pluginconfig(int argc, char *const argv[])
DEBUG_LOG("Plugin uses default ims header: %s", pc->ims_header.c_str());
}
+ if (pc->consider_ident_header && pc->ident_header.empty()) {
+ pc->ident_header = DefaultIdentHeader;
+ DEBUG_LOG("Plugin uses default ident header: %s",
pc->ident_header.c_str());
+ }
+
return pc;
}
@@ -253,6 +277,7 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
// Consider config options
if (nullptr != pc) {
+ txn_state->config = pc;
char cache_key_url[16384] = {0};
int cache_key_url_len = 0;
@@ -292,6 +317,7 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
}
// optionally consider an ims header
+ bool ims_active = false;
if (pc->consider_ims_header) {
TSMLoc const imsloc = TSMimeHdrFieldFind(hdr_buf, hdr_loc,
pc->ims_header.data(), pc->ims_header.size());
if (TS_NULL_MLOC != imsloc) {
@@ -300,10 +326,21 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
TSHandleMLocRelease(hdr_buf, hdr_loc, imsloc);
if (0 < itime) {
txn_state->ims_time = itime;
+ ims_active = true;
}
}
}
+ // If not revalidating then consider the identity header
+ if (!ims_active && pc->consider_ident_header) {
+ TSMLoc const identloc = TSMimeHdrFieldFind(hdr_buf, hdr_loc,
pc->ident_header.data(), pc->ident_header.size());
+ if (TS_NULL_MLOC != identloc) {
+ DEBUG_LOG("Servicing the '%s' header", pc->ident_header.c_str());
+ TSHandleMLocRelease(hdr_buf, hdr_loc, identloc);
+ txn_state->ident_check = true;
+ };
+ }
+
txn_state->verify_cacheability = pc->verify_cacheability;
txn_state->cache_complete_responses = pc->cache_complete_responses;
}
@@ -321,7 +358,7 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, txn_contp);
DEBUG_LOG("Added TS_HTTP_SEND_REQUEST_HDR_HOOK,
TS_HTTP_SEND_RESPONSE_HDR_HOOK, and TS_HTTP_TXN_CLOSE_HOOK");
- if (0 < txn_state->ims_time) {
+ if (0 < txn_state->ims_time || txn_state->ident_check) {
TSHttpTxnHookAdd(txnp, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK,
txn_contp);
DEBUG_LOG("Also Added TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK");
}
@@ -581,7 +618,7 @@ get_date_from_cached_hdr(TSHttpTxn txn)
}
/**
- * Handle a special IMS request.
+ * Handle a special IMS request or identity check on stale asset
*/
void
handle_cache_lookup_complete(TSHttpTxn txnp, txndata *const txn_state)
@@ -605,6 +642,61 @@ handle_cache_lookup_complete(TSHttpTxn txnp, txndata
*const txn_state)
}
}
}
+ } else if (TS_CACHE_LOOKUP_HIT_STALE == cachestat &&
txn_state->ident_check) {
+ pluginconfig const *const pc = txn_state->config;
+ DEBUG_LOG("Stale asset ident check");
+
+ TSMBuffer resp_buf = nullptr;
+ TSMLoc resp_loc = TS_NULL_MLOC;
+
+ if (TS_SUCCESS == TSHttpTxnCachedRespGet(txnp, &resp_buf, &resp_loc)) {
+ if (TS_HTTP_STATUS_OK == TSHttpHdrStatusGet(resp_buf, resp_loc)) {
+ // get the request identifier
+ TSMBuffer req_buf = nullptr;
+ TSMLoc req_loc = TS_NULL_MLOC;
+ if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc)) {
+ TSMLoc const ident_loc = TSMimeHdrFieldFind(req_buf, req_loc,
pc->ident_header.data(), pc->ident_header.size());
+ if (TS_NULL_MLOC != ident_loc) {
+ DEBUG_LOG("Checking identifier against the '%s' header",
pc->ident_header.c_str());
+
+ int len = 0;
+ char const *const str = TSMimeHdrFieldValueStringGet(req_buf,
req_loc, ident_loc, -1, &len);
+
+ // determine which identifier has been provided
+ std::string_view const svreq(str, len);
+ std::string_view tag;
+ if (svreq.substr(0, Etag.length()) == Etag) {
+ DEBUG_LOG("Etag identifier provided in '%.*s'", len, str);
+ tag = Etag;
+ } else if (svreq.substr(0, LastModified.length()) ==
LastModified) {
+ DEBUG_LOG("Last-Modified indentifier provided in '%.*s'", len,
str);
+ tag = LastModified;
+ }
+
+ if (!tag.empty()) {
+ TSMLoc const id_loc = TSMimeHdrFieldFind(resp_buf, resp_loc,
tag.data(), tag.size());
+ if (TS_NULL_MLOC != id_loc) {
+ int len = 0;
+ char const *const str =
TSMimeHdrFieldValueStringGet(resp_buf, resp_loc, id_loc, 0, &len);
+ std::string_view const sv(str, len);
+
+ DEBUG_LOG("Checking cached '%.*s' against request '%.*s'",
len, str, (int)svreq.size(), svreq.data());
+
+ if (std::string_view::npos != svreq.rfind(sv)) {
+ DEBUG_LOG("Flipping cache lookup status from STALE to
FRESH");
+ TSHttpTxnCacheLookupStatusSet(txnp,
TS_CACHE_LOOKUP_HIT_FRESH);
+ }
+ TSHandleMLocRelease(resp_buf, resp_loc, id_loc);
+ }
+ }
+
+ TSHandleMLocRelease(req_buf, req_loc, ident_loc);
+ }
+ TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
+ }
+ }
+ TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc);
+ }
}
}
}
diff --git a/plugins/header_rewrite/parser.cc b/plugins/header_rewrite/parser.cc
index 921ec2f83a..3dbd63de4a 100644
--- a/plugins/header_rewrite/parser.cc
+++ b/plugins/header_rewrite/parser.cc
@@ -196,6 +196,7 @@ Parser::preprocess(std::vector<std::string> tokens)
}
} else {
TSError("[%s] conditions must be embraced in %%{}", PLUGIN_NAME);
+ TSError("[%s] token: '%s'", PLUGIN_NAME, tokens[0].c_str());
return false;
}
} else {
diff --git a/plugins/slice/Config.cc b/plugins/slice/Config.cc
index 9d62dd71f8..98da5bd410 100644
--- a/plugins/slice/Config.cc
+++ b/plugins/slice/Config.cc
@@ -29,6 +29,7 @@ namespace
{
constexpr std::string_view DefaultSliceSkipHeader = {"X-Slicer-Info"};
constexpr std::string_view DefaultCrrImsHeader = {"X-Crr-Ims"};
+constexpr std::string_view DefaultCrrIdentHeader = {"X-Crr-Ident"};
} // namespace
Config::~Config()
@@ -116,6 +117,7 @@ Config::fromArgs(int const argc, char const *const argv[])
{const_cast<char *>("crr-ims-header"), required_argument, nullptr,
'c'},
{const_cast<char *>("disable-errorlog"), no_argument, nullptr,
'd'},
{const_cast<char *>("exclude-regex"), required_argument, nullptr,
'e'},
+ {const_cast<char *>("crr-ident-header"), required_argument, nullptr,
'g'},
{const_cast<char *>("include-regex"), required_argument, nullptr,
'i'},
{const_cast<char *>("ref-relative"), no_argument, nullptr,
'l'},
{const_cast<char *>("pace-errorlog"), required_argument, nullptr,
'p'},
@@ -133,7 +135,7 @@ Config::fromArgs(int const argc, char const *const argv[])
// getopt assumes args start at '1' so this hack is needed
char *const *argvp = (const_cast<char *const *>(argv) - 1);
for (;;) {
- int const opt = getopt_long(argc + 1, argvp, "b:dc:e:i:lm:p:r:s:t:x:z:",
longopts, nullptr);
+ int const opt = getopt_long(argc + 1, argvp, "b:dc:e:g:i:lm:p:r:s:t:x:z:",
longopts, nullptr);
if (-1 == opt) {
break;
}
@@ -175,6 +177,10 @@ Config::fromArgs(int const argc, char const *const argv[])
DEBUG_LOG("Using regex for url exclude: '%s'", m_regexstr.c_str());
}
} break;
+ case 'g': {
+ m_crr_ident_header.assign(optarg);
+ DEBUG_LOG("Using override crr ident header %s", optarg);
+ } break;
case 'i': {
if (None != m_regex_type) {
ERROR_LOG("Regex already specified!");
@@ -278,6 +284,10 @@ Config::fromArgs(int const argc, char const *const argv[])
m_crr_ims_header = DefaultCrrImsHeader;
DEBUG_LOG("Using default crr ims header %s", m_crr_ims_header.c_str());
}
+ if (m_crr_ident_header.empty()) {
+ m_crr_ident_header = DefaultCrrIdentHeader;
+ DEBUG_LOG("Using default crr ident header %s", m_crr_ident_header.c_str());
+ }
if (m_skip_header.empty()) {
m_skip_header = DefaultSliceSkipHeader;
DEBUG_LOG("Using default slice skip header %s", m_skip_header.c_str());
diff --git a/plugins/slice/Config.h b/plugins/slice/Config.h
index b7694a861a..824ffddd40 100644
--- a/plugins/slice/Config.h
+++ b/plugins/slice/Config.h
@@ -52,6 +52,7 @@ struct Config {
std::string m_skip_header;
std::string m_crr_ims_header;
+ std::string m_crr_ident_header;
// Convert optarg to bytes
static int64_t bytesFrom(char const *const valstr);
diff --git a/plugins/slice/slice.cc b/plugins/slice/slice.cc
index 9be1a8b729..04bd1086ce 100644
--- a/plugins/slice/slice.cc
+++ b/plugins/slice/slice.cc
@@ -53,7 +53,10 @@ should_skip_this_obj(TSHttpTxn txnp, Config *const config)
int len = 0;
char *const urlstr = TSHttpTxnEffectiveUrlStringGet(txnp, &len);
- if (!config->isKnownLargeObj({urlstr, static_cast<size_t>(len)})) {
+ bool const known_large = config->isKnownLargeObj({urlstr,
static_cast<size_t>(len)});
+ TSfree(urlstr);
+
+ if (!known_large) {
DEBUG_LOG("Not a known large object, not slicing: %.*s", len, urlstr);
return true;
}
diff --git a/plugins/slice/util.cc b/plugins/slice/util.cc
index 2e12b00eeb..7967185b81 100644
--- a/plugins/slice/util.cc
+++ b/plugins/slice/util.cc
@@ -98,6 +98,26 @@ request_block(TSCont contp, Data *const data)
header.setKeyVal(SLICE_CRR_HEADER.data(), SLICE_CRR_HEADER.size(),
SLICE_CRR_VAL.data(), SLICE_CRR_VAL.size());
}
+ // Attach the identifier header
+ Config const *const cfg = data->m_config;
+ if (!cfg->m_crr_ident_header.empty() &&
!header.hasKey(cfg->m_crr_ident_header.data(), cfg->m_crr_ident_header.size()))
{
+ swoc::LocalBufferWriter<8192> idbuf;
+ if (0 < data->m_etaglen) {
+ idbuf.write(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
+ idbuf.write(": ");
+ idbuf.write(data->m_etag, data->m_etaglen);
+ } else if (0 < data->m_lastmodifiedlen) {
+ idbuf.write(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED);
+ idbuf.write(": ");
+ idbuf.write(data->m_lastmodified, data->m_lastmodifiedlen);
+ }
+
+ if (0 < idbuf.size()) {
+ DEBUG_LOG("Adding identity '%.*s'", (int)idbuf.size(), idbuf.data());
+ header.setKeyVal(cfg->m_crr_ident_header.data(),
cfg->m_crr_ident_header.size(), idbuf.data(), idbuf.size());
+ }
+ }
+
// create virtual connection back into ATS
TSHttpConnectOptions options = TSHttpConnectOptionsGet(TS_CONNECT_PLUGIN);
options.addr = reinterpret_cast<sockaddr
*>(&data->m_client_ip);
diff --git
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ident.test.py
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ident.test.py
new file mode 100644
index 0000000000..d64e268aee
--- /dev/null
+++
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ident.test.py
@@ -0,0 +1,195 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+Test.Summary = '''
+cache_range_requests X-Crr-Ident plugin test
+'''
+
+# Test description:
+# Preload the cache with the entire asset to be range requested.
+# Ensure asset is stale and request with properly formed header.
+
+Test.SkipUnless(
+ Condition.PluginExists('cache_range_requests.so'),
+ Condition.PluginExists('xdebug.so'),
+)
+#Test.ContinueOnFail = False
+Test.ContinueOnFail = True
+Test.testName = "cache_range_requests_ident"
+
+# Define and configure ATS
+ts = Test.MakeATSProcess("ts")
+
+# Define and configure origin server
+server = Test.MakeOriginServer("server")
+
+# default root
+req_chk = {
+ "headers": "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "uuid:
none\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": ""
+}
+
+res_chk = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" +
"\r\n", "timestamp": "1469733493.993", "body": ""}
+
+server.addResponse("sessionlog.json", req_chk, res_chk)
+
+body = "lets go surfin now"
+bodylen = len(body)
+
+req_full = {
+ "headers": "GET /path HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": ""
+}
+
+last_modified = "Fri, 07 Mar 2025 18:06:58 GMT"
+etag = '"772102f4-56f4bc1e6d417"'
+
+res_full = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=1\r\n" +
+ "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + "Connection:
close\r\n" + 'Etag: ' + etag + '\r\n' +
+ 'Last-Modified: ' + last_modified + '\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": body
+}
+
+server.addResponse("sessionlog.json", req_full, res_full)
+
+req_custom = {
+ "headers": "GET /pathheader HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": ""
+}
+
+etag_custom = 'foo'
+
+res_custom = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=1\r\n" +
+ "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + "Connection:
close\r\n" + 'Etag: ' + etag_custom + '\r\n' + '\r\n',
+ "timestamp": "1469733493.993",
+ "body": body
+}
+
+server.addResponse("sessionlog.json", req_custom, res_custom)
+
+# cache range requests plugin remap
+ts.Disk.remap_config.AddLines(
+ [
+ f'map http://ident http://127.0.0.1:{server.Variables.Port}' + '
@plugin=cache_range_requests.so @pparam=--consider-ident',
+ f'map http://identheader http://127.0.0.1:{server.Variables.Port}' +
+ ' @plugin=cache_range_requests.so @pparam=--consider-ident' + '
@pparam=--ident-header=CrrIdent',
+ ])
+
+# cache debug
+ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache')
+
+# minimal configuration
+ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'cache_range_requests',
+})
+
+curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H
"x-debug: x-cache"'.format(ts.Variables.port)
+
+# 0 Test - Fetch asset into cache
+tr = Test.AddTestRun("0- range cache load")
+ps = tr.Processes.Default
+ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
+ps.StartBefore(Test.Processes.ts)
+ps.Command = curl_and_args + ' http://ident/path -r 0-'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss for load")
+tr.StillRunningAfter = ts
+
+# 1 Test - Fetch asset into cache
+tr = Test.AddTestRun("0- range cache load")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://identheader/pathheader -r 0-'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss for load")
+tr.StillRunningAfter = ts
+
+# 2 Test - Ensure range is fetched
+tr = Test.AddTestRun("0- cache hit check")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://ident/path -r 0-'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit",
"expected cache hit")
+tr.StillRunningAfter = ts
+
+# 3 Test - Ensure range is fetched
+tr = Test.AddTestRun("0- cache hit check")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://identheader/pathheader -r 0-'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit",
"expected cache hit")
+tr.StillRunningAfter = ts
+
+# These requests should flip from STALE to FRESH
+
+# 4 Test - Ensure X-Crr-Ident Etag header results in hit-fresh
+tr = Test.AddTestRun("0- range X-Crr-Ident Etag check")
+tr.DelayStart = 2
+ps = tr.Processes.Default
+ps.Command = curl_and_args + f" http://ident/path -r 0- -H 'X-Crr-Ident: Etag:
{etag}'"
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache hit-fresh")
+tr.StillRunningAfter = ts
+
+# 5 Test - Ensure X-Crr-Ident Etag header results in hit-fresh, custom header
+tr = Test.AddTestRun("0- range CrrIdent Etag check")
+tr.DelayStart = 2
+ps = tr.Processes.Default
+ps.Command = curl_and_args + f" http://identheader/pathheader -r 0- -H
'CrrIdent: Etag: {etag_custom}'"
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache hit-fresh")
+tr.StillRunningAfter = ts
+
+# 6 Test - Ensure X-Crr-Ident Last-Modified header results in hit-fresh
+tr = Test.AddTestRun("0- range X-Crr-Ident Last-Modified check")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + f' http://ident/path -r 0- -H "X-Crr-Ident:
Last-Modified: {last_modified}"'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache hit-fresh")
+tr.StillRunningAfter = ts
+
+# 7 Test - Provide a mismatch Etag force IMS request
+tr = Test.AddTestRun("0- range X-Crr-Ident check")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + f' http://ident/path -r 0- -H "X-Crr-Ident:
Last-Modified: foo"'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
+tr.StillRunningAfter = ts
+
+# post checks for traffic.out
+
+ts.Disk.traffic_out.Content = Testers.ContainsExpression(
+ """Checking cached '"772102f4-56f4bc1e6d417"' against request 'Etag:
"772102f4-56f4bc1e6d417"'""",
+ "Etag is correctly considered")
+
+ts.Disk.traffic_out.Content = Testers.ContainsExpression(
+ """Checking cached 'foo' against request 'Etag: foo'""", "Etag custom
header is correctly considered")
+
+ts.Disk.traffic_out.Content = Testers.ContainsExpression(
+ """Checking cached 'Fri, 07 Mar 2025 18:06:58 GMT' against request
'Last-Modified: Fri, 07 Mar 2025 18:06:58 GMT'""",
+ "Last-Modified is correctly considered")
diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold
b/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold
new file mode 100644
index 0000000000..f81ffa57f8
--- /dev/null
+++ b/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold
@@ -0,0 +1,13 @@
+cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2::
x-crr-ident=::-:: uid=::plain 0:: crc=TCP_MISS
+cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=3-5::
x-crr-ident=::Etag: "plain":: uid=::plain 1:: crc=TCP_MISS
+cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::plain:: crc=TCP_MISS
+cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2::
x-crr-ident=::-:: uid=::plain 0:: crc=TCP_REFRESH_MISS
+cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5::
x-crr-ident=::Etag: "plain":: uid=::-:: crc=TCP_HIT
+cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::plain:: crc=TCP_MISS
+cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2::
x-crr-ident=::-:: uid=::chg 0:: crc=TCP_REFRESH_MISS
+cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=3-5::
x-crr-ident=::Etag: "chg":: uid=::chg 1:: crc=TCP_REFRESH_MISS
+cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::chg:: crc=TCP_MISS
+cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=0-2:: x-crr-ident=::-::
uid=::-:: crc=TCP_HIT
+cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5::
x-crr-ident=::Etag: "chg":: uid=::-:: crc=TCP_HIT
+cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::chg:: crc=TCP_MISS
+cpuup=/404.txt sssc=404 pssc=404 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::-:: crc=TCP_MISS
diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_ident.gold
b/tests/gold_tests/pluginTest/slice/gold/slice_ident.gold
new file mode 100644
index 0000000000..ed44f114db
--- /dev/null
+++ b/tests/gold_tests/pluginTest/slice/gold/slice_ident.gold
@@ -0,0 +1,15 @@
+/etag 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
+/lm 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
+/etag 000 206 range=::bytes=0-10:: x-crr-ident=::-:: crrident=::-::
+/etag 000 206 range=::bytes=11-21:: x-crr-ident=::Etag: "foo":: crrident=::-::
+/etag 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
+/etag 000 206 range=::bytes=0-10:: x-crr-ident=::-:: crrident=::-::
+/etag 000 206 range=::bytes=11-21:: x-crr-ident=::-:: crrident=::Etag: "foo"::
+/etag 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
+/lm 000 206 range=::bytes=0-10:: x-crr-ident=::-:: crrident=::-::
+/lm 000 206 range=::bytes=11-21:: x-crr-ident=::Last-Modified: Fri, 07 Mar
2025 18:06:58 GMT:: crrident=::-::
+/lm 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
+/lm 000 206 range=::bytes=0-10:: x-crr-ident=::-:: crrident=::-::
+/lm 000 206 range=::bytes=11-21:: x-crr-ident=::-:: crrident=::Last-Modified:
Fri, 07 Mar 2025 18:06:58 GMT::
+/lm 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
+/404.txt 000 404 range=::-:: x-crr-ident=::-:: crrident=::-::
diff --git a/tests/gold_tests/pluginTest/slice/slice_crr_ident.test.py
b/tests/gold_tests/pluginTest/slice/slice_crr_ident.test.py
new file mode 100644
index 0000000000..5c789de76d
--- /dev/null
+++ b/tests/gold_tests/pluginTest/slice/slice_crr_ident.test.py
@@ -0,0 +1,222 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+Test.Summary = '''
+Combined Slice/crr ident test
+'''
+
+# Test description:
+# Preload the cache with the entire asset to be range requested.
+# Reload remap rule with slice plugin
+# Request content through the slice plugin
+
+Test.SkipUnless(
+ Condition.PluginExists('cache_range_requests.so'),
+ Condition.PluginExists('header_rewrite.so'),
+ Condition.PluginExists('slice.so'),
+ Condition.PluginExists('xdebug.so'),
+)
+Test.ContinueOnFail = False
+
+# configure origin server
+server = Test.MakeOriginServer("server", lookup_key="{%UID}")
+
+# default root
+req_header_chk = {
+ "headers": "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "UID:
none\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+res_header_chk = {
+ "headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+server.addResponse("sessionlog.json", req_header_chk, res_header_chk)
+
+# Define ATS and configure
+ts = Test.MakeATSProcess("ts")
+
+# set up slice plugin with remap host into cache_range_requests
+ts.Disk.remap_config.AddLines(
+ [
+ f'map http://slice/ http://127.0.0.1:{server.Variables.Port}/' +
+ ' @plugin=slice.so @pparam=--blockbytes-test=3
@pparam=--remap-host=crr',
+ f'map http://crr/ http://127.0.0.1:{server.Variables.Port}/' +
+ ' @plugin=cache_range_requests.so @pparam=--consider-ims
@pparam=--consider-ident' +
+ ' @plugin=header_rewrite.so @pparam=hdr_rw.conf',
+ ])
+
+ts.Disk.logging_yaml.AddLines(
+ '''
+logging:
+ formats:
+ - name: custom
+ format: 'cpuup=%<cquup> sssc=%<sssc> pssc=%<pssc> phr=%<phr>
range=::%<{Range}cqh>:: x-crr-ident=::%<{X-Crr-Ident}cqh>:: uid=::%<{UID}pqh>::
crc=%<crc>'
+ logs:
+ - filename: transaction
+ format: custom
+'''.split("\n"))
+
+ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache')
+
+ts.Disk.records_config.update(
+ {
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags':
'cache_range_requests|header_rewrite|slice|log',
+ })
+
+ts.Disk.MakeConfigFile("hdr_rw.conf").AddLines(
+ [
+ 'cond %{SEND_REQUEST_HDR_HOOK}', 'cond %{HEADER:Range} ="bytes=0-2"
[AND]', 'set-header UID %{CLIENT-HEADER:UID} 0'
+ '', 'cond %{SEND_REQUEST_HDR_HOOK}', 'cond %{HEADER:Range}
="bytes=3-5" [AND]', 'set-header UID %{CLIENT-HEADER:UID} 1'
+ ])
+
+# Test case: short lived asset. second slice should be HIT_FRESH
+
+req_header_plain_0 = {
+ "headers": "GET /plain HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "UID:
plain 0\r\n" + "Range: bytes=0-2\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+res_header_plain_0 = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=1\r\n" + "Connection: close\r\n" +
+ "Content-Range: bytes 0-2/5\r\n" + 'Etag: "plain"\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "aaa"
+}
+
+server.addResponse("sessionlog.json", req_header_plain_0, res_header_plain_0)
+
+req_header_plain_1 = {
+ "headers": "GET /plain HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "UID:
plain 1\r\n" + "Range: bytes=3-5\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+res_header_plain_1 = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=1\r\n" + "Connection: close\r\n" +
+ "Content-Range: bytes 3-4/5\r\n" + 'Etag: "plain"\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "BB"
+}
+
+server.addResponse("sessionlog.json", req_header_plain_1, res_header_plain_1)
+
+# curl helper
+curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x
localhost:{}'.format(ts.Variables.port) + ' -H "x-debug: x-cache"'
+
+# 0 Test - Preload plain asset
+tr = Test.AddTestRun("Preload plain")
+ps = tr.Processes.Default
+ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
+ps.StartBefore(Test.Processes.ts)
+ps.Command = curl_and_args + ' http://slice/plain -H "UID: plain"'
+ps.ReturnCode = 0
+ps.Streams.stderr.Content = Testers.ContainsExpression("aaaBB", "expected
aaaBB")
+ps.Streams.stdout.Content = Testers.ContainsExpression('Etag: "plain"',
"expected etag plain")
+tr.StillRunningAfter = ts
+
+# 2 Test - Request again, should result in stale asset
+tr = Test.AddTestRun("Request 2nd slice (expect slice1 to be fresh)")
+tr.DelayStart = 2 # ensure its really stale
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://slice/plain -H "UID: plain"'
+ps.ReturnCode = 0
+ps.Streams.stderr = Testers.ContainsExpression("aaaBB", "expected aaaBB")
+ps.Streams.stdout.Content = Testers.ContainsExpression('Etag: "plain"',
"expected etag plain")
+tr.StillRunningAfter = ts
+
+# change out the asset
+
+req_header_chg_0 = {
+ "headers": "GET /plain HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "UID:
chg 0\r\n" + "Range: bytes=0-2\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+res_header_chg_0 = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=60\r\n" +
+ "Connection: close\r\n" + "Content-Range: bytes 0-2/5\r\n" + 'Etag:
"chg"\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "AAA"
+}
+
+server.addResponse("sessionlog.json", req_header_chg_0, res_header_chg_0)
+
+req_header_chg_1 = {
+ "headers": "GET /plain HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "UID:
chg 1\r\n" + "Range: bytes=3-5\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+res_header_chg_1 = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=60\r\n" +
+ "Connection: close\r\n" + "Content-Range: bytes 3-4/5\r\n" + 'Etag:
"chg"\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "bb"
+}
+
+server.addResponse("sessionlog.json", req_header_chg_1, res_header_chg_1)
+
+# 3 Test - Request again, should result in new asset
+tr = Test.AddTestRun("Request again, asset replaced")
+tr.DelayStart = 2 # ensure its really stale
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://slice/plain -H "UID: chg"'
+ps.ReturnCode = 0
+ps.Streams.stderr = Testers.ContainsExpression("AAAbb", "expected AAAbb")
+ps.Streams.stdout.Content = Testers.ContainsExpression('Etag: "chg"',
"expected etag chg")
+tr.StillRunningAfter = ts
+
+# 4 Test - Request again, should all be hit
+tr = Test.AddTestRun("Request again, asset replaced but hit")
+tr.DelayStart = 2 # ensure its really stale
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://slice/plain -H "UID: chg"'
+ps.ReturnCode = 0
+ps.Streams.stderr = Testers.ContainsExpression("AAAbb", "expected AAAbb")
+ps.Streams.stdout.Content = Testers.ContainsExpression('Etag: "chg"',
"expected etag chg")
+tr.StillRunningAfter = ts
+
+# 5 Test - add token to transaction log
+tr = Test.AddTestRun("Fetch 404.txt asset")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://crr/404.txt'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("404", "expected 404
Not Found response")
+tr.StillRunningAfter = ts
+
+condwaitpath = os.path.join(Test.Variables.AtsTestToolsDir, 'condwait')
+
+tslog = os.path.join(ts.Variables.LOGDIR, 'transaction.log')
+Test.AddAwaitFileContainsTestRun('Await ts transactions to finish logging.',
tslog, '404.txt')
+
+# 6 Check logs
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = (f"cat {tslog}")
+tr.Streams.stdout = "gold/slice_crr_ident.gold"
+tr.Processes.Default.ReturnCode = 0
diff --git a/tests/gold_tests/pluginTest/slice/slice_ident.test.py
b/tests/gold_tests/pluginTest/slice/slice_ident.test.py
new file mode 100644
index 0000000000..b10635f01e
--- /dev/null
+++ b/tests/gold_tests/pluginTest/slice/slice_ident.test.py
@@ -0,0 +1,188 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+Test.Summary = '''
+Slice plugin test for sending ident header
+'''
+
+# Test description:
+# Preload the cache with the entire asset to be range requested.
+# Reload remap rule with slice plugin
+# Request content through the slice plugin
+
+Test.SkipUnless(Condition.PluginExists('slice.so'),)
+Test.ContinueOnFail = False
+
+# configure origin server
+server = Test.MakeOriginServer("server")
+
+# default root
+request_header_chk = {
+ "headers": "GET / HTTP/1.1\r\n" + "Host: ats\r\n" + "Range: none\r\n" +
"\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+response_header_chk = {
+ "headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+server.addResponse("sessionlog.json", request_header_chk, response_header_chk)
+
+block_bytes = 11
+body = "lets go surfin now"
+
+etag = '"foo"'
+last_modified = "Fri, 07 Mar 2025 18:06:58 GMT"
+
+request_etag = {
+ "headers": "GET /etag HTTP/1.1\r\n" + "Host: origin\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+response_etag = {
+ "headers":
+ "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + f'Etag: {etag}\r\n'
+ f'Last-Modified: {last_modified}\r\n' +
+ "Cache-Control: max-age=500\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": body,
+}
+
+server.addResponse("sessionlog.json", request_etag, response_etag)
+
+request_lm = {
+ "headers": "GET /lm HTTP/1.1\r\n" + "Host: origin\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+response_lm = {
+ "headers":
+ "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + f'Last-Modified:
{last_modified}\r\n' + "Cache-Control: max-age=500\r\n" +
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": body,
+}
+
+server.addResponse("sessionlog.json", request_lm, response_lm)
+
+# use this second ats instance to serve up the slices
+
+# Define ATS and configure
+ts = Test.MakeATSProcess("ts")
+ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'slice',
+})
+
+ts.Disk.remap_config.AddLines(
+ [
+ f"map http://preload/ http://127.0.0.1:{server.Variables.Port}";,
+ f'map http://slice/ http://127.0.0.1:{server.Variables.Port}' +
+ f' @plugin=slice.so @pparam=--blockbytes-test={block_bytes}',
+ f'map http://slicecustom/ http://127.0.0.1:{server.Variables.Port}' +
+ f' @plugin=slice.so @pparam=--blockbytes-test={block_bytes}' + '
@pparam=--crr-ident-header=CrrIdent',
+ ])
+
+ts.Disk.logging_yaml.AddLines(
+ '''
+logging:
+ formats:
+ - name: custom
+ format: '%<cquup> %<sssc> %<pssc> range=::%<{Range}cqh>::
x-crr-ident=::%<{X-Crr-Ident}cqh>:: crrident=::%<{CrrIdent}cqh>::'
+ logs:
+ - filename: transaction
+ format: custom
+'''.split("\n"))
+
+# helpers for curl
+curl_and_args = f'curl -s -D /dev/stdout -o /dev/stderr -x
localhost:{ts.Variables.port}'
+
+# 0 Test - Preload etag asset
+tr = Test.AddTestRun("Preload etag asset")
+ps = tr.Processes.Default
+ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
+ps.StartBefore(Test.Processes.ts)
+ps.Command = curl_and_args + ' http://preload/etag'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200
OK response")
+tr.StillRunningAfter = ts
+
+# 1 Test - Preload last-modified asset
+tr = Test.AddTestRun("Preload last-modified asset")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://preload/lm'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200
OK response")
+tr.StillRunningAfter = ts
+
+# 2 Test - Fetch etag asset
+tr = Test.AddTestRun("Fetch etag asset")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://slice/etag'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200
OK response")
+tr.StillRunningAfter = ts
+
+# 3 Test - Fetch etag asset
+tr = Test.AddTestRun("Fetch etag asset, custom")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://slicecustom/etag'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200
OK response")
+tr.StillRunningAfter = ts
+
+# 4 Test - Fetch last-modified asset
+tr = Test.AddTestRun("Fetch last-modified asset, custom")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://slice/lm'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200
OK response")
+tr.StillRunningAfter = ts
+
+# 5 Test - Fetch last-modified asset, custom
+tr = Test.AddTestRun("Fetch last-modified asset")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://slicecustom/lm'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200
OK response")
+tr.StillRunningAfter = ts
+
+# 6 Test - add token to transaction log
+tr = Test.AddTestRun("Fetch last-modified asset")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://prefetch/404.txt'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("404", "expected 404
Not Found response")
+tr.StillRunningAfter = ts
+
+condwaitpath = os.path.join(Test.Variables.AtsTestToolsDir, 'condwait')
+
+tslog = os.path.join(ts.Variables.LOGDIR, 'transaction.log')
+Test.AddAwaitFileContainsTestRun('Await ts transactions to finish logging.',
tslog, '404.txt')
+
+# 6 Check logs
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = (f"cat {tslog}")
+tr.Streams.stdout = "gold/slice_ident.gold"
+tr.Processes.Default.ReturnCode = 0
diff --git a/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py
b/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py
index 8716de8837..4248905e61 100644
--- a/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py
+++ b/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py
@@ -68,7 +68,8 @@ ts.Disk.remap_config.AddLines(
[
f'map http://slice/ http://127.0.0.1:{server.Variables.Port}/' +
' @plugin=slice.so @pparam=--blockbytes-test=3
@pparam=--remap-host=crr',
- f'map http://crr/ http://127.0.0.1:{server.Variables.Port}/' + '
@plugin=cache_range_requests.so @pparam=--consider-ims',
+ f'map http://crr/ http://127.0.0.1:{server.Variables.Port}/' +
+ ' @plugin=cache_range_requests.so @pparam=--consider-ims
@pparam=--consider-ident',
f'map http://slicehdr/ http://127.0.0.1:{server.Variables.Port}/' + '
@plugin=slice.so @pparam=--blockbytes-test=3' +
' @pparam=--remap-host=crrhdr @pparam=--crr-ims-header=crr-foo',
f'map http://crrhdr/ http://127.0.0.1:{server.Variables.Port}/'
@@ -79,7 +80,7 @@ ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache')
ts.Disk.records_config.update(
{
- 'proxy.config.diags.debug.enabled': 0,
+ 'proxy.config.diags.debug.enabled': 1,
'proxy.config.diags.debug.tags': 'cache_range_requests|slice',
})
@@ -340,27 +341,83 @@ ps.Command = curl_and_args + ' http://slice/assetgone'
ps.Streams.stdout.Content = Testers.ContainsExpression("404 Not Found",
"Expected 404")
tr.StillRunningAfter = ts
-# custom headers
+## custom headers
+
+# Test case: 2nd slice out of date (refetch and continue)
+
+req_header_custom_2ndold1 = {
+ "headers":
+ "GET /second-custom HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"uuid: etagold-custom-1\r\n" + "Range: bytes=3-5\r\n"
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+res_header_custom_2ndold1 = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=5000\r\n" +
+ "Connection: close\r\n" + "Content-Range: bytes 3-4/5\r\n" + 'Etag:
"etagold-custom"\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "aa"
+}
+
+server.addResponse("sessionlog.json", req_header_custom_2ndold1,
res_header_custom_2ndold1)
+
+req_header_custom_2ndnew0 = {
+ "headers":
+ "GET /second-custom HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"uuid: etagnew-custom-0\r\n" + "Range: bytes=0-2\r\n"
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+res_header_custom_2ndnew0 = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=5000\r\n" +
+ "Connection: close\r\n" + "Content-Range: bytes 0-2/5\r\n" + 'Etag:
"etagnew-custom"\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "bbb"
+}
+
+server.addResponse("sessionlog.json", req_header_custom_2ndnew0,
res_header_custom_2ndnew0)
+
+req_header_custom_2ndnew1 = {
+ "headers":
+ "GET /second-custom HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"uuid: etagnew-custom-1\r\n" + "Range: bytes=3-5\r\n"
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+res_header_custom_2ndnew1 = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=5000\r\n" +
+ "Connection: close\r\n" + "Content-Range: bytes 3-4/5\r\n" + 'Etag:
"etagnew-custom"\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "bb"
+}
+
+server.addResponse("sessionlog.json", req_header_custom_2ndnew1,
res_header_custom_2ndnew1)
edt = datetime.datetime.fromtimestamp(time.time() + 100)
edate = to_httpdate(edt)
# 12 Test - Preload reference etagold-1
-tr = Test.AddTestRun("Preload slice etagold-1")
+tr = Test.AddTestRun("Preload slice etagold-custom-1")
ps = tr.Processes.Default
-ps.Command = curl_and_args + f' http://crrhdr/second -r 3-5 -H "uuid:
etagold-1" -H "crr-foo: {edate}"'
+ps.Command = curl_and_args + f' http://crrhdr/second-custom -r 3-5 -H "uuid:
etagold-custom-1" -H "crr-foo: {edate}"'
ps.ReturnCode = 0
ps.Streams.stderr = "gold/aa.gold"
-ps.Streams.stdout.Content = Testers.ContainsExpression("etagold", "expected
etagold")
+ps.Streams.stdout.Content = Testers.ContainsExpression("etagold-custom",
"expected etagold-custom")
tr.StillRunningAfter = ts
# 13 Test - Request second slice via slice plugin, with instructions to fetch
new 2nd slice
tr = Test.AddTestRun("Request 2nd slice (expect refetch)")
ps = tr.Processes.Default
-ps.Command = curl_and_args + ' http://slicehdr/second -r 3- -H "uuid:
etagnew-1"'
+ps.Command = curl_and_args + ' http://slicehdr/second-custom -r 3- -H "uuid:
etagnew-custom-1"'
ps.ReturnCode = 0
ps.Streams.stderr = "gold/bb.gold"
-ps.Streams.stdout.Content = Testers.ContainsExpression("etagnew", "expected
etagnew")
+ps.Streams.stdout.Content = Testers.ContainsExpression("etagnew-custom",
"expected etagnew-custom")
tr.StillRunningAfter = ts
# Over riding the built in ERROR check since we expect to see logSliceErrors
