[trafficserver] branch master updated: Normalizes function names to match hook names in intercept plugins
This is an automated email from the ASF dual-hosted git repository.
rrm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 7320d5c Normalizes function names to match hook names in intercept
plugins
7320d5c is described below
commit 7320d5c5333ea2c346c92a80d276dc68762ad493
Author: Randall Meyer
AuthorDate: Tue Apr 21 12:28:59 2020 -0700
Normalizes function names to match hook names in intercept plugins
---
example/plugins/c-api/intercept/intercept.cc | 8
plugins/generator/generator.cc | 6 +++---
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/example/plugins/c-api/intercept/intercept.cc
b/example/plugins/c-api/intercept/intercept.cc
index 4d68be7..03bf86d 100644
--- a/example/plugins/c-api/intercept/intercept.cc
+++ b/example/plugins/c-api/intercept/intercept.cc
@@ -60,7 +60,7 @@
static TSCont TxnHook;
static TSCont InterceptHook;
-static int InterceptInterceptionHook(TSCont contp, TSEvent event, void *edata);
+static int InterceptInterceptHook(TSCont contp, TSEvent event, void *edata);
static int InterceptTxnHook(TSCont contp, TSEvent event, void *edata);
// We are going to stream data between Traffic Server and an
@@ -266,7 +266,7 @@ InterceptTransferData(InterceptIO *from, InterceptIO *to)
// starts with TS_EVENT_NET_ACCEPT, and then continues with
// TSVConn events.
static int
-InterceptInterceptionHook(TSCont contp, TSEvent event, void *edata)
+InterceptInterceptHook(TSCont contp, TSEvent event, void *edata)
{
argument_type arg(edata);
@@ -514,7 +514,7 @@ InterceptTxnHook(TSCont contp, TSEvent event, void *edata)
switch (event) {
case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: {
if (InterceptShouldInterceptRequest(arg.txn)) {
- TSCont c = InterceptContCreate(InterceptInterceptionHook,
TSMutexCreate(), arg.txn);
+ TSCont c = InterceptContCreate(InterceptInterceptHook, TSMutexCreate(),
arg.txn);
VDEBUG("intercepting origin server request for txn=%p, cont=%p",
arg.txn, c);
TSHttpTxnServerIntercept(c, arg.txn);
@@ -548,7 +548,7 @@ TSPluginInit(int /* argc */, const char * /* argv */[])
// XXX accept hostname and port arguments
TxnHook = InterceptContCreate(InterceptTxnHook, nullptr, nullptr);
- InterceptHook = InterceptContCreate(InterceptInterceptionHook, nullptr,
nullptr);
+ InterceptHook = InterceptContCreate(InterceptInterceptHook, nullptr,
nullptr);
// Wait until after the cache lookup to decide whether to
// intercept a request. For cache hits, we will never intercept.
diff --git a/plugins/generator/generator.cc b/plugins/generator/generator.cc
index 743ad51..0401fc7 100644
--- a/plugins/generator/generator.cc
+++ b/plugins/generator/generator.cc
@@ -80,7 +80,7 @@ static uint8_t GeneratorData[32 * 1024];
static int StatCountBytes = -1;
static int StatCountResponses = -1;
-static int GeneratorInterceptionHook(TSCont contp, TSEvent event, void *edata);
+static int GeneratorInterceptHook(TSCont contp, TSEvent event, void *edata);
static int GeneratorTxnHook(TSCont contp, TSEvent event, void *edata);
struct GeneratorRequest;
@@ -406,7 +406,7 @@ fail:
// starts with TS_EVENT_NET_ACCEPT, and then continues with
// TSVConn events.
static int
-GeneratorInterceptionHook(TSCont contp, TSEvent event, void *edata)
+GeneratorInterceptHook(TSCont contp, TSEvent event, void *edata)
{
argument_type arg(edata);
@@ -610,7 +610,7 @@ GeneratorTxnHook(TSCont contp, TSEvent event, void *edata)
if (status != TS_CACHE_LOOKUP_HIT_FRESH) {
// This transaction is going to be a cache miss, so intercept it.
VDEBUG("intercepting origin server request for txn=%p", arg.txn);
- TSHttpTxnServerIntercept(TSContCreate(GeneratorInterceptionHook,
TSMutexCreate()), arg.txn);
+ TSHttpTxnServerIntercept(TSContCreate(GeneratorInterceptHook,
TSMutexCreate()), arg.txn);
}
break;
[trafficserver] branch master updated: Added Apache notifications file
This is an automated email from the ASF dual-hosted git repository. bcall pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/trafficserver.git The following commit(s) were added to refs/heads/master by this push: new 1c7ef0d Added Apache notifications file 1c7ef0d is described below commit 1c7ef0dafb0569fadab986a0c4f4a106616ac764 Author: Bryan Call AuthorDate: Tue Apr 21 17:23:55 2020 -0700 Added Apache notifications file --- .asf.yaml | 4 1 file changed, 4 insertions(+) diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 000..cc85e9e --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,4 @@ +notifications: + commits: commits@trafficserver.apache.org + issues: iss...@trafficserver.apache.org + pullrequests: git...@trafficserver.apache.org
[trafficserver] branch master updated: Test PUSHing an object into the cache and the GETting it with a few variations on the client connection protocol.
This is an automated email from the ASF dual-hosted git repository.
rrm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new ce2bccf Test PUSHing an object into the cache and the GETting it with
a few variations on the client connection protocol.
ce2bccf is described below
commit ce2bccfac3bab47573240a85ffb78d59c510c536
Author: Walter Karas
AuthorDate: Fri Apr 17 11:02:49 2020 -0500
Test PUSHing an object into the cache and the GETting it with a few
variations on the client connection protocol.
The new test can also be modified temporarily and used to test client-side
communication when GETting multi-GB objects.
---
tests/gold_tests/bigobj/bigobj.test.py | 131 +
tests/gold_tests/bigobj/check_ramp.c | 57 ++
tests/gold_tests/bigobj/log2.gold | 4 +
tests/gold_tests/bigobj/push_request.c | 79
tests/gold_tests/bigobj/ssl/server.key | 15
tests/gold_tests/bigobj/ssl/server.pem | 32
6 files changed, 318 insertions(+)
diff --git a/tests/gold_tests/bigobj/bigobj.test.py
b/tests/gold_tests/bigobj/bigobj.test.py
new file mode 100644
index 000..21ae164
--- /dev/null
+++ b/tests/gold_tests/bigobj/bigobj.test.py
@@ -0,0 +1,131 @@
+'''
+Test PUSHing an object into the cache and the GETting it with a few variations
on the client connection protocol.
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = '''
+Test PUSHing an object into the cache and the GETting it with a few variations
on the client connection protocol.
+'''
+
+# NOTE: You can also use this to test client-side communication when GETting
very large (multi-GB) objects
+# by increasing the value of the obj_kilobytes variable below. (But do not
increase it on any shared branch
+# that we do CI runs on.)
+
+Test.SkipUnless(
+Condition.HasCurlFeature('http2')
+)
+
+ts = Test.MakeATSProcess("ts", enable_tls=True)
+
+ts.addSSLfile("ssl/server.pem")
+ts.addSSLfile("ssl/server.key")
+
+ts.Disk.records_config.update({
+# Do not accept connections from clients until cache subsystem is
operational.
+'proxy.config.http.wait_for_cache': 1,
+
+'proxy.config.diags.debug.enabled': 1,
+'proxy.config.diags.debug.tags': 'http|dns|cache',
+'proxy.config.http.cache.http': 1, # enable caching.
+'proxy.config.http.cache.required_headers': 0, # No required headers for
caching
+'proxy.config.http.push_method_enabled': 1,
+'proxy.config.proxy_name': 'Poxy_Proxy', # This will be the server name.
+'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir),
+'proxy.config.ssl.server.private_key.path':
'{0}'.format(ts.Variables.SSLDir),
+'proxy.config.url_remap.remap_required': 0
+})
+
+ts.Disk.ssl_multicert_config.AddLine(
+'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key'
+)
+
+ts.Disk.remap_config.AddLine(
+'map https://localhost http://localhost'
+)
+
+# Set up to check the output after the tests have run.
+#
+log_id = Test.Disk.File("log2.txt")
+log_id.Content = "log2.gold"
+
+# Size of object to get. (NOTE: If you increase this significantly you may
also have to increase cache
+# capacity in tests/gold_tests/autest-size/min_cfg/storage.config. Also, for
very large objects, if
+# proxy.config.diags.debug.enabled is 1, the PUSH request will timeout and
fail.)
+#
+obj_kilobytes = 10 * 1024
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'cc ' + Test.TestDirectory + '/push_request.c
-o push_request'
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'cc ' + Test.TestDirectory + '/check_ramp.c -o
check_ramp'
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+# Delay on readiness of TS IPv4 ssl port
+tr.Processes.Default.StartBefore(Test.Processes.ts,
ready=When.PortOpen(ts.Variables.ssl_port))
+#
+# Put object with URL http://localhost/bigobj in cache using PUSH request.
+tr.Processes.Default.Command = (
+'./push_request {} | nc localhost {}'.format(obj_kilobytes,
ts.Variables.port)
+)
[trafficserver] branch 9.0.x updated: Updated ChangeLog
This is an automated email from the ASF dual-hosted git repository. zwoop pushed a commit to branch 9.0.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git The following commit(s) were added to refs/heads/9.0.x by this push: new a5362cd Updated ChangeLog a5362cd is described below commit a5362cd8f24a094983e5401d1a3b8f7604cf607f Author: Leif Hedstrom AuthorDate: Tue Apr 21 13:49:14 2020 -0600 Updated ChangeLog --- CHANGELOG-9.0.0 | 9 +++-- 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG-9.0.0 b/CHANGELOG-9.0.0 index 31a964d..08bfb8a 100644 --- a/CHANGELOG-9.0.0 +++ b/CHANGELOG-9.0.0 @@ -639,7 +639,6 @@ Changes with Apache Traffic Server 9.0.0 #5839 - Fix inactivity timeout on QUIC #5841 - Cleanup: unifdef WRITE_AND_TRANSFER #5847 - Cleanup: Remove unused empty files - #5848 - Convert check programs in iocore/eventsystem/ into Catch based unit tests #5852 - Replaced ProxyTxn::outbound vars with accept::options #5853 - correct the size of DNS buffers #5856 - Fixes 'traffic_ctl server restart' to restart @@ -652,12 +651,9 @@ Changes with Apache Traffic Server 9.0.0 #5870 - Fix bad limit in poll loop. jtest -c1 now works again. #5879 - Weak mutex locking macros #5880 - ProxyTxn Refactor move code to cc - #5883 - Add unit tests for MIOBuffer:write() #5885 - Add the ability to static link ASAN, TSAN and LSAN #5892 - ProxySsn renaming member vars - #5893 - Cleanup: Remove unused AllocType and unused functions #5902 - Ran clang-format - #5903 - Reduce unnecesary IOBufferBlock allocation #5905 - Update Server IP in Transaction when attaching a session from the pool #5906 - Cleanup: unifdef TRACK_BUFFER_USER #5907 - Fix macOS build @@ -716,7 +712,6 @@ Changes with Apache Traffic Server 9.0.0 #6099 - doc + unittest TSRemap(Init|NewInstance) failures #6106 - Removes proxy.config.cache.storage_filename #6116 - Updated to clang-format v9.0.0 - #6118 - Avoid IOBufferReader::read_avail() call from MIOBuffer::high_water() #6120 - Fix null pointer dereference reported by clang-analyzer #6125 - Add Metrics to track SSL Handshake attempts #6126 - Fix the thread ready check @@ -868,6 +863,7 @@ Changes with Apache Traffic Server 9.0.0 #6576 - Ensure TSContSchedule API family are called from an EThread. #6577 - When using TSContSchedule() and TSContScheduleAPI() set the calling thread as the thread affinity when not already set #6578 - Fix migrate use after free + #6586 - traffic_dump: don't dump cookies from the wire #6590 - Skip compressible content type check with null strings. #6591 - Cleanup: fix a inline function style #6606 - Making client session id unique across HTTP/1 and 2 sessions @@ -878,10 +874,11 @@ Changes with Apache Traffic Server 9.0.0 #6618 - Fix missing virtual destructor for PluginUserArgsMixin. #6628 - Use default rwlock attributes on initialize #6632 - Fixes a bug where the nexthop markNextHop method to mark a host down is not called. - #6643 - Bug fixes to h2 buffering + #6645 - Traffic Dump: Adding an SNI filtering option. #6655 - Fix origin scheme selection with partial-blind addition #6656 - Check sni against SSL object #6658 - Update TSStatFindName to check that sync callback is set on the stat + #6663 - Fixes memory leak during log configuration #6664 - cache_range_requests: remove unnecessary Last-Modified header from tests #6677 - Format to match perferred if/else formatting for sh scripts #6678 - Removes commented out code from example
[trafficserver] 02/03: Traffic Dump: Adding an SNI filtering option. (#6645)
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 149bc661c53daa8b5069e462d7f89b495b70126a
Author: Brian Neradt
AuthorDate: Tue Apr 21 08:44:41 2020 -0500
Traffic Dump: Adding an SNI filtering option. (#6645)
Co-authored-by: bneradt
(cherry picked from commit eed1f74d81b583941370eb00784ba819fd78421a)
---
doc/admin-guide/plugins/traffic_dump.en.rst| 4 +
plugins/experimental/traffic_dump/traffic_dump.cc | 55 ++--
.../pluginTest/traffic_dump/gold/200_sni_bob.gold | 11 ++
.../pluginTest/traffic_dump/gold/200_sni_dave.gold | 12 ++
.../pluginTest/traffic_dump/ssl/server.key | 28
.../pluginTest/traffic_dump/ssl/server.pem | 21 +++
.../pluginTest/traffic_dump/ssl/signed-foo.key | 28
.../pluginTest/traffic_dump/ssl/signed-foo.pem | 19 +++
.../pluginTest/traffic_dump/ssl/signer.pem | 17 +++
.../traffic_dump/traffic_dump_sni_filter.test.py | 144 +
10 files changed, 326 insertions(+), 13 deletions(-)
diff --git a/doc/admin-guide/plugins/traffic_dump.en.rst
b/doc/admin-guide/plugins/traffic_dump.en.rst
index eece215..76bedf3 100644
--- a/doc/admin-guide/plugins/traffic_dump.en.rst
+++ b/doc/admin-guide/plugins/traffic_dump.en.rst
@@ -50,6 +50,10 @@ Plugin Configuration
(`optional`) - a comma seperatated list of HTTP case-insensitive field
names whose values are considered sensitive information. Traffic Dump will not
dump the incoming field values for any of these fields but will instead dump a
generic value for them of the same length as the original. If this option is
not used, a default list of "Cookie,Set-Cookie" is used. Providing this option
overwrites that default list with whatever values the user provides. Pass a
quoted empty string as the a [...]
+ .. option:: --sni-filter
+
+ (`optional`) - an SNI with which to filter sessions. Only HTTPS sessions
with the provided SNI will be dumped. The sample option will apply a sampling
rate to these filtered sessions. Thus, with a sample value of 2, 1/2 of all
sessions with the specified SNI will be dumped.
+
``traffic_ctl``
* ``traffic_ctl plugin msg traffic_dump.sample N`` - changes the sampling
ratio N as mentioned above.
* ``traffic_ctl plugin msg traffic_dump.reset`` - resets the disk usage
counter.
diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc
b/plugins/experimental/traffic_dump/traffic_dump.cc
index cfe5b36..34ca15d 100644
--- a/plugins/experimental/traffic_dump/traffic_dump.cc
+++ b/plugins/experimental/traffic_dump/traffic_dump.cc
@@ -31,6 +31,7 @@
#include
#include
#include
+#include
#include
#include
@@ -49,6 +50,8 @@ namespace
{
const char *PLUGIN_NAME = "traffic_dump";
const std::string closing = "]}]}";
+uint64_t session_counter = 0;
+
std::string defaut_sensitive_field_value;
// A case-insensitive comparitor used for comparing HTTP field names.
@@ -87,6 +90,7 @@ std::unordered_set default_s
std::unordered_set
sensitive_fields;
ts::file::path log_path{"dump"}; // default log directory
+std::string sni_filter;// The SNI requested for
filtering against.
int s_arg_idx = 0; // Session Arg Index to pass on
session data
std::atomic sample_pool_size(1000); // Sampling ratio
std::atomic max_disk_usage(1000); //< Max disk space for logs
(approximate)
@@ -597,9 +601,29 @@ global_ssn_handler(TSCont contp, TSEvent event, void
*edata)
return TS_SUCCESS;
}
case TS_EVENT_HTTP_SSN_START: {
-// Grab session id to do sampling
+// Grab session id for logging against a global value rather than the local
+// session_counter.
int64_t id = TSHttpSsnIdGet(ssnp);
-if (id % sample_pool_size != 0) {
+
+// If the user has asked for SNI filtering, filter on that first because
+// any sampling will apply just to that subset of connections that match
+// that SNI.
+if (!sni_filter.empty()) {
+ TSVConn ssn_vc = TSHttpSsnClientVConnGet(ssnp);
+ TSSslConnection ssl_conn = TSVConnSslConnectionGet(ssn_vc);
+ SSL *ssl_obj = (SSL *)ssl_conn;
+ if (ssl_obj == nullptr) {
+TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore non-HTTPS session
%" PRId64 "...", id);
+break;
+ }
+ const std::string sni = SSL_get_servername(ssl_obj,
TLSEXT_NAMETYPE_host_name);
+ if (sni != sni_filter) {
+TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore HTTPS session with
non-filtered SNI: %s", sni.c_str());
+break;
+ }
+}
+const auto this_session_count = session_counter++;
+if (this_session_count % sample_pool_size != 0) {
TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore session %" PRId64
"...", id);
break;
} else if (disk_usage >=
[trafficserver] 03/03: Fixes memory leak during log configuration
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 18b3197c2a8ff66aa2030113009720d92cda848f
Author: Randall Meyer
AuthorDate: Wed Apr 15 09:11:44 2020 -0700
Fixes memory leak during log configuration
This was introduced in fa6e573176
(cherry picked from commit 7f0c8fff8b530811fb6636d2c1b194d93fa4e9e2)
---
proxy/logging/LogConfig.cc | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc
index 41371f5..9a8c602 100644
--- a/proxy/logging/LogConfig.cc
+++ b/proxy/logging/LogConfig.cc
@@ -208,12 +208,14 @@ LogConfig::read_configuration_variables()
register_rolled_log_auto_delete(MANAGER_LOG_FILENAME, val);
// For traffic.out
-const char
*configured_name(REC_ConfigReadString("proxy.config.output.logfile"));
+char *configured_name(REC_ConfigReadString("proxy.config.output.logfile"));
const char *traffic_logname = configured_name ? configured_name :
"traffic.out";
val =
static_cast(REC_ConfigReadInteger("proxy.config.output.logfile.rolling_min_count"));
register_rolled_log_auto_delete(traffic_logname, val);
rolling_max_count =
static_cast(REC_ConfigReadInteger("proxy.config.log.rolling_max_count"));
+
+ats_free(configured_name);
}
// PERFORMANCE
val =
static_cast(REC_ConfigReadInteger("proxy.config.log.sampling_frequency"));
[trafficserver] branch 9.0.x updated (5f8fdec -> 18b3197)
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a change to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git.
from 5f8fdec Fix link issue of test_libhttp2 on FreeBSD 12 with
--enable-debug
new 0ab8b34 traffic_dump: don't dump cookies from the wire (#6586)
new 149bc66 Traffic Dump: Adding an SNI filtering option. (#6645)
new 18b3197 Fixes memory leak during log configuration
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
doc/admin-guide/plugins/traffic_dump.en.rst| 10 +-
plugins/experimental/traffic_dump/traffic_dump.cc | 193 +++--
proxy/logging/LogConfig.cc | 4 +-
.../pluginTest/traffic_dump/gold/200.gold | 1 +
.../pluginTest/traffic_dump/gold/200_sni_bob.gold | 11 ++
.../pluginTest/traffic_dump/gold/200_sni_dave.gold | 12 ++
.../traffic_dump}/ssl/server.key | 0
.../traffic_dump}/ssl/server.pem | 0
.../traffic_dump}/ssl/signed-foo.key | 0
.../traffic_dump}/ssl/signed-foo.pem | 0
.../traffic_dump}/ssl/signer.pem | 15 --
.../pluginTest/traffic_dump/traffic_dump.test.py | 46 +++--
.../traffic_dump/traffic_dump_sni_filter.test.py | 144 +++
.../pluginTest/traffic_dump/verify_replay.py | 55 ++
14 files changed, 441 insertions(+), 50 deletions(-)
create mode 100644
tests/gold_tests/pluginTest/traffic_dump/gold/200_sni_bob.gold
create mode 100644
tests/gold_tests/pluginTest/traffic_dump/gold/200_sni_dave.gold
copy tests/gold_tests/{tls => pluginTest/traffic_dump}/ssl/server.key (100%)
copy tests/gold_tests/{tls => pluginTest/traffic_dump}/ssl/server.pem (100%)
copy tests/gold_tests/{tls => pluginTest/traffic_dump}/ssl/signed-foo.key
(100%)
copy tests/gold_tests/{tls => pluginTest/traffic_dump}/ssl/signed-foo.pem
(100%)
copy tests/gold_tests/{tls => pluginTest/traffic_dump}/ssl/signer.pem (52%)
create mode 100644
tests/gold_tests/pluginTest/traffic_dump/traffic_dump_sni_filter.test.py
[trafficserver] 01/03: traffic_dump: don't dump cookies from the wire (#6586)
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 0ab8b341fecdd68dbe1de3b48428d25aac0d180c
Author: Brian Neradt
AuthorDate: Fri Apr 10 15:00:04 2020 -0400
traffic_dump: don't dump cookies from the wire (#6586)
Cookies are often very sensitive information and should not be logged.
This change modifies traffic_dump to not dump the actual cookies from
the wire.
--promiscuous-mode is added in case the user wants to dump sensitive
headers.
--sensitive-fields is added so the user can treat custom fields as
sensitive.
Co-authored-by: bneradt
(cherry picked from commit 4d7aeb09f83e447b2d3112d23bb0a98ba32ca51d)
---
doc/admin-guide/plugins/traffic_dump.en.rst| 6 +-
plugins/experimental/traffic_dump/traffic_dump.cc | 140 -
.../pluginTest/traffic_dump/gold/200.gold | 1 +
.../pluginTest/traffic_dump/traffic_dump.test.py | 46 ---
.../pluginTest/traffic_dump/verify_replay.py | 55
5 files changed, 226 insertions(+), 22 deletions(-)
diff --git a/doc/admin-guide/plugins/traffic_dump.en.rst
b/doc/admin-guide/plugins/traffic_dump.en.rst
index 45c7d51..eece215 100644
--- a/doc/admin-guide/plugins/traffic_dump.en.rst
+++ b/doc/admin-guide/plugins/traffic_dump.en.rst
@@ -46,6 +46,10 @@ Plugin Configuration
(`required`) - specifies the max disk usage N bytes (approximate). Traffic
Dump will stop capturing new sessions once disk usage exceeds this limit.
+ .. option:: --sensitive-fields
+
+ (`optional`) - a comma seperatated list of HTTP case-insensitive field
names whose values are considered sensitive information. Traffic Dump will not
dump the incoming field values for any of these fields but will instead dump a
generic value for them of the same length as the original. If this option is
not used, a default list of "Cookie,Set-Cookie" is used. Providing this option
overwrites that default list with whatever values the user provides. Pass a
quoted empty string as the a [...]
+
``traffic_ctl``
* ``traffic_ctl plugin msg traffic_dump.sample N`` - changes the sampling
ratio N as mentioned above.
* ``traffic_ctl plugin msg traffic_dump.reset`` - resets the disk usage
counter.
@@ -57,7 +61,7 @@ This format contains traffic data including:
* Each session and transactions in the session.
* Timestamps.
-* The four headers (ua request, proxy request, origin server response, proxy
response).
+* The four sets of headers (user agent request, proxy request, origin server
response, proxy response).
* The protocol stack for the user agent.
* The transaction count for the outbound session.
* The content block sizes.
diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc
b/plugins/experimental/traffic_dump/traffic_dump.cc
index 31563b6..cfe5b36 100644
--- a/plugins/experimental/traffic_dump/traffic_dump.cc
+++ b/plugins/experimental/traffic_dump/traffic_dump.cc
@@ -32,20 +32,59 @@
#include
#include
+#include
#include
#include
#include
#include
#include
#include
+#include
#include "tscore/ts_file.h"
+#include "tscpp/util/TextView.h"
#include "ts/ts.h"
namespace
{
const char *PLUGIN_NAME = "traffic_dump";
const std::string closing = "]}]}";
+std::string defaut_sensitive_field_value;
+
+// A case-insensitive comparitor used for comparing HTTP field names.
+struct InsensitiveCompare {
+ bool
+ operator()(std::string_view a, std::string_view b) const
+ {
+return strcasecmp(a, b) == 0;
+ }
+};
+
+struct StringHashByLower {
+public:
+ size_t
+ operator()(const std::string ) const
+ {
+std::string lower;
+std::transform(str.begin(), str.end(), lower.begin(), [](unsigned char c)
-> unsigned char { return std::tolower(c); });
+return std::hash()(lower);
+ }
+};
+
+/// Fields considered sensitive because they may contain user-private
+/// information. These fields are replaced with auto-generated generic content
+/// by default. To turn off this behavior, the user should add the
+/// --promiscuous-mode flag as a commandline argument.
+///
+/// While these are specified with case, they are matched case-insensitively.
+std::unordered_set
default_sensitive_fields = {
+ "Set-Cookie",
+ "Cookie",
+};
+
+/// The set of fields, default and user-specified, that are sensitive and whose
+/// values will be replaced with auto-generated generic content.
+std::unordered_set
sensitive_fields;
ts::file::path log_path{"dump"}; // default log directory
int s_arg_idx = 0; // Session Arg Index to pass on
session data
@@ -183,10 +222,10 @@ esc_json_out(const char *buf, int64_t len, std::ostream
)
/// escape_json(): escape chars in a string and returns json string
std::string
-escape_json(std::string const )
+escape_json(std::string_view s)
{
[trafficserver] 01/05: Convert check programs in iocore/eventsystem/ into Catch based unit tests
This is an automated email from the ASF dual-hosted git repository. zwoop pushed a commit to branch 8.1.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git commit 18dee3ba07a16ff489f6bd599824274048b14e04 Author: Masaori Koshiba AuthorDate: Tue Aug 20 10:11:04 2019 +0900 Convert check programs in iocore/eventsystem/ into Catch based unit tests (cherry picked from commit 39522703d7b9fdf4834b797e2d812f37f8970f71) Conflicts: iocore/eventsystem/Makefile.am iocore/eventsystem/test_Buffer.cc iocore/eventsystem/test_Event.cc --- .gitignore| 4 +- iocore/eventsystem/Makefile.am| 40 +++-- iocore/eventsystem/test_Buffer.cc | 64 -- iocore/eventsystem/test_Event.cc | 82 -- iocore/eventsystem/unit_tests/test_EventSystem.cc | 100 ++ iocore/eventsystem/unit_tests/test_IOBuffer.cc| 72 6 files changed, 188 insertions(+), 174 deletions(-) diff --git a/.gitignore b/.gitignore index b86d970..4306346 100644 --- a/.gitignore +++ b/.gitignore @@ -91,8 +91,8 @@ lib/perl/lib/Apache/TS.pm iocore/net/test_certlookup iocore/net/test_UDPNet iocore/aio/test_AIO -iocore/eventsystem/test_Buffer -iocore/eventsystem/test_Event +iocore/eventsystem/test_IOBuffer +iocore/eventsystem/test_EventSystem iocore/eventsystem/test_MIOBufferWriter iocore/hostdb/test_RefCountCache diff --git a/iocore/eventsystem/Makefile.am b/iocore/eventsystem/Makefile.am index 4a76772..292da6f 100644 --- a/iocore/eventsystem/Makefile.am +++ b/iocore/eventsystem/Makefile.am @@ -69,7 +69,8 @@ libinkevent_a_SOURCES = \ UnixEvent.cc \ UnixEventProcessor.cc -check_PROGRAMS = test_Buffer test_Event \ +check_PROGRAMS = test_IOBuffer \ + test_EventSystem \ test_MIOBufferWriter test_LD_FLAGS = \ @@ -85,6 +86,7 @@ test_CPP_FLAGS = \ -I$(abs_top_srcdir)/proxy/logging \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ + -I$(abs_top_srcdir)/tests/include \ @OPENSSL_INCLUDES@ test_LD_ADD = \ @@ -92,37 +94,23 @@ test_LD_ADD = \ $(top_builddir)/lib/records/librecords_p.a \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ - $(top_builddir)/src/tscore/libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ @LIBTCL@ @HWLOC_LIBS@ -test_Buffer_SOURCES = \ - test_Buffer.cc - -#test_UNUSED_SOURCES = \ -# test_I_Buffer.cc \ -# test_P_Buffer.cc - -test_Event_SOURCES = \ - test_Event.cc - -#test_UNUSED_SOURCES = \ -# test_I_Event.cc \ -# test_P_Event.cc - -test_Buffer_CPPFLAGS = $(test_CPP_FLAGS) -test_Event_CPPFLAGS = $(test_CPP_FLAGS) - -test_Buffer_LDFLAGS = $(test_LD_FLAGS) -test_Event_LDFLAGS = $(test_LD_FLAGS) - -test_Buffer_LDADD = $(test_LD_ADD) -test_Event_LDADD = $(test_LD_ADD) +test_EventSystem_SOURCES = unit_tests/test_EventSystem.cc +test_EventSystem_CPPFLAGS = $(test_CPP_FLAGS) +test_EventSystem_LDFLAGS = $(test_LD_FLAGS) +test_EventSystem_LDADD = $(test_LD_ADD) +test_IOBuffer_SOURCES = unit_tests/test_IOBuffer.cc +test_IOBuffer_CPPFLAGS = $(test_CPP_FLAGS) +test_IOBuffer_LDFLAGS = $(test_LD_FLAGS) +test_IOBuffer_LDADD = $(test_LD_ADD) test_MIOBufferWriter_SOURCES = unit_tests/test_MIOBufferWriter.cc - -test_MIOBufferWriter_CPPFLAGS = $(test_CPP_FLAGS) -I$(abs_top_srcdir)/tests/include +test_MIOBufferWriter_CPPFLAGS = $(test_CPP_FLAGS) test_MIOBufferWriter_LDFLAGS = $(test_LD_FLAGS) test_MIOBufferWriter_LDADD = $(test_LD_ADD) diff --git a/iocore/eventsystem/test_Buffer.cc b/iocore/eventsystem/test_Buffer.cc deleted file mode 100644 index 3e31a49..000 --- a/iocore/eventsystem/test_Buffer.cc +++ /dev/null @@ -1,64 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "I_EventSystem.h" -#include "tscore/I_Layout.h" -#include "tscore/ink_string.h" -
[trafficserver] 03/05: Reduce unnecesary IOBufferBlock allocation
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 8.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 3e00a0a4dd9a3994bfe4284062e2362af61a4e2c
Author: Masaori Koshiba
AuthorDate: Tue Sep 3 16:23:00 2019 +0900
Reduce unnecesary IOBufferBlock allocation
- Add unit tests for MIOBuffer::write_avail()
- Do nothing when the next block of the current writer exists
(cherry picked from commit c40271ae7543a87bc90b4278954b79d304bb70b0)
Conflicts:
iocore/eventsystem/IOBuffer.cc
---
iocore/eventsystem/IOBuffer.cc | 10 +-
iocore/eventsystem/I_IOBuffer.h| 5 +-
iocore/eventsystem/P_IOBuffer.h| 4 +-
iocore/eventsystem/unit_tests/test_IOBuffer.cc | 162 +
4 files changed, 176 insertions(+), 5 deletions(-)
diff --git a/iocore/eventsystem/IOBuffer.cc b/iocore/eventsystem/IOBuffer.cc
index d5f5207..a45548f 100644
--- a/iocore/eventsystem/IOBuffer.cc
+++ b/iocore/eventsystem/IOBuffer.cc
@@ -62,6 +62,9 @@ init_buffer_allocators(int iobuffer_advice)
}
}
+//
+// MIOBuffer
+//
int64_t
MIOBuffer::remove_append(IOBufferReader *r)
{
@@ -190,6 +193,9 @@ MIOBuffer::puts(char *s, int64_t len)
return 0;
}
+//
+// IOBufferReader
+//
int64_t
IOBufferReader::read(void *ab, int64_t len)
{
@@ -279,7 +285,9 @@ IOBufferReader::memcpy(const void *ap, int64_t len, int64_t
offset)
return p;
}
-//-- MIOBufferWriter
+//
+// MIOBufferWriter
+//
MIOBufferWriter &
MIOBufferWriter::write(const void *data_, size_t length)
{
diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h
index 852ccee..c249e46 100644
--- a/iocore/eventsystem/I_IOBuffer.h
+++ b/iocore/eventsystem/I_IOBuffer.h
@@ -839,9 +839,8 @@ public:
void append_block(int64_t asize_index);
/**
-Adds new block to the end of block list using the block size for
-the buffer specified when the buffer was allocated.
-
+Adds a new block to the end of the block list. Note that this does nothing
when the next block of the current writer exists.
+The block size is the same as specified size when the buffer was allocated.
*/
void add_block();
diff --git a/iocore/eventsystem/P_IOBuffer.h b/iocore/eventsystem/P_IOBuffer.h
index 0820859..05457d2 100644
--- a/iocore/eventsystem/P_IOBuffer.h
+++ b/iocore/eventsystem/P_IOBuffer.h
@@ -987,7 +987,9 @@ MIOBuffer::append_block(int64_t asize_index)
TS_INLINE void
MIOBuffer::add_block()
{
- append_block(size_index);
+ if (this->_writer == nullptr || this->_writer->next == nullptr) {
+append_block(size_index);
+ }
}
TS_INLINE void
diff --git a/iocore/eventsystem/unit_tests/test_IOBuffer.cc
b/iocore/eventsystem/unit_tests/test_IOBuffer.cc
index c83f68a..9868889 100644
--- a/iocore/eventsystem/unit_tests/test_IOBuffer.cc
+++ b/iocore/eventsystem/unit_tests/test_IOBuffer.cc
@@ -173,6 +173,168 @@ TEST_CASE("MIOBuffer", "[iocore]")
free_MIOBuffer(miob);
}
+
+ SECTION("write_avail")
+ {
+MIOBuffer *miob= new_MIOBuffer();
+IOBufferReader *miob_r = miob->alloc_reader();
+uint8_t buf[8192];
+memset(buf, 0xAA, sizeof(buf));
+
+// initial state
+CHECK(miob->block_size() == 4096);
+CHECK(miob->current_write_avail() == 4096);
+CHECK(miob->write_avail() == 4096);
+
+SECTION("water_mark == 0 (default)")
+{
+ REQUIRE(miob->water_mark == 0);
+
+ // fill half of the current buffer
+ miob->write(buf, 2048);
+ CHECK(miob->max_read_avail() == 2048);
+ CHECK(miob->current_write_avail() == 2048);
+ CHECK(miob->high_water() == true);
+ CHECK(miob->current_low_water() == false);
+ CHECK(miob->write_avail() == 2048); ///< should have no side effect
+
+ // fill all of the current buffer
+ miob->write(buf, 2048);
+ CHECK(miob->max_read_avail() == 4096);
+ CHECK(miob->current_write_avail() == 0);
+ CHECK(miob->high_water() == true);
+ CHECK(miob->current_low_water() == true);
+ CHECK(miob->write_avail() == 0); ///< should have no side effect
+
+ // consume half of the data
+ miob_r->consume(2048);
+ CHECK(miob->max_read_avail() == 2048);
+ CHECK(miob->current_write_avail() == 0);
+ CHECK(miob->high_water() == true);
+ CHECK(miob->current_low_water() == true);
+ CHECK(miob->write_avail() == 0); ///< should have no side effect
+
+ // consume all of the data
+ miob_r->consume(2048);
+ CHECK(miob->max_read_avail() == 0);
+ CHECK(miob->current_write_avail() == 0);
+ CHECK(miob->high_water() == false);
+ CHECK(miob->current_low_water() == true);
+ CHECK(miob->write_avail() == 4096); ///< should have a side effect: add
a new block
+
+ CHECK(miob->max_read_avail() == 0);
+ CHECK(miob->current_write_avail() == 4096);
+ CHECK(miob->high_water() == false);
+
[trafficserver] branch 8.1.x updated (f62e880 -> cad531b)
This is an automated email from the ASF dual-hosted git repository. zwoop pushed a change to branch 8.1.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git. from f62e880 cache_range_requests autests remove unnecessary Last-Modified header from tests new 18dee3b Convert check programs in iocore/eventsystem/ into Catch based unit tests new 2ba0224 Add unit tests for MIOBuffer:write() new 3e00a0a Reduce unnecesary IOBufferBlock allocation new b3d4254 Avoid IOBufferReader::read_avail() call from MIOBuffer::high_water() new cad531b Cleanup: Remove unused AllocType and unused functions The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .gitignore| 4 +- iocore/eventsystem/IOBuffer.cc| 30 +- iocore/eventsystem/I_IOBuffer.h | 66 ++-- iocore/eventsystem/Makefile.am| 40 +-- iocore/eventsystem/P_IOBuffer.h | 49 +-- iocore/eventsystem/test_Buffer.cc | 64 iocore/eventsystem/test_Event.cc | 82 - iocore/eventsystem/unit_tests/test_EventSystem.cc | 100 ++ iocore/eventsystem/unit_tests/test_IOBuffer.cc| 361 ++ 9 files changed, 534 insertions(+), 262 deletions(-) delete mode 100644 iocore/eventsystem/test_Buffer.cc delete mode 100644 iocore/eventsystem/test_Event.cc create mode 100644 iocore/eventsystem/unit_tests/test_EventSystem.cc create mode 100644 iocore/eventsystem/unit_tests/test_IOBuffer.cc
[trafficserver] 04/05: Avoid IOBufferReader::read_avail() call from MIOBuffer::high_water()
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 8.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit b3d4254b17a0ef9a3416067afdb96e441d27c6c8
Author: Masaori Koshiba
AuthorDate: Fri Nov 1 11:31:01 2019 +0900
Avoid IOBufferReader::read_avail() call from MIOBuffer::high_water()
(cherry picked from commit 5e639715a52c6e0c09bdf8842989fbd83850c81a)
---
iocore/eventsystem/IOBuffer.cc | 20
iocore/eventsystem/I_IOBuffer.h | 19 ++-
2 files changed, 38 insertions(+), 1 deletion(-)
diff --git a/iocore/eventsystem/IOBuffer.cc b/iocore/eventsystem/IOBuffer.cc
index a45548f..b56b825 100644
--- a/iocore/eventsystem/IOBuffer.cc
+++ b/iocore/eventsystem/IOBuffer.cc
@@ -193,6 +193,26 @@ MIOBuffer::puts(char *s, int64_t len)
return 0;
}
+bool
+MIOBuffer::is_max_read_avail_more_than(int64_t size)
+{
+ bool no_reader = true;
+ for (auto : this->readers) {
+if (reader.allocated()) {
+ if (reader.is_read_avail_more_than(size)) {
+return true;
+ }
+ no_reader = false;
+}
+ }
+
+ if (no_reader && this->_writer) {
+return (this->_writer->read_avail() > size);
+ }
+
+ return false;
+}
+
//
// IOBufferReader
//
diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h
index c249e46..e3712be 100644
--- a/iocore/eventsystem/I_IOBuffer.h
+++ b/iocore/eventsystem/I_IOBuffer.h
@@ -1003,7 +1003,7 @@ public:
bool
high_water()
{
-return max_read_avail() > water_mark;
+return is_max_read_avail_more_than(this->water_mark);
}
/**
@@ -1086,8 +1086,25 @@ public:
{
return !_writer;
}
+
+ /**
+Get the maximum amount of available data across all of the readers.
+If there're no allocated reader, return available data size of current
writer.
+
+This calls IOBufferReader::read_avail() and it could be expensive when it
has a ton of IOBufferBlock.
+The `is_max_read_avail(int64_t size)` is preferred if possible.
+
+@return maximum amount of available data
+ */
int64_t max_read_avail();
+ /**
+Check if there is more than @a size bytes available to read.
+
+@return @c true if more than @a size byte are available.
+ */
+ bool is_max_read_avail_more_than(int64_t size);
+
int max_block_count();
void check_add_block();
[trafficserver] 05/05: Cleanup: Remove unused AllocType and unused functions
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 8.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit cad531b8c56eabcae63824427ad2950a7102a9b6
Author: Masaori Koshiba
AuthorDate: Fri Aug 30 09:52:22 2019 +0900
Cleanup: Remove unused AllocType and unused functions
(cherry picked from commit c65f0a1fbe4838cf56ba422f0a197895bf3c498e)
---
iocore/eventsystem/I_IOBuffer.h | 42 +-
iocore/eventsystem/P_IOBuffer.h | 45 -
2 files changed, 5 insertions(+), 82 deletions(-)
diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h
index e3712be..ebef058 100644
--- a/iocore/eventsystem/I_IOBuffer.h
+++ b/iocore/eventsystem/I_IOBuffer.h
@@ -65,11 +65,8 @@ extern int64_t default_large_iobuffer_size; // matched to
size of OS buffers
enum AllocType {
NO_ALLOC,
- FAST_ALLOCATED,
- XMALLOCED,
MEMALIGNED,
DEFAULT_ALLOC,
- CONSTANT,
};
#define DEFAULT_BUFFER_NUMBER 128
@@ -147,14 +144,6 @@ void init_buffer_allocators(int iobuffer_advice);
- FAST_ALLOCATED
-
-
-
- XMALLOCED
-
-
-
MEMALIGNED
@@ -162,10 +151,6 @@ void init_buffer_allocators(int iobuffer_advice);
DEFAULT_ALLOC
-
- CONSTANT
-
-
*/
@@ -477,8 +462,6 @@ public:
void realloc_set_internal(void *b, int64_t buf_size, int64_t asize_index);
void realloc(void *b, int64_t buf_size);
void realloc(int64_t i);
- void realloc_xmalloc(void *b, int64_t buf_size);
- void realloc_xmalloc(int64_t buf_size);
/**
Frees the IOBufferBlock object and its underlying memory.
@@ -845,6 +828,10 @@ public:
void add_block();
/**
+Deprecated
+
+TODO: remove this function. Because ats_xmalloc() doesn't exist anymore.
+
Adds by reference len bytes of data pointed to by b to the end
of the buffer. b MUST be a pointer to the beginning of block
allocated from the ats_xmalloc() routine. The data will be deallocated
@@ -1073,9 +1060,7 @@ public:
void dealloc_all_readers();
void set(void *b, int64_t len);
- void set_xmalloced(void *b, int64_t len);
void alloc(int64_t i = default_large_iobuffer_size);
- void alloc_xmalloc(int64_t buf_size);
void append_block_internal(IOBufferBlock *b);
int64_t puts(char *buf, int64_t len);
@@ -1158,16 +1143,6 @@ public:
{
_writer->realloc(b, buf_size);
}
- void
- realloc_xmalloc(void *b, int64_t buf_size)
- {
-_writer->realloc_xmalloc(b, buf_size);
- }
- void
- realloc_xmalloc(int64_t buf_size)
- {
-_writer->realloc_xmalloc(buf_size);
- }
int64_t size_index;
@@ -1364,12 +1339,6 @@ extern IOBufferData *new_xmalloc_IOBufferData_internal(
#endif
void *b, int64_t size);
-extern IOBufferData *new_constant_IOBufferData_internal(
-#ifdef TRACK_BUFFER_USER
- const char *locaction,
-#endif
- void *b, int64_t size);
-
#ifdef TRACK_BUFFER_USER
class IOBufferData_tracker
{
@@ -1385,14 +1354,13 @@ public:
};
#endif
+// TODO: remove new_xmalloc_IOBufferData. Because ats_xmalloc() doesn't exist
anymore.
#ifdef TRACK_BUFFER_USER
#define new_IOBufferData IOBufferData_tracker(RES_PATH("memory/IOBuffer/"))
#define new_xmalloc_IOBufferData(b, size)
new_xmalloc_IOBufferData_internal(RES_PATH("memory/IOBuffer/"), (b), (size))
-#define new_constant_IOBufferData(b, size)
new_constant_IOBufferData_internal(RES_PATH("memory/IOBuffer/"), (b), (size))
#else
#define new_IOBufferData new_IOBufferData_internal
#define new_xmalloc_IOBufferData new_xmalloc_IOBufferData_internal
-#define new_constant_IOBufferData new_constant_IOBufferData_internal
#endif
extern int64_t iobuffer_size_to_index(int64_t size, int64_t max =
max_iobuffer_size);
diff --git a/iocore/eventsystem/P_IOBuffer.h b/iocore/eventsystem/P_IOBuffer.h
index 05457d2..c3a62bf 100644
--- a/iocore/eventsystem/P_IOBuffer.h
+++ b/iocore/eventsystem/P_IOBuffer.h
@@ -217,20 +217,6 @@ new_IOBufferData_internal(
}
TS_INLINE IOBufferData *
-new_constant_IOBufferData_internal(
-#ifdef TRACK_BUFFER_USER
- const char *loc,
-#endif
- void *b, int64_t size)
-{
- return new_IOBufferData_internal(
-#ifdef TRACK_BUFFER_USER
-loc,
-#endif
-b, size, BUFFER_SIZE_INDEX_FOR_CONSTANT_SIZE(size));
-}
-
-TS_INLINE IOBufferData *
new_xmalloc_IOBufferData_internal(
#ifdef TRACK_BUFFER_USER
const char *location,
@@ -526,18 +512,6 @@ IOBufferBlock::realloc(void *b, int64_t buf_size)
}
TS_INLINE void
-IOBufferBlock::realloc_xmalloc(void *b, int64_t buf_size)
-{
- realloc_set_internal(b, buf_size, -buf_size);
-}
-
-TS_INLINE void
-IOBufferBlock::realloc_xmalloc(int64_t buf_size)
-{
- realloc_set_internal(ats_malloc(buf_size), buf_size, -buf_size);
-}
-
-TS_INLINE void
IOBufferBlock::realloc(int64_t i)
{
if ((i == data->_size_index) || (i >=
[trafficserver] 02/05: Add unit tests for MIOBuffer:write()
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 8.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 2ba0224481e0dc17953c38ddecc82c4d890275bb
Author: Masaori Koshiba
AuthorDate: Fri Aug 23 15:06:03 2019 +0900
Add unit tests for MIOBuffer:write()
(cherry picked from commit 6751fd0d41f083d6481a1387b89bb55f0d4c7261)
Conflicts:
iocore/eventsystem/unit_tests/test_IOBuffer.cc
---
iocore/eventsystem/unit_tests/test_IOBuffer.cc | 159 ++---
1 file changed, 143 insertions(+), 16 deletions(-)
diff --git a/iocore/eventsystem/unit_tests/test_IOBuffer.cc
b/iocore/eventsystem/unit_tests/test_IOBuffer.cc
index 8368b0f..c83f68a 100644
--- a/iocore/eventsystem/unit_tests/test_IOBuffer.cc
+++ b/iocore/eventsystem/unit_tests/test_IOBuffer.cc
@@ -24,30 +24,154 @@
#define CATCH_CONFIG_MAIN
#include "catch.hpp"
-#include "I_EventSystem.h"
#include "tscore/I_Layout.h"
+#include "I_EventSystem.h"
+#include "RecordsConfig.h"
+
#include "diags.i"
#define TEST_THREADS 1
TEST_CASE("MIOBuffer", "[iocore]")
{
- for (unsigned i = 0; i < 100; ++i) {
-MIOBuffer *b1= new_MIOBuffer(default_small_iobuffer_size);
-int64_t len1 = b1->write_avail();
-IOBufferReader *b1reader = b1->alloc_reader();
-b1->fill(len1);
-CHECK(b1reader->read_avail() == len1);
-
-MIOBuffer *b2= new_MIOBuffer(default_large_iobuffer_size);
-int64_t len2 = b1->write_avail();
-IOBufferReader *b2reader = b2->alloc_reader();
-b2->fill(len2);
-CHECK(b2reader->read_avail() == len2);
-
-free_MIOBuffer(b2);
-free_MIOBuffer(b1);
+ // These value could be tweaked by `ink_event_system_init()` using
`proxy.config.io.max_buffer_size`
+ REQUIRE(default_small_iobuffer_size == DEFAULT_SMALL_BUFFER_SIZE);
+ REQUIRE(default_large_iobuffer_size == DEFAULT_LARGE_BUFFER_SIZE);
+
+ REQUIRE(BUFFER_SIZE_FOR_INDEX(default_small_iobuffer_size) == 512);
+ REQUIRE(BUFFER_SIZE_FOR_INDEX(default_large_iobuffer_size) == 4096);
+
+ SECTION("new_MIOBuffer 100 times")
+ {
+int64_t read_avail_len1 = 0;
+int64_t read_avail_len2 = 0;
+
+for (unsigned i = 0; i < 100; ++i) {
+ MIOBuffer *b1= new_MIOBuffer(default_small_iobuffer_size);
+ int64_t len1 = b1->write_avail();
+ IOBufferReader *b1reader = b1->alloc_reader();
+ b1->fill(len1);
+ read_avail_len1 += b1reader->read_avail();
+
+ MIOBuffer *b2= new_MIOBuffer(default_large_iobuffer_size);
+ int64_t len2 = b2->write_avail();
+ IOBufferReader *b2reader = b2->alloc_reader();
+ b2->fill(len2);
+ read_avail_len2 += b2reader->read_avail();
+
+ free_MIOBuffer(b2);
+ free_MIOBuffer(b1);
+}
+
+CHECK(read_avail_len1 == 100 *
BUFFER_SIZE_FOR_INDEX(default_small_iobuffer_size));
+CHECK(read_avail_len2 == 100 *
BUFFER_SIZE_FOR_INDEX(default_large_iobuffer_size));
+ }
+
+ SECTION("write")
+ {
+MIOBuffer *miob= new_MIOBuffer();
+IOBufferReader *miob_r = miob->alloc_reader();
+const IOBufferBlock *block = miob->first_write_block();
+
+SECTION("initial state")
+{
+ CHECK(miob->size_index == default_large_iobuffer_size);
+ CHECK(miob->water_mark == 0);
+ CHECK(miob->first_write_block() != nullptr);
+ CHECK(miob->block_size() == 4096);
+ CHECK(miob->block_write_avail() == 4096);
+ CHECK(miob->current_write_avail() == 4096);
+ CHECK(miob->write_avail() == 4096);
+
+ CHECK(miob->max_read_avail() == 0);
+ CHECK(miob_r->read_avail() == 0);
+}
+
+SECTION("write(const void *rbuf, int64_t nbytes)")
+{
+ SECTION("1K")
+ {
+uint8_t buf[1024];
+memset(buf, 0xAA, sizeof(buf));
+
+int64_t written = miob->write(buf, sizeof(buf));
+
+REQUIRE(written == sizeof(buf));
+
+CHECK(miob->block_size() == 4096);
+CHECK(miob->block_write_avail() == 3072);
+CHECK(miob->current_write_avail() == 3072);
+CHECK(miob->write_avail() == 3072);
+
+CHECK(miob->first_write_block() == block);
+
+CHECK(miob->max_read_avail() == sizeof(buf));
+CHECK(miob_r->read_avail() == sizeof(buf));
+ }
+
+ SECTION("4K")
+ {
+uint8_t buf[4096];
+memset(buf, 0xAA, sizeof(buf));
+
+int64_t written = miob->write(buf, sizeof(buf));
+
+REQUIRE(written == sizeof(buf));
+
+CHECK(miob->block_size() == 4096);
+CHECK(miob->block_write_avail() == 0);
+CHECK(miob->current_write_avail() == 0);
+CHECK(miob->write_avail() == 0);
+
+CHECK(miob->first_write_block() == block);
+
+CHECK(miob->max_read_avail() == sizeof(buf));
+CHECK(miob_r->read_avail() == sizeof(buf));
+ }
+
+ SECTION("5K")
+ {
+uint8_t buf[5120];
+memset(buf, 0xAA,
[trafficserver] branch 8.1.x updated: cache_range_requests autests remove unnecessary Last-Modified header from tests
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 8.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/8.1.x by this push:
new f62e880 cache_range_requests autests remove unnecessary Last-Modified
header from tests
f62e880 is described below
commit f62e880c8a6c821a0bb4e34b8a940f9c48b6979d
Author: Brian Olsen
AuthorDate: Wed Apr 15 18:11:21 2020 +
cache_range_requests autests remove unnecessary Last-Modified header from
tests
(cherry picked from commit d3c3cc5f5a16f5cb77929fb809768343d1539042)
---
.../cache_range_requests.test.py | 27 +-
.../cache_range_requests/gold/frange.stdout.gold | 3 +--
.../cache_range_requests/gold/full.stdout.gold | 3 +--
.../cache_range_requests/gold/inner.stdout.gold| 3 +--
.../cache_range_requests/gold/last.stdout.gold | 3 +--
.../cache_range_requests/gold/pselect.stdout.gold | 3 +--
.../gold/pselect_none.stdout.gold | 1 -
7 files changed, 16 insertions(+), 27 deletions(-)
diff --git
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py
index 77fd29e..da4df7c 100644
---
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py
+++
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py
@@ -76,8 +76,7 @@ res_full = {"headers":
"HTTP/1.1 200 OK\r\n" +
"Cache-Control: max-age=500\r\n" +
"Connection: close\r\n" +
- 'Etag: "772102f4-56f4bc1e6d417"\r\n' +
- "Last-Modified: Sat, 23 Jun 2018 09:27:29 GMT\r\n" +
+ 'Etag: "path"\r\n' +
"\r\n",
"timestamp": "1469733493.993",
"body": body
@@ -107,8 +106,7 @@ res_inner = {"headers":
"Cache-Control: max-age=500\r\n" +
"Content-Range: bytes {0}/{1}\r\n".format(inner_str, bodylen) +
"Connection: close\r\n" +
- 'Etag: "772102f4-56f4bc1e6d417"\r\n' +
- "Last-Modified: Sat, 23 Jun 2018 09:27:29 GMT\r\n" +
+ 'Etag: "path"\r\n' +
"\r\n",
"timestamp": "1469733493.993",
"body": body[7:15]
@@ -135,8 +133,7 @@ res_frange = {"headers":
"Cache-Control: max-age=500\r\n" +
"Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) +
"Connection: close\r\n" +
- 'Etag: "772102f4-56f4bc1e6d417"\r\n' +
- "Last-Modified: Sat, 23 Jun 2018 09:27:29 GMT\r\n" +
+ 'Etag: "path"\r\n' +
"\r\n",
"timestamp": "1469733493.993",
"body": body
@@ -163,8 +160,7 @@ res_last = {"headers":
"Cache-Control: max-age=200\r\n" +
"Content-Range: bytes {0}-{1}/{1}\r\n".format(bodylen - 5, bodylen) +
"Connection: close\r\n" +
- 'Etag: "772102f4-56f4bc1e6d417"\r\n' +
- "Last-Modified: Sat, 23 Jun 2018 09:27:29 GMT\r\n" +
+ 'Etag: "path"\r\n' +
"\r\n",
"timestamp": "1469733493.993",
"body": body[-5:]
@@ -191,8 +187,7 @@ res_pselect = {"headers":
"Cache-Control: max-age=200\r\n" +
"Content-Range: bytes {}/19\r\n".format(pselect_str) +
"Connection: close\r\n" +
- 'Etag: "772102f4-56f4bc1e6d417"\r\n' +
- "Last-Modified: Sat, 23 Jun 2018 09:27:29 GMT\r\n" +
+ 'Etag: "path"\r\n' +
"\r\n",
"timestamp": "1469733493.993",
"body": body[1:10]
@@ -238,7 +233,7 @@ tr = Test.AddTestRun("inner range cache miss")
ps = tr.Processes.Default
ps.Command = curl_and_args + ' http://www.example.com/path -r {} -H "uuid:
inner"'.format(inner_str)
ps.ReturnCode = 0
-ps.Streams.stdout = "gold/inner.stdout.gold"
+#ps.Streams.stdout = "gold/inner.stdout.gold"
ps.Streams.stderr = "gold/inner.stderr.gold"
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss")
tr.StillRunningAfter = ts
@@ -248,7 +243,7 @@ tr = Test.AddTestRun("inner range cache hit")
ps = tr.Processes.Default
ps.Command = curl_and_args + ' http://www.example.com/path -r
{}'.format(inner_str)
ps.ReturnCode = 0
-ps.Streams.stdout = "gold/inner.stdout.gold"
+#ps.Streams.stdout = "gold/inner.stdout.gold"
ps.Streams.stderr = "gold/inner.stderr.gold"
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit",
"expected cache hit")
tr.StillRunningAfter = ts
@@ -260,7 +255,7 @@ tr = Test.AddTestRun("0- request miss")
ps = tr.Processes.Default
ps.Command = curl_and_args + ' http://www.example.com/path -r {} -H "uuid:
frange"'.format(frange_str)
ps.ReturnCode = 0
-ps.Streams.stdout = "gold/frange.stdout.gold"
+#ps.Streams.stdout = "gold/frange.stdout.gold"
ps.Streams.stderr = "gold/frange.stderr.gold"
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss")
tr.StillRunningAfter = ts
@@ -270,7 +265,7 @@ tr = Test.AddTestRun("0- request hit")
ps = tr.Processes.Default
ps.Command = curl_and_args + ' http://www.example.com/path -r
{}'.format(frange_str)
ps.ReturnCode = 0
-ps.Streams.stdout = "gold/frange.stdout.gold"
+#ps.Streams.stdout = "gold/frange.stdout.gold"
[trafficserver] branch 9.0.x updated: Updated ChangeLog
This is an automated email from the ASF dual-hosted git repository. zwoop pushed a commit to branch 9.0.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git The following commit(s) were added to refs/heads/9.0.x by this push: new e64c472 Updated ChangeLog e64c472 is described below commit e64c472851e1917284cd34682d6f83d9a95f2f15 Author: Leif Hedstrom AuthorDate: Tue Apr 21 09:13:25 2020 -0600 Updated ChangeLog --- CHANGELOG-9.0.0 | 13 +++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-9.0.0 b/CHANGELOG-9.0.0 index 3d601bb..31a964d 100644 --- a/CHANGELOG-9.0.0 +++ b/CHANGELOG-9.0.0 @@ -286,7 +286,6 @@ Changes with Apache Traffic Server 9.0.0 #4965 - HdrHeap: Remove pointless code and misleading comment. #4966 - Some tidying up of the global namespace. #4972 - Fix tls_client_verify curl command missing certificate - #4973 - Adjust tls_client_versions to be more resilient for different curl versions #4974 - Fix tls_check_cert_selection test #4975 - URL: remove undefined function declaration. #4976 - Fix tls_forward_nonhttp test requirement @@ -623,7 +622,6 @@ Changes with Apache Traffic Server 9.0.0 #5772 - Remove ssl_error stats that aren't really errors. #5774 - Updating the default cipher-suite lists for the 9.x release. #5778 - Issue 5604 - fix memory leaks for http parser. - #5780 - Fixed std::array initialization for older compilers #5786 - Mark debug logs of polling as verbose #5789 - Add protection against null pointer access #5792 - Rename remaining references to ip_allow.config to ip_allow.yaml @@ -734,6 +732,7 @@ Changes with Apache Traffic Server 9.0.0 #6144 - Cleans up some of the filenames mess #6147 - Reverse debug specific check. #6148 - Disable the most expensive "make check" tests by default + #6149 - For per-transaction config override, crossing the const-correctness event horizon. #6159 - Detect bogus jemalloc version #6160 - Fixes misc. spelling and whitespace #6163 - Update yaml-cpp to 0.6.3 @@ -802,10 +801,12 @@ Changes with Apache Traffic Server 9.0.0 #6347 - Fix localstatedir and runtimedir for Debian layout #6358 - Add header guard - issue #6357 #6359 - Cleanup: Remove unused accessors of HttpVCTableEntry + #6362 - Change localtime/gmtime usages to use the threadsafe versions with local storage #6363 - Removing always true/false comparisons #6364 - Fix rare SSN/TXN Start/Close Hook misorderings #6369 - Improve config_describe logs to print a better output format. #6370 - Change default matching for connection value from ip to both + #6371 - Fixing shadowed variables, both global and local #6372 - Change alloca usage to malloc/free #6373 - Change bitfields to be unsigned explicitly #6374 - Add exception to throw since there is no context and could cause a crash @@ -860,6 +861,7 @@ Changes with Apache Traffic Server 9.0.0 #6556 - Removes copypasta curl text from tests and removes checks for curl #6558 - Support body factory template suppression for internal requests #6565 - Fixed build issue with Ubuntu 16 debug + #6566 - Add more options to session_sharing.match #6567 - Moved printing the incoming headers for debug before remapping #6569 - AuTest for server_push_preload plugin #6573 - SSL: Always renew TLS Session Tickets iff TLSv1.3 is being used @@ -876,3 +878,10 @@ Changes with Apache Traffic Server 9.0.0 #6618 - Fix missing virtual destructor for PluginUserArgsMixin. #6628 - Use default rwlock attributes on initialize #6632 - Fixes a bug where the nexthop markNextHop method to mark a host down is not called. + #6643 - Bug fixes to h2 buffering + #6655 - Fix origin scheme selection with partial-blind addition + #6656 - Check sni against SSL object + #6658 - Update TSStatFindName to check that sync callback is set on the stat + #6664 - cache_range_requests: remove unnecessary Last-Modified header from tests + #6677 - Format to match perferred if/else formatting for sh scripts + #6678 - Removes commented out code from example
[trafficserver] branch master updated: Traffic Dump: Adding an SNI filtering option. (#6645)
This is an automated email from the ASF dual-hosted git repository.
shinrich pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new eed1f74 Traffic Dump: Adding an SNI filtering option. (#6645)
eed1f74 is described below
commit eed1f74d81b583941370eb00784ba819fd78421a
Author: Brian Neradt
AuthorDate: Tue Apr 21 08:44:41 2020 -0500
Traffic Dump: Adding an SNI filtering option. (#6645)
Co-authored-by: bneradt
---
doc/admin-guide/plugins/traffic_dump.en.rst| 4 +
plugins/experimental/traffic_dump/traffic_dump.cc | 55 ++--
.../pluginTest/traffic_dump/gold/200_sni_bob.gold | 11 ++
.../pluginTest/traffic_dump/gold/200_sni_dave.gold | 12 ++
.../pluginTest/traffic_dump/ssl/server.key | 28
.../pluginTest/traffic_dump/ssl/server.pem | 21 +++
.../pluginTest/traffic_dump/ssl/signed-foo.key | 28
.../pluginTest/traffic_dump/ssl/signed-foo.pem | 19 +++
.../pluginTest/traffic_dump/ssl/signer.pem | 17 +++
.../traffic_dump/traffic_dump_sni_filter.test.py | 144 +
10 files changed, 326 insertions(+), 13 deletions(-)
diff --git a/doc/admin-guide/plugins/traffic_dump.en.rst
b/doc/admin-guide/plugins/traffic_dump.en.rst
index eece215..76bedf3 100644
--- a/doc/admin-guide/plugins/traffic_dump.en.rst
+++ b/doc/admin-guide/plugins/traffic_dump.en.rst
@@ -50,6 +50,10 @@ Plugin Configuration
(`optional`) - a comma seperatated list of HTTP case-insensitive field
names whose values are considered sensitive information. Traffic Dump will not
dump the incoming field values for any of these fields but will instead dump a
generic value for them of the same length as the original. If this option is
not used, a default list of "Cookie,Set-Cookie" is used. Providing this option
overwrites that default list with whatever values the user provides. Pass a
quoted empty string as the a [...]
+ .. option:: --sni-filter
+
+ (`optional`) - an SNI with which to filter sessions. Only HTTPS sessions
with the provided SNI will be dumped. The sample option will apply a sampling
rate to these filtered sessions. Thus, with a sample value of 2, 1/2 of all
sessions with the specified SNI will be dumped.
+
``traffic_ctl``
* ``traffic_ctl plugin msg traffic_dump.sample N`` - changes the sampling
ratio N as mentioned above.
* ``traffic_ctl plugin msg traffic_dump.reset`` - resets the disk usage
counter.
diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc
b/plugins/experimental/traffic_dump/traffic_dump.cc
index cfe5b36..34ca15d 100644
--- a/plugins/experimental/traffic_dump/traffic_dump.cc
+++ b/plugins/experimental/traffic_dump/traffic_dump.cc
@@ -31,6 +31,7 @@
#include
#include
#include
+#include
#include
#include
@@ -49,6 +50,8 @@ namespace
{
const char *PLUGIN_NAME = "traffic_dump";
const std::string closing = "]}]}";
+uint64_t session_counter = 0;
+
std::string defaut_sensitive_field_value;
// A case-insensitive comparitor used for comparing HTTP field names.
@@ -87,6 +90,7 @@ std::unordered_set default_s
std::unordered_set
sensitive_fields;
ts::file::path log_path{"dump"}; // default log directory
+std::string sni_filter;// The SNI requested for
filtering against.
int s_arg_idx = 0; // Session Arg Index to pass on
session data
std::atomic sample_pool_size(1000); // Sampling ratio
std::atomic max_disk_usage(1000); //< Max disk space for logs
(approximate)
@@ -597,9 +601,29 @@ global_ssn_handler(TSCont contp, TSEvent event, void
*edata)
return TS_SUCCESS;
}
case TS_EVENT_HTTP_SSN_START: {
-// Grab session id to do sampling
+// Grab session id for logging against a global value rather than the local
+// session_counter.
int64_t id = TSHttpSsnIdGet(ssnp);
-if (id % sample_pool_size != 0) {
+
+// If the user has asked for SNI filtering, filter on that first because
+// any sampling will apply just to that subset of connections that match
+// that SNI.
+if (!sni_filter.empty()) {
+ TSVConn ssn_vc = TSHttpSsnClientVConnGet(ssnp);
+ TSSslConnection ssl_conn = TSVConnSslConnectionGet(ssn_vc);
+ SSL *ssl_obj = (SSL *)ssl_conn;
+ if (ssl_obj == nullptr) {
+TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore non-HTTPS session
%" PRId64 "...", id);
+break;
+ }
+ const std::string sni = SSL_get_servername(ssl_obj,
TLSEXT_NAMETYPE_host_name);
+ if (sni != sni_filter) {
+TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore HTTPS session with
non-filtered SNI: %s", sni.c_str());
+break;
+ }
+}
+const auto this_session_count = session_counter++;
+if (this_session_count % sample_pool_size != 0) {
TSDebug(PLUGIN_NAME,
