This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/9.0.x by this push:
new a3d9bea Adding a log pipe buffer size test.
a3d9bea is described below
commit a3d9bead4101f54b750d9f8c8facc76315eaf1fb
Author: bneradt <bner...@verizonmedia.com>
AuthorDate: Fri Apr 3 19:49:38 2020 +0000
Adding a log pipe buffer size test.
(cherry picked from commit 15e749ea4cf3b5e66ffd0ad53d20fa4c611e2ca5)
---
proxy/logging/LogFile.cc | 10 +-
src/traffic_layout/info.cc | 6 +
tests/gold_tests/autest-site/when.test.ext | 36 ++++
tests/gold_tests/logging/log_pipe.test.py | 186 +++++++++++++++++++++
.../logging/pipe_buffer_is_larger_than.py | 69 ++++++++
5 files changed, 302 insertions(+), 5 deletions(-)
diff --git a/proxy/logging/LogFile.cc b/proxy/logging/LogFile.cc
index 59c6bde..26fcca6 100644
--- a/proxy/logging/LogFile.cc
+++ b/proxy/logging/LogFile.cc
@@ -199,21 +199,21 @@ LogFile::open_file()
if (m_pipe_buffer_size) {
long pipe_size = (long)fcntl(m_fd, F_GETPIPE_SZ);
if (pipe_size == -1) {
- Error("get pipe size failed for pipe %s", m_name);
+ Error("Get pipe size failed for pipe %s: %s", m_name, strerror(errno));
} else {
- Debug("log-file", "Default pipe size for pipe %s = %ld", m_name,
pipe_size);
+ Debug("log-file", "Previous buffer size for pipe %s: %ld", m_name,
pipe_size);
}
int ret = fcntl(m_fd, F_SETPIPE_SZ, m_pipe_buffer_size);
if (ret == -1) {
- Error("set pipe size failed for pipe %s", m_name);
+ Error("Set pipe size failed for pipe %s to size %d: %s", m_name,
m_pipe_buffer_size, strerror(errno));
}
pipe_size = (long)fcntl(m_fd, F_GETPIPE_SZ);
if (pipe_size == -1) {
- Error("get pipe size failed for pipe %s", m_name);
+ Error("Get pipe size after setting it failed for pipe %s: %s", m_name,
strerror(errno));
} else {
- Debug("log-file", "NEW pipe size for pipe %s = %ld", m_name,
pipe_size);
+ Debug("log-file", "New buffer size for pipe %s: %ld", m_name,
pipe_size);
}
}
#endif // F_GETPIPE_SZ
diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc
index 062aa7a..4cb6195 100644
--- a/src/traffic_layout/info.cc
+++ b/src/traffic_layout/info.cc
@@ -21,6 +21,7 @@
limitations under the License.
*/
+#include <fcntl.h>
#include <openssl/crypto.h>
#include "tscore/I_Layout.h"
#include "tscore/Filenames.h"
@@ -88,6 +89,11 @@ produce_features(bool json)
#else
print_feature("TS_HAS_BROTLI", 0, json);
#endif
+#ifdef F_GETPIPE_SZ
+ print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 1, json);
+#else
+ print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 0, json);
+#endif /* F_GETPIPE_SZ */
print_feature("TS_HAS_JEMALLOC", TS_HAS_JEMALLOC, json);
print_feature("TS_HAS_TCMALLOC", TS_HAS_TCMALLOC, json);
print_feature("TS_HAS_IN6_IS_ADDR_UNSPECIFIED",
TS_HAS_IN6_IS_ADDR_UNSPECIFIED, json);
diff --git a/tests/gold_tests/autest-site/when.test.ext
b/tests/gold_tests/autest-site/when.test.ext
new file mode 100644
index 0000000..d44c9ba
--- /dev/null
+++ b/tests/gold_tests/autest-site/when.test.ext
@@ -0,0 +1,36 @@
+'''
+When extensions.
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from autest.api import AddWhenFunction
+import hosts.output as host
+
+
+def FileContains(haystack, needle):
+ with open(haystack) as f:
+ result = needle in f.read()
+
+ host.WriteDebug(
+ ['FileExists', 'when'],
+ "Testing for file content '{0}' in '{1}' : {2}".format(
+ needle, haystack, result))
+
+ return result
+
+
+AddWhenFunction(FileContains)
diff --git a/tests/gold_tests/logging/log_pipe.test.py
b/tests/gold_tests/logging/log_pipe.test.py
new file mode 100644
index 0000000..cdadbd0
--- /dev/null
+++ b/tests/gold_tests/logging/log_pipe.test.py
@@ -0,0 +1,186 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+Test.Summary = '''
+Test custom log file format
+'''
+Test.SkipUnless(
+ Condition.HasATSFeature('TS_HAS_PIPE_BUFFER_SIZE_CONFIG')
+)
+
+ts_counter = 1
+
+
+def get_ts(logging_config):
+ """
+ Create a Traffic Server process.
+ """
+ global ts_counter
+ ts = Test.MakeATSProcess("ts{}".format(ts_counter))
+ ts_counter += 1
+
+ ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'log-file',
+ })
+
+ # Since we're only verifying logs and not traffic, we don't need an origin
+ # server. The following will simply deny the requests and emit a log
+ # message.
+ ts.Disk.remap_config.AddLine(
+ 'map / http://www.linkedin.com/ @action=deny'
+ )
+
+ ts.Disk.logging_yaml.AddLines(logging_config)
+
+ return ts
+
+
+#
+# Test 1: Default configured log pipe size.
+#
+tr = Test.AddTestRun()
+pipe_name = "default_pipe_size.pipe"
+ts = get_ts(
+ '''
+logging:
+ formats:
+ - name: custom
+ format: "%<hii> %<hiih>"
+ logs:
+ - filename: '{}'
+ mode: ascii_pipe
+ format: custom
+'''.format(pipe_name).split("\n")
+)
+
+pipe_path = os.path.join(ts.Variables.LOGDIR, pipe_name)
+
+ts.Streams.All += Testers.ContainsExpression(
+ "Created named pipe .*{}".format(pipe_name),
+ "Verify that the named pipe was created")
+
+ts.Streams.All += Testers.ContainsExpression(
+ "no readers for pipe .*{}".format(pipe_name),
+ "Verify that no readers for the pipe was detected.")
+
+ts.Streams.All += Testers.ExcludesExpression(
+ "New buffer size for pipe".format(pipe_name),
+ "Verify that the default pipe size was used.")
+
+curl = tr.Processes.Process("client_request", 'curl "http://127.0.0.1:{0}";
--verbose'.format(
+ ts.Variables.port))
+
+reader_output = os.path.join(ts.Variables.LOGDIR, "reader_output")
+pipe_reader = tr.Processes.Process("pipe_reader", 'cat {} | tee
{}'.format(pipe_path, reader_output))
+curl_ready = tr.Processes.Process("curl_ready", 'sleep 30')
+# In the AuTest environment, it can take more than 10 seconds for the log file
+# to be created.
+curl_ready.StartupTimeout = 30
+curl_ready.Ready = When.FileContains(reader_output, '127.0.0.1')
+
+tr.Processes.Default.Command = "sleep 10"
+tr.Processes.Default.Return = 0
+
+# Process ordering.
+tr.Processes.Default.StartBefore(curl_ready)
+curl_ready.StartBefore(curl)
+curl.StartBefore(pipe_reader)
+pipe_reader.StartBefore(ts)
+
+
+#
+# Test 2: Change the log's buffer size.
+#
+tr = Test.AddTestRun()
+pipe_name = "change_pipe_size.pipe"
+# 64 KB is the default, so set the size larger than that to verify we can
+# increase the size.
+pipe_size = 75000
+ts = get_ts(
+ '''
+logging:
+ formats:
+ - name: custom
+ format: "%<hii> %<hiih>"
+ logs:
+ - filename: '{}'
+ mode: ascii_pipe
+ format: custom
+ pipe_buffer_size: {}
+ '''.format(pipe_name, pipe_size).split("\n")
+)
+
+pipe_path = os.path.join(ts.Variables.LOGDIR, pipe_name)
+
+ts.Streams.All += Testers.ContainsExpression(
+ "Created named pipe .*{}".format(pipe_name),
+ "Verify that the named pipe was created")
+
+ts.Streams.All += Testers.ContainsExpression(
+ "no readers for pipe .*{}".format(pipe_name),
+ "Verify that no readers for the pipe was detected.")
+
+ts.Streams.All += Testers.ContainsExpression(
+ "Previous buffer size for pipe .*{}".format(pipe_name),
+ "Verify that the named pipe's size was adjusted")
+
+# See fcntl:
+# "Attempts to set the pipe capacity below the page size
+# are silently rounded up to the page size."
+#
+# As a result of this, we cannot check that the pipe size is the exact size we
+# requested, but it should be at least that big. We use the
+# pipe_buffer_is_larger_than.py helper script to verify that the pipe grew in
+# size.
+ts.Streams.All += Testers.ContainsExpression(
+ "New buffer size for pipe.*{}".format(pipe_name),
+ "Verify that the named pipe's size was adjusted")
+buffer_verifier = "pipe_buffer_is_larger_than.py"
+tr.Setup.Copy(buffer_verifier)
+verify_buffer_size = tr.Processes.Process(
+ "verify_buffer_size",
+ "python3 {} {} {}".format(buffer_verifier, pipe_path, pipe_size))
+verify_buffer_size.Return = 0
+verify_buffer_size.Streams.All += Testers.ContainsExpression(
+ "Success",
+ "The buffer size verifier should report success.")
+
+curl = tr.Processes.Process("client_request", 'curl "http://127.0.0.1:{0}";
--verbose'.format(
+ ts.Variables.port))
+
+reader_output = os.path.join(ts.Variables.LOGDIR, "reader_output")
+pipe_reader = tr.Processes.Process("pipe_reader", 'cat {} | tee
{}'.format(pipe_path, reader_output))
+curl_ready = tr.Processes.Process("curl_ready", 'sleep 30')
+# In the AuTest environment, it can take more than 10 seconds for the log file
+# to be created.
+curl_ready.StartupTimeout = 30
+curl_ready.Ready = When.FileContains(reader_output, '127.0.0.1')
+
+tr.Processes.Default.Command = "sleep 10"
+tr.Processes.Default.Return = 0
+
+
+# Process ordering.
+tr.Processes.Default.StartBefore(verify_buffer_size)
+verify_buffer_size.StartBefore(curl_ready)
+curl_ready.StartBefore(curl)
+curl.StartBefore(pipe_reader)
+pipe_reader.StartBefore(ts)
diff --git a/tests/gold_tests/logging/pipe_buffer_is_larger_than.py
b/tests/gold_tests/logging/pipe_buffer_is_larger_than.py
new file mode 100644
index 0000000..d598505
--- /dev/null
+++ b/tests/gold_tests/logging/pipe_buffer_is_larger_than.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import fcntl
+import sys
+
+F_SETPIPE_SZ = 1031 # Linux 2.6.35+
+F_GETPIPE_SZ = 1032 # Linux 2.6.35+
+
+
+def parse_args():
+ parser = parser = argparse.ArgumentParser(
+ description='Verify that a FIFO has a buffer of at least a certain
size')
+
+ parser.add_argument(
+ 'pipe_name',
+ help='The pipe name upon which to verify the size is large
enough.')
+
+ parser.add_argument(
+ 'minimum_buffer_size',
+ help='The minimu buffer size for the pipe to expect.')
+
+ return parser.parse_args()
+
+
+def test_fifo(fifo, minimum_buffer_size):
+ try:
+ fifo_fd = open(fifo, "rb+", buffering=0)
+ buffer_size = fcntl.fcntl(fifo_fd, F_GETPIPE_SZ)
+
+ if buffer_size >= int(minimum_buffer_size):
+ print("Success. Size is: {} which is larger than: {}".format(
+ buffer_size,
+ minimum_buffer_size))
+ return 0
+ else:
+ print("Fail. Size is: {} which is smaller than: {}".format(
+ buffer_size,
+ minimum_buffer_size))
+ return 1
+ except Exception as e:
+ print("Unable to open fifo, error: {}".format(str(e)))
+ return 2
+
+
+def main():
+ args = parse_args()
+ return test_fifo(args.pipe_name, args.minimum_buffer_size)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
