Compress /var/log/messages during client test sysinfo collection. Compress all log messages saved by the autoserv logfile_monitor system before writing them to disk.
Both of these help alleviate the problem of an insane amount of disk space being used by copies of kernel logs that occur during some stress tests that are intentionally triggering verbose error logs from the kernel. Side effect: there will be multiple logfile_monitor.log.nnnn-nnn.gz files instead of a single logfile_monitor.log file as the logfile monitor is stopped and started a few times during some autoserv runs. Signed-off-by: Gregory Smith <[email protected]> --- autotest/client/bin/base_sysinfo.py 2011-01-06 14:57:07.000000000 -0800 +++ autotest/client/bin/base_sysinfo.py 2011-07-18 16:27:20.000000000 -0700 @@ -1,4 +1,4 @@ -import os, shutil, re, glob, subprocess, logging +import os, shutil, re, glob, subprocess, logging, gzip from autotest_lib.client.common_lib import log from autotest_lib.client.bin import utils, package @@ -324,11 +324,19 @@ if current_inode == self._messages_inode: bytes_to_skip = self._messages_size in_messages = open("/var/log/messages") - in_messages.seek(bytes_to_skip) - out_messages = open(os.path.join(logdir, "messages"), "w") - out_messages.write(in_messages.read()) - in_messages.close() - out_messages.close() + out_file_name = os.path.join(logdir, "messages.gz") + out_messages = gzip.GzipFile(out_file_name, "w") + try: + in_messages.seek(bytes_to_skip) + while True: + # Read data in managable chunks rather than all at once. + in_data = in_messages.read(200000) + if not in_data: + break + out_messages.write(in_data) + finally: + out_messages.close() + in_messages.close() except Exception, e: logging.error("/var/log/messages collection failed with %s", e) --- autotest/server/crashcollect.py 2011-02-03 22:24:17.000000000 -0800 +++ autotest/server/crashcollect.py 2011-07-18 16:27:20.000000000 -0700 @@ -1,4 +1,4 @@ -import os, time, pickle, logging, shutil +import os, time, pickle, logging, shutil, gzip from autotest_lib.server import utils @@ -175,11 +175,13 @@ else: size_at_start = 0 raw_messages_file = open(messages_raw) - messages_file = open(messages, "w") - raw_messages_file.seek(size_at_start) - shutil.copyfileobj(raw_messages_file, messages_file) - raw_messages_file.close() - messages_file.close() + messages_file = gzip.GzipFile(messages+".gz", "w") + try: + raw_messages_file.seek(size_at_start) + shutil.copyfileobj(raw_messages_file, messages_file) + raw_messages_file.close() + finally: + messages_file.close() # get rid of the "raw" versions of messages os.remove(messages_raw) --- autotest/server/hosts/logfile_monitor.py 2010-03-12 14:43:13.000000000 -0800 +++ autotest/server/hosts/logfile_monitor.py 2011-07-18 16:27:20.000000000 -0700 @@ -133,7 +133,6 @@ # Setup warning stream before we actually launch warning_stream = os.fdopen(r, 'r', 0) - devnull_r = open(os.devnull, 'r') devnull_w = open(os.devnull, 'w') # Launch console.py locally console_proc = subprocess.Popen( --- autotest/server/hosts/monitors/console.py 2009-11-19 11:23:31.000000000 -0800 +++ autotest/server/hosts/monitors/console.py 2011-07-18 16:27:20.000000000 -0700 @@ -3,8 +3,9 @@ # Script for translating console output (from STDIN) into Autotest # warning messages. -import optparse, os, sys -import monitors_util +import gzip, optparse, os, signal, sys, time +import common +from autotest_lib.server.hosts.monitors import monitors_util PATTERNS_PATH = os.path.join(os.path.dirname(__file__), 'console_patterns') @@ -20,13 +21,53 @@ help='Path to alert hook patterns file') +def _open_logfile(logfile_base_name): + """Opens an output file using the given name. + + A timestamp and compression is added to the name. + + @param logfile_base_name - The log file path without a compression suffix. + @returns An open file like object. Its close method must be called before + exiting or data may be lost due to internal buffering. + """ + timestamp = int(time.time()) + while True: + logfile_name = '%s.%d-%d.gz' % (logfile_base_name, + timestamp, os.getpid()) + if not os.path.exists(logfile_name): + break + timestamp += 1 + logfile = gzip.GzipFile(logfile_name, 'w') + return logfile + + +def _set_logfile_close_signal_handler(logfile): + """Setup a signal handler to explicitly call logfile.close() and exit. + + Because we are writing a compressed file we need to make sure we properly + close to flush our internal buffer on exit. logfile_monitor.py sends us + a SIGTERM and waits 5 seconds for before sending a SIGKILL so we have + plenty of time to do this. + + @param logfile - An open file object to be closed on SIGTERM. + """ + def _on_signal_close_logfile_before_exit(unused_signal_no, unused_frame): + logfile.close() + os.exit(1) + signal.signal(signal.SIGTERM, _on_signal_close_logfile_before_exit) + + +def _unset_signal_handler(): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + def main(): (options, args) = parser.parse_args() if len(args) != 2: parser.print_help() sys.exit(1) - logfile = open(args[0], 'a', 0) + logfile = _open_logfile(args[0]) warnfile = os.fdopen(int(args[1]), 'w', 0) # For now we aggregate all the alert_hooks. alert_hooks = [] @@ -34,8 +75,13 @@ alert_hooks.extend(monitors_util.build_alert_hooks_from_path( patterns_path, warnfile)) - monitors_util.process_input( - sys.stdin, logfile, options.log_timestamp_format, alert_hooks) + _set_logfile_close_signal_handler(logfile) + try: + monitors_util.process_input( + sys.stdin, logfile, options.log_timestamp_format, alert_hooks) + finally: + logfile.close() + _unset_signal_handler() if __name__ == '__main__': ==== (added) //depot/google_vendor_src_branch/autotest/server/hosts/monitors/console_unittest.py#1 ==== --- /dev/null 2011-05-19 20:30:53.561792601 -0700 +++ autotest/server/hosts/monitors/console_unittest.py 2011-07-18 16:27:20.000000000 -0700 @@ -0,0 +1,56 @@ +#!/usr/bin/python + +"""Tests for console.py""" + +import os, shutil, signal, StringIO, tempfile + +import common +from autotest_lib.client.common_lib.test_utils import mock +from autotest_lib.client.common_lib.test_utils import unittest +from autotest_lib.server.hosts.monitors import console + +class console_test(unittest.TestCase): + def setUp(self): + self.god = mock.mock_god() + self.tempdir = tempfile.mkdtemp() + + + def tearDown(self): + shutil.rmtree(self.tempdir) + + + def test_open_logfile(self): + path = os.path.join(self.tempdir, 'its-log-log') + fun_log = console._open_logfile(path) + fun_log.write("it's big it's heavy it's wood.\n") + fun_log.close() + + # Open it again to ensure the original gets renamed. + fun_log = console._open_logfile(path) + fun_log.write("it's better than bad, it's good!\n") + fun_log.close() + + log_file_list = os.listdir(self.tempdir) + self.assertEqual(2, len(log_file_list)) + for name in log_file_list: + self.assertTrue(name.startswith('its-log-log.'), + 'unexpected name %s' % name) + self.assertTrue(name.endswith('.gz'), 'unexpected name %s' % name) + + + def test_logfile_close_signal_handler(self): + self.god.stub_function(os, 'exit') + os.exit.expect_call(1) + logfile = StringIO.StringIO() + console._set_logfile_close_signal_handler(logfile) + try: + self.assertFalse(logfile.closed) + os.kill(os.getpid(), signal.SIGTERM) + finally: + console._unset_signal_handler() + self.god.check_playback() + self.assertTrue(logfile.closed) + + +if __name__ == '__main__': + unittest.main() _______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
