IMPALA-4775: minor adjustments to python test infra logging - Set up log handler to append, not truncate. This was the cause of IMPALA-4775.
Other improvements: - Log a thread name, not thread ID. Thread names are more useful. - Use ISO 8601-like timestamps I tested that running disrepancy_searcher.py doesn't overwrite its logs anymore. One such command that could reproduce it is: tests/comparison/discrepancy_searcher.py \ --use-postgresql \ --query-count 1 \ --db-name tpch_kudu I also ensured the stress test (concurrent_select.py) still logged to its file. Change-Id: I2b7af5b2be20f3c6f38d25612f6888433c62d693 Reviewed-on: http://gerrit.cloudera.org:8080/5746 Reviewed-by: Alex Behm <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/fe2be252 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/fe2be252 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/fe2be252 Branch: refs/heads/master Commit: fe2be25245280492c9a6b5b4e20e066b244fb808 Parents: 7438730 Author: Michael Brown <[email protected]> Authored: Tue Jan 17 12:25:03 2017 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Fri Jan 20 02:24:47 2017 +0000 ---------------------------------------------------------------------- tests/comparison/cli_options.py | 12 ++++++------ tests/comparison/discrepancy_searcher.py | 6 ++++-- tests/stress/concurrent_select.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fe2be252/tests/comparison/cli_options.py ---------------------------------------------------------------------- diff --git a/tests/comparison/cli_options.py b/tests/comparison/cli_options.py index 06dacb1..714d899 100644 --- a/tests/comparison/cli_options.py +++ b/tests/comparison/cli_options.py @@ -48,7 +48,7 @@ def add_logging_options(parser, default_debug_log_file=None): help='Path to debug log file.') -def configure_logging(log_level, debug_log_file=None, log_thread_id=False, +def configure_logging(log_level, debug_log_file=None, log_thread_name=False, log_process_id=False): root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) @@ -59,16 +59,16 @@ def configure_logging(log_level, debug_log_file=None, log_thread_id=False, format = "%(asctime)s" if log_process_id: format += " %(process)d" - if log_thread_id: - format += " %(thread)d" + if log_thread_name: + format += " %(threadName)s" format += " %(levelname)s:%(module)s[%(lineno)s]:%(message)s" - console_logger.setFormatter(logging.Formatter(format, "%H:%M:%S")) + console_logger.setFormatter(logging.Formatter(format)) root_logger.addHandler(console_logger) if debug_log_file: - file_logger = logging.FileHandler(debug_log_file, mode="w") + file_logger = logging.FileHandler(debug_log_file) file_logger.name = "file" - file_logger.setFormatter(logging.Formatter(format, "%H:%M:%S")) + file_logger.setFormatter(logging.Formatter(format)) file_logger.setLevel(logging.DEBUG) root_logger.addHandler(file_logger) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fe2be252/tests/comparison/discrepancy_searcher.py ---------------------------------------------------------------------- diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py index a88dd2c..4b0bce9 100755 --- a/tests/comparison/discrepancy_searcher.py +++ b/tests/comparison/discrepancy_searcher.py @@ -377,7 +377,8 @@ class QueryExecutor(object): query_thread = Thread( target=self._fetch_sql_results, args=[query, cursor, sql_writer, log_file], - name='Statement execution thread {0}'.format(current_thread().name)) + name='{db_type}-exec-{id_}'.format( + db_type=cursor.db_type, id_=id(query))) query_thread.daemon = True query_thread.sql = '' query_thread.data_set = None @@ -914,7 +915,8 @@ if __name__ == '__main__': # TODO: Seed the random query generator for repeatable queries? args = parser.parse_args() - cli_options.configure_logging(args.log_level, debug_log_file=args.debug_log_file) + cli_options.configure_logging( + args.log_level, debug_log_file=args.debug_log_file, log_thread_name=True) cluster = cli_options.create_cluster(args) ref_conn = cli_options.create_connection(args, args.ref_db_type, db_name=args.db_name) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fe2be252/tests/stress/concurrent_select.py ---------------------------------------------------------------------- diff --git a/tests/stress/concurrent_select.py b/tests/stress/concurrent_select.py index c66fa34..2283608 100755 --- a/tests/stress/concurrent_select.py +++ b/tests/stress/concurrent_select.py @@ -1777,7 +1777,7 @@ def main(): args = parser.parse_args() cli_options.configure_logging( - args.log_level, debug_log_file=args.debug_log_file, log_thread_id=True, + args.log_level, debug_log_file=args.debug_log_file, log_thread_name=True, log_process_id=True) LOG.debug("CLI args: %s" % (args, ))
