Repository: incubator-impala Updated Branches: refs/heads/master 1a611b393 -> a81ad5eaa
IMPALA-4722: Disable log caching in test_scratch_disk test_scratch_disk fails sporadically when trying to assert the presence of log messages. This is probably caused by log caching, since after such failures the log files do contains the lines in question. I manually tested this by running the tests repeatedly for 2 days (10k runs). To make future diagnosis of similar problems easier, this change also adds more output to assert_impalad_log_contains(). Change-Id: I9f21284338ee7b4374aca249b6556282b0148389 Reviewed-on: http://gerrit.cloudera.org:8080/5669 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8b7f8766 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8b7f8766 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8b7f8766 Branch: refs/heads/master Commit: 8b7f876649c34dec7fee1dea34f5acf4db1c037d Parents: 1a611b3 Author: Lars Volker <[email protected]> Authored: Tue Jan 10 11:57:19 2017 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Thu Jan 12 18:58:48 2017 +0000 ---------------------------------------------------------------------- tests/common/custom_cluster_test_suite.py | 12 ++++++++---- tests/custom_cluster/test_scratch_disk.py | 12 +++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b7f8766/tests/common/custom_cluster_test_suite.py ---------------------------------------------------------------------- diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py index 4bbef14..365776a 100644 --- a/tests/common/custom_cluster_test_suite.py +++ b/tests/common/custom_cluster_test_suite.py @@ -127,8 +127,11 @@ class CustomClusterTestSuite(ImpalaTestSuite): def assert_impalad_log_contains(self, level, line_regex, expected_count=1): """ - Assert that impalad log with specified level (e.g. ERROR, WARNING, INFO) - contains expected_count lines with a substring matching the regex. + Assert that impalad log with specified level (e.g. ERROR, WARNING, INFO) contains + expected_count lines with a substring matching the regex. When using this method to + check log files of running processes, the caller should make sure that log buffering + has been disabled, for example by adding '-logbuflevel=-1' to the daemon startup + options. """ pattern = re.compile(line_regex) found = 0 @@ -139,5 +142,6 @@ class CustomClusterTestSuite(ImpalaTestSuite): for line in log_file: if pattern.search(line): found += 1 - assert found == expected_count, ("Expected %d lines in file %s matching regex '%s'"\ - + ", but found %d lines") % (expected_count, log_file_path, line_regex, found) + assert found == expected_count, ("Expected %d lines in file %s matching regex '%s'" + + ", but found %d lines. Last line was: \n%s") % (expected_count, log_file_path, + line_regex, found, line) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b7f8766/tests/custom_cluster/test_scratch_disk.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py index f523dbe..7e02de5 100644 --- a/tests/custom_cluster/test_scratch_disk.py +++ b/tests/custom_cluster/test_scratch_disk.py @@ -83,7 +83,7 @@ class TestScratchDir(CustomClusterTestSuite): scratch because all directories are on same disk.""" normal_dirs = self.generate_dirs(5) self._start_impala_cluster([ - '--impalad_args="-scratch_dirs={0}"'.format(','.join(normal_dirs))]) + '--impalad_args="-logbuflevel=-1 -scratch_dirs={0}"'.format(','.join(normal_dirs))]) self.assert_impalad_log_contains("INFO", "Using scratch directory ", expected_count=1) exec_option = vector.get_value('exec_option') @@ -96,7 +96,7 @@ class TestScratchDir(CustomClusterTestSuite): @pytest.mark.execute_serially def test_no_dirs(self, vector): """ Test we can execute a query with no scratch dirs """ - self._start_impala_cluster(['--impalad_args="-scratch_dirs="']) + self._start_impala_cluster(['--impalad_args="-logbuflevel=-1 -scratch_dirs="']) self.assert_impalad_log_contains("WARNING", "Running without spill to disk: no scratch directories provided\.") exec_option = vector.get_value('exec_option') @@ -113,7 +113,8 @@ class TestScratchDir(CustomClusterTestSuite): """ Test we can execute a query with only bad non-writable scratch """ non_writable_dirs = self.generate_dirs(5, writable=False) self._start_impala_cluster([ - '--impalad_args="-scratch_dirs={0}"'.format(','.join(non_writable_dirs))]) + '--impalad_args="-logbuflevel=-1 -scratch_dirs={0}"'.format( + ','.join(non_writable_dirs))]) self.assert_impalad_log_contains("ERROR", "Running without spill to disk: could " + "not use any scratch directories in list:.*. See previous " + "warnings for information on causes.") @@ -134,7 +135,8 @@ class TestScratchDir(CustomClusterTestSuite): """ Test that non-existing directories are not created or used """ non_existing_dirs = self.generate_dirs(5, non_existing=True) self._start_impala_cluster([ - '--impalad_args="-scratch_dirs={0}"'.format(','.join(non_existing_dirs))]) + '--impalad_args="-logbuflevel=-1 -scratch_dirs={0}"'.format( + ','.join(non_existing_dirs))]) self.assert_impalad_log_contains("ERROR", "Running without spill to disk: could " + "not use any scratch directories in list:.*. See previous " + "warnings for information on causes.") @@ -157,7 +159,7 @@ class TestScratchDir(CustomClusterTestSuite): have permissions changed or are removed after impalad startup.""" dirs = self.generate_dirs(3); self._start_impala_cluster([ - '--impalad_args="-scratch_dirs={0}"'.format(','.join(dirs)), + '--impalad_args="-logbuflevel=-1 -scratch_dirs={0}"'.format(','.join(dirs)), '--impalad_args=--allow_multiple_scratch_dirs_per_device=true']) self.assert_impalad_log_contains("INFO", "Using scratch directory ", expected_count=len(dirs))
