IMPALA-3608: Updates Impala E2E test framework to allow multiple exception messages
Some of our tests which are expected to fail due to low query memory limits can fail non-deterministically with different error messages. In addition, some tests may throw different error messages when running with the legacy join nodes. This change updates the test infrastructure to allow multiple exception messages to be specified by using adding "ANY_OF" to the "CATCH" subsection. Change-Id: Ie6d81fd3ae601f565b575edfeefff7c5a6c07974 Reviewed-on: http://gerrit.cloudera.org:8080/3205 Reviewed-by: Michael Ho <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/3a4a7752 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/3a4a7752 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/3a4a7752 Branch: refs/heads/master Commit: 3a4a77521e31e28d139472809ae3642c6c912e2f Parents: e26dc85 Author: Michael Ho <[email protected]> Authored: Tue May 24 14:26:40 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Tue May 31 23:32:10 2016 -0700 ---------------------------------------------------------------------- .../tpch/queries/tpch-outer-joins.test | 3 +- tests/common/impala_test_suite.py | 36 ++++++++++++-------- tests/util/test_file_parser.py | 15 +++++++- 3 files changed, 38 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3a4a7752/testdata/workloads/tpch/queries/tpch-outer-joins.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/tpch/queries/tpch-outer-joins.test b/testdata/workloads/tpch/queries/tpch-outer-joins.test index efba5e8..0da2850 100644 --- a/testdata/workloads/tpch/queries/tpch-outer-joins.test +++ b/testdata/workloads/tpch/queries/tpch-outer-joins.test @@ -29,8 +29,9 @@ SELECT straight_join * FROM orders o RIGHT OUTER JOIN lineitem l ON o.o_orderkey = if(l.l_orderkey % 2 = 0, 0, l.l_orderkey) ORDER BY l_receiptdate, l_orderkey, l_shipdate limit 10 ----- CATCH +---- CATCH: ANY_OF Repartitioning did not reduce the size of a spilled partition +Memory limit exceeded ==== ---- QUERY # Regression test for IMPALA-2612. The following query will cause CastToChar http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3a4a7752/tests/common/impala_test_suite.py ---------------------------------------------------------------------- diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index 35c77df..ca7e168 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -163,8 +163,7 @@ class ImpalaTestSuite(BaseTestSuite): self.client.set_configuration({'sync_ddl': sync_ddl}) self.client.execute("drop database if exists `" + db_name + "` cascade") - @classmethod - def restore_query_options(self, query_options_changed): + def __restore_query_options(self, query_options_changed): """ Restore the list of modified query options to their default values. """ @@ -189,6 +188,25 @@ class ImpalaTestSuite(BaseTestSuite): except Exception as e: LOG.info('Unexpected exception when executing ' + query_str + ' : ' + str(e)) + def __verify_exceptions(self, expected_strs, actual_str, use_db): + """ + Verifies that at least one of the strings in 'expected_str' is a substring of the + actual exception string 'actual_str'. + """ + actual_str = actual_str.replace('\n', '') + for expected_str in expected_strs: + # In error messages, some paths are always qualified and some are not. + # So, allow both $NAMENODE and $FILESYSTEM_PREFIX to be used in CATCH. + expected_str = expected_str.strip() \ + .replace('$FILESYSTEM_PREFIX', FILESYSTEM_PREFIX) \ + .replace('$NAMENODE', NAMENODE) \ + .replace('$IMPALA_HOME', IMPALA_HOME) + if use_db: expected_str = expected_str.replace('$DATABASE', use_db) + # Strip newlines so we can split error message into multiple lines + expected_str = expected_str.replace('\n', '') + if expected_str in actual_str: return + assert False, 'Unexpected exception string: %s' % actual_str + def run_test_case(self, test_file_name, vector, use_db=None, multiple_impalad=False, encoding=None, wait_secs_between_stmts=None): """ @@ -275,22 +293,12 @@ class ImpalaTestSuite(BaseTestSuite): time.sleep(wait_secs_between_stmts) except Exception as e: if 'CATCH' in test_section: - # In error messages, some paths are always qualified and some are not. - # So, allow both $NAMENODE and $FILESYSTEM_PREFIX to be used in CATCH. - expected_str = test_section['CATCH'].strip() \ - .replace('$FILESYSTEM_PREFIX', FILESYSTEM_PREFIX) \ - .replace('$NAMENODE', NAMENODE) \ - .replace('$IMPALA_HOME', IMPALA_HOME) - if use_db: expected_str = expected_str.replace('$DATABASE', use_db) - # Strip newlines so we can split error message into multiple lines - expected_str = expected_str.replace('\n', '') - actual_str = str(e).replace('\n', '') - assert expected_str in actual_str + self.__verify_exceptions(test_section['CATCH'], str(e), use_db) continue raise finally: if len(query_options_changed) > 0: - self.restore_query_options(query_options_changed) + self.__restore_query_options(query_options_changed) if 'CATCH' in test_section: assert test_section['CATCH'].strip() == '' http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3a4a7752/tests/util/test_file_parser.py ---------------------------------------------------------------------- diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py index f608aa7..6c3ffc2 100644 --- a/tests/util/test_file_parser.py +++ b/tests/util/test_file_parser.py @@ -174,6 +174,9 @@ def parse_test_file_text(text, valid_section_names, skip_unknown_sections=True): if(len(subsection_info) == 2): subsection_name, subsection_comment = subsection_info + lines_content = lines[1:-1] + subsection_str = '\n'.join([line for line in lines_content]) + if subsection_name not in valid_section_names: if skip_unknown_sections or not subsection_name: print sub_section @@ -194,7 +197,17 @@ def parse_test_file_text(text, valid_section_names, skip_unknown_sections=True): else: raise RuntimeError, 'Unknown subsection comment: %s' % comment - parsed_sections[subsection_name] = '\n'.join([line for line in lines[1:-1]]) + if subsection_name == 'CATCH': + parsed_sections['CATCH'] = list() + if subsection_comment == None: + parsed_sections['CATCH'].append(subsection_str) + elif subsection_comment == 'ANY_OF': + parsed_sections['CATCH'].extend(lines_content) + else: + raise RuntimeError, 'Unknown subsection comment: %s' % subsection_comment + continue + + parsed_sections[subsection_name] = subsection_str if parsed_sections: sections.append(parsed_sections)
