This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7369ebb8ba02edfedcef071029b7bcd62157f452 Author: Joe McDonnell <[email protected]> AuthorDate: Thu Aug 29 13:22:35 2024 -0700 IMPALA-13415: Add a special testing mode to track Calcite progress This introduces a Calcite report mode for pytests. The mode changes the behavior for run_test_case() so that it continues past failures and exports information about the test results to JSON. The JSON files can then be processed into an HTML summary by the bin/calcite_report_generator.py. The HTML has multiple layers of reporting: 1. Reporting on individual tests shows the test section along with the reported result 2. There are multiple aggregations of these individual results to have clear summaries and organization for browing: a. Level 1 is a report at the test function level (e.g. query_test/test_foo.py::TestFoo::test_foo) with links to the individaul results. b. Level 2 is at the test file level (e.g. query_test/test_foo.py) with links down to the test function level. c. Level 3 is a top level view with a summary across all the tests with links down to the test file level. The errors are classified into different categories (e.g. parse failures, analysis failures, result differences, etc). In general, parse failures and unsupported features are lower priority issues while result differences and runtime failures are higher priority. The report is designed to compare two different points in time to see differences. For example, someone can run tests for a baseline and then do a comparison run with a new commit. Testing: - Ran on the code change for IMPALA-13468 and browsed the results - Ran tests normally and verified that they continue to work Change-Id: I453c219c22b6cbc253574e0467d2c0d7b1fac092 Reviewed-on: http://gerrit.cloudera.org:8080/21866 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Michael Smith <[email protected]> Reviewed-by: Joe McDonnell <[email protected]> --- bin/calcite_report_generator.py | 374 ++++++++++++++++++++++++++++++++++++++ tests/common/impala_test_suite.py | 274 ++++++++++++++++------------ tests/conftest.py | 13 ++ 3 files changed, 546 insertions(+), 115 deletions(-) diff --git a/bin/calcite_report_generator.py b/bin/calcite_report_generator.py new file mode 100755 index 000000000..959e78d98 --- /dev/null +++ b/bin/calcite_report_generator.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This processes the JSON files produces by the pytest +# "calcite_report_mode" option to produce a set of HTML pages +# with layers of aggregation / navigation. It produces the following +# layers: +# Level 0: Base results +# - Individual HTML file for each test +# - Leaf nodes in the directory structure +# - e.g. query_test/test_foo.py::TestFoo::test_single_foo[test_dimension: x] +# - Directory location: {out_dir}/{test_file_dir}/{test_function_dir}/{unique} +# Level 1: Aggregation of results for single test function +# - Summary HTML file for each test function +# - e.g. query_test/test_foo.py::TestFoo::test_single_foo +# - Directory location: {out_dir}/{test_file_dir}/{test_function_dir}/index.html +# Level 2: Aggregation of results for single test class +# - Summary HTML file for each test file +# - e.g. query_test/test_foo.py +# - Directory location: {out_dir}/{test_file_dir}/index.html +# Level 3: Top level aggregation of results +# - Summary HTML file across all test files +# - Directory location: {out_dir}/index.html +# +# It is designed to compare two separate runs to show the differences. + +import glob +import json +import os +import sys +from argparse import ArgumentParser + +HEADER_TEMPLATE = """ +<!DOCTYPE html> +<html> + <head> + <link rel="stylesheet" href="{0}"> + </head> +<body> +""" + +FOOTER = """ +</body> +</html> +""" + +RESULT_CATEGORIES = ["Success", "Parse Failure", "Analysis Failure", + "Unsupported Feature", "Result Difference", + "Profile Difference", "Other"] + +RESULT_CATEGORY_STYLE_MAP = { + "Success": "success", + "Parse Failure": "expected_fail", + "Analysis Failure": "fail", + "Unsupported Feature": "expected_fail", + "Result Difference": "fail", + "Profile Difference": "fail", + "Other": "fail" +} + + +# This steals the logic in tests_file_parser.py to produce +# a single section string +def produce_section_string(test_case): + SUBSECTION_DELIMITER = "----" + s = "" + for section_name, section_value in test_case.items(): + if section_name in ['QUERY_NAME', 'VERIFIER']: + continue + full_section_name = section_name + if section_name == 'QUERY' and test_case.get('QUERY_NAME'): + full_section_name = '%s: %s' % (section_name, test_case['QUERY_NAME']) + if section_name == 'RESULTS' and test_case.get('VERIFIER'): + full_section_name = '%s: %s' % (section_name, test_case['VERIFIER']) + s += ("%s %s\n" % (SUBSECTION_DELIMITER, full_section_name)) + section_value = ''.join(test_case[section_name]).strip() + if section_value: + s += section_value + s += "\n" + return s + + +def categorize_error_string(error_string): + if error_string is None: + return "Success" + elif "ParseException" in error_string: + return "Parse Failure" + elif "AnalysisException" in error_string: + return "Analysis Failure" + elif "UnsupportedFeatureException" in error_string: + return "Unsupported Feature" + elif "Comparing QueryTestResults" in error_string: + return "Result Difference" + elif "PROFILE" in error_string: + return "Profile Difference" + else: + return "Other" + + +def process_single_test_node(before_json_contents, after_json_contents, out_filename): + test_node_id = before_json_contents["test_node_id"] + test_file, test_class, _, test_function = test_node_id.split("[")[0].split("::") + result_category_counts = {} + for result_category in RESULT_CATEGORIES: + # Total count, number increased, number decreased + result_category_counts[result_category] = [0, 0, 0] + with open(out_filename, "w") as f: + f.write(HEADER_TEMPLATE.format("../../style.css")) + parent_node = "{0}::{1}::{2}".format(test_file, test_class, test_function) + f.write('<a href="index.html">Up to {0}</a>'.format(parent_node)) + f.write("<p>{0}</p>\n".format(before_json_contents["test_node_id"])) + f.write("<p>{0}</p>\n".format(before_json_contents["test_file"])) + f.write("<table>\n") + # Table header + f.write("<tr>\n") + f.write("<th>Test Section</th>\n") + f.write("<th>Before Result</th>\n") + f.write("<th>After Result</th>\n") + f.write("</tr>\n") + # All the result rows + for before_result, after_result in zip(before_json_contents["results"], + after_json_contents["results"]): + before_section = before_result["section"] + after_section = after_result["section"] + if "QUERY" in before_section: + if before_section["QUERY"] != after_section["QUERY"]: + raise Exception("Mismatch in test sections: BEFORE: {0} AFTER: {1}".format( + before_section, after_section)) + f.write("<tr>\n") + section_string = produce_section_string(before_section) + f.write("<td><pre>{0}</pre></td>".format(section_string)) + before_error_category = categorize_error_string(before_result["error"]) + f.write('<td id="{0}"><pre>{1}</pre></td>'.format( + RESULT_CATEGORY_STYLE_MAP[before_error_category], + "Success" if before_error_category == "Success" else before_result["error"])) + + after_error_category = categorize_error_string(after_result["error"]) + f.write('<td id="{0}"><pre>{1}</pre></td>'.format( + RESULT_CATEGORY_STYLE_MAP[after_error_category], + "Success" if after_error_category == "Success" else after_result["error"])) + + after_error_counts = result_category_counts[after_error_category] + # Always bump the first counter to count the total + after_error_counts[0] += 1 + if after_error_category != before_error_category: + before_error_counts = result_category_counts[before_error_category] + # Bump before's counter for number decreased + before_error_counts[2] += 1 + # Bump after's counter for number increased + after_error_counts[1] += 1 + f.write("</tr>") + + f.write("</table>") + f.write(FOOTER) + + return result_category_counts + + +def produce_function_index(out_filename, description, parent_description, stylesheet_link, + values): + result_category_counts = {} + for result_category in RESULT_CATEGORIES: + # Total count, number increased, number decreased + result_category_counts[result_category] = [0, 0, 0] + with open(out_filename, "w") as f: + f.write(HEADER_TEMPLATE.format(stylesheet_link)) + if parent_description is not None: + f.write('<a href="../index.html">Up to {0}</a>'.format(parent_description)) + f.write("<p>{0}</p>".format(description)) + f.write("<table>\n") + # Table header + f.write("<tr>\n") + f.write("<th>Name</th>\n") + for result_category in RESULT_CATEGORIES: + f.write("<th>{0}</th>".format(result_category)) + f.write("</tr>\n") + for value in sorted(values): + item_description, filename, stats = value + f.write("<tr>\n") + f.write('<td><a href="{0}">{1}</a></td>'.format(filename, item_description)) + for result_category in stats: + result_counts = stats[result_category] + if result_counts[1] == 0 and result_counts[2] == 0: + f.write("<td>{0}</td>".format(result_counts[0])) + else: + f.write("<td>{0} (+{1}, -{2}) </td>".format(*result_counts)) + total_result_counts = result_category_counts[result_category] + for i, val in enumerate(result_counts): + total_result_counts[i] += val + f.write("</tr>\n") + + # Add summary + f.write("<tr>\n") + f.write("<td>Total</td>") + for result_category in stats: + total_result_counts = result_category_counts[result_category] + if total_result_counts[1] == 0 and total_result_counts[2] == 0: + f.write("<td>{0}</td>".format(total_result_counts[0])) + else: + f.write("<td>{0} (+{1}, -{2}) </td>".format(*total_result_counts)) + f.write("</tr>\n") + f.write("</table>\n") + f.write(FOOTER) + + return result_category_counts + + +def get_output_files_set(directory): + glob_list = glob.glob(os.path.join(directory, "output_*.json")) + return set([os.path.basename(x) for x in glob_list]) + + +def main(): + parser = ArgumentParser() + parser.add_argument("--before_directory", required=True) + parser.add_argument("--after_directory", required=True) + parser.add_argument("--output_directory", required=True) + parser.add_argument("--allow_file_differences", default=False, action="store_true") + args = parser.parse_args() + + # Right now, only cover the simplest possible case: we have the same set of files in + # the before and after directories. That lets us pair them up easily. + # This assumption would be violated if we add/remove/change the test dimensions. + # Hopefully, that won't be necessary for Calcite reports for a while. + before_files = get_output_files_set(args.before_directory) + after_files = get_output_files_set(args.after_directory) + if before_files == after_files: + files_intersection = before_files + elif args.allow_file_differences: + files_intersection = before_files.intersection(after_files) + if len(files_intersection) == 0: + print("ERROR: there are no files in common for the directories") + else: + print("There are file differences between the directories. Ignoring these files:") + for f in before_files - after_files: + print(os.path.join(args.before_directory, f)) + for f in after_files - before_files: + print(os.path.join(args.after_directory, f)) + else: + print("ERROR: the directories contain different sets of files") + sys.exit(1) + + if not os.path.exists(args.output_directory): + os.mkdir(args.output_directory) + + # Write out CSS to root directory. + # Note: This needs to be in its own file separate from the HTML to avoid issues with + # Content-Security-Policy. + with open(os.path.join(args.output_directory, "style.css"), "w") as css: + css.write("table, th, td { border: 1px solid black; border-collapse: collapse; }\n") + css.write("#success { background-color: #d2ffd2; }\n") + css.write("#fail { background-color: #ffd2d2; }\n") + css.write("#expected_fail { background-color: #ffffa0; }\n") + + # Multiple levels of information that build up from the individual tests + # to higher levels. + # Level 0: Base results + # - Individual HTML file for each test + # - Leaf nodes in the directory structure + # - e.g. query_test/test_foo.py::TestFoo::test_single_foo[test_dimension: x] + # - Directory location: {out_dir}/{test_file_dir}/{test_function_dir}/{unique} + # Level 1: Aggregation of results for single test function + # - Summary HTML file for each test function + # - e.g. query_test/test_foo.py::TestFoo::test_single_foo + # - Directory location: {out_dir}/{test_file_dir}/{test_function_dir}/index.html + # Level 2: Aggregation of results for single test class + # - Summary HTML file for each test file + # - e.g. query_test/test_foo.py + # - Directory location: {out_dir}/{test_file_dir}/index.html + # Level 3: Top level aggregation of results + # - Summary HTML file across all test files + # - Directory location: {out_dir}/index.html + + # Iterate over all the files and write out the level 0 individual test results. + # While doing the iteration, also build the data structure for the level 1 + # aggregation. + level1_index = {} + for filename in files_intersection: + before_filename = os.path.join(args.before_directory, filename) + with open(before_filename) as f: + after_filename = os.path.join(args.after_directory, filename) + with open(after_filename) as g: + before_json_contents = json.load(f) + after_json_contents = json.load(g) + test_node_id = before_json_contents["test_node_id"] + # We are expecting the test files to match, so bail out if the files don't + # match. + if test_node_id != after_json_contents["test_node_id"]: + raise Exception("File {0} does not have the same test node id as {1}".format( + before_filename, after_filename)) + if len(before_json_contents["results"]) != len(after_json_contents["results"]): + raise Exception("File {0} has different number of tests from file {1}".format( + before_filename, after_filename)) + + # Break apart the test node id to allow aggregating at various levels and + # organizing the directory structure + test_file, test_class, _, test_function = test_node_id.split("[")[0].split("::") + + # Step 1: Write out individual test html files + # (When this becomes a diff, we'll have pairs of files to put into this) + out_subdir = os.path.join(args.output_directory, test_file.replace("/", "_"), + "{0}_{1}".format(test_class, test_function)) + if not os.path.exists(out_subdir): + os.makedirs(out_subdir) + output_filename = os.path.join(out_subdir, + os.path.basename(before_filename).replace(".json", ".html")) + out_stats = process_single_test_node(before_json_contents, after_json_contents, + output_filename) + + # Build the data structure for the level 1 aggregation + level1_id = (test_file, test_class, test_function) + if level1_id not in level1_index: + level1_index[level1_id] = [] + level1_index[level1_id].append( + [test_node_id, os.path.basename(output_filename), out_stats]) + + # Write out level 1 (aggregation per test function) while also building the data + # structure for level 2 (aggregation per test file). + level2_index = {} + for key, value in level1_index.items(): + out_filename = os.path.join(args.output_directory, key[0].replace("/", "_"), + "{0}_{1}".format(key[1], key[2]), "index.html") + relative_filename = os.path.join("{0}_{1}".format(key[1], key[2]), "index.html") + out_description = "{0}::{1}::{2}".format(key[0], key[1], key[2]) + parent_description = key[0] + out_stats = produce_function_index(out_filename, out_description, parent_description, + "../../style.css", value) + # Grab the python file level key + level2_key = key[0] + if level2_key not in level2_index: + level2_index[level2_key] = [] + level2_index[level2_key].append([out_description, relative_filename, out_stats]) + + # Write out level 2 (aggregation per test file) while also building the data + # structure for level 3 (top level aggregation) + level3_index = {} + level3_index["Top"] = [] + for key, value in level2_index.items(): + out_filename = os.path.join(args.output_directory, key.replace("/", "_"), + "index.html") + relative_filename = os.path.join(key.replace("/", "_"), "index.html") + out_description = key + parent_description = "Top" + out_stats = produce_function_index(out_filename, out_description, + parent_description, "../style.css", value) + level3_index["Top"].append([out_description, relative_filename, out_stats]) + + # Write out level 3 (top level aggregation) + for key, value in level3_index.items(): + out_filename = os.path.join(args.output_directory, "index.html") + out_description = "Top" + parent_description = None + out_stats = produce_function_index(out_filename, out_description, parent_description, + "style.css", value) + + +if __name__ == "__main__": + main() diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index 95a5591b4..bab057481 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -21,6 +21,7 @@ from __future__ import absolute_import, division, print_function from builtins import range, round import glob import grp +import hashlib import json import logging import os @@ -38,6 +39,7 @@ from getpass import getuser from impala.hiveserver2 import HiveServer2Cursor from random import choice from subprocess import check_call +import tests.common from tests.common.base_test_suite import BaseTestSuite from tests.common.environ import ( HIVE_MAJOR_VERSION, @@ -155,6 +157,7 @@ INTERNAL_LISTEN_HOST = os.getenv("INTERNAL_LISTEN_HOST") # Some tests use the IP instead of the host. INTERNAL_LISTEN_IP = socket.gethostbyname_ex(INTERNAL_LISTEN_HOST)[2][0] EE_TEST_LOGS_DIR = os.getenv("IMPALA_EE_TEST_LOGS_DIR") +IMPALA_LOGS_DIR = os.getenv("IMPALA_LOGS_DIR") # Match any SET statement. Assume that query options' names # only contain alphabets, underscores and digits after position 1. # The statement may include SQL line comments starting with --, which we need to @@ -725,132 +728,173 @@ class ImpalaTestSuite(BaseTestSuite): encoding=encoding) # Assumes that it is same across all the coordinators. lineage_log_dir = self.get_var_current_val('lineage_event_log_dir') + failed_count = 0 + total_count = 0 + result_list = [] for test_section in sections: - if 'HIVE_MAJOR_VERSION' in test_section: - needed_hive_major_version = int(test_section['HIVE_MAJOR_VERSION']) - assert needed_hive_major_version in [2, 3] - assert HIVE_MAJOR_VERSION in [2, 3] - if needed_hive_major_version != HIVE_MAJOR_VERSION: + current_error = None + try: + if 'HIVE_MAJOR_VERSION' in test_section: + needed_hive_major_version = int(test_section['HIVE_MAJOR_VERSION']) + assert needed_hive_major_version in [2, 3] + assert HIVE_MAJOR_VERSION in [2, 3] + if needed_hive_major_version != HIVE_MAJOR_VERSION: + continue + + if 'IS_HDFS_ONLY' in test_section and not IS_HDFS: continue - if 'IS_HDFS_ONLY' in test_section and not IS_HDFS: - continue - - if 'SHELL' in test_section: - assert len(test_section) == 1, \ - "SHELL test sections can't contain other sections" - cmd = self.__do_replacements(test_section['SHELL'], use_db=use_db, - extra=test_file_vars) - LOG.info("Shell command: " + cmd) - check_call(cmd, shell=True) - continue + if 'SHELL' in test_section: + assert len(test_section) == 1, \ + "SHELL test sections can't contain other sections" + cmd = self.__do_replacements(test_section['SHELL'], use_db=use_db, + extra=test_file_vars) + LOG.info("Shell command: " + cmd) + check_call(cmd, shell=True) + continue - if 'QUERY' in test_section: - query_section = test_section['QUERY'] - exec_fn = __exec_in_impala - elif 'HIVE_QUERY' in test_section: - query_section = test_section['HIVE_QUERY'] - exec_fn = __exec_in_hive - else: - assert 0, ('Error in test file {}. Test cases require a ' - '-- QUERY or HIVE_QUERY section.\n{}').format( - test_file_name, pprint.pformat(test_section)) + if 'QUERY' in test_section: + query_section = test_section['QUERY'] + exec_fn = __exec_in_impala + elif 'HIVE_QUERY' in test_section: + query_section = test_section['HIVE_QUERY'] + exec_fn = __exec_in_hive + else: + assert 0, ('Error in test file {}. Test cases require a ' + '-- QUERY or HIVE_QUERY section.\n{}').format( + test_file_name, pprint.pformat(test_section)) - # TODO: support running query tests against different scale factors - query = QueryTestSectionReader.build_query( - self.__do_replacements(query_section, use_db=use_db, extra=test_file_vars)) + # TODO: support running query tests against different scale factors + query = QueryTestSectionReader.build_query( + self.__do_replacements(query_section, use_db=use_db, extra=test_file_vars)) - if 'QUERY_NAME' in test_section: - LOG.info('Query Name: \n%s\n' % test_section['QUERY_NAME']) + if 'QUERY_NAME' in test_section: + LOG.info('Query Name: \n%s\n' % test_section['QUERY_NAME']) - result = None - try: - result = exec_fn(query, user=test_section.get('USER', '').strip() or None) - except Exception as e: - if 'CATCH' in test_section: - self.__verify_exceptions(test_section['CATCH'], str(e), use_db) - assert error_msg_expected(str(e)) # Only checks if message contains query id - continue - raise + result = None + try: + result = exec_fn(query, user=test_section.get('USER', '').strip() or None) + except Exception as e: + if 'CATCH' in test_section: + self.__verify_exceptions(test_section['CATCH'], str(e), use_db) + assert error_msg_expected(str(e)) + continue + raise - if 'CATCH' in test_section and '__NO_ERROR__' not in test_section['CATCH']: - expected_str = self.__do_replacements(" or ".join(test_section['CATCH']).strip(), + if 'CATCH' in test_section and '__NO_ERROR__' not in test_section['CATCH']: + expected_str = self.__do_replacements( + " or ".join(test_section['CATCH']).strip(), use_db=use_db, extra=test_file_vars) - assert False, "Expected exception: {0}\n\nwhen running:\n\n{1}".format( - expected_str, query) - - assert result is not None - assert result.success, "Query failed: {0}".format(result.data) - - # Decode the results read back if the data is stored with a specific encoding. - if encoding: result.data = [row.decode(encoding) for row in result.data] - # Replace $NAMENODE in the expected results with the actual namenode URI. - if 'RESULTS' in test_section: - # Combining 'RESULTS' with 'DML_RESULTS" is currently unsupported because - # __verify_results_and_errors calls verify_raw_results which always checks - # ERRORS, TYPES, LABELS, etc. which doesn't make sense if there are two - # different result sets to consider (IMPALA-4471). - assert 'DML_RESULTS' not in test_section - test_section['RESULTS'] = self.__do_replacements( - test_section['RESULTS'], use_db=use_db, extra=test_file_vars) - self.__verify_results_and_errors(vector, test_section, result, use_db) - else: - # TODO: Can't validate errors without expected results for now. - assert 'ERRORS' not in test_section,\ - "'ERRORS' sections must have accompanying 'RESULTS' sections" - # If --update_results, then replace references to the namenode URI with $NAMENODE. - # TODO(todd) consider running do_replacements in reverse, though that may cause - # some false replacements for things like username. - if pytest.config.option.update_results and 'RESULTS' in test_section: - test_section['RESULTS'] = test_section['RESULTS'] \ - .replace(NAMENODE, '$NAMENODE') \ - .replace(IMPALA_HOME, '$IMPALA_HOME') \ - .replace(INTERNAL_LISTEN_HOST, '$INTERNAL_LISTEN_HOST') \ - .replace(INTERNAL_LISTEN_IP, '$INTERNAL_LISTEN_IP') - rt_profile_info = None - if 'RUNTIME_PROFILE_%s' % table_format_info.file_format in test_section: - # If this table format has a RUNTIME_PROFILE section specifically for it, evaluate - # that section and ignore any general RUNTIME_PROFILE sections. - rt_profile_info = 'RUNTIME_PROFILE_%s' % table_format_info.file_format - elif 'RUNTIME_PROFILE' in test_section: - rt_profile_info = 'RUNTIME_PROFILE' - - if rt_profile_info is not None: - if test_file_vars: - # only do test_file_vars replacement if it exist. - test_section[rt_profile_info] = self.__do_replacements( - test_section[rt_profile_info], extra=test_file_vars) - rt_profile = verify_runtime_profile(test_section[rt_profile_info], - result.runtime_profile, - update_section=pytest.config.option.update_results) - if pytest.config.option.update_results: - test_section[rt_profile_info] = "".join(rt_profile) - - if 'LINEAGE' in test_section: - # Lineage flusher thread runs every 5s by default and is not configurable. Wait - # for that period. (TODO) Get rid of this for faster test execution. - time.sleep(5) - current_query_lineage = self.get_query_lineage(result.query_id, lineage_log_dir) - assert current_query_lineage != "", ( - "No lineage found for query {} in dir {}".format( - result.query_id, lineage_log_dir)) - if pytest.config.option.update_results: - test_section['LINEAGE'] = json.dumps(current_query_lineage, indent=2, - separators=(',', ': ')) + assert False, "Expected exception: {0}\n\nwhen running:\n\n{1}".format( + expected_str, query) + + assert result is not None + assert result.success, "Query failed: {0}".format(result.data) + + # Decode the results read back if the data is stored with a specific encoding. + if encoding: result.data = [row.decode(encoding) for row in result.data] + # Replace $NAMENODE in the expected results with the actual namenode URI. + if 'RESULTS' in test_section: + # Combining 'RESULTS' with 'DML_RESULTS" is currently unsupported because + # __verify_results_and_errors calls verify_raw_results which always checks + # ERRORS, TYPES, LABELS, etc. which doesn't make sense if there are two + # different result sets to consider (IMPALA-4471). + assert 'DML_RESULTS' not in test_section + test_section['RESULTS'] = self.__do_replacements( + test_section['RESULTS'], use_db=use_db, extra=test_file_vars) + self.__verify_results_and_errors(vector, test_section, result, use_db) else: - verify_lineage(json.loads(test_section['LINEAGE']), current_query_lineage) - - if 'DML_RESULTS' in test_section: - assert 'ERRORS' not in test_section - # The limit is specified to ensure the queries aren't unbounded. We shouldn't have - # test files that are checking the contents of tables larger than that anyways. - dml_results_query = "select * from %s limit 1000" % \ - test_section['DML_RESULTS_TABLE'] - dml_result = exec_fn(dml_results_query) - verify_raw_results(test_section, dml_result, - vector.get_value('table_format').file_format, result_section='DML_RESULTS', - update_section=pytest.config.option.update_results) + # TODO: Can't validate errors without expected results for now. + assert 'ERRORS' not in test_section,\ + "'ERRORS' sections must have accompanying 'RESULTS' sections" + # If --update_results, then replace references to the namenode URI with $NAMENODE. + # TODO(todd) consider running do_replacements in reverse, though that may cause + # some false replacements for things like username. + if pytest.config.option.update_results and 'RESULTS' in test_section: + test_section['RESULTS'] = test_section['RESULTS'] \ + .replace(NAMENODE, '$NAMENODE') \ + .replace(IMPALA_HOME, '$IMPALA_HOME') \ + .replace(INTERNAL_LISTEN_HOST, '$INTERNAL_LISTEN_HOST') \ + .replace(INTERNAL_LISTEN_IP, '$INTERNAL_LISTEN_IP') + rt_profile_info = None + if 'RUNTIME_PROFILE_%s' % table_format_info.file_format in test_section: + # If this table format has a RUNTIME_PROFILE section specifically for it, + # evaluate that section and ignore any general RUNTIME_PROFILE sections. + rt_profile_info = 'RUNTIME_PROFILE_%s' % table_format_info.file_format + elif 'RUNTIME_PROFILE' in test_section: + rt_profile_info = 'RUNTIME_PROFILE' + + if rt_profile_info is not None: + if test_file_vars: + # only do test_file_vars replacement if it exist. + test_section[rt_profile_info] = self.__do_replacements( + test_section[rt_profile_info], extra=test_file_vars) + rt_profile = verify_runtime_profile(test_section[rt_profile_info], + result.runtime_profile, + update_section=pytest.config.option.update_results) + if pytest.config.option.update_results: + test_section[rt_profile_info] = "".join(rt_profile) + + if 'LINEAGE' in test_section: + # Lineage flusher thread runs every 5s by default and is not configurable. Wait + # for that period. (TODO) Get rid of this for faster test execution. + time.sleep(5) + current_query_lineage = self.get_query_lineage(result.query_id, lineage_log_dir) + assert current_query_lineage != "", ( + "No lineage found for query {} in dir {}".format( + result.query_id, lineage_log_dir)) + if pytest.config.option.update_results: + test_section['LINEAGE'] = json.dumps(current_query_lineage, indent=2, + separators=(',', ': ')) + else: + verify_lineage(json.loads(test_section['LINEAGE']), current_query_lineage) + + if 'DML_RESULTS' in test_section: + assert 'ERRORS' not in test_section + # The limit is specified to ensure the queries aren't unbounded. We shouldn't + # have test files that are checking the contents of tables larger than that + # anyways. + dml_results_query = "select * from %s limit 1000" % \ + test_section['DML_RESULTS_TABLE'] + dml_result = exec_fn(dml_results_query) + verify_raw_results(test_section, dml_result, + vector.get_value('table_format').file_format, result_section='DML_RESULTS', + update_section=pytest.config.option.update_results) + except Exception as e: + # When the calcite report mode is off, fail fast when hitting an error. + if not pytest.config.option.calcite_report_mode: + raise + current_error = str(e) + failed_count += 1 + finally: + if pytest.config.option.calcite_report_mode: + result_list.append({"section": test_section, "error": current_error}) + total_count += 1 + + # Write out the output information + if pytest.config.option.calcite_report_mode: + report = {} + report["test_node_id"] = tests.common.nodeid + report["test_file"] = os.path.join("testdata", "workloads", self.get_workload(), + 'queries', test_file_name + '.test') + report["results"] = result_list + # The node ids are unique, so there should not be hash collisions + nodeid_hash = hashlib.sha256(tests.common.nodeid.encode()).hexdigest() + output_file = "output_{0}.json".format(nodeid_hash) + output_directory = pytest.config.option.calcite_report_output_dir + if output_directory is None: + output_directory = os.path.join(IMPALA_LOGS_DIR, "calcite_report") + if not os.path.exists(output_directory): + os.makedirs(output_directory) + with open(os.path.join(output_directory, output_file), "w") as f: + json.dump(report, f, indent=2) + + # Since the report mode continues after error, we should return an error at + # the end if it hit failures + if failed_count != 0: + raise Exception("{0} out of {1} tests failed".format(failed_count, total_count)) + if pytest.config.option.update_results: # Print updated test results to path like # $EE_TEST_LOGS_DIR/impala_updated_results/tpcds/queries/tpcds-decimal_v2-q98.test diff --git a/tests/conftest.py b/tests/conftest.py index eca1471c2..9820df701 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -175,6 +175,16 @@ def pytest_addoption(parser): "version used to execute the tests. " "(See $IMPALA_HOME/bin/set-pythonpath.sh.)") + parser.addoption("--calcite_report_mode", action="store_true", default=False, + help="Mode designed to provide coverage for the Calcite planner. " + "Produces a JSON file for each run_test_case() invocation and " + "continues past errors. These JSON files can be processed to produce " + "a report to detect improvements and protect against regressions.") + + parser.addoption("--calcite_report_output_dir", default=None, + help="Location to store the output JSON files for " + "calcite_report_mode. Defaults to ${IMPALA_LOGS_DIR}/calcite_report.") + def pytest_assertrepr_compare(op, left, right): """ @@ -694,3 +704,6 @@ def pytest_runtest_logstart(nodeid, location): # than being elided. tests.common.current_node = \ nodeid.replace(",", ";").replace(" ", "").replace("=", "-")[0:255] + + # Store the unaltered nodeid as well + tests.common.nodeid = nodeid
