Stamatis Zampetakis created IMPALA-15761:
--------------------------------------------
Summary: TestIcebergWriterLimit fails intermittently with
different file count
Key: IMPALA-15761
URL: https://issues.apache.org/jira/browse/IMPALA-15761
Project: IMPALA
Issue Type: Bug
Reporter: Stamatis Zampetakis
The TestIcebergWriterLimit failed in a [recent Jenkins
run|https://jenkins.impala.io/job/ubuntu-20.04-from-scratch/10933/] due to a
different file count generated by a [CTAS
command|https://github.com/apache/impala/blob/3126fa631c7f7b8945af1c3656ae16472508c87d/testdata/workloads/functional-query/queries/QueryTest/iceberg-writer-limit.test#L6].
{noformat}
---- QUERY
# -- CTAS (unpartitioned, no limit) --
# Sanity: without MAX_FS_WRITERS, mt_dop=4 produces multiple data files.
set mt_dop=4;
CREATE TABLE ins_no_limit STORED BY ICEBERG
AS SELECT id, string_col FROM functional_parquet.alltypes ORDER BY id;
====
---- QUERY
SELECT count(*) FROM $DATABASE.ins_no_limit.`files` WHERE content = 0;
---- RESULTS
12
{noformat}
The test expects 12 files to be created but it seems that in the Jenkins run
only 11 were created.
{noformat}
query_test/test_iceberg.py:3463: in test_writer_limit
self.run_test_case('QueryTest/iceberg-writer-limit', vector,
self = <test_iceberg.TestIcebergWriterLimit object at
0x7fc738696b50>
unique_database = 'test_writer_limit_80ff6c8e'
vector = "table_format: parquet/none" | "protocol: hs2" |
"exec_option: {'abort_on_error': 1, 'batch_size': 0, 'disable_codegen': False,
'disable_codegen_rows_threshold': 5000, 'exec_single_node_rows_threshold': 0,
'num_nodes': 0, 'test_replan': 1}"
common/impala_test_suite.py:1090: in run_test_case
self.__verify_results_and_errors(vector, test_section,
_ImpalaTestSuite__exec_in_hive = <function
ImpalaTestSuite.run_test_case.<locals>.__exec_in_hive at 0x7fc738898d60>
_ImpalaTestSuite__exec_in_impala = <function
ImpalaTestSuite.run_test_case.<locals>.__exec_in_impala at 0x7fc738898ae0>
_ImpalaTestSuite__exec_in_trino = <function
ImpalaTestSuite.run_test_case.<locals>.__exec_in_trino at 0x7fc738898e00>
catch_section_name = 'CATCH'
current_error = None
encoding = None
exec_fn = <function
ImpalaTestSuite.run_test_case.<locals>.__exec_in_impala at 0x7fc738898ae0>
exec_options = {'abort_on_error': 1, 'batch_size': 0,
'disable_codegen': False, 'disable_codegen_rows_threshold': 5000, ...}
failed_count = 0
impalad_client = <tests.common.impala_connection.ImpylaHS2Connection
object at 0x7fc738694090>
lineage_log_dir = ''
multiple_impalad = False
protocol = 'hs2'
query = 'SELECT count(*) FROM
test_writer_limit_80ff6c8e.ins_no_limit.`files` WHERE content = 0'
query_section = 'SELECT count(*) FROM $DATABASE.ins_no_limit.`files`
WHERE content = 0;\n'
remaining = 0
result = <tests.common.impala_connection.ImpylaHS2ResultSet object
at 0x7fc738629050>
result_list = []
results_section_name = 'RESULTS'
rt_profile_info = None
sections = [defaultdict(<class 'str'>, {'QUERY': '# -- CTAS
(unpartitioned, no limit) --\n# Sanity: without MAX_FS_WRITERS, mt_do...
exactly 1 position delete file.\nset mt_dop=4;\nset max_fs_writers=1;\nDELETE
FROM del_v2 WHERE id % 2 = 0;\n'}), ...]
self = <test_iceberg.TestIcebergWriterLimit object at
0x7fc738696b50>
table_format_info = <tests.common.test_dimensions.TableFormatInfo
object at 0x7fc73a5d7050>
target_impalad_clients =
[<tests.common.impala_connection.ImpylaHS2Connection object at 0x7fc738694090>]
test_file_name = 'QueryTest/iceberg-writer-limit'
test_file_vars = None
test_section = defaultdict(<class 'str'>, {'QUERY': 'SELECT count(*)
FROM $DATABASE.ins_no_limit.`files` WHERE content = 0;\n', 'RESULTS': '12\n',
'TYPES': 'BIGINT\n'})
total_count = 2
use_db = 'test_writer_limit_80ff6c8e'
user = None
vector = "table_format: parquet/none" | "protocol: hs2" |
"exec_option: {'abort_on_error': 1, 'batch_size': 0, 'disable_codegen': False,
'disable_codegen_rows_threshold': 5000, 'exec_single_node_rows_threshold': 0,
'num_nodes': 0, 'test_replan': 1}"
common/impala_test_suite.py:852: in __verify_results_and_errors
verify_raw_results(test_section, result, vector,
replace_filenames_with_placeholder = True
result = <tests.common.impala_connection.ImpylaHS2ResultSet object
at 0x7fc738629050>
result_section = 'RESULTS'
result_section_name = 'RESULTS'
section_name = 'ERRORS'
self = <test_iceberg.TestIcebergWriterLimit object at
0x7fc738696b50>
test_section = defaultdict(<class 'str'>, {'QUERY': 'SELECT count(*)
FROM $DATABASE.ins_no_limit.`files` WHERE content = 0;\n', 'RESULTS': '12\n',
'TYPES': 'BIGINT\n'})
type_section = 'TYPES'
use_db = 'test_writer_limit_80ff6c8e'
vector = "table_format: parquet/none" | "protocol: hs2" |
"exec_option: {'abort_on_error': 1, 'batch_size': 0, 'disable_codegen': False,
'disable_codegen_rows_threshold': 5000, 'exec_single_node_rows_threshold': 0,
'num_nodes': 0, 'test_replan': 1}"
common/test_result_verifier.py:505: in verify_raw_results
VERIFIER_MAP[verifier](expected, actual)
actual = <tests.common.test_result_verifier.QueryTestResult object
at 0x7fc7387cea10>
actual_labels = ['COUNT(*)']
actual_types = ['BIGINT']
exec_result = <tests.common.impala_connection.ImpylaHS2ResultSet object
at 0x7fc738629050>
expected = <tests.common.test_result_verifier.QueryTestResult object
at 0x7fc7387ce310>
expected_results = '12\n'
expected_results_list = ['12']
expected_types = ['BIGINT']
file_format = 'parquet'
is_raw_string = False
order_matters = False
protocol = 'hs2'
replace_filenames = True
result_section = 'RESULTS'
section = 'BIGINT\n'
test_section = defaultdict(<class 'str'>, {'QUERY': 'SELECT count(*)
FROM $DATABASE.ins_no_limit.`files` WHERE content = 0;\n', 'RESULTS': '12\n',
'TYPES': 'BIGINT\n'})
type_section = 'TYPES'
update_section = False
vector = "table_format: parquet/none" | "protocol: hs2" |
"exec_option: {'abort_on_error': 1, 'batch_size': 0, 'disable_codegen': False,
'disable_codegen_rows_threshold': 5000, 'exec_single_node_rows_threshold': 0,
'num_nodes': 0, 'test_replan': 1}"
verifier = None
common/test_result_verifier.py:296: in verify_query_result_is_equal
assert expected_results == actual_results
E assert Comparing QueryTestResults (expected vs actual):
E 12 != 11
actual_results = <tests.common.test_result_verifier.QueryTestResult
object at 0x7fc7387cea10>
expected_results = <tests.common.test_result_verifier.QueryTestResult
object at 0x7fc7387ce310>
{noformat}
--
This message was sent by Atlassian Jira
(v10.3.26#10030026)
