bharath v created IMPALA-7983:
---------------------------------
Summary: Failed verification/decryption while reading spilled data
causing test failure
Key: IMPALA-7983
URL: https://issues.apache.org/jira/browse/IMPALA-7983
Project: IMPALA
Issue Type: Bug
Components: Backend
Affects Versions: Impala 3.1.0
Reporter: bharath v
Failed test:
{noformat}
query_test.test_spilling.TestSpillingDebugActionDimensions.test_spilling[protocol:
beeswax | exec_option: {'debug_action':
'-1:OPEN:[email protected]',
'default_spillable_buffer_size': '256k'} | table_format: parquet/none]
{noformat}
Build type: {{exhaustive/centos6}}
Build logs:
{noformat}
query_test/test_spilling.py:59: in test_spilling
self.run_test_case('QueryTest/spilling', vector)
common/impala_test_suite.py:467: in run_test_case result =
self.__execute_query(target_impalad_client, query, user=user)
common/impala_test_suite.py:688: in __execute_query return
impalad_client.execute(query, user=user) common/impala_connection.py:170: in
execute return self.__beeswax_client.execute(sql_stmt, user=user)
beeswax/impala_beeswax.py:182: in execute handle =
self.__execute_query(query_string.strip(), user=user)
beeswax/impala_beeswax.py:359: in __execute_query
self.wait_for_finished(handle) beeswax/impala_beeswax.py:380: in
wait_for_finished raise ImpalaBeeswaxException("Query aborted:" +
error_log, None) E ImpalaBeeswaxException: ImpalaBeeswaxException: E Query
aborted:Error reading 524288 bytes from scratch file
'/tmp/impala-scratch/4b417a5b42d94b12:2ba2321000000000_d1ff959f-84da-4176-8b29-ff8e1560e941'
on backend shared-centos64-ec2-m2-4xlarge-ondemand-1d0a.vpc.cloudera.com:22001
at offset 806092800: verification of read data failed. E OpenSSL error in
EVP_DecryptFinal decrypting:
{noformat}
Stacktrace
{noformat}
query_test/test_spilling.py:59: in test_spilling
self.run_test_case('QueryTest/spilling', vector)
common/impala_test_suite.py:467: in run_test_case
result = self.__execute_query(target_impalad_client, query, user=user)
common/impala_test_suite.py:688: in __execute_query
return impalad_client.execute(query, user=user)
common/impala_connection.py:170: in execute
return self.__beeswax_client.execute(sql_stmt, user=user)
beeswax/impala_beeswax.py:182: in execute
handle = self.__execute_query(query_string.strip(), user=user)
beeswax/impala_beeswax.py:359: in __execute_query
self.wait_for_finished(handle)
beeswax/impala_beeswax.py:380: in wait_for_finished
raise ImpalaBeeswaxException("Query aborted:" + error_log, None)
E ImpalaBeeswaxException: ImpalaBeeswaxException:
E Query aborted:Error reading 524288 bytes from scratch file
'/tmp/impala-scratch/4b417a5b42d94b12:2ba2321000000000_d1ff959f-84da-4176-8b29-ff8e1560e941'
on backend shared-centos64-ec2-m2-4xlarge-ondemand-1d0a.vpc.cloudera.com:22001
at offset 806092800: verification of read data failed.
E OpenSSL error in EVP_DecryptFinal decrypting:
{noformat}
Build stderr:
{noformat}
-- executing against localhost:21000
use tpch_parquet;
-- 2018-12-14 00:17:21,945 INFO MainThread: Started query
244b8ebf1a985cca:df5b0e3a00000000
SET debug_action=-1:OPEN:[email protected];
SET default_spillable_buffer_size=256k;
-- executing against localhost:21000
set buffer_pool_limit=215m;
-- 2018-12-14 00:17:21,990 INFO MainThread: Started query
1a46eec9e244c88a:49a6b2df00000000
-- executing against localhost:21000
select count(l1.l_tax)
from
lineitem l1,
lineitem l2,
lineitem l3
where
l1.l_tax < 0.01 and
l2.l_tax < 0.04 and
l1.l_orderkey = l2.l_orderkey and
l1.l_orderkey = l3.l_orderkey and
l1.l_comment = l3.l_comment and
l1.l_shipdate = l3.l_shipdate;
-- 2018-12-14 00:17:22,059 INFO MainThread: Started query
ef4546e5750f98b3:d67f771600000000
-- executing against localhost:21000
SET BUFFER_POOL_LIMIT="";
-- 2018-12-14 00:17:47,105 INFO MainThread: Started query
b84dc37674cb5ab9:e080595b00000000
-- executing against localhost:21000
set buffer_pool_limit=50m;
-- 2018-12-14 00:17:47,132 INFO MainThread: Started query
be4b201e2b52176a:2f88f48300000000
-- executing against localhost:21000
select max(t1.total_count), max(t1.l_shipinstruct), max(t1.l_comment) from
(select l_shipinstruct, l_comment, count(*) over () total_count from lineitem)
t1;
-- 2018-12-14 00:17:47,213 INFO MainThread: Started query
1d4f0a32074b6174:5f21f35700000000
-- executing against localhost:21000
SET BUFFER_POOL_LIMIT="";
-- 2018-12-14 00:18:11,983 INFO MainThread: Started query
fa4364af8ab07926:b2cde42000000000
-- executing against localhost:21000
set buffer_pool_limit=20m;
-- 2018-12-14 00:18:11,997 INFO MainThread: Started query
1143a7d66b2b11df:c04d2fdf00000000
-- executing against localhost:21000
select a.int_col, count(*)
from functional.alltypessmall a, functional.alltypessmall b,
functional.alltypessmall c
where a.id = b.id and b.id = c.id group by a.int_col;
-- 2018-12-14 00:18:12,015 INFO MainThread: Started query
c94b5a6145d5b924:60b5939100000000
-- executing against localhost:21000
SET BUFFER_POOL_LIMIT="";
-- 2018-12-14 00:18:12,340 INFO MainThread: Started query
c74fe6cefcd9dd86:64e37ce100000000
-- 2018-12-14 00:18:12,348 INFO MainThread: Query Name:
TPCH-Q21
-- executing against localhost:21000
set buffer_pool_limit=100m;
-- 2018-12-14 00:18:12,356 INFO MainThread: Started query
5f4300a672e4b48f:6c6477ea00000000
-- executing against localhost:21000
select
s_name,
count(*) as numwait
from
supplier,
lineitem l1 join [BROADCAST]
orders,
nation
where
s_suppkey = l1.l_suppkey
and o_orderkey = l1.l_orderkey
and o_orderstatus = 'F'
and l1.l_receiptdate > l1.l_commitdate
and exists (
select
*
from
lineitem l2
where
l2.l_orderkey = l1.l_orderkey
and l2.l_suppkey <> l1.l_suppkey
)
and not exists (
select
*
from
lineitem l3
where
l3.l_orderkey = l1.l_orderkey
and l3.l_suppkey <> l1.l_suppkey
and l3.l_receiptdate > l3.l_commitdate
)
and s_nationkey = n_nationkey
and n_name = 'SAUDI ARABIA'
group by
s_name
order by
numwait desc,
s_name
limit 100;
-- 2018-12-14 00:18:12,452 INFO MainThread: Started query
134aa693a8e4fed5:e4e21c7000000000
-- executing against localhost:21000
SET BUFFER_POOL_LIMIT="";
-- 2018-12-14 00:18:47,992 INFO MainThread: Started query
3c419c8cc9242655:956aae7500000000
-- executing against localhost:21000
set buffer_pool_limit=170m;
-- 2018-12-14 00:18:48,104 INFO MainThread: Started query
8646a7bfbe58b96f:1acdb73000000000
-- executing against localhost:21000
set disable_outermost_topn=1;
-- 2018-12-14 00:18:48,132 INFO MainThread: Started query
f14f5ee0e6988761:7f5815c200000000
-- executing against localhost:21000
select * from lineitem
inner join orders on l_orderkey = o_orderkey
order by l_linenumber, l_suppkey, l_partkey, l_orderkey
limit 20;
-- 2018-12-14 00:18:48,203 INFO MainThread: Started query
4b417a5b42d94b12:2ba2321000000000
-- executing against localhost:21000
SET BUFFER_POOL_LIMIT="";
-- 2018-12-14 00:25:37,632 INFO MainThread: Started query
9d4396efceba3f14:df6e2b6200000000
-- executing against localhost:21000
SET DISABLE_OUTERMOST_TOPN="0";
-- 2018-12-14 00:25:37,707 INFO MainThread: Started query
1244360bf6a6b531:bfb5d65b00000000
{noformat}
Couldn't find a smoking gun in the logs.
{noformat}
I1214 00:25:37.534629 31548 status.cc:128] OpenSSL error in EVP_DecryptFinal
decrypting:
@ 0x1a36b7a
@ 0x21843bd
@ 0x2184f15
@ 0x2184697
@ 0x1ef7f95
@ 0x1ef617a
@ 0x21f4162
@ 0x21f071b
@ 0x272446f
@ 0x271bbc9
@ 0x272916f
@ 0x271b4c9
@ 0x27308f3
@ 0x2730587
@ 0x272fe48
@ 0x272f51b
@ 0x1ee06d7
@ 0x1ee026b
@ 0x1edfb7b
@ 0x1edf780
@ 0x2723378
@ 0x2722b49
@ 0x27219c9
@ 0x2371c0d
@ 0x2370853
@ 0x1f1e53f
@ 0x1f1b773
@ 0x1f2c148
@ 0x1f2a451
@ 0x1f2cef0
@ 0x1d44705
@ 0x21ca1c2
I1214 00:25:37.534849 31548 status.cc:77] Error reading 524288 bytes from
scratch file
'/tmp/impala-scratch/4b417a5b42d94b12:2ba2321000000000_d1ff959f-84da-4176-8b29-ff8e1560e941'
on backend shared-centos64-ec2-m2-4xlarge-ondemand-1d0a.vpc.cloudera.com:22001
at offset 806092800: verification of read data failed.
@ 0x1a35d7b
@ 0x1ef8089
@ 0x1ef617a
@ 0x21f4162
@ 0x21f071b
@ 0x272446f
@ 0x271bbc9
@ 0x272916f
@ 0x271b4c9
@ 0x27308f3
@ 0x2730587
@ 0x272fe48
@ 0x272f51b
@ 0x1ee06d7
@ 0x1ee026b
@ 0x1edfb7b
@ 0x1edf780
@ 0x2723378
@ 0x2722b49
@ 0x27219c9
@ 0x2371c0d
@ 0x2370853
@ 0x1f1e53f
@ 0x1f1b773
@ 0x1f2c148
@ 0x1f2a451
@ 0x1f2cef0
@ 0x1d44705
@ 0x21ca1c2
@ 0x21d24e2
@ 0x21d2406
@ 0x21d23c9
I1214 00:25:37.554332 31545 query-state.cc:249] UpdateBackendExecState(): last
report for 4b417a5b42d94b12:2ba2321000000000
I1214 00:25:37.558511 31548 query-state.cc:576] Instance completed.
instance_id=4b417a5b42d94b12:2ba2321000000005 #in-flight=3
status=SCRATCH_READ_VERIFY_FAILED: Error reading 524288 bytes from scratch file
'/tmp/impala-scratch/4b417a5b42d94b12:2ba2321000000000_d1ff959f-84da-4176-8b29-ff8e1560e941'
on backend shared-centos64-ec2-m2-4xlarge-ondemand-1d0a.vpc.cloudera.com:22001
at offset 806092800: verification of read data failed.
OpenSSL error in EVP_DecryptFinal decrypting:
I1214 00:25:37.568645 31545 query-state.cc:383] Cancelling fragment instances
as directed by the coordinator. Returned status: Cancelled
{noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
