Indhumathi Muthumurugesh created HIVE-30043:
-----------------------------------------------
Summary: FULL OUTER MapJoin (dynamic partitioned hash join) throws
ClassCastException: LongWritable cannot be cast to ShortWritable
Key: HIVE-30043
URL: https://issues.apache.org/jira/browse/HIVE-30043
Project: Hive
Issue Type: Bug
Reporter: Indhumathi Muthumurugesh
Assignee: Indhumathi Muthumurugesh
With {{hive.optimize.dynamic.partition.hashjoin=true}} and a query shape that
forces row-mode (non-vectorized) execution of a {{MapJoinOperator}}
implementing a {{{}FULL OUTER JOIN{}}}, the query fails with:
{code:java}
java.lang.ClassCastException: class org.apache.hadoop.io.LongWritable cannot be
cast to
class org.apache.hadoop.hive.serde2.io.ShortWritable {code}
The crash lands on a BIGINT column (derived from count(1)) that is being
misinterpreted as SMALLINT. The same query, same data, and same settings run
correctly when the plan is fully vectorized instead
(hive.vectorized.adaptor.usage.mode=all), or when the
dynamic-partitioned-hash-join conversion is disabled.
h2. Stack Trace
{code:java}
ERROR : FAILED: Execution Error, return code 2 from
org.apache.hadoop.hive.ql.exec.tez.TezTask.Vertex failed, vertexName=Reducer 8,
...Caused by: java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:313)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:291)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:293)
... 16 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException:
Hive Runtime Error while processing row at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:387)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:303)
... 18 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException:
Unexpected exception from MapJoinOperator : Error evaluating if(_col22 is not
null, _col22, 0L) at
org.apache.hadoop.hive.ql.exec.MapJoinOperator.process(MapJoinOperator.java:599)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:372)
... 19 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException:
Error evaluating if(_col22 is not null, _col22, 0L) at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:92)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:888) at
org.apache.hadoop.hive.ql.exec.CommonJoinOperator.internalForward(CommonJoinOperator.java:889)
at
org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:921)
at
org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:1015)
at
org.apache.hadoop.hive.ql.exec.MapJoinOperator.process(MapJoinOperator.java:586)
... 20 moreCaused by: java.lang.ClassCastException: class
org.apache.hadoop.io.LongWritable cannot be cast to class
org.apache.hadoop.hive.serde2.io.ShortWritable
(org.apache.hadoop.io.LongWritable and
org.apache.hadoop.hive.serde2.io.ShortWritable are in unnamed module of loader
'app') at
org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector.get(WritableShortObjectInspector.java:36)
at
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.getLong(PrimitiveObjectInspectorUtils.java:777)
at
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter$LongConverter.convert(PrimitiveObjectInspectorConverter.java:183)
at
org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils$ReturnObjectInspectorResolver.convertIfNecessary(GenericUDFUtils.java:247)
at
org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils$ReturnObjectInspectorResolver.convertIfNecessary(GenericUDFUtils.java:213)
at
org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf.evaluate(GenericUDFIf.java:168)
at
org.apache.hadoop.hive.ql.exec.ExprNodeGenericFuncEvaluator._evaluate(ExprNodeGenericFuncEvaluator.java:231)
at
org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:80)
at
org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorHead._evaluate(ExprNodeEvaluatorHead.java:44)
at
org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:80)
at
org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:68)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:87)
... 25 more {code}
h2. Steps to Reproduce
{code:java}
set hive.execution.engine=tez;
set hive.auto.convert.join=true;
set hive.optimize.dynamic.partition.hashjoin=true;
set hive.vectorized.adaptor.usage.mode=chosen;DROP TABLE IF EXISTS t_base;
CREATE TABLE t_base (
entity_id STRING,
proc_type STRING,
event_dt STRING,
code_list STRING,
notif_cd STRING,
category_cd STRING,
status_cd STRING,
amount_usd DOUBLE,
flag_ind STRING,
flag_cnt SMALLINT
) STORED AS ORC;INSERT INTO TABLE t_base VALUES
('BIN001','EDIT','20260101','045','1','0','C',100.00,'N',CAST(0 AS SMALLINT)),
('BIN001','EDIT','20260101','045','1','1','C',200.00,'N',CAST(0 AS SMALLINT)),
('BIN001','EDIT','20260101','000','3','2','W',50.00,'N',CAST(0 AS SMALLINT)),
('BIN001','EDIT','20260101','000','1','0','A',75.00,'N',CAST(0 AS SMALLINT)),
('BIN001','EDIT','20260101','000','1','3','S',10.00,'N',CAST(0 AS SMALLINT)),
('BIN002','EDIT','20260101','000','1','1','C',300.00,'N',CAST(0 AS SMALLINT)),
('BIN002','EDIT','20260101','000','1','0','U',20.00,'N',CAST(0 AS SMALLINT)),
('BIN002','EDIT','20260101','000','1','4','T',5.00,'N',CAST(0 AS SMALLINT)),
('BIN003','EDIT','20260101','000','1','5','C',400.00,'Y',CAST(1 AS SMALLINT)),
('BIN003','EDIT','20260101','000','1','6','C',150.00,'Y',CAST(1 AS SMALLINT)),
('BIN003','EDIT','20260101','000','1','0','A',60.00,'N',CAST(0 AS SMALLINT)),
('BIN003','EDIT','20260101','000','1','1','W',30.00,'N',CAST(0 AS
SMALLINT));CREATE TEMPORARY FUNCTION test_uuid AS
'org.apache.hadoop.hive.ql.udf.UDFUUID';SELECT coalesce(join_a.entity_id,
join_b.entity_id) as entity_id,
coalesce(join_a.warn_code_1, '000') as warn_code_1,
coalesce(join_a.warn_1_total, 0) as warn_1_total,
coalesce(join_a.notif_1_total, 0) as notif_1_total,
coalesce(join_b.cat_0_tran_count, 0) as cat_0_tran_count,
coalesce(if(join_b.cat_0_dollar_total > 9999999999.99, 9999999999.99,
join_b.cat_0_dollar_total), 0) as cat_0_dollar_total,
COALESCE(join_b.flag_cnt, 0) as flag_cnt,
test_uuid() as row_id
FROM
(select warn_agg.entity_id,
warn_agg.warn_code_1,
warn_agg.warn_1_total,
notif_agg.notif_1_total
from
(Select d.entity_id,
COALESCE(d.code_list[0], '000') as warn_code_1,
cast(COALESCE(d.count_list[0], 0) as int) as warn_1_total
from
(SELECT c.entity_id, collect_list(c.code) as code_list,
collect_list(c.count) as count_list
FROM
(select b.entity_id, b.code, count(1) as count
from (
select entity_id, trim(code) as code
from (
select entity_id,
split(CONCAT(SUBSTR(RPAD(TRIM(regexp_replace(code_list,'=','0')),6,'0'),0,3),'-',SUBSTR(RPAD(TRIM(regexp_replace(code_list,'=','0')),6,'0'),4,3)),
'-') as code_array
from t_base where proc_type = 'EDIT' and event_dt
in ('20260101')) a
lateral view explode(a.code_array) exploded as code) b
where b.code != '000'
group by b.entity_id, b.code) c
GROUP BY c.entity_id order by c.entity_id) d) warn_agg
join
(Select e.entity_id,
cast(COALESCE(e.count_list[0], 0) as int) as notif_1_total
from
(SELECT d.entity_id, collect_list(d.notif_cd) as notif_cd_list,
collect_list(d.count) as count_list
FROM
(select b.entity_id, b.notif_cd, COALESCE(a.count, 0) as
count
from
(select entity_id, trim(notif_cd) as notif_cd
from (
select distinct entity_id,
split(CONCAT(SUBSTR('13',1,1),'-',SUBSTR('13',2,1)), '-') as notif_cd_array
from t_base where proc_type = 'EDIT' and event_dt
in ('20260101')) a
lateral view explode(a.notif_cd_array) exploded as
notif_cd) b
left outer join
(SELECT c.entity_id, c.notif_cd, count(1) as count
FROM (select entity_id, notif_cd from t_base
where proc_type = 'EDIT' and event_dt in
('20260101')) c
group by c.entity_id, c.notif_cd order by c.notif_cd) a
on a.entity_id = b.entity_id and a.notif_cd = b.notif_cd
order by b.entity_id, b.notif_cd) d
GROUP BY d.entity_id) e) notif_agg
on warn_agg.entity_id = notif_agg.entity_id) join_a
full outer join
(Select d.entity_id,
cast(d.count_list[0] as int) as cat_0_tran_count,
cast(d.amount_usd_list[0] as double) as cat_0_dollar_total,
flag_agg.flag_cnt
FROM
(SELECT c.entity_id, collect_list(c.category_code) as
category_code_list, collect_list(c.count) as count_list,
collect_list(cast(c.amount_usd as string)) as amount_usd_list
FROM
(select b.entity_id, b.category_code, COALESCE(a.count, 0) as
count, COALESCE(a.amount_usd, 0.0) as amount_usd
from
(select entity_id, trim(category_code) as category_code
from (
select distinct entity_id,
split(CONCAT(SUBSTR('01',1,1),'-',SUBSTR('01',2,1)), '-') as category_code_array
from t_base where proc_type = 'EDIT' and event_dt in
('20260101')) a
lateral view explode(a.category_code_array) exploded as
category_code) b
left outer join
(select entity_id, category_cd, count(1) as count,
sum(case when status_cd in ('C','W','A','S','U','T')
then amount_usd else 0 end) as amount_usd
from t_base where proc_type = 'EDIT' and event_dt in
('20260101')
group by entity_id, category_cd order by category_cd) a
on a.entity_id = b.entity_id and a.category_cd = b.category_code
order by b.entity_id, b.category_code) c
GROUP BY c.entity_id) d
left outer join
(select entity_id, count(1) as flag_cnt from t_base
where proc_type = 'EDIT' and event_dt in ('20260101') and flag_ind =
'Y'
group by entity_id) flag_agg
on d.entity_id = flag_agg.entity_id
) join_b
on join_a.entity_id = join_b.entity_id; {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
