[
https://issues.apache.org/jira/browse/HIVE-30043?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
ASF GitHub Bot updated HIVE-30043:
----------------------------------
Labels: pull-request-available (was: )
> FULL OUTER MapJoin (dynamic partitioned hash join) throws ClassCastException:
> LongWritable cannot be cast to ShortWritable
> --------------------------------------------------------------------------------------------------------------------------
>
> Key: HIVE-30043
> URL: https://issues.apache.org/jira/browse/HIVE-30043
> Project: Hive
> Issue Type: Bug
> Reporter: Indhumathi Muthumurugesh
> Assignee: Indhumathi Muthumurugesh
> Priority: Major
> Labels: pull-request-available
>
> With {{hive.optimize.dynamic.partition.hashjoin=true}} and a query shape that
> forces row-mode (non-vectorized) execution of a {{MapJoinOperator}}
> implementing a {{{}FULL OUTER JOIN{}}}, the query fails with:
> {code:java}
> java.lang.ClassCastException: class org.apache.hadoop.io.LongWritable cannot
> be cast to
> class org.apache.hadoop.hive.serde2.io.ShortWritable {code}
> The crash lands on a BIGINT column (derived from count(1)) that is being
> misinterpreted as SMALLINT. The same query, same data, and same settings run
> correctly when the plan is fully vectorized instead
> (hive.vectorized.adaptor.usage.mode=all), or when the
> dynamic-partitioned-hash-join conversion is disabled.
> h2. Stack Trace
> {code:java}
> ERROR : FAILED: Execution Error, return code 2 from
> org.apache.hadoop.hive.ql.exec.tez.TezTask.Vertex failed, vertexName=Reducer
> 8, ...Caused by: java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:313)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:291)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:293)
> ... 16 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException:
> Hive Runtime Error while processing row at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:387)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:303)
> ... 18 moreCaused by:
> org.apache.hadoop.hive.ql.metadata.HiveException: Unexpected exception from
> MapJoinOperator : Error evaluating if(_col22 is not null, _col22, 0L) at
> org.apache.hadoop.hive.ql.exec.MapJoinOperator.process(MapJoinOperator.java:599)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:372)
> ... 19 moreCaused by:
> org.apache.hadoop.hive.ql.metadata.HiveException: Error evaluating if(_col22
> is not null, _col22, 0L) at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:92)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:888)
> at
> org.apache.hadoop.hive.ql.exec.CommonJoinOperator.internalForward(CommonJoinOperator.java:889)
> at
> org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:921)
> at
> org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:1015)
> at
> org.apache.hadoop.hive.ql.exec.MapJoinOperator.process(MapJoinOperator.java:586)
> ... 20 moreCaused by: java.lang.ClassCastException: class
> org.apache.hadoop.io.LongWritable cannot be cast to class
> org.apache.hadoop.hive.serde2.io.ShortWritable
> (org.apache.hadoop.io.LongWritable and
> org.apache.hadoop.hive.serde2.io.ShortWritable are in unnamed module of
> loader 'app') at
> org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector.get(WritableShortObjectInspector.java:36)
> at
> org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.getLong(PrimitiveObjectInspectorUtils.java:777)
> at
> org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter$LongConverter.convert(PrimitiveObjectInspectorConverter.java:183)
> at
> org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils$ReturnObjectInspectorResolver.convertIfNecessary(GenericUDFUtils.java:247)
> at
> org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils$ReturnObjectInspectorResolver.convertIfNecessary(GenericUDFUtils.java:213)
> at
> org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf.evaluate(GenericUDFIf.java:168)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeGenericFuncEvaluator._evaluate(ExprNodeGenericFuncEvaluator.java:231)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:80)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorHead._evaluate(ExprNodeEvaluatorHead.java:44)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:80)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:68)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:87)
> ... 25 more {code}
> h2. Steps to Reproduce
> {code:java}
> set hive.execution.engine=tez;
> set hive.auto.convert.join=true;
> set hive.optimize.dynamic.partition.hashjoin=true;
> set hive.vectorized.adaptor.usage.mode=chosen;DROP TABLE IF EXISTS t_base;
> CREATE TABLE t_base (
> entity_id STRING,
> proc_type STRING,
> event_dt STRING,
> code_list STRING,
> notif_cd STRING,
> category_cd STRING,
> status_cd STRING,
> amount_usd DOUBLE,
> flag_ind STRING,
> flag_cnt SMALLINT
> ) STORED AS ORC;INSERT INTO TABLE t_base VALUES
> ('BIN001','EDIT','20260101','045','1','0','C',100.00,'N',CAST(0 AS
> SMALLINT)),
> ('BIN001','EDIT','20260101','045','1','1','C',200.00,'N',CAST(0 AS
> SMALLINT)),
> ('BIN001','EDIT','20260101','000','3','2','W',50.00,'N',CAST(0 AS
> SMALLINT)),
> ('BIN001','EDIT','20260101','000','1','0','A',75.00,'N',CAST(0 AS
> SMALLINT)),
> ('BIN001','EDIT','20260101','000','1','3','S',10.00,'N',CAST(0 AS
> SMALLINT)),
> ('BIN002','EDIT','20260101','000','1','1','C',300.00,'N',CAST(0 AS
> SMALLINT)),
> ('BIN002','EDIT','20260101','000','1','0','U',20.00,'N',CAST(0 AS
> SMALLINT)),
> ('BIN002','EDIT','20260101','000','1','4','T',5.00,'N',CAST(0 AS SMALLINT)),
> ('BIN003','EDIT','20260101','000','1','5','C',400.00,'Y',CAST(1 AS
> SMALLINT)),
> ('BIN003','EDIT','20260101','000','1','6','C',150.00,'Y',CAST(1 AS
> SMALLINT)),
> ('BIN003','EDIT','20260101','000','1','0','A',60.00,'N',CAST(0 AS
> SMALLINT)),
> ('BIN003','EDIT','20260101','000','1','1','W',30.00,'N',CAST(0 AS
> SMALLINT));CREATE TEMPORARY FUNCTION test_uuid AS
> 'org.apache.hadoop.hive.ql.udf.UDFUUID';SELECT coalesce(join_a.entity_id,
> join_b.entity_id) as entity_id,
> coalesce(join_a.warn_code_1, '000') as warn_code_1,
> coalesce(join_a.warn_1_total, 0) as warn_1_total,
> coalesce(join_a.notif_1_total, 0) as notif_1_total,
> coalesce(join_b.cat_0_tran_count, 0) as cat_0_tran_count,
> coalesce(if(join_b.cat_0_dollar_total > 9999999999.99, 9999999999.99,
> join_b.cat_0_dollar_total), 0) as cat_0_dollar_total,
> COALESCE(join_b.flag_cnt, 0) as flag_cnt,
> test_uuid() as row_id
> FROM
> (select warn_agg.entity_id,
> warn_agg.warn_code_1,
> warn_agg.warn_1_total,
> notif_agg.notif_1_total
> from
> (Select d.entity_id,
> COALESCE(d.code_list[0], '000') as warn_code_1,
> cast(COALESCE(d.count_list[0], 0) as int) as warn_1_total
> from
> (SELECT c.entity_id, collect_list(c.code) as code_list,
> collect_list(c.count) as count_list
> FROM
> (select b.entity_id, b.code, count(1) as count
> from (
> select entity_id, trim(code) as code
> from (
> select entity_id,
> split(CONCAT(SUBSTR(RPAD(TRIM(regexp_replace(code_list,'=','0')),6,'0'),0,3),'-',SUBSTR(RPAD(TRIM(regexp_replace(code_list,'=','0')),6,'0'),4,3)),
> '-') as code_array
> from t_base where proc_type = 'EDIT' and event_dt
> in ('20260101')) a
> lateral view explode(a.code_array) exploded as code) b
> where b.code != '000'
> group by b.entity_id, b.code) c
> GROUP BY c.entity_id order by c.entity_id) d) warn_agg
> join
> (Select e.entity_id,
> cast(COALESCE(e.count_list[0], 0) as int) as notif_1_total
> from
> (SELECT d.entity_id, collect_list(d.notif_cd) as notif_cd_list,
> collect_list(d.count) as count_list
> FROM
> (select b.entity_id, b.notif_cd, COALESCE(a.count, 0) as
> count
> from
> (select entity_id, trim(notif_cd) as notif_cd
> from (
> select distinct entity_id,
> split(CONCAT(SUBSTR('13',1,1),'-',SUBSTR('13',2,1)), '-') as notif_cd_array
> from t_base where proc_type = 'EDIT' and
> event_dt in ('20260101')) a
> lateral view explode(a.notif_cd_array) exploded as
> notif_cd) b
> left outer join
> (SELECT c.entity_id, c.notif_cd, count(1) as count
> FROM (select entity_id, notif_cd from t_base
> where proc_type = 'EDIT' and event_dt in
> ('20260101')) c
> group by c.entity_id, c.notif_cd order by
> c.notif_cd) a
> on a.entity_id = b.entity_id and a.notif_cd = b.notif_cd
> order by b.entity_id, b.notif_cd) d
> GROUP BY d.entity_id) e) notif_agg
> on warn_agg.entity_id = notif_agg.entity_id) join_a
> full outer join
> (Select d.entity_id,
> cast(d.count_list[0] as int) as cat_0_tran_count,
> cast(d.amount_usd_list[0] as double) as cat_0_dollar_total,
> flag_agg.flag_cnt
> FROM
> (SELECT c.entity_id, collect_list(c.category_code) as
> category_code_list, collect_list(c.count) as count_list,
> collect_list(cast(c.amount_usd as string)) as amount_usd_list
> FROM
> (select b.entity_id, b.category_code, COALESCE(a.count, 0) as
> count, COALESCE(a.amount_usd, 0.0) as amount_usd
> from
> (select entity_id, trim(category_code) as category_code
> from (
> select distinct entity_id,
> split(CONCAT(SUBSTR('01',1,1),'-',SUBSTR('01',2,1)), '-') as
> category_code_array
> from t_base where proc_type = 'EDIT' and event_dt in
> ('20260101')) a
> lateral view explode(a.category_code_array) exploded as
> category_code) b
> left outer join
> (select entity_id, category_cd, count(1) as count,
> sum(case when status_cd in
> ('C','W','A','S','U','T') then amount_usd else 0 end) as amount_usd
> from t_base where proc_type = 'EDIT' and event_dt in
> ('20260101')
> group by entity_id, category_cd order by category_cd) a
> on a.entity_id = b.entity_id and a.category_cd =
> b.category_code
> order by b.entity_id, b.category_code) c
> GROUP BY c.entity_id) d
> left outer join
> (select entity_id, count(1) as flag_cnt from t_base
> where proc_type = 'EDIT' and event_dt in ('20260101') and flag_ind
> = 'Y'
> group by entity_id) flag_agg
> on d.entity_id = flag_agg.entity_id
> ) join_b
> on join_a.entity_id = join_b.entity_id; {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
