[
https://issues.apache.org/jira/browse/HIVE-14564?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15426343#comment-15426343
]
Hive QA commented on HIVE-14564:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12824260/HIVE-14564.000.patch
{color:red}ERROR:{color} -1 due to no test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 132 failed/errored test(s), 10442 tests
executed
*Failed tests:*
{noformat}
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[bucket_groupby]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[columnstats_partlvl]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[columnstats_partlvl_dp]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[columnstats_tbllvl]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[complex_alias]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[constant_prop_3]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[correlationoptimizer13]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[decimal_udf]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[display_colstats_tbllvl]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[dynamic_rdd_cache]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[groupby9]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[groupby_join_pushdown]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[groupby_position]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[having2]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[index_auto_update]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[lateral_view]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[limit_pushdown3]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[limit_pushdown]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[limit_pushdown_negative]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[multi_insert_gby3]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[multi_insert_lateral_view]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[multigroupby_singlemr]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[offset_limit_ppd_optimizer]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_gby2]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_gby]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_windowing1]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ptf]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ptfgroupbyjoin]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[reduce_deduplicate_extended2]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_in_having]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_notexists]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_notexists_having]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_unqualcolumnrefs]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[temp_table_display_colstats_tbllvl]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_aggregate]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_round_2]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_udf]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_groupby_3]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_groupby_reduce]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_interval_2]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_orderby_5]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_0]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_13]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_15]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_limit]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_short_regress]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorized_parquet]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorized_parquet_types]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorized_ptf]
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[windowing_gby2]
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[dynamic_partition_pruning_2]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[explainuser_1]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[explainuser_2]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[limit_pushdown]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[load_dyn_part1]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[load_dyn_part2]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[ptf]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[transform_ppr1]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vector_decimal_aggregate]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vector_decimal_round_2]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vector_decimal_udf]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vector_groupby_3]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vector_groupby_reduce]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vector_interval_2]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vector_orderby_5]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorization_0]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorization_13]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorization_15]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorization_limit]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorization_short_regress]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorized_parquet]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorized_parquet_types]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorized_ptf]
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[windowing_gby]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query12]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query15]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query17]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query18]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query19]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query20]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query22]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query25]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query26]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query27]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query29]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query31]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query39]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query3]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query42]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query45]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query46]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query51]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query52]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query54]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query55]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query58]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query64]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query65]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query66]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query67]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query68]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query70]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query71]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query72]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query79]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query7]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query85]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query87]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query89]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query91]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query92]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query97]
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query98]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[dynamic_rdd_cache]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby9]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby_position]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[limit_pushdown]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multi_insert_gby3]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multi_insert_lateral_view]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multigroupby_singlemr]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[ptf]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vector_decimal_aggregate]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vector_groupby_3]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vector_orderby_5]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_0]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_13]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_15]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_short_regress]
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorized_ptf]
org.apache.hive.beeline.TestBeeLineWithArgs.testEmbeddedBeelineOutputs
org.apache.hive.jdbc.TestJdbcWithMiniHS2.testSelectThriftSerializeInTasks
org.apache.hive.service.cli.operation.TestOperationLoggingLayout.testSwitchLogLayout
{noformat}
Test results:
https://builds.apache.org/job/PreCommit-HIVE-MASTER-Build/925/testReport
Console output:
https://builds.apache.org/job/PreCommit-HIVE-MASTER-Build/925/console
Test logs:
http://ec2-204-236-174-241.us-west-1.compute.amazonaws.com/logs/PreCommit-HIVE-MASTER-Build-925/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 132 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 12824260 - PreCommit-HIVE-MASTER-Build
> Column Pruning generates out of order columns in SelectOperator which cause
> ArrayIndexOutOfBoundsException.
> -----------------------------------------------------------------------------------------------------------
>
> Key: HIVE-14564
> URL: https://issues.apache.org/jira/browse/HIVE-14564
> Project: Hive
> Issue Type: Bug
> Components: Query Planning
> Affects Versions: 2.1.0
> Reporter: zhihai xu
> Assignee: zhihai xu
> Priority: Critical
> Attachments: HIVE-14564.000.patch
>
>
> Column Pruning generates out of order columns in SelectOperator which cause
> ArrayIndexOutOfBoundsException.
> {code}
> 2016-07-26 21:49:24,390 FATAL [main]
> org.apache.hadoop.hive.ql.exec.mr.ExecMapper:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row
> at
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:507)
> at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:170)
> at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:54)
> at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:453)
> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
> at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:415)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException:
> java.lang.ArrayIndexOutOfBoundsException
> at
> org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.processOp(ReduceSinkOperator.java:397)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:815)
> at
> org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:157)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:497)
> ... 9 more
> Caused by: java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at org.apache.hadoop.io.Text.set(Text.java:225)
> at
> org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryString.init(LazyBinaryString.java:48)
> at
> org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct.uncheckedGetField(LazyBinaryStruct.java:264)
> at
> org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct.getField(LazyBinaryStruct.java:201)
> at
> org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector.getStructFieldData(LazyBinaryStructObjectInspector.java:64)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator._evaluate(ExprNodeColumnEvaluator.java:94)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:77)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:65)
> at
> org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.makeValueWritable(ReduceSinkOperator.java:550)
> at
> org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.processOp(ReduceSinkOperator.java:377)
> ... 13 more
> {code}
> The exception is because the serialization and deserialization doesn't match.
> The serialization by LazyBinarySerDe from previous MapReduce job used
> different order of columns. When the current MapReduce job deserialized the
> intermediate sequence file generated by previous MapReduce job, it will get
> corrupted data from the deserialization using wrong order of columns by
> LazyBinaryStruct. The unmatched columns between serialization and
> deserialization is caused by SelectOperator's Column Pruning
> {{ColumnPrunerSelectProc}}.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
