[
https://issues.apache.org/jira/browse/HIVE-22435?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Krisztian Kasa updated HIVE-22435:
----------------------------------
Description:
Steps to reproduce:
1. Apply the attached patch
{code:java}
git apply -3 -p0
{code}
HIVE-20150
{code:java}
.15.patch{code}
2. rebuild project
{code:java}
mvn clean install -DskipTests
{code}
3. Run the following test
{code:java}
mvn test -DskipSparkTests -Dtest=TestMiniLlapLocalCliDriver
-Dqfile=limit_pushdown3.q -pl itests/qtest -Pitests
{code}
Query execution fails with exception
{code:java}
select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint
order by ctinyint limit 20
{code}
{code:java}
[ERROR] Failures:
[ERROR] TestMiniLlapLocalCliDriver.testCliDriver:59 Client execution failed
with error code = 2
running
select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint
order by ctinyint limit 20
fname=limit_pushdown3.q
See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or
check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/
for specific test cases logs.
org.apache.hadoop.hive.ql.metadata.HiveException: Vertex failed,
vertexName=Reducer 2, vertexId=vertex_1572454329409_0001_9_01,
diagnostics=[Task failed, taskId=task_1572454329409_0001_9_01_000000,
diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task (
failure ) :
attempt_1572454329409_0001_9_01_000000_0:java.lang.RuntimeException:
java.lang.RuntimeException: cannot find field key from [0:key._col0,
1:key._col1]
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:296)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:250)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:374)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1682)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at
org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:118)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: cannot find field key from [0:key._col0,
1:key._col1]
at
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:538)
at
org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:153)
at
org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:80)
at
org.apache.hadoop.hive.ql.exec.TopNKeyOperator.initializeOp(TopNKeyOperator.java:106)
at
org.apache.hadoop.hive.ql.exec.vector.VectorTopNKeyOperator.initializeOp(VectorTopNKeyOperator.java:71)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:191)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:266)
... 15 more
], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
attempt_1572454329409_0001_9_01_000000_1:java.lang.RuntimeException:
java.lang.RuntimeException: cannot find field key from [0:key._col0,
1:key._col1]
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:296)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:250)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:374)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1682)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at
org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:118)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: cannot find field key from [0:key._col0,
1:key._col1]
at
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:538)
at
org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:153)
at
org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:80)
at
org.apache.hadoop.hive.ql.exec.TopNKeyOperator.initializeOp(TopNKeyOperator.java:106)
at
org.apache.hadoop.hive.ql.exec.vector.VectorTopNKeyOperator.initializeOp(VectorTopNKeyOperator.java:71)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:191)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:266)
... 15 more
]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
killedTasks:0, Vertex vertex_1572454329409_0001_9_01 [Reducer 2] killed/failed
due to:OWN_TASK_FAILURE]Vertex killed, vertexName=Reducer 3,
vertexId=vertex_1572454329409_0001_9_02, diagnostics=[Vertex received Kill
while in RUNNING state., Vertex did not succeed due to OTHER_VERTEX_FAILURE,
failedTasks:0 killedTasks:1, Vertex vertex_1572454329409_0001_9_02 [Reducer 3]
killed/failed due to:OTHER_VERTEX_FAILURE]DAG did not succeed due to
VERTEX_FAILURE. failedVertices:1 killedVertices:1
at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:242)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:212)
at
org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2129)
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1784)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1456)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1230)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1224)
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:164)
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:230)
at
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:249)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:193)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:415)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:346)
at
org.apache.hadoop.hive.ql.QTestUtil.executeClientInternal(QTestUtil.java:684)
at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:654)
at
org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:183)
at
org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:104)
at
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver(TestMiniLlapLocalCliDriver.java:59)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.apache.hadoop.hive.cli.control.CliAdapter$2$1.evaluate(CliAdapter.java:92)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at org.junit.runners.Suite.runChild(Suite.java:127)
at org.junit.runners.Suite.runChild(Suite.java:26)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at
org.apache.hadoop.hive.cli.control.CliAdapter$1$1.evaluate(CliAdapter.java:73)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:379)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:340)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:125)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:413)
{code}
VectorTopNKeyOperator extends TopNKeyOperator and it calls it's
super.initializeOp method
[https://github.com/apache/hive/blob/5c8392468cb581f53b6cb55d201fc933dca025e3/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorTopNKeyOperator.java#L71]
which is focusing on non-vectorized execution.
cc. [~jcamachorodriguez]
was:
Steps to reproduce:
1. Apply the attached patch
{code}
git apply -3 -p0 HIVE-20150.15.patch
{code}
2. rebuild project
{code}
mvn clean install -DskipTests
{code}
3. Run the following test
{code}
mvn test -DskipSparkTests -Dtest=TestMiniLlapLocalCliDriver
-Dqfile=limit_pushdown3.q -pl itests/qtest -Pitests
{code}
Query execution fails with exception
{code}
select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint
order by ctinyint limit 20
{code}
{code}
[ERROR] Failures:
[ERROR] TestMiniLlapLocalCliDriver.testCliDriver:59 Client execution failed
with error code = 2
running
select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint
order by ctinyint limit 20
fname=limit_pushdown3.q
See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or
check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/
for specific test cases logs.
org.apache.hadoop.hive.ql.metadata.HiveException: Vertex failed,
vertexName=Reducer 2, vertexId=vertex_1572454329409_0001_9_01,
diagnostics=[Task failed, taskId=task_1572454329409_0001_9_01_000000,
diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task (
failure ) :
attempt_1572454329409_0001_9_01_000000_0:java.lang.RuntimeException:
java.lang.RuntimeException: cannot find field key from [0:key._col0,
1:key._col1]
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:296)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:250)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:374)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1682)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at
org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:118)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: cannot find field key from [0:key._col0,
1:key._col1]
at
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:538)
at
org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:153)
at
org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:80)
at
org.apache.hadoop.hive.ql.exec.TopNKeyOperator.initializeOp(TopNKeyOperator.java:106)
at
org.apache.hadoop.hive.ql.exec.vector.VectorTopNKeyOperator.initializeOp(VectorTopNKeyOperator.java:71)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:191)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:266)
... 15 more
], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
attempt_1572454329409_0001_9_01_000000_1:java.lang.RuntimeException:
java.lang.RuntimeException: cannot find field key from [0:key._col0,
1:key._col1]
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:296)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:250)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:374)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1682)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at
org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:118)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: cannot find field key from [0:key._col0,
1:key._col1]
at
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:538)
at
org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:153)
at
org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:80)
at
org.apache.hadoop.hive.ql.exec.TopNKeyOperator.initializeOp(TopNKeyOperator.java:106)
at
org.apache.hadoop.hive.ql.exec.vector.VectorTopNKeyOperator.initializeOp(VectorTopNKeyOperator.java:71)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:191)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:266)
... 15 more
]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
killedTasks:0, Vertex vertex_1572454329409_0001_9_01 [Reducer 2] killed/failed
due to:OWN_TASK_FAILURE]Vertex killed, vertexName=Reducer 3,
vertexId=vertex_1572454329409_0001_9_02, diagnostics=[Vertex received Kill
while in RUNNING state., Vertex did not succeed due to OTHER_VERTEX_FAILURE,
failedTasks:0 killedTasks:1, Vertex vertex_1572454329409_0001_9_02 [Reducer 3]
killed/failed due to:OTHER_VERTEX_FAILURE]DAG did not succeed due to
VERTEX_FAILURE. failedVertices:1 killedVertices:1
at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:242)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:212)
at
org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2129)
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1784)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1456)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1230)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1224)
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:164)
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:230)
at
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:249)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:193)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:415)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:346)
at
org.apache.hadoop.hive.ql.QTestUtil.executeClientInternal(QTestUtil.java:684)
at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:654)
at
org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:183)
at
org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:104)
at
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver(TestMiniLlapLocalCliDriver.java:59)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.apache.hadoop.hive.cli.control.CliAdapter$2$1.evaluate(CliAdapter.java:92)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at org.junit.runners.Suite.runChild(Suite.java:127)
at org.junit.runners.Suite.runChild(Suite.java:26)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at
org.apache.hadoop.hive.cli.control.CliAdapter$1$1.evaluate(CliAdapter.java:73)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:379)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:340)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:125)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:413)
{code}
VectorTopNKeyOperator extends TopNKeyOperator and it calls it's
super.initializeOp method
https://github.com/apache/hive/blob/5c8392468cb581f53b6cb55d201fc933dca025e3/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorTopNKeyOperator.java#L71
which is focusing on non-vectorized execution.
> Exception when using VectorTopNKeyOperator operator
> ---------------------------------------------------
>
> Key: HIVE-22435
> URL: https://issues.apache.org/jira/browse/HIVE-22435
> Project: Hive
> Issue Type: Bug
> Reporter: Krisztian Kasa
> Priority: Major
> Attachments: HIVE-20150.15.patch
>
>
> Steps to reproduce:
> 1. Apply the attached patch
> {code:java}
> git apply -3 -p0
> {code}
> HIVE-20150
> {code:java}
> .15.patch{code}
> 2. rebuild project
> {code:java}
> mvn clean install -DskipTests
> {code}
> 3. Run the following test
> {code:java}
> mvn test -DskipSparkTests -Dtest=TestMiniLlapLocalCliDriver
> -Dqfile=limit_pushdown3.q -pl itests/qtest -Pitests
> {code}
> Query execution fails with exception
> {code:java}
> select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint
> order by ctinyint limit 20
> {code}
> {code:java}
> [ERROR] Failures:
> [ERROR] TestMiniLlapLocalCliDriver.testCliDriver:59 Client execution failed
> with error code = 2
> running
> select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint
> order by ctinyint limit 20
> fname=limit_pushdown3.q
> See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log,
> or check ./ql/target/surefire-reports or
> ./itests/qtest/target/surefire-reports/ for specific test cases logs.
> org.apache.hadoop.hive.ql.metadata.HiveException: Vertex failed,
> vertexName=Reducer 2, vertexId=vertex_1572454329409_0001_9_01,
> diagnostics=[Task failed, taskId=task_1572454329409_0001_9_01_000000,
> diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task (
> failure ) :
> attempt_1572454329409_0001_9_01_000000_0:java.lang.RuntimeException:
> java.lang.RuntimeException: cannot find field key from [0:key._col0,
> 1:key._col1]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:296)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:250)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:374)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1682)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at
> org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:118)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.RuntimeException: cannot find field key from
> [0:key._col0, 1:key._col1]
> at
> org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:538)
> at
> org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:153)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:80)
> at
> org.apache.hadoop.hive.ql.exec.TopNKeyOperator.initializeOp(TopNKeyOperator.java:106)
> at
> org.apache.hadoop.hive.ql.exec.vector.VectorTopNKeyOperator.initializeOp(VectorTopNKeyOperator.java:71)
> at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:191)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:266)
> ... 15 more
> ], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
> attempt_1572454329409_0001_9_01_000000_1:java.lang.RuntimeException:
> java.lang.RuntimeException: cannot find field key from [0:key._col0,
> 1:key._col1]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:296)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:250)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:374)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1682)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at
> org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:118)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.RuntimeException: cannot find field key from
> [0:key._col0, 1:key._col1]
> at
> org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:538)
> at
> org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:153)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:80)
> at
> org.apache.hadoop.hive.ql.exec.TopNKeyOperator.initializeOp(TopNKeyOperator.java:106)
> at
> org.apache.hadoop.hive.ql.exec.vector.VectorTopNKeyOperator.initializeOp(VectorTopNKeyOperator.java:71)
> at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:191)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:266)
> ... 15 more
> ]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
> killedTasks:0, Vertex vertex_1572454329409_0001_9_01 [Reducer 2]
> killed/failed due to:OWN_TASK_FAILURE]Vertex killed, vertexName=Reducer 3,
> vertexId=vertex_1572454329409_0001_9_02, diagnostics=[Vertex received Kill
> while in RUNNING state., Vertex did not succeed due to OTHER_VERTEX_FAILURE,
> failedTasks:0 killedTasks:1, Vertex vertex_1572454329409_0001_9_02 [Reducer
> 3] killed/failed due to:OTHER_VERTEX_FAILURE]DAG did not succeed due to
> VERTEX_FAILURE. failedVertices:1 killedVertices:1
> at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:242)
> at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:212)
> at
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
> at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2129)
> at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1784)
> at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1456)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1230)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1224)
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:164)
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:230)
> at
> org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:249)
> at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:193)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:415)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:346)
> at
> org.apache.hadoop.hive.ql.QTestUtil.executeClientInternal(QTestUtil.java:684)
> at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:654)
> at
> org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:183)
> at
> org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:104)
> at
> org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver(TestMiniLlapLocalCliDriver.java:59)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.apache.hadoop.hive.cli.control.CliAdapter$2$1.evaluate(CliAdapter.java:92)
> at org.junit.rules.RunRules.evaluate(RunRules.java:20)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
> at org.junit.runners.Suite.runChild(Suite.java:127)
> at org.junit.runners.Suite.runChild(Suite.java:26)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
> at
> org.apache.hadoop.hive.cli.control.CliAdapter$1$1.evaluate(CliAdapter.java:73)
> at org.junit.rules.RunRules.evaluate(RunRules.java:20)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
> at
> org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:379)
> at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:340)
> at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:125)
> at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:413)
> {code}
> VectorTopNKeyOperator extends TopNKeyOperator and it calls it's
> super.initializeOp method
>
> [https://github.com/apache/hive/blob/5c8392468cb581f53b6cb55d201fc933dca025e3/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorTopNKeyOperator.java#L71]
> which is focusing on non-vectorized execution.
> cc. [~jcamachorodriguez]
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
