[
https://issues.apache.org/jira/browse/HIVE-17631?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16257560#comment-16257560
]
Sergey Shelukhin commented on HIVE-17631:
-----------------------------------------
The error is {noformat}
2017-11-17T12:54:53,234 ERROR [Dispatcher thread {Central}] impl.VertexImpl:
Vertex Input: srcpart_date initializer failed,
vertex=vertex_1510952066792_0001_40_02 [Map 1]
org.apache.tez.dag.app.dag.impl.AMUserCodeException:
java.lang.RuntimeException: ORC split generation failed with exception:
java.lang.IllegalStateException: Failed to retrieve dynamic value for
RS_7_srcpart_small_key1_min
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallback.onFailure(RootInputInitializerManager.java:328)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at com.google.common.util.concurrent.Futures$4.run(Futures.java:1140)
~[guava-14.0.1.jar:?]
at
com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:293)
~[guava-14.0.1.jar:?]
at
com.google.common.util.concurrent.ExecutionList$RunnableExecutorPair.execute(ExecutionList.java:150)
~[guava-14.0.1.jar:?]
at
com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:135)
~[guava-14.0.1.jar:?]
at
com.google.common.util.concurrent.ListenableFutureTask.done(ListenableFutureTask.java:91)
~[guava-14.0.1.jar:?]
at
java.util.concurrent.FutureTask.finishCompletion(FutureTask.java:384)
~[?:1.8.0_45]
at java.util.concurrent.FutureTask.setException(FutureTask.java:251)
~[?:1.8.0_45]
at java.util.concurrent.FutureTask.run(FutureTask.java:271)
~[?:1.8.0_45]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
~[?:1.8.0_45]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
~[?:1.8.0_45]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_45]
Caused by: java.lang.RuntimeException: ORC split generation failed with
exception: java.lang.IllegalStateException: Failed to retrieve dynamic value
for RS_7_srcpart_small_key1_min
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.generateSplitsInfo(OrcInputFormat.java:1745)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getSplits(OrcInputFormat.java:1832)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.HiveInputFormat.addSplitsForGroup(HiveInputFormat.java:497)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.HiveInputFormat.getSplits(HiveInputFormat.java:675)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator.initialize(HiveSplitGenerator.java:196)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable$1.run(RootInputInitializerManager.java:278)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable$1.run(RootInputInitializerManager.java:269)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_45]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_45]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1962)
~[hadoop-common-3.0.0-beta1.jar:?]
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable.call(RootInputInitializerManager.java:269)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable.call(RootInputInitializerManager.java:253)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_45]
... 3 more
Caused by: java.util.concurrent.ExecutionException:
java.lang.IllegalStateException: Failed to retrieve dynamic value for
RS_7_srcpart_small_key1_min
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
~[?:1.8.0_45]
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
~[?:1.8.0_45]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.generateSplitsInfo(OrcInputFormat.java:1739)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getSplits(OrcInputFormat.java:1832)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.HiveInputFormat.addSplitsForGroup(HiveInputFormat.java:497)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.HiveInputFormat.getSplits(HiveInputFormat.java:675)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator.initialize(HiveSplitGenerator.java:196)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable$1.run(RootInputInitializerManager.java:278)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable$1.run(RootInputInitializerManager.java:269)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_45]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_45]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1962)
~[hadoop-common-3.0.0-beta1.jar:?]
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable.call(RootInputInitializerManager.java:269)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at
org.apache.tez.dag.app.dag.RootInputInitializerManager$InputInitializerCallable.call(RootInputInitializerManager.java:253)
~[tez-dag-0.9.1-SNAPSHOT.jar:0.9.1-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_45]
... 3 more
Caused by: java.lang.IllegalStateException: Failed to retrieve dynamic value
for RS_7_srcpart_small_key1_min
at
org.apache.hadoop.hive.ql.plan.DynamicValue.getValue(DynamicValue.java:130)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.plan.DynamicValue.getJavaValue(DynamicValue.java:97)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.plan.DynamicValue.getLiteral(DynamicValue.java:93)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl$PredicateLeafImpl.getLiteralList(SearchArgumentImpl.java:120)
~[hive-storage-api-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.orc.impl.RecordReaderImpl.evaluatePredicateMinMax(RecordReaderImpl.java:553)
~[orc-core-1.4.0.jar:1.4.0]
at
org.apache.orc.impl.RecordReaderImpl.evaluatePredicateRange(RecordReaderImpl.java:463)
~[orc-core-1.4.0.jar:1.4.0]
at
org.apache.orc.impl.RecordReaderImpl.evaluatePredicate(RecordReaderImpl.java:440)
~[orc-core-1.4.0.jar:1.4.0]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.isStripeSatisfyPredicate(OrcInputFormat.java:2118)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.pickStripesInternal(OrcInputFormat.java:2097)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.pickStripes(OrcInputFormat.java:2088)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.access$3000(OrcInputFormat.java:153)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.callInternal(OrcInputFormat.java:1440)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.access$2700(OrcInputFormat.java:1225)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator$1.run(OrcInputFormat.java:1409)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator$1.run(OrcInputFormat.java:1406)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_45]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_45]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1962)
~[hadoop-common-3.0.0-beta1.jar:?]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.call(OrcInputFormat.java:1406)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.call(OrcInputFormat.java:1225)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_45]
... 3 more
Caused by: java.lang.IllegalStateException: DynamicValueRegistry not available
at
org.apache.hadoop.hive.ql.plan.DynamicValue.getValue(DynamicValue.java:125)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.plan.DynamicValue.getJavaValue(DynamicValue.java:97)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.plan.DynamicValue.getLiteral(DynamicValue.java:93)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl$PredicateLeafImpl.getLiteralList(SearchArgumentImpl.java:120)
~[hive-storage-api-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.orc.impl.RecordReaderImpl.evaluatePredicateMinMax(RecordReaderImpl.java:553)
~[orc-core-1.4.0.jar:1.4.0]
at
org.apache.orc.impl.RecordReaderImpl.evaluatePredicateRange(RecordReaderImpl.java:463)
~[orc-core-1.4.0.jar:1.4.0]
at
org.apache.orc.impl.RecordReaderImpl.evaluatePredicate(RecordReaderImpl.java:440)
~[orc-core-1.4.0.jar:1.4.0]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.isStripeSatisfyPredicate(OrcInputFormat.java:2118)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.pickStripesInternal(OrcInputFormat.java:2097)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.pickStripes(OrcInputFormat.java:2088)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.access$3000(OrcInputFormat.java:153)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.callInternal(OrcInputFormat.java:1440)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.access$2700(OrcInputFormat.java:1225)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator$1.run(OrcInputFormat.java:1409)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator$1.run(OrcInputFormat.java:1406)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_45]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_45]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1962)
~[hadoop-common-3.0.0-beta1.jar:?]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.call(OrcInputFormat.java:1406)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$SplitGenerator.call(OrcInputFormat.java:1225)
~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_45]
... 3 more
{noformat}
Looks like something changed in ORC split generation that is now causing
dynamic values to be queries.
[~jdere] does this ring a bell? Should dynamic values be accessed during split
generation?
> upgrade orc to 1.4.0
> --------------------
>
> Key: HIVE-17631
> URL: https://issues.apache.org/jira/browse/HIVE-17631
> Project: Hive
> Issue Type: Improvement
> Affects Versions: 3.0.0
> Reporter: Saijin Huang
> Assignee: Sergey Shelukhin
> Attachments: HIVE-17631.patch
>
>
> It seems like orc 1.4.0 has a latest and stable version:
> https://orc.apache.org/docs/releases.html
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
