[
https://issues.apache.org/jira/browse/DRILL-4109?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15023323#comment-15023323
]
Deneche A. Hakim commented on DRILL-4109:
-----------------------------------------
DEBUG logs shows the exception is in MergeJoinBatch, and more precisely in
RecordIterator:
{noformat}
org.apache.drill.common.exceptions.UserException: SYSTEM ERROR:
NullPointerException
Fragment 3:4
[Error Id: ba52b9cb-6500-4278-9786-85d71d2654e2 on atsqa4-133.qa.lab:31010]
at
org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:534)
~[drill-common-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.sendFinalState(FragmentExecutor.java:321)
[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.cleanup(FragmentExecutor.java:184)
[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:290)
[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.common.SelfCleaningRunnable.run(SelfCleaningRunnable.java:38)
[drill-common-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[na:1.7.0_71]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[na:1.7.0_71]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_71]
Caused by: java.lang.NullPointerException: null
at
org.apache.drill.exec.record.RecordIterator.mark(RecordIterator.java:96)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.test.generated.JoinWorkerGen152.doJoin(JoinTemplate.java:80)
~[na:na]
at
org.apache.drill.exec.physical.impl.join.MergeJoinBatch.innerNext(MergeJoinBatch.java:206)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:109)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:109)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch.innerNext(RemovingRecordBatch.java:94)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:109)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.innerNext(ProjectRecordBatch.java:132)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:109)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch.innerNext(ExternalSortBatch.java:276)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:109)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch.innerNext(RemovingRecordBatch.java:94)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.RecordIterator.nextBatch(RecordIterator.java:89)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.RecordIterator.next(RecordIterator.java:164)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.RecordIterator.prepare(RecordIterator.java:148)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.join.JoinStatus.prepare(JoinStatus.java:85)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.join.MergeJoinBatch.innerNext(MergeJoinBatch.java:161)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:109)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.innerNext(ProjectRecordBatch.java:132)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.test.generated.HashAggregatorGen55.doWork(HashAggTemplate.java:314)
~[na:na]
at
org.apache.drill.exec.physical.impl.aggregate.HashAggBatch.innerNext(HashAggBatch.java:133)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:109)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.innerNext(ProjectRecordBatch.java:132)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:162)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:104)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.SingleSenderCreator$SingleSenderRootExec.innerNext(SingleSenderCreator.java:93)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:94)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:256)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:250)
~[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[na:1.7.0_71]
at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_71]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1595)
~[hadoop-common-2.7.0-mapr-1506.jar:na]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:250)
[drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
... 4 common frames omitted
{noformat}
> NPE in external sort
> ---------------------
>
> Key: DRILL-4109
> URL: https://issues.apache.org/jira/browse/DRILL-4109
> Project: Apache Drill
> Issue Type: Bug
> Affects Versions: 1.4.0
> Reporter: Victoria Markman
> Priority: Blocker
> Attachments: 29ac6c1b-9b33-3457-8bc8-9e2dff6ad438.sys.drill,
> 29b41f37-4803-d7ce-e05f-912d1f65da79.sys.drill, drillbit.log,
> drillbit.log.debug
>
>
> 4 node cluster
> 36GB of direct memory
> 4GB heap memory
> planner.memory.max_query_memory_per_node=2GB (default)
> planner.enable_hashjoin = false
> Spill directory has 6.4T of memory available:
> {noformat}
> [Tue Nov 17 18:23:18 /tmp/drill ] # df -H .
> Filesystem Size Used Avail Use% Mounted on
> localhost:/mapr 7.7T 1.4T 6.4T 18% /mapr
> {noformat}
> Run query below:
> framework/resources/Advanced/tpcds/tpcds_sf100/original/query15.sql
> drillbit.log
> {code}
> 2015-11-18 02:22:12,639 [29b41f37-4803-d7ce-e05f-912d1f65da79:frag:3:9] INFO
> o.a.d.e.p.i.xsort.ExternalSortBatch - Merging and spilling to
> /tmp/drill/spill/29b41f37-4803-d7ce-e05f-912d1f65da79/major_fragment_3/minor_fragment_9/operator_17/7
> 2015-11-18 02:22:12,770 [29b41f37-4803-d7ce-e05f-912d1f65da79:frag:3:5] INFO
> o.a.d.e.p.i.xsort.ExternalSortBatch - Merging and spilling to
> /tmp/drill/spill/29b41f37-4803-d7ce-e05f-912d1f65da79/major_fragment_3/minor_fragment_5/operator_17/7
> 2015-11-18 02:22:13,345 [29b41f37-4803-d7ce-e05f-912d1f65da79:frag:3:17] INFO
> o.a.d.e.p.i.xsort.ExternalSortBatch - Completed spilling to
> /tmp/drill/spill/29b41f37-4803-d7ce-e05f-912d1f65da79/major_fragment_3/minor_fragment_17/operator_17/7
> 2015-11-18 02:22:13,346 [29b41f37-4803-d7ce-e05f-912d1f65da79:frag:3:13] INFO
> o.a.d.e.p.i.xsort.ExternalSortBatch - Completed spilling to
> /tmp/drill/spill/29b41f37-4803-d7ce-e05f-912d1f65da79/major_fragment_3/minor_fragment_13/operator_16/1
> 2015-11-18 02:22:13,346 [29b41f37-4803-d7ce-e05f-912d1f65da79:frag:3:13] WARN
> o.a.d.e.p.i.xsort.ExternalSortBatch - Starting to merge. 34 batch groups.
> Current allocated memory: 2252186
> 2015-11-18 02:22:13,363 [29b41f37-4803-d7ce-e05f-912d1f65da79:frag:3:13] INFO
> o.a.d.e.w.fragment.FragmentExecutor -
> 29b41f37-4803-d7ce-e05f-912d1f65da79:3:13: State change requested RUNNING -->
> FAILED
> 2015-11-18 02:22:13,370 [29b41f37-4803-d7ce-e05f-912d1f65da79:frag:3:13] INFO
> o.a.d.e.w.fragment.FragmentExecutor -
> 29b41f37-4803-d7ce-e05f-912d1f65da79:3:13: State change requested FAILED -->
> FINISHED
> 2015-11-18 02:22:13,371 [29b41f37-4803-d7ce-e05f-912d1f65da79:frag:3:13]
> ERROR o.a.d.e.w.fragment.FragmentExecutor - SYSTEM ERROR: NullPointerException
> Fragment 3:13
> [Error Id: c5d67dcb-16aa-4951-89f5-599b4b4eb54d on atsqa4-133.qa.lab:31010]
> org.apache.drill.common.exceptions.UserException: SYSTEM ERROR:
> NullPointerException
> Fragment 3:13
> [Error Id: c5d67dcb-16aa-4951-89f5-599b4b4eb54d on atsqa4-133.qa.lab:31010]
> at
> org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:534)
> ~[drill-common-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
> at
> org.apache.drill.exec.work.fragment.FragmentExecutor.sendFinalState(FragmentExecutor.java:321)
> [drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
> at
> org.apache.drill.exec.work.fragment.FragmentExecutor.cleanup(FragmentExecutor.java:184)
> [drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
> at
> org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:290)
> [drill-java-exec-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
> at
> org.apache.drill.common.SelfCleaningRunnable.run(SelfCleaningRunnable.java:38)
> [drill-common-1.4.0-SNAPSHOT.jar:1.4.0-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> [na:1.7.0_71]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> [na:1.7.0_71]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_71]
> java.lang.NullPointerException: null
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
