Hi, Wiewiorka
according to the stack trace, it seems that the MR job has succeeded,
however when kylin try to get the JobCounters for statistical information,
the job history server has thrown NPE.
Is there any problem with JobHistory Server? or can you please manually
check the job history to see if you can get the JobCounters successfully?
On Fri, Apr 3, 2015 at 10:22 PM Marek Wiewiorka <[email protected]>
wrote:
> Hi All - when I'm trying to build a new cube I'm reaching the
> aforementioned step of the pipeline and constantly getting the following
> null pointer exception.
> This happens during a reduce task building a cube from a table containing
> 5*10^6 rows .
> The same cube from a table with 1000 rows is building successfully.
> Do you have any idea what might be here the problem?
>
> [pool-7-thread-10]:[2015-04-03
> 10:02:30,630][ERROR][org.apache.kylin.job.common.HadoopCmdOutput.
> updateJobCounter(HadoopCmdOutput.java:100)]
> - org.apache.hadoop.ipc.RemoteException(java.lang.
> NullPointerException): java.lang.NullPointerException
> at
> org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$
> HSClientProtocolHandler.getCounters(HistoryClientService.java:228)
> at
> org.apache.hadoop.mapreduce.v2.api.impl.pb.service.
> MRClientProtocolPBServiceImpl.getCounters(MRClientProtocolPBServiceImpl.
> java:159)
> at
> org.apache.hadoop.yarn.proto.MRClientProtocol$MRClientProtocolService$2.
> callBlockingMethod(MRClientProtocol.java:281)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(
> ProtobufRpcEngine.java:585)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:928)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2013)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2009)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:415)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1594)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2007)
>
> java.io.IOException:
> org.apache.hadoop.ipc.RemoteException(java.lang.NullPointerException):
> java.lang.NullPointerException
> at
> org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$
> HSClientProtocolHandler.getCounters(HistoryClientService.java:228)
> at
> org.apache.hadoop.mapreduce.v2.api.impl.pb.service.
> MRClientProtocolPBServiceImpl.getCounters(MRClientProtocolPBServiceImpl.
> java:159)
> at
> org.apache.hadoop.yarn.proto.MRClientProtocol$MRClientProtocolService$2.
> callBlockingMethod(MRClientProtocol.java:281)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(
> ProtobufRpcEngine.java:585)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:928)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2013)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2009)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:415)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1594)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2007)
>
> at
> org.apache.hadoop.mapred.ClientServiceDelegate.invoke(
> ClientServiceDelegate.java:334)
> at
> org.apache.hadoop.mapred.ClientServiceDelegate.getJobCounters(
> ClientServiceDelegate.java:371)
> at
> org.apache.hadoop.mapred.YARNRunner.getJobCounters(YARNRunner.java:521)
> at org.apache.hadoop.mapreduce.Job$7.run(Job.java:756)
> at org.apache.hadoop.mapreduce.Job$7.run(Job.java:753)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:415)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1594)
> at org.apache.hadoop.mapreduce.Job.getCounters(Job.java:753)
> at
> org.apache.kylin.job.common.HadoopCmdOutput.updateJobCounter(
> HadoopCmdOutput.java:86)
> at
> org.apache.kylin.job.common.MapReduceExecutable.doWork(
> MapReduceExecutable.java:144)
> at
> org.apache.kylin.job.execution.AbstractExecutable.
> execute(AbstractExecutable.java:107)
> at
> org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(
> DefaultChainedExecutable.java:50)
> at
> org.apache.kylin.job.execution.AbstractExecutable.
> execute(AbstractExecutable.java:107)
> at
> org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(
> DefaultScheduler.java:132)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(
> ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(
> ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> Caused by:
> org.apache.hadoop.ipc.RemoteException(java.lang.NullPointerException):
> java.lang.NullPointerException
> at
> org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$
> HSClientProtocolHandler.getCounters(HistoryClientService.java:228)
> at
> org.apache.hadoop.mapreduce.v2.api.impl.pb.service.
> MRClientProtocolPBServiceImpl.getCounters(MRClientProtocolPBServiceImpl.
> java:159)
> at
> org.apache.hadoop.yarn.proto.MRClientProtocol$MRClientProtocolService$2.
> callBlockingMethod(MRClientProtocol.java:281)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(
> ProtobufRpcEngine.java:585)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:928)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2013)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2009)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:415)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1594)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2007)
>
> at org.apache.hadoop.ipc.Client.call(Client.java:1410)
> at org.apache.hadoop.ipc.Client.call(Client.java:1363)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.
> invoke(ProtobufRpcEngine.java:206)
> at com.sun.proxy.$Proxy64.getCounters(Unknown Source)
> at
> org.apache.hadoop.mapreduce.v2.api.impl.pb.client.
> MRClientProtocolPBClientImpl.getCounters(MRClientProtocolPBClientImpl.
> java:166)
> at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at
> org.apache.hadoop.mapred.ClientServiceDelegate.invoke(
> ClientServiceDelegate.java:320)
> ... 17 more
>
