[ https://issues.apache.org/jira/browse/APEXCORE-807?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Pramod Immaneni updated APEXCORE-807: ------------------------------------- Issue Type: Bug (was: Task) > In secure mode containers are failing after one day and the application is > failing after seven days > --------------------------------------------------------------------------------------------------- > > Key: APEXCORE-807 > URL: https://issues.apache.org/jira/browse/APEXCORE-807 > Project: Apache Apex Core > Issue Type: Bug > Reporter: Pramod Immaneni > Assignee: Pramod Immaneni > Priority: Major > > java.lang.RuntimeException: java.util.concurrent.ExecutionException: > java.lang.RuntimeException: > org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): > token (HDFS_DELEGATION_TOKEN token nnnnnn for xxxxxx) can't be found in cache > at com.google.common.base.Throwables.propagate(Throwables.java:156) > at com.datatorrent.stram.engine.Node.reportStats(Node.java:489) > at com.datatorrent.stram.engine.GenericNode.reportStats(GenericNode.java:825) > at > com.datatorrent.stram.engine.GenericNode.processEndWindow(GenericNode.java:184) > at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:397) > at > com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1465) > Caused by: java.util.concurrent.ExecutionException: > java.lang.RuntimeException: > org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): > token (HDFS_DELEGATION_TOKEN token nnnnnn for xxxxxx) can't be found in cache > at java.util.concurrent.FutureTask.report(FutureTask.java:122) > at java.util.concurrent.FutureTask.get(FutureTask.java:192) > at com.datatorrent.stram.engine.Node.reportStats(Node.java:482) > ... 4 more > Caused by: java.lang.RuntimeException: > org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): > token (HDFS_DELEGATION_TOKEN token nnnnnn for xxxxxx) can't be found in cache > at com.google.common.base.Throwables.propagate(Throwables.java:156) > at > com.datatorrent.common.util.AsyncFSStorageAgent.copyToHDFS(AsyncFSStorageAgent.java:131) > at > com.datatorrent.common.util.AsyncFSStorageAgent.flush(AsyncFSStorageAgent.java:156) > at com.datatorrent.stram.engine.Node$CheckpointHandler.call(Node.java:706) > at com.datatorrent.stram.engine.Node$CheckpointHandler.call(Node.java:696) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: > org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): > token (HDFS_DELEGATION_TOKEN token nnnnnn for xxxxxx) can't be found in cache > at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1554) > at org.apache.hadoop.ipc.Client.call(Client.java:1498) > at org.apache.hadoop.ipc.Client.call(Client.java:1398) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233) > at com.sun.proxy.$Proxy10.create(Unknown Source) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:313) > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:291) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:203) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:185) > at com.sun.proxy.$Proxy11.create(Unknown Source) > at > org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1822) > at org.apache.hadoop.hdfs.DFSClient.primitiveCreate(DFSClient.java:1762) > at org.apache.hadoop.fs.Hdfs.createInternal(Hdfs.java:104) > at org.apache.hadoop.fs.Hdfs.createInternal(Hdfs.java:60) > at > org.apache.hadoop.fs.AbstractFileSystem.create(AbstractFileSystem.java:585) > at org.apache.hadoop.fs.FileContext$3.next(FileContext.java:688) > at org.apache.hadoop.fs.FileContext$3.next(FileContext.java:684) > at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) > at org.apache.hadoop.fs.FileContext.create(FileContext.java:684) > at > com.datatorrent.common.util.AsyncFSStorageAgent.copyToHDFS(AsyncFSStorageAgent.java:119) > ... 9 more -- This message was sent by Atlassian JIRA (v7.6.3#76005)