> Caused by: java.net.ConnectException: Connection refused Seems hbase config on the node was not in good shape.
On Tue, Jul 12, 2016 at 11:40 PM, Ahmed Mahran <[email protected]> wrote: > Hi, > > Kylin 1.5.2.1-cdh5.7 > Hadoop 2.6.0-cdh5.7.1 > Hive 1.1.0-cdh5.7.1 > HBase 1.2.0-cdh5.7.1 > > kylin.log > > 2016-07-12 08:10:03,894 ERROR [pool-5-thread-1] dao.ExecutableDao:145 : > error get all Jobs: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > location > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:316) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:156) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) > at > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) > at > org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320) > at > org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:295) > at > org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:160) > at > org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:155) > at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:867) > at > org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:137) > at > org.apache.kylin.storage.hbase.HBaseResourceStore.listResourcesImpl(HBaseResourceStore.java:107) > at > org.apache.kylin.common.persistence.ResourceStore.listResources(ResourceStore.java:123) > at org.apache.kylin.job.dao.ExecutableDao.getJobIds(ExecutableDao.java:135) > at > org.apache.kylin.job.manager.ExecutableManager.getAllJobIds(ExecutableManager.java:204) > at > org.apache.kylin.job.impl.threadpool.DefaultScheduler$FetcherRunner.run(DefaultScheduler.java:81) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:416) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:906) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873) > at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1242) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:34070) > at > org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRowOrBefore(ProtobufUtil.java:1582) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1398) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1199) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:305) > ... 21 more > 2016-07-12 08:10:03,900 ERROR [pool-5-thread-1] > manager.ExecutableManager:206 : error get All Job Ids > org.apache.kylin.job.exception.PersistentException: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > location > at org.apache.kylin.job.dao.ExecutableDao.getJobIds(ExecutableDao.java:146) > at > org.apache.kylin.job.manager.ExecutableManager.getAllJobIds(ExecutableManager.java:204) > at > org.apache.kylin.job.impl.threadpool.DefaultScheduler$FetcherRunner.run(DefaultScheduler.java:81) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't > get the location > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:316) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:156) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) > at > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) > at > org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320) > at > org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:295) > at > org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:160) > at > org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:155) > at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:867) > at > org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:137) > at > org.apache.kylin.storage.hbase.HBaseResourceStore.listResourcesImpl(HBaseResourceStore.java:107) > at > org.apache.kylin.common.persistence.ResourceStore.listResources(ResourceStore.java:123) > at org.apache.kylin.job.dao.ExecutableDao.getJobIds(ExecutableDao.java:135) > ... 9 more > Caused by: java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:416) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:906) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873) > at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1242) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:34070) > at > org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRowOrBefore(ProtobufUtil.java:1582) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1398) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1199) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:305) > ... 21 more > 2016-07-12 08:10:03,901 WARN [pool-5-thread-1] > threadpool.DefaultScheduler:108 : Job Fetcher caught a exception > java.lang.RuntimeException: > org.apache.kylin.job.exception.PersistentException: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > location > 2016-07-12 08:10:11,955 INFO [http-bio-7070-exec-7] > controller.QueryController:174 : Using project: CDR_Demo_Project > 2016-07-12 08:10:11,958 INFO [http-bio-7070-exec-7] > controller.QueryController:175 : The original query: select * from > CDR_AGGREGATION > 2016-07-12 08:10:11,960 INFO [http-bio-7070-exec-7] > service.QueryService:266 : The corrected query: select * from > CDR_AGGREGATION > LIMIT 50000 > 2016-07-12 08:10:12,053 INFO [http-bio-7070-exec-7] > routing.QueryRouter:48 : The project manager's reference is > org.apache.kylin.metadata.project.ProjectManager@5759a069 > 2016-07-12 08:10:12,054 INFO [http-bio-7070-exec-7] > routing.QueryRouter:60 : Find candidates by table DEFAULT.CDR_AGGREGATION > and project=CDR_DEMO_PROJECT : > org.apache.kylin.query.routing.Candidate@44071cfe > 2016-07-12 08:10:12,055 INFO [http-bio-7070-exec-7] > routing.QueryRouter:49 : Applying rule: class > org.apache.kylin.query.routing.rules.RemoveUncapableRealizationsRule, > realizations before: [CDR_Demo_Cube_1(CUBE)], realizations after: > [CDR_Demo_Cube_1(CUBE)] > 2016-07-12 08:10:12,055 INFO [http-bio-7070-exec-7] > routing.QueryRouter:49 : Applying rule: class > org.apache.kylin.query.routing.rules.RealizationSortRule, realizations > before: [CDR_Demo_Cube_1(CUBE)], realizations after: [CDR_Demo_Cube_1(CUBE)] > 2016-07-12 08:10:12,056 INFO [http-bio-7070-exec-7] > routing.QueryRouter:72 : The realizations remaining: > [CDR_Demo_Cube_1(CUBE)] And the final chosen one is the first one > 2016-07-12 08:10:12,077 DEBUG [http-bio-7070-exec-7] > enumerator.OLAPEnumerator:107 : query storage... > 2016-07-12 08:10:12,078 INFO [http-bio-7070-exec-7] > enumerator.OLAPEnumerator:181 : No group by and aggregation found in this > query, will hack some result for better look of output... > 2016-07-12 08:10:12,078 INFO [http-bio-7070-exec-7] > v2.CubeStorageQuery:239 : exactAggregation is true > 2016-07-12 08:10:12,079 INFO [http-bio-7070-exec-7] > v2.CubeStorageQuery:357 : Enable limit 50000 > 2016-07-12 08:10:12,087 DEBUG [http-bio-7070-exec-7] > v2.CubeHBaseEndpointRPC:257 : New scanner for current segment > CDR_Demo_Cube_1[20150101000000_20160101000000] will use > SCAN_FILTER_AGGR_CHECKMEM as endpoint's behavior > 2016-07-12 08:10:12,088 DEBUG [http-bio-7070-exec-7] > v2.CubeHBaseEndpointRPC:313 : Serialized scanRequestBytes 660 bytes, > rawScanBytesString 106 bytes > 2016-07-12 08:10:12,088 INFO [http-bio-7070-exec-7] > v2.CubeHBaseEndpointRPC:315 : The scan 416215cb for segment > CDR_Demo_Cube_1[20150101000000_20160101000000] is as below with 1 separate > raw scans, shard part of start/end key is set to 0 > 2016-07-12 08:10:12,090 INFO [http-bio-7070-exec-7] v2.CubeHBaseRPC:271 : > Visiting hbase table KYLIN_HX9PP90NMQ: cuboid exact match, from 15 to 15 > Start: > \x00\x00\x00\x00\x00\x00\x00\x00\x00\x0F\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 > (\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0F\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00) > Stop: > > \x00\x00\x00\x00\x00\x00\x00\x00\x00\x0F\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00 > (\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0F\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00), > No Fuzzy Key > 2016-07-12 08:10:12,090 DEBUG [http-bio-7070-exec-7] > v2.CubeHBaseEndpointRPC:320 : Submitting rpc to 2 shards starting from > shard 0, scan range count 1 > 2016-07-12 08:10:12,163 INFO [http-bio-7070-exec-7] > v2.CubeHBaseEndpointRPC:103 : Timeout for ExpectedSizeIterator is: 66000 > 2016-07-12 08:10:12,164 DEBUG [http-bio-7070-exec-7] > enumerator.OLAPEnumerator:127 : return TupleIterator... > 2016-07-12 08:10:45,430 ERROR [pool-11-thread-5] > util.LoggableCachedThreadPool:44 : Execution exception when running task in > pool-11-thread-5 > 2016-07-12 08:10:45,432 ERROR [pool-11-thread-5] > util.LoggableCachedThreadPool:54 : Caught exception in thread > pool-11-thread-5: > java.lang.RuntimeException: <sub-thread for GTScanRequest 416215cb> Error > when visiting cubes by endpoint > at > org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC$1.run(CubeHBaseEndpointRPC.java:345) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:416) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:906) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873) > at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1242) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:34070) > at > org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRowOrBefore(ProtobufUtil.java:1582) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1398) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1199) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1179) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1136) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getRegionLocation(ConnectionManager.java:971) > at > org.apache.hadoop.hbase.client.HRegionLocator.getRegionLocation(HRegionLocator.java:83) > at org.apache.hadoop.hbase.client.HTable.getRegionLocation(HTable.java:569) > at > org.apache.hadoop.hbase.client.HTable.getKeysAndRegionsInRange(HTable.java:793) > at > org.apache.hadoop.hbase.client.HTable.getKeysAndRegionsInRange(HTable.java:763) > at > org.apache.hadoop.hbase.client.HTable.getStartKeysInRange(HTable.java:1830) > at > org.apache.hadoop.hbase.client.HTable.coprocessorService(HTable.java:1785) > at > org.apache.hadoop.hbase.client.HTable.coprocessorService(HTable.java:1765) > at > org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC.getResults(CubeHBaseEndpointRPC.java:389) > at > org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC.access$200(CubeHBaseEndpointRPC.java:75) > at > org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC$1.run(CubeHBaseEndpointRPC.java:343) > ... 5 more > 2016-07-12 08:11:03,839 ERROR [pool-5-thread-1] dao.ExecutableDao:145 : > error get all Jobs: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > location > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:316) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:156) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) > at > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) > at > org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320) > at > org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:295) > at > org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:160) > at > org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:155) > at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:867) > at > org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:137) > at > org.apache.kylin.storage.hbase.HBaseResourceStore.listResourcesImpl(HBaseResourceStore.java:107) > at > org.apache.kylin.common.persistence.ResourceStore.listResources(ResourceStore.java:123) > at org.apache.kylin.job.dao.ExecutableDao.getJobIds(ExecutableDao.java:135) > at > org.apache.kylin.job.manager.ExecutableManager.getAllJobIds(ExecutableManager.java:204) > at > org.apache.kylin.job.impl.threadpool.DefaultScheduler$FetcherRunner.run(DefaultScheduler.java:81) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:416) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:906) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873) > at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1242) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:34070) > at > org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRowOrBefore(ProtobufUtil.java:1582) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1398) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1199) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:305) > ... 21 more > 2016-07-12 08:11:03,842 ERROR [pool-5-thread-1] > manager.ExecutableManager:206 : error get All Job Ids > org.apache.kylin.job.exception.PersistentException: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > location > at org.apache.kylin.job.dao.ExecutableDao.getJobIds(ExecutableDao.java:146) > at > org.apache.kylin.job.manager.ExecutableManager.getAllJobIds(ExecutableManager.java:204) > at > org.apache.kylin.job.impl.threadpool.DefaultScheduler$FetcherRunner.run(DefaultScheduler.java:81) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't > get the location > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:316) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:156) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) > at > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) > at > org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320) > at > org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:295) > at > org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:160) > at > org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:155) > at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:867) > at > org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:137) > at > org.apache.kylin.storage.hbase.HBaseResourceStore.listResourcesImpl(HBaseResourceStore.java:107) > at > org.apache.kylin.common.persistence.ResourceStore.listResources(ResourceStore.java:123) > at org.apache.kylin.job.dao.ExecutableDao.getJobIds(ExecutableDao.java:135) > ... 9 more > Caused by: java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:416) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:906) > at > org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873) > at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1242) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:34070) > at > org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRowOrBefore(ProtobufUtil.java:1582) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1398) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1199) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:305) > ... 21 more > 2016-07-12 08:11:03,843 WARN [pool-5-thread-1] > threadpool.DefaultScheduler:108 : Job Fetcher caught a exception > java.lang.RuntimeException: > org.apache.kylin.job.exception.PersistentException: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > location > 2016-07-12 08:11:18,165 ERROR [http-bio-7070-exec-7] > controller.QueryController:209 : Exception when execute sql > java.sql.SQLException: Error while executing SQL "select * from > CDR_AGGREGATION > LIMIT 50000": Timeout visiting cube! > at org.apache.calcite.avatica.Helper.createException(Helper.java:56) > at org.apache.calcite.avatica.Helper.createException(Helper.java:41) > at > org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:143) > at > org.apache.calcite.avatica.AvaticaStatement.executeQuery(AvaticaStatement.java:186) > at > org.apache.kylin.rest.service.QueryService.execute(QueryService.java:361) > at > org.apache.kylin.rest.service.QueryService.queryWithSqlMassage(QueryService.java:273) > at org.apache.kylin.rest.service.QueryService.query(QueryService.java:121) > at > org.apache.kylin.rest.service.QueryService$$FastClassByCGLIB$$4957273f.invoke(<generated>) > at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) > at > org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:618) > at > org.apache.kylin.rest.service.QueryService$$EnhancerByCGLIB$$2b43fc30.query(<generated>) > at > org.apache.kylin.rest.controller.QueryController.doQueryWithCache(QueryController.java:192) > at > org.apache.kylin.rest.controller.QueryController.query(QueryController.java:94) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.springframework.web.method.support.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:213) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:126) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:96) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:617) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:578) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:80) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:923) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:789) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:646) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) > at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:118) > at > org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:113) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:103) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:113) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:54) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:45) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(BasicAuthenticationFilter.java:201) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.ui.DefaultLoginPageGeneratingFilter.doFilter(DefaultLoginPageGeneratingFilter.java:91) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:183) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:105) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:192) > at > org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:160) > at > org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) > at > org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) > at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:195) > at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:266) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) > at > org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) > at > org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) > at > org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:504) > at > org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170) > at > org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) > at > org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950) > at > org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) > at > org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421) > at > org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1074) > at > org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611) > at > org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:314) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at > org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.lang.RuntimeException: Timeout visiting cube! > at > org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC$ExpectedSizeIterator.next(CubeHBaseEndpointRPC.java:127) > at > org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC$ExpectedSizeIterator.next(CubeHBaseEndpointRPC.java:81) > at > com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48) > at com.google.common.collect.Iterators$6.hasNext(Iterators.java:583) > at > org.apache.kylin.storage.hbase.cube.v2.SequentialCubeTupleIterator.hasNext(SequentialCubeTupleIterator.java:96) > at > org.apache.kylin.query.enumerator.OLAPEnumerator.moveNext(OLAPEnumerator.java:74) > at > org.apache.calcite.linq4j.EnumerableDefaults$TakeWhileEnumerator.moveNext(EnumerableDefaults.java:2818) > at > org.apache.calcite.linq4j.Linq4j$EnumeratorIterator.<init>(Linq4j.java:664) > at org.apache.calcite.linq4j.Linq4j.enumeratorIterator(Linq4j.java:98) > at > org.apache.calcite.linq4j.AbstractEnumerable.iterator(AbstractEnumerable.java:33) > at org.apache.calcite.avatica.MetaImpl.createCursor(MetaImpl.java:85) > at > org.apache.calcite.avatica.AvaticaResultSet.execute(AvaticaResultSet.java:190) > at > org.apache.calcite.jdbc.CalciteResultSet.execute(CalciteResultSet.java:65) > at > org.apache.calcite.jdbc.CalciteResultSet.execute(CalciteResultSet.java:44) > at > org.apache.calcite.avatica.AvaticaConnection$1.execute(AvaticaConnection.java:566) > at > org.apache.calcite.jdbc.CalciteMetaImpl.prepareAndExecute(CalciteMetaImpl.java:578) > at > org.apache.calcite.avatica.AvaticaConnection.prepareAndExecuteInternal(AvaticaConnection.java:571) > at > org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:135) > ... 80 more > 2016-07-12 08:11:18,173 INFO [http-bio-7070-exec-7] > service.QueryService:250 : > ==========================[QUERY]=============================== > SQL: select * from CDR_AGGREGATION > User: ADMIN > Success: false > Duration: 0.0 > Project: CDR_Demo_Project > Realization Names: [CDR_Demo_Cube_1] > Cuboid Ids: [15] > Total scan count: 0 > Result row count: 0 > Accept Partial: true > Is Partial Result: false > Hit Exception Cache: false > Storage cache used: false > Message: Error while executing SQL "select * from CDR_AGGREGATION LIMIT > 50000": Timeout visiting cube! > ==========================[QUERY]=============================== > > 2016-07-12 08:11:18,174 ERROR [http-bio-7070-exec-7] > controller.BasicController:44 : > org.apache.kylin.rest.exception.InternalErrorException: Error while > executing SQL "select * from CDR_AGGREGATION LIMIT 50000": Timeout visiting > cube! > at > org.apache.kylin.rest.controller.QueryController.doQueryWithCache(QueryController.java:224) > at > org.apache.kylin.rest.controller.QueryController.query(QueryController.java:94) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.springframework.web.method.support.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:213) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:126) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:96) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:617) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:578) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:80) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:923) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:789) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:646) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) > at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:118) > at > org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:113) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:103) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:113) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:54) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:45) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(BasicAuthenticationFilter.java:201) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.ui.DefaultLoginPageGeneratingFilter.doFilter(DefaultLoginPageGeneratingFilter.java:91) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:183) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:105) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) > at > org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:192) > at > org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:160) > at > org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) > at > org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) > at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:195) > at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:266) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) > at > org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) > at > org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) > at > org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:504) > at > org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170) > at > org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) > at > org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950) > at > org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) > at > org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421) > at > org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1074) > at > org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611) > at > org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:314) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at > org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) > at java.lang.Thread.run(Thread.java:745) > > > >
