Hi, Upgrading hadoop-lzo compression to 0.4.10 from 0.4.9 solved the problem.
Thanks and regards, - Ashish On Wednesday, July 13, 2011 22:00 IST, Stack <[email protected]> wrote: > Try upping your heap to see if that makes a difference. Your usage > and data may not suit what we have configured for default. > St.Ack > > On Tue, Jul 12, 2011 at 11:51 PM, Ashish Shinde <[email protected]> wrote: > > Hi, > > > > I get an out of memory error while running a MR job on a 42 GB hbase > > table. The cluster size is 5 and the region servers start with an Xmx = > > 1GB. > > > > I guess the Xmx is too low. But is that the only problem or is this due > > to a bug. > > > > I am using hadoop-lzo-0.4.9.jar and associated 64 bit native libs. > > > > Please find the stack trace below. > > > > Thanks and regards, > > - Ashish > > > > > > > > Caused by: java.lang.RuntimeException: > > org.apache.hadoop.hbase.client.RetriesExhaustedException: Trying to > > contact region server ip-10-2-33-71.ec2.internal:60020 for region > > 88_AlignedSample_asample,\x00\x00\x00\x00\x00\x00\x03\x83,1310470380666.61bc91374b963cdb99a63d99b2f007f7., > > row '\x00\x00\x00m\x00\x00\x00\x08', but failed after 10 attempts. > > Exceptions: org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > is closing at > > org.apache.hadoop.hbase.regionserver.HRegion.startRegionOperation(HRegion.java:3334) > > at > > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1106) > > at > > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1102) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1781) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: Region is not > > online: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:2340) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1780) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: Region is not > > online: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:2340) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1780) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: Region is not > > online: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:2340) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1780) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: Region is not > > online: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:2340) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1780) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > java.io.IOException: java.io.IOException: java.lang.OutOfMemoryError: > > Java heap space at > > org.apache.hadoop.hbase.regionserver.HRegionServer.convertThrowableToIOE(HRegionServer.java:992) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.convertThrowableToIOE(HRegionServer.java:981) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1783) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > Caused by: java.lang.OutOfMemoryError: Java heap space at > > java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:39) at > > java.nio.ByteBuffer.allocate(ByteBuffer.java:312) at > > org.apache.hadoop.hbase.io.hfile.HFile$Reader.decompress(HFile.java:1093) > > at > > org.apache.hadoop.hbase.io.hfile.HFile$Reader.readBlock(HFile.java:1036) > > at > > org.apache.hadoop.hbase.io.hfile.HFile$Reader$Scanner.seekTo(HFile.java:1433) > > at > > org.apache.hadoop.hbase.io.HalfStoreFileReader$1.seekTo(HalfStoreFileReader.java:160) > > at > > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:139) > > at > > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:96) > > at > > org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:77) > > at > > org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:1341) > > at > > org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.<init>(HRegion.java:2269) > > at > > org.apache.hadoop.hbase.regionserver.HRegion.instantiateInternalScanner(HRegion.java:1126) > > at > > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1118) > > at > > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1102) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1781) > > ... > > 5 more > > > > java.net.ConnectException: Connection refused > > java.net.ConnectException: Connection refused > > java.net.ConnectException: Connection refused > > java.net.ConnectException: Connection refused > > > > at > > > > org.apache.hadoop.hbase.client.HTable$ClientScanner$1.hasNext(HTable.java:1213) > > at > > > > com.strandgenomics.ngs.hadoop.sample.io.ReadEnumeration.fillCache(ReadEnumeration.java:182) > > at > > > > com.strandgenomics.ngs.hadoop.sample.io.ReadEnumeration.hasMoreElements(ReadEnumeration.java:245) > > ... > > 15 more Caused by: > > org.apache.hadoop.hbase.client.RetriesExhaustedException: > > Trying to contact region server > > ip-10-2-33-71.ec2.internal:60020 for region > > > > 88_AlignedSample_asample,\x00\x00\x00\x00\x00\x00\x03\x83,1310470380666.61bc91374b963cdb99a63d99b2f007f7., > > row '\x00\x00\x00m\x00\x00\x00\x08', but failed after 10 > > attempts. Exceptions: > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: > > > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > is closing at > > > > org.apache.hadoop.hbase.regionserver.HRegion.startRegionOperation(HRegion.java:3334) > > at > > > > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1106) > > at > > > > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1102) > > at > > > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1781) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) > > at > > > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) > > at > > > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: Region is not > > online: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:2340) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1780) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: Region is not > > online: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:2340) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1780) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: Region is not > > online: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:2340) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1780) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > org.apache.hadoop.hbase.NotServingRegionException: > > org.apache.hadoop.hbase.NotServingRegionException: Region is not > > online: > > 88_AlignedSample_asample,,1310400775439.3a8e41d085bdc47e5f0776c4b9c5b7c8. > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:2340) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1780) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > > > java.io.IOException: java.io.IOException: java.lang.OutOfMemoryError: > > Java heap space at > > org.apache.hadoop.hbase.regionserver.HRegionServer.convertThrowableToIOE(HRegionServer.java:992) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.convertThrowableToIOE(HRegionServer.java:981) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1783) > > at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at > > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > > at java.lang.reflect.Method.invoke(Method.java:597) at > > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at > > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) > > Caused by: java.lang.OutOfMemoryError: Java heap space at > > java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:39) at > > java.nio.ByteBuffer.allocate(ByteBuffer.java:312) at > > org.apache.hadoop.hbase.io.hfile.HFile$Reader.decompress(HFile.java:1093) > > at > > org.apache.hadoop.hbase.io.hfile.HFile$Reader.readBlock(HFile.java:1036) > > at > > org.apache.hadoop.hbase.io.hfile.HFile$Reader$Scanner.seekTo(HFile.java:1433) > > at > > org.apache.hadoop.hbase.io.HalfStoreFileReader$1.seekTo(HalfStoreFileReader.java:160) > > at > > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:139) > > at > > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:96) > > at > > org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:77) > > at > > org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:1341) > > at > > org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.<init>(HRegion.java:2269) > > at > > org.apache.hadoop.hbase.regionserver.HRegion.instantiateInternalScanner(HRegion.java:1126) > > at > > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1118) > > at > > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1102) > > at > > org.apache.hadoop.hbase.regionserver.HRegionServer.openScanner(HRegionServer.java:1781) > > ... > > 5 more > > > > java.net.ConnectException: Connection refused > > java.net.ConnectException: Connection refused > > java.net.ConnectException: Connection refused > > java.net.ConnectException: Connection refused > > > > at > > > > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getRegionServerWithRetries(HConnectionManager.java:1008) > > at > > > > org.apache.hadoop.hbase.client.HTable$ClientScanner.nextScanner(HTable.java:1061) > > at > > > > org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1150) > > at > > > > org.apache.hadoop.hbase.client.HTable$ClientScanner$1.hasNext(HTable.java:1210) > > ... > > 17 more > >
