[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Attachment: HBASE-16993.master.005.patch > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch, > HBASE-16993.master.001.patch, HBASE-16993.master.002.patch, > HBASE-16993.master.003.patch, HBASE-16993.master.004.patch, > HBASE-16993.master.005.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Attachment: HBASE-16993.master.004.patch > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch, > HBASE-16993.master.001.patch, HBASE-16993.master.002.patch, > HBASE-16993.master.003.patch, HBASE-16993.master.004.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Attachment: HBASE-16993.master.003.patch > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch, > HBASE-16993.master.001.patch, HBASE-16993.master.002.patch, > HBASE-16993.master.003.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Attachment: HBASE-16993.master.002.patch > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch, > HBASE-16993.master.001.patch, HBASE-16993.master.002.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Release Note: Make it so bucket sizes no longer have to be exact multiple of 256 (side effect is we now can support caches > 256TB -- smile). Incompatible change as the bucket entry format has changed. Means we cannot read persisted cache written in the old format. On restart, if present, the old persisted cache will be removed and we continue w/ startup; i.e. cache will not be unpopulated after startup (This behavior is 'standard' when we are unable to find or read the the persisted file -- so no change here). Persisted file works in release 1.2.4 and 1.1.7. Previous to HBASE-16460, persisted file operation didn't work was: Make it so bucket sizes no longer have to be exact multiple of 256 (side effect is we now can support caches > 256TB -- smile). Incompatible change as the bucket entry format has changed. Means we cannot read persisted cache written in the old format. On restart, if present, the old persisted cache will be removed and we continue w/ startup; i.e. cache will not be unpopulated after startup (This behavior is 'standard' when we are unable to find or read the the persisted file -- so no change here). Persisted file works in release 1.2.4 and 1.1.7. Previous to this > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch, > HBASE-16993.master.001.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Release Note: Make it so bucket sizes no longer have to be exact multiple of 256 (side effect is we now can support caches > 256TB -- smile). Incompatible change as the bucket entry format has changed. Means we cannot read persisted cache written in the old format. On restart, if present, the old persisted cache will be removed and we continue w/ startup; i.e. cache will not be unpopulated after startup (This behavior is 'standard' when we are unable to find or read the the persisted file -- so no change here). Persisted file works in release 1.2.4 and 1.1.7. Previous to this was: Make it so bucket sizes no longer have to be exact multiple of 256 (side effect is we now can support caches > 256TB -- smile). Incompatible change as the bucket entry format has changed. Means we cannot read persisted cache written in the old format. On restart, if present, we remove the old persisted file and continue w/ startup; i.e. cache will be unpopulated after startup. > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch, > HBASE-16993.master.001.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Attachment: HBASE-16993.master.001.patch > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch, > HBASE-16993.master.001.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Fix Version/s: 2.0.0 Component/s: BucketCache > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] stack updated HBASE-16993: -- Hadoop Flags: Incompatible change,Reviewed Release Note: Make it so bucket sizes no longer have to be exact multiple of 256 (side effect is we now can support caches > 256TB -- smile). Incompatible change as the bucket entry format has changed. Means we cannot read persisted cache written in the old format. On restart, if present, we remove the old persisted file and continue w/ startup; i.e. cache will be unpopulated after startup. was:directly use offset value in setOffset of calss BucketEntry > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: BucketCache, io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Fix For: 2.0.0 > > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] liubangchen updated HBASE-16993: Attachment: HBASE-16993.001.patch > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Attachments: HBASE-16993.000.patch, HBASE-16993.001.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] liubangchen updated HBASE-16993: Attachment: HBASE-16993.000.patch > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Attachments: HBASE-16993.000.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32381) > at
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] liubangchen updated HBASE-16993: Assignee: liubangchen Release Note: directly use offset value in setOffset of calss BucketEntry (was: directly use offset value in setOffset of calss BucketEntry) Status: Patch Available (was: Open) directly use offset value in setOffset of calss BucketEntry,otherwise bucket.sizes must be a multiple of 1024 else you will run into 'java.io.IOException: Invalid HFile block magic' when you go to read from cache > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen >Assignee: liubangchen > Attachments: HBASE-16993.000.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] liubangchen updated HBASE-16993: Attachment: (was: HBASE-16993.patch) > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32381) > at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2117) > at
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] liubangchen updated HBASE-16993: Status: Open (was: Patch Available) > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32381) > at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2117) > at
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] liubangchen updated HBASE-16993: Attachment: HBASE-16993.patch fix bucketsizes must be multiple of 1024 > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen > Attachments: HBASE-16993.patch > > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at > org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32381) > at
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] liubangchen updated HBASE-16993: Release Note: directly use offset value in setOffset of calss BucketEntry Status: Patch Available (was: Reopened) directly use offset value in setOffset of calss BucketEntry > BucketCache throw java.io.IOException: Invalid HFile block magic when > DATA_BLOCK_ENCODING set to DIFF > - > > Key: HBASE-16993 > URL: https://issues.apache.org/jira/browse/HBASE-16993 > Project: HBase > Issue Type: Bug > Components: io >Affects Versions: 1.1.3 > Environment: hbase version 1.1.3 >Reporter: liubangchen > Original Estimate: 336h > Remaining Estimate: 336h > > hbase-site.xml setting > > hbase.bucketcache.bucket.sizes > 16384,32768,40960, > 46000,49152,51200,65536,131072,524288 > > > hbase.bucketcache.size > 16384 > > > hbase.bucketcache.ioengine > offheap > > > hfile.block.cache.size > 0.3 > > > hfile.block.bloom.cacheonwrite > true > > > hbase.rs.cacheblocksonwrite > true > > > hfile.block.index.cacheonwrite > true > n_splits = 200 > create 'usertable',{NAME =>'family', COMPRESSION => 'snappy', VERSIONS => > 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => > {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => > 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| > "user#{1000+i*(-1000)/n_splits}"}} > load data > bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > recordcount=2 -p insertorder=hashed -p insertstart=0 -p > clientbuffering=true -p durability=SKIP_WAL -threads 20 -s > run > bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p > columnfamily=family -p fieldcount=10 -p fieldlength=100 -p > operationcount=2000 -p readallfields=true -p clientbuffering=true -p > requestdistribution=zipfian -threads 10 -s > log info > 2016-11-02 20:20:20,261 ERROR > [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: > Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket > cache > java.io.IOException: Invalid HFile block magic: > \x00\x00\x00\x00\x00\x00\x00\x00 > at > org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) > at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) > at > org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) > at > org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) > at > org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) > at > org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) > at > org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) > at > org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) > at > org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) > at > org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) > at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) > at >
[jira] [Updated] (HBASE-16993) BucketCache throw java.io.IOException: Invalid HFile block magic when DATA_BLOCK_ENCODING set to DIFF
[ https://issues.apache.org/jira/browse/HBASE-16993?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] liubangchen updated HBASE-16993: Environment: hbase version 1.1.3 (was: hbase-site.xml setting hbase.bucketcache.bucket.sizes 16384,32768,40960, 46000,49152,51200,65536,131072,524288 hbase.bucketcache.size 16384 hbase.bucketcache.ioengine offheap hfile.block.cache.size 0.3 hfile.block.bloom.cacheonwrite true hbase.rs.cacheblocksonwrite true hfile.block.index.cacheonwrite true 'family', COMPRESSION => 'snappy', VERSIONS => 1,DATA_BLOCK_ENCODING => 'DIFF',CONFIGURATION => {'hbase.hregion.memstore.block.multiplier' => 5}},{DURABILITY => 'SKIP_WAL'},{SPLITS => (1..n_splits).map {|i| "user#{1000+i*(-1000)/n_splits}"}} load data bin/ycsb load hbase10 -P workloads/workloada -p table=usertable -p columnfamily=family -p fieldcount=10 -p fieldlength=100 -p recordcount=2 -p insertorder=hashed -p insertstart=0 -p clientbuffering=true -p durability=SKIP_WAL -threads 20 -s run bin/ycsb run hbase10 -P workloads/workloadb -p table=usertable -p columnfamily=family -p fieldcount=10 -p fieldlength=100 -p operationcount=2000 -p readallfields=true -p clientbuffering=true -p requestdistribution=zipfian -threads 10 -s log info 2016-11-02 20:20:20,261 ERROR [RW.default.readRpcServer.handler=36,queue=21,port=6020] bucket.BucketCache: Failed reading block fdcc7ed6f3b2498b9ef316cc8206c233_44819759 from bucket cache java.io.IOException: Invalid HFile block magic: \x00\x00\x00\x00\x00\x00\x00\x00 at org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) at org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) at org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:134) at org.apache.hadoop.hbase.io.hfile.HFileBlock$1.deserialize(HFileBlock.java:121) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getBlock(BucketCache.java:427) at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.getBlock(CombinedBlockCache.java:85) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.getCachedBlock(HFileReaderV2.java:266) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:403) at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:584) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:247) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:156) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:363) at org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:217) at org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2071) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:5369) at org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2546) at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2532) at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2514) at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6558) at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:6537) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:1935) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32381) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2117) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:104) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108) at java.lang.Thread.run(Thread.java:745) 2016-11-02 20:20:20,263 ERROR [RW.default.readRpcServer.handler=50,queue=20,port=6020] bucket.BucketCache: Failed reading block c45d6b14789546b785bae94c69c683d5_34198622 from bucket cache java.io.IOException: Invalid HFile block magic: \x00\x00\x00\x00\x00\x00\x00\x00 at org.apache.hadoop.hbase.io.hfile.BlockType.parse(BlockType.java:154) at org.apache.hadoop.hbase.io.hfile.BlockType.read(BlockType.java:167) at org.apache.hadoop.hbase.io.hfile.HFileBlock.(HFileBlock.java:273) at