[
https://issues.apache.org/jira/browse/HBASE-28283?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Haiping lv updated HBASE-28283:
-------------------------------
Description:
During the execution of warmUpRegion in HRegionServer, the process of loading
HFiles begins with the identification of the HFiles that need to be loaded,
followed by the actual loading process. If an HFile is moved or deleted before
it is loaded, the loading process will fail due to the missing HFile, leading
to the failure of the entire warmUpRegion process.
The reason an HFile might be moved before being loaded is that the region has
not been closed yet. While the source RegionServer is still operational, if a
compaction or split occurs, the original HFiles can be archived.
The logs are as followed:
{code:java}
2023-12-16T22:47:00,989 ERROR
[RpcServer.default.FPBQ.Fifo.handler=7,queue=1,port=16020]
regionserver.HRegion: Could not initialize all stores for the
region=itbll,\xE9x\xD4\xFD\xF3\xB6E8,1702703598150.4eb5a64755a98500833e3daae883d3a3.
2023-12-16T22:47:00,990 ERROR
[RpcServer.default.FPBQ.Fifo.handler=7,queue=1,port=16020]
regionserver.RSRpcServices: Failed warmup of
itbll,\xE9x\xD4\xFD\xF3\xB6E8,1702703598150.4eb5a64755a98500833e3daae883d3a3.
java.io.IOException: java.io.IOException: java.io.FileNotFoundException: File
does not exist:
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeStores(HRegion.java:1207)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeWarmup(HRegion.java:1231)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion.warmupHRegion(HRegion.java:7841)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.warmupRegion(RSRpcServices.java:2014)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:35678)
~[hbase-protocol-shaded-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
Caused by: java.io.IOException: java.io.FileNotFoundException: File does not
exist:
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
at
org.apache.hadoop.hbase.regionserver.StoreEngine.openStoreFiles(StoreEngine.java:289)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreEngine.initialize(StoreEngine.java:339)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.regionserver.HStore.<init>(HStore.java:300)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:6904)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1173)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1170)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_372]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_372]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_372]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_372]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by: java.io.FileNotFoundException: File does not exist:
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
at
org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1606)
~[hadoop-hdfs-client-3.2.4.jar:?]
at
org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1599)
~[hadoop-hdfs-client-3.2.4.jar:?]
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
~[hadoop-common-3.2.4.jar:?]
at
org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1614)
~[hadoop-hdfs-client-3.2.4.jar:?]
at
org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:458)
~[hadoop-common-3.2.4.jar:?]
at
org.apache.hadoop.hbase.regionserver.StoreFileInfo.getReferencedFileStatus(StoreFileInfo.java:399)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreFileInfo.computeHDFSBlocksDistributionInternal(StoreFileInfo.java:352)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreFileInfo.computeHDFSBlocksDistribution(StoreFileInfo.java:346)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreFileInfo.initHDFSBlocksDistribution(StoreFileInfo.java:690)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HStoreFile.open(HStoreFile.java:381)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HStoreFile.initReader(HStoreFile.java:512)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreEngine.createStoreFileAndReader(StoreEngine.java:226)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreEngine.lambda$openStoreFiles$0(StoreEngine.java:267)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_372]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_372]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_372]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_372]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372] {code}
was:
During the execution of warmUpRegion in HRegionServer, the process of loading
HFiles begins with the identification of the HFiles that need to be loaded,
followed by the actual loading process. If an HFile is moved or deleted before
it is loaded, the loading process will fail due to the missing HFile, leading
to the failure of the entire warmUpRegion process.
The reason an HFile might be moved before being loaded is that the region has
not been closed yet. While the source RegionServer is still operational, if a
compaction or split occurs, the original HFiles can be archived.
The logs are as followed:
{code:java}
2023-12-16T22:47:00,989 ERROR
[RpcServer.default.FPBQ.Fifo.handler=7,queue=1,port=16020]
regionserver.HRegion: Could not initialize all stores for the
region=itbll,\xE9x\xD4\xFD\xF3\xB6E8,1702703598150.4eb5a64755a98500833e3daae883d3a3.
2023-12-16T22:47:00,990 ERROR
[RpcServer.default.FPBQ.Fifo.handler=7,queue=1,port=16020]
regionserver.RSRpcServices: Failed warmup of
itbll,\xE9x\xD4\xFD\xF3\xB6E8,1702703598150.4eb5a64755a98500833e3daae883d3a3.
java.io.IOException: java.io.IOException: java.io.FileNotFoundException: File
does not exist:
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeStores(HRegion.java:1207)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeWarmup(HRegion.java:1231)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion.warmupHRegion(HRegion.java:7841)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.warmupRegion(RSRpcServices.java:2014)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:35678)
~[hbase-protocol-shaded-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
Caused by: java.io.IOException: java.io.FileNotFoundException: File does not
exist:
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
at
org.apache.hadoop.hbase.regionserver.StoreEngine.openStoreFiles(StoreEngine.java:289)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreEngine.initialize(StoreEngine.java:339)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at org.apache.hadoop.hbase.regionserver.HStore.<init>(HStore.java:300)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:6904)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1173)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1170)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_372]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_372]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_372]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_372]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by: java.io.FileNotFoundException: File does not exist:
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
at
org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1606)
~[hadoop-hdfs-client-3.2.4.jar:?]
at
org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1599)
~[hadoop-hdfs-client-3.2.4.jar:?]
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
~[hadoop-common-3.2.4.jar:?]
at
org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1614)
~[hadoop-hdfs-client-3.2.4.jar:?]
at
org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:458)
~[hadoop-common-3.2.4.jar:?]
at
org.apache.hadoop.hbase.regionserver.StoreFileInfo.getReferencedFileStatus(StoreFileInfo.java:399)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreFileInfo.computeHDFSBlocksDistributionInternal(StoreFileInfo.java:352)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreFileInfo.computeHDFSBlocksDistribution(StoreFileInfo.java:346)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreFileInfo.initHDFSBlocksDistribution(StoreFileInfo.java:690)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HStoreFile.open(HStoreFile.java:381)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.HStoreFile.initReader(HStoreFile.java:512)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreEngine.createStoreFileAndReader(StoreEngine.java:226)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at
org.apache.hadoop.hbase.regionserver.StoreEngine.lambda$openStoreFiles$0(StoreEngine.java:267)
~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_372]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_372]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_372]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_372]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372] {code}
{code:java}
2023-12-16T22:46:56,451 INFO [regionserver/core-1-4:16020-shortCompactions-0]
regionserver.HStore: Starting compaction of
[hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/139a3688cfac4f32b29feb48b62f8ce9,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/a9285dad2eb44a7d8a0bbeb49232b434,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/e54ec6ddb95d4651920951cc30a9432b,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/4521b2df0a4b4191a1736b99453db6dd]
into
tmpdir=hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/.tmp,
totalSize=2.4 M
2023-12-16T22:46:56,451 DEBUG [regionserver/core-1-4:16020-shortCompactions-0]
compactions.Compactor: Compacting 6dc12f7a46914e89a4e97d2d90a23e8d,
keycount=70891, bloomtype=ROWPREFIX_FIXED_LENGTH, size=1.5 M,
encoding=FAST_DIFF, compression=LZ4, seqNum=1567586
2023-12-16T22:47:00,567 DEBUG
[StoreCloser-itbll,\xE9x\xD4\xFD\xF3\xB6E8,1702703598150.4eb5a64755a98500833e3daae883d3a3.-1]
regionserver.HStore: Moving the files
[hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/139a3688cfac4f32b29feb48b62f8ce9,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/a9285dad2eb44a7d8a0bbeb49232b434,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/e54ec6ddb95d4651920951cc30a9432b,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/4521b2df0a4b4191a1736b99453db6dd,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/d9c9d500d36f46f295ede09f857ab1aa,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/e79f2c8de1b3497e9cd2419abd96cde0,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/1cd3513db5204235b89abc0c8ea90754]
to archive
2023-12-16T22:47:00,580 DEBUG
[StoreCloser-itbll,\xE9x\xD4\xFD\xF3\xB6E8,1702703598150.4eb5a64755a98500833e3daae883d3a3.-1]
backup.HFileArchiver: Archived from FileableStoreFile,
hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
to
hdfs://master-1-1:9000/v3/hbase/archive/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
{code}
> The warmup of the region failed because the HFile was archived due to the
> execution of a compaction.
> ----------------------------------------------------------------------------------------------------
>
> Key: HBASE-28283
> URL: https://issues.apache.org/jira/browse/HBASE-28283
> Project: HBase
> Issue Type: Bug
> Reporter: Haiping lv
> Assignee: Haiping lv
> Priority: Major
>
> During the execution of warmUpRegion in HRegionServer, the process of loading
> HFiles begins with the identification of the HFiles that need to be loaded,
> followed by the actual loading process. If an HFile is moved or deleted
> before it is loaded, the loading process will fail due to the missing HFile,
> leading to the failure of the entire warmUpRegion process.
> The reason an HFile might be moved before being loaded is that the region has
> not been closed yet. While the source RegionServer is still operational, if a
> compaction or split occurs, the original HFiles can be archived.
>
> The logs are as followed:
> {code:java}
> 2023-12-16T22:47:00,989 ERROR
> [RpcServer.default.FPBQ.Fifo.handler=7,queue=1,port=16020]
> regionserver.HRegion: Could not initialize all stores for the
> region=itbll,\xE9x\xD4\xFD\xF3\xB6E8,1702703598150.4eb5a64755a98500833e3daae883d3a3.
> 2023-12-16T22:47:00,990 ERROR
> [RpcServer.default.FPBQ.Fifo.handler=7,queue=1,port=16020]
> regionserver.RSRpcServices: Failed warmup of
> itbll,\xE9x\xD4\xFD\xF3\xB6E8,1702703598150.4eb5a64755a98500833e3daae883d3a3.
> java.io.IOException: java.io.IOException: java.io.FileNotFoundException: File
> does not exist:
> hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initializeStores(HRegion.java:1207)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initializeWarmup(HRegion.java:1231)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.warmupHRegion(HRegion.java:7841)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.warmupRegion(RSRpcServices.java:2014)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:35678)
> ~[hbase-protocol-shaded-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> Caused by: java.io.IOException: java.io.FileNotFoundException: File does not
> exist:
> hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
> at
> org.apache.hadoop.hbase.regionserver.StoreEngine.openStoreFiles(StoreEngine.java:289)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.StoreEngine.initialize(StoreEngine.java:339)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.HStore.<init>(HStore.java:300)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:6904)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1173)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1170)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> ~[?:1.8.0_372]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> ~[?:1.8.0_372]
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> ~[?:1.8.0_372]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> ~[?:1.8.0_372]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> ~[?:1.8.0_372]
> at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
> Caused by: java.io.FileNotFoundException: File does not exist:
> hdfs://master-1-1:9000/v3/hbase/data/default/itbll/4eb5a64755a98500833e3daae883d3a3/tiny/6dc12f7a46914e89a4e97d2d90a23e8d
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1606)
> ~[hadoop-hdfs-client-3.2.4.jar:?]
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1599)
> ~[hadoop-hdfs-client-3.2.4.jar:?]
> at
> org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
> ~[hadoop-common-3.2.4.jar:?]
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1614)
> ~[hadoop-hdfs-client-3.2.4.jar:?]
> at
> org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:458)
> ~[hadoop-common-3.2.4.jar:?]
> at
> org.apache.hadoop.hbase.regionserver.StoreFileInfo.getReferencedFileStatus(StoreFileInfo.java:399)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.StoreFileInfo.computeHDFSBlocksDistributionInternal(StoreFileInfo.java:352)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.StoreFileInfo.computeHDFSBlocksDistribution(StoreFileInfo.java:346)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.StoreFileInfo.initHDFSBlocksDistribution(StoreFileInfo.java:690)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.HStoreFile.open(HStoreFile.java:381)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.HStoreFile.initReader(HStoreFile.java:512)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.StoreEngine.createStoreFileAndReader(StoreEngine.java:226)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.regionserver.StoreEngine.lambda$openStoreFiles$0(StoreEngine.java:267)
> ~[hbase-server-3.0.0-beta-1-SNAPSHOT.jar:3.0.0-beta-1-SNAPSHOT]
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> ~[?:1.8.0_372]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> ~[?:1.8.0_372]
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> ~[?:1.8.0_372]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> ~[?:1.8.0_372]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> ~[?:1.8.0_372]
> at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372] {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
