[
https://issues.apache.org/jira/browse/HBASE-24372?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Lijin Bin updated HBASE-24372:
------------------------------
Description:
{code}
2020-03-25 14:58:58,375 INFO
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000] master.HMaster:
Client=hbaseadmin//10.196.142.227 create 'extra_500000039', {NAME => 'extra',
VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR =>
'false', KEEP_DELETED_CELLS => 'false', CACHE_DATA_ON_WRITE => 'false',
DATA_BLOCK_ENCODING => 'DIFF', TTL => '2592000 SECONDS (30 DAYS)', MIN_VERSIONS
=> '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE =>
'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false',
PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'SNAPPY', BLOCKCACHE =>
'true', BLOCKSIZE => '65536'}
2020-03-25 14:58:58,482 INFO
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
rsgroup.RSGroupAdminServer: Moving table extra_500000039 to RSGroup default
2020-03-25 14:58:58,485 ERROR
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
master.TableStateManager: Unable to get table extra_500000039 state
org.apache.hadoop.hbase.master.TableStateManager$TableStateNotFoundException:
extra_500000039
at
org.apache.hadoop.hbase.master.TableStateManager.getTableState(TableStateManager.java:215)
at
org.apache.hadoop.hbase.master.TableStateManager.isTableState(TableStateManager.java:147)
at
org.apache.hadoop.hbase.master.assignment.AssignmentManager.isTableDisabled(AssignmentManager.java:388)
at
org.apache.hadoop.hbase.rsgroup.RSGroupAdminServer.moveTableRegionsToGroup(RSGroupAdminServer.java:233)
at
org.apache.hadoop.hbase.rsgroup.RSGroupAdminServer.moveTables(RSGroupAdminServer.java:347)
at
org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint.assignTableToGroup(RSGroupAdminEndpoint.java:456)
at
org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint.postCreateTable(RSGroupAdminEndpoint.java:479)
at
org.apache.hadoop.hbase.master.MasterCoprocessorHost$13.call(MasterCoprocessorHost.java:350)
at
org.apache.hadoop.hbase.master.MasterCoprocessorHost$13.call(MasterCoprocessorHost.java:347)
at
org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:551)
at
org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:625)
at
org.apache.hadoop.hbase.master.MasterCoprocessorHost.postCreateTable(MasterCoprocessorHost.java:347)
at org.apache.hadoop.hbase.master.HMaster$4.run(HMaster.java:2048)
at
org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:134)
at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2031)
at
org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:658)
at
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)
2020-03-25 14:58:58,486 INFO
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
rsgroup.RSGroupAdminServer: Moving region(s) for table extra_500000039 to
RSGroup default
2020-03-25 14:58:58,486 INFO
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
master.MasterRpcServices: Client=hbaseadmin//10.196.142.227 procedure request
for creating table: namespace: "default"
qualifier: "extra_500000039"
procId is: 14019
{code}
Latch is released when execute prepareCreate, so
MasterCoprocessorHost#postCreateTable can run before the table actually
created, when RSGroup enable, it will throw exception like the above.
Looks like the issue will not happen in master, but can happen in branch-2.
was:
{code}
2020-03-25 14:58:58,375 INFO
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000] master.HMaster:
Client=hbaseadmin//10.196.142.227 create 'extra_500000039', {NAME => 'extra',
VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR =>
'false', KEEP_DELETED_CELLS => 'false', CACHE_DATA_ON_WRITE => 'false',
DATA_BLOCK_ENCODING => 'DIFF', TTL => '2592000 SECONDS (30 DAYS)', MIN_VERSIONS
=> '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE =>
'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false',
PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'SNAPPY', BLOCKCACHE =>
'true', BLOCKSIZE => '65536'}
2020-03-25 14:58:58,482 INFO
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
rsgroup.RSGroupAdminServer: Moving table extra_500000039 to RSGroup default
2020-03-25 14:58:58,485 ERROR
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
master.TableStateManager: Unable to get table extra_500000039 state
org.apache.hadoop.hbase.master.TableStateManager$TableStateNotFoundException:
extra_500000039
at
org.apache.hadoop.hbase.master.TableStateManager.getTableState(TableStateManager.java:215)
at
org.apache.hadoop.hbase.master.TableStateManager.isTableState(TableStateManager.java:147)
at
org.apache.hadoop.hbase.master.assignment.AssignmentManager.isTableDisabled(AssignmentManager.java:388)
at
org.apache.hadoop.hbase.rsgroup.RSGroupAdminServer.moveTableRegionsToGroup(RSGroupAdminServer.java:233)
at
org.apache.hadoop.hbase.rsgroup.RSGroupAdminServer.moveTables(RSGroupAdminServer.java:347)
at
org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint.assignTableToGroup(RSGroupAdminEndpoint.java:456)
at
org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint.postCreateTable(RSGroupAdminEndpoint.java:479)
at
org.apache.hadoop.hbase.master.MasterCoprocessorHost$13.call(MasterCoprocessorHost.java:350)
at
org.apache.hadoop.hbase.master.MasterCoprocessorHost$13.call(MasterCoprocessorHost.java:347)
at
org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:551)
at
org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:625)
at
org.apache.hadoop.hbase.master.MasterCoprocessorHost.postCreateTable(MasterCoprocessorHost.java:347)
at org.apache.hadoop.hbase.master.HMaster$4.run(HMaster.java:2048)
at
org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:134)
at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2031)
at
org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:658)
at
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)
2020-03-25 14:58:58,486 INFO
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
rsgroup.RSGroupAdminServer: Moving region(s) for table extra_500000039 to
RSGroup default
2020-03-25 14:58:58,486 INFO
[RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
master.MasterRpcServices: Client=hbaseadmin//10.196.142.227 procedure request
for creating table: namespace: "default"
qualifier: "extra_500000039"
procId is: 14019
{code}
Latch is released when execute prepareCreate, so
MasterCoprocessorHost#postCreateTable can run and may
> HBASE-20658 Release latch later for CreateTable procedure.
> ----------------------------------------------------------
>
> Key: HBASE-24372
> URL: https://issues.apache.org/jira/browse/HBASE-24372
> Project: HBase
> Issue Type: Bug
> Reporter: Lijin Bin
> Priority: Major
>
> {code}
> 2020-03-25 14:58:58,375 INFO
> [RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000] master.HMaster:
> Client=hbaseadmin//10.196.142.227 create 'extra_500000039', {NAME => 'extra',
> VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR =>
> 'false', KEEP_DELETED_CELLS => 'false', CACHE_DATA_ON_WRITE => 'false',
> DATA_BLOCK_ENCODING => 'DIFF', TTL => '2592000 SECONDS (30 DAYS)',
> MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW',
> CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE
> => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'SNAPPY',
> BLOCKCACHE => 'true', BLOCKSIZE => '65536'}
> 2020-03-25 14:58:58,482 INFO
> [RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
> rsgroup.RSGroupAdminServer: Moving table extra_500000039 to RSGroup default
> 2020-03-25 14:58:58,485 ERROR
> [RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
> master.TableStateManager: Unable to get table extra_500000039 state
> org.apache.hadoop.hbase.master.TableStateManager$TableStateNotFoundException:
> extra_500000039
> at
> org.apache.hadoop.hbase.master.TableStateManager.getTableState(TableStateManager.java:215)
> at
> org.apache.hadoop.hbase.master.TableStateManager.isTableState(TableStateManager.java:147)
> at
> org.apache.hadoop.hbase.master.assignment.AssignmentManager.isTableDisabled(AssignmentManager.java:388)
> at
> org.apache.hadoop.hbase.rsgroup.RSGroupAdminServer.moveTableRegionsToGroup(RSGroupAdminServer.java:233)
> at
> org.apache.hadoop.hbase.rsgroup.RSGroupAdminServer.moveTables(RSGroupAdminServer.java:347)
> at
> org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint.assignTableToGroup(RSGroupAdminEndpoint.java:456)
> at
> org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint.postCreateTable(RSGroupAdminEndpoint.java:479)
> at
> org.apache.hadoop.hbase.master.MasterCoprocessorHost$13.call(MasterCoprocessorHost.java:350)
> at
> org.apache.hadoop.hbase.master.MasterCoprocessorHost$13.call(MasterCoprocessorHost.java:347)
> at
> org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:551)
> at
> org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:625)
> at
> org.apache.hadoop.hbase.master.MasterCoprocessorHost.postCreateTable(MasterCoprocessorHost.java:347)
> at org.apache.hadoop.hbase.master.HMaster$4.run(HMaster.java:2048)
> at
> org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:134)
> at
> org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2031)
> at
> org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:658)
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)
> 2020-03-25 14:58:58,486 INFO
> [RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
> rsgroup.RSGroupAdminServer: Moving region(s) for table extra_500000039 to
> RSGroup default
> 2020-03-25 14:58:58,486 INFO
> [RpcServer.default.FPBQ.Fifo.handler=397,queue=77,port=60000]
> master.MasterRpcServices: Client=hbaseadmin//10.196.142.227 procedure request
> for creating table: namespace: "default"
> qualifier: "extra_500000039"
> procId is: 14019
> {code}
> Latch is released when execute prepareCreate, so
> MasterCoprocessorHost#postCreateTable can run before the table actually
> created, when RSGroup enable, it will throw exception like the above.
> Looks like the issue will not happen in master, but can happen in branch-2.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
