[
https://issues.apache.org/jira/browse/HBASE-29007?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Chandra Sekhar K resolved HBASE-29007.
--------------------------------------
Assignee: (was: Chandra Sekhar K)
Resolution: Duplicate
> Master Failed to Start after upgrading from 2.5.10 to 3.0.0-beta-1
> -------------------------------------------------------------------
>
> Key: HBASE-29007
> URL: https://issues.apache.org/jira/browse/HBASE-29007
> Project: HBase
> Issue Type: Bug
> Reporter: Chandra Sekhar K
> Priority: Major
> Labels: duplicate
> Attachments: hbase-hdoop-master-master.log,
> hbase-hdoop-regionserver-master.log
>
>
> Master Failed to Start after upgrading from 2.5.10 to 3.0.0-beta-1
> {code:java}
> 2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster]
> master.HMaster: Failed to become active master
> java.lang.IllegalStateException: Expected the service
> ClusterSchemaServiceImpl [FAILED] to be RUNNING, but the service has FAILED
> at
> org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:384)
> ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
> at
> org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:324)
> ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
> at
> org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1535)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1204)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2494)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155)
> ~[hbase-common-3.0.0-beta-1.jar:3.0.0-beta-1]
> at java.lang.Thread.run(Thread.java:840) ~[?:?]
> Caused by:
> org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException: Failed
> 18 actions: RetriesExhaustedException: 18 times, servers with issues:
> at
> org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.makeError(BufferedMutatorOverAsyncBufferedMutator.java:107)
> ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.internalFlush(BufferedMutatorOverAsyncBufferedMutator.java:122)
> ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.close(BufferedMutatorOverAsyncBufferedMutator.java:166)
> ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.TableNamespaceManager.migrateNamespaceTable(TableNamespaceManager.java:92)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:122)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.ClusterSchemaServiceImpl.doStart(ClusterSchemaServiceImpl.java:61)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.startAsync(AbstractService.java:252)
> ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
> at
> org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1532)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> ... 5 more
> 2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster]
> master.HMaster: Master server abort: loaded coprocessors are: []
> 2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster]
> master.HMaster: ***** ABORTING master master,16000,1733059831148: Unhandled
> exception. Starting shutdown. *****
> java.lang.IllegalStateException: Expected the service
> ClusterSchemaServiceImpl [FAILED] to be RUNNING, but the service has FAILED
> at
> org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:384)
> ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
> at
> org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:324)
> ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
> at
> org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1535)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1204)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2494)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155)
> ~[hbase-common-3.0.0-beta-1.jar:3.0.0-beta-1]
> at java.lang.Thread.run(Thread.java:840) ~[?:?]{code}
> Region Server has below warning log
>
> {code:java}
> 2024-12-01T19:00:47,354 WARN
> [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020]
> regionserver.HRegion: No such column family in batch mutation in region
> org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation@74bf756d
> org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException: Column
> family ns does not exist in region hbase:meta,,1.1588230740 in table
> 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 =>
> '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|',
> METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME =>
> 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS
> => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER',
> MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL',
> IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE
> => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE',
> VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING
> => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE
> => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE',
> BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table',
> INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS =>
> 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER',
> MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL',
> IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE
> => '8192 B (8KB)'}
> at
> org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5344)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5331)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.checkFamilies(HRegion.java:5325)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.access$100(HRegion.java:239)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.checkAndPreparePut(HRegion.java:3849)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.checkAndPrepareMutation(HRegion.java:3486)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.checkAndPrepareMutation(HRegion.java:3498)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation$1.visit(HRegion.java:3878)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.visitBatchOperations(HRegion.java:3377)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.checkAndPrepare(HRegion.java:3857)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4753)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4673)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:984)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:902)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:867)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2830)
> ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45018)
> ~[hbase-protocol-shaded-3.0.0-beta-1.jar:3.0.0-beta-1]{code}
> complete log files are attached
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
