unsubscribe On Sun, Dec 1, 2024 at 8:41 AM Chandra Sekhar K (Jira) <j...@apache.org> wrote:
> Chandra Sekhar K created HBASE-29007: > ---------------------------------------- > > Summary: Master Failed to Start after upgrading from 2.5.10 > to 3.0.0-beta-1 > Key: HBASE-29007 > URL: https://issues.apache.org/jira/browse/HBASE-29007 > Project: HBase > Issue Type: Bug > Reporter: Chandra Sekhar K > Assignee: Chandra Sekhar K > Attachments: hbase-hdoop-master-master.log, > hbase-hdoop-regionserver-master.log > > Master Failed to Start after upgrading from 2.5.10 to 3.0.0-beta-1 > {code:java} > 2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster] > master.HMaster: Failed to become active master > java.lang.IllegalStateException: Expected the service > ClusterSchemaServiceImpl [FAILED] to be RUNNING, but the service has FAILED > at > org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:384) > ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5] > at > org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:324) > ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5] > at > org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1535) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1204) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2494) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) > ~[hbase-common-3.0.0-beta-1.jar:3.0.0-beta-1] > at java.lang.Thread.run(Thread.java:840) ~[?:?] > Caused by: > org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException: Failed > 18 actions: RetriesExhaustedException: 18 times, servers with issues: > at > org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.makeError(BufferedMutatorOverAsyncBufferedMutator.java:107) > ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.internalFlush(BufferedMutatorOverAsyncBufferedMutator.java:122) > ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.close(BufferedMutatorOverAsyncBufferedMutator.java:166) > ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.TableNamespaceManager.migrateNamespaceTable(TableNamespaceManager.java:92) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:122) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.ClusterSchemaServiceImpl.doStart(ClusterSchemaServiceImpl.java:61) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.startAsync(AbstractService.java:252) > ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5] > at > org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1532) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > ... 5 more > 2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster] > master.HMaster: Master server abort: loaded coprocessors are: [] > 2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster] > master.HMaster: ***** ABORTING master master,16000,1733059831148: Unhandled > exception. Starting shutdown. ***** > java.lang.IllegalStateException: Expected the service > ClusterSchemaServiceImpl [FAILED] to be RUNNING, but the service has FAILED > at > org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:384) > ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5] > at > org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:324) > ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5] > at > org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1535) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1204) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2494) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) > ~[hbase-common-3.0.0-beta-1.jar:3.0.0-beta-1] > at java.lang.Thread.run(Thread.java:840) ~[?:?]{code} > Region Server has below warning log > > > {code:java} > 2024-12-01T19:00:47,354 WARN > [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020] > regionserver.HRegion: No such column family in batch mutation in region > org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation@74bf756d > org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException: Column > family ns does not exist in region hbase:meta,,1.1588230740 in table > 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => > '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', > METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => > 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS > => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', > MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', > IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE > => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', > VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', > DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => > '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => > 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B > (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => > '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', > TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', > BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', > BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} > at > org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5344) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5331) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion.checkFamilies(HRegion.java:5325) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion.access$100(HRegion.java:239) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.checkAndPreparePut(HRegion.java:3849) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.checkAndPrepareMutation(HRegion.java:3486) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.checkAndPrepareMutation(HRegion.java:3498) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation$1.visit(HRegion.java:3878) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.visitBatchOperations(HRegion.java:3377) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.checkAndPrepare(HRegion.java:3857) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4753) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4673) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:984) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:902) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:867) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2830) > ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1] > at > org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45018) > ~[hbase-protocol-shaded-3.0.0-beta-1.jar:3.0.0-beta-1]{code} > complete log files are attached > > > > -- > This message was sent by Atlassian Jira > (v8.20.10#820010) >
