[
https://issues.apache.org/jira/browse/HDDS-11375?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Tsz-wo Sze reassigned HDDS-11375:
---------------------------------
Assignee: Tsz-wo Sze
Description:
This is a problem if Ozone is upgraded to the latest (unreleased) Ratis code
base. Ozone currently is using Ratis 3.1.0 which does not have this problem.
All Ozone DN startup is failing with below error:
{code:java}
2024-08-27 15:54:46,040 ERROR
[main]-org.apache.hadoop.ozone.HddsDatanodeService: Exception in
HddsDatanodeService.
java.lang.RuntimeException: Can't start the HDDS datanode plugin
at
org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:336)
at
org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:209)
at
org.apache.hadoop.ozone.HddsDatanodeService.call(HddsDatanodeService.java:177)
at
org.apache.hadoop.ozone.HddsDatanodeService.call(HddsDatanodeService.java:95)
at picocli.CommandLine.executeUserObject(CommandLine.java:2045)
at picocli.CommandLine.access$1500(CommandLine.java:148)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2465)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2457)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2419)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2277)
at picocli.CommandLine$RunLast.execute(CommandLine.java:2421)
at picocli.CommandLine.execute(CommandLine.java:2174)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
at
org.apache.hadoop.ozone.HddsDatanodeService.main(HddsDatanodeService.java:159)
Caused by: java.io.IOException: java.lang.IllegalArgumentException: Illegal
configuration: raft.grpc.message.size.max(= 32MB (=33554432)) must be 1m
(=1048576) larger than raft.server.log.appender.buffer.byte-limit(= 33554432).
at org.apache.ratis.util.IOUtils.asIOException(IOUtils.java:56)
at
org.apache.ratis.server.RaftServer$Builder.newRaftServer(RaftServer.java:196)
at org.apache.ratis.server.RaftServer$Builder.build(RaftServer.java:210)
at
org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis.<init>(XceiverServerRatis.java:214)
at
org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis.newXceiverServerRatis(XceiverServerRatis.java:533)
at
org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer.<init>(OzoneContainer.java:209)
at
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.<init>(DatanodeStateMachine.java:183)
at
org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:291)
... 14 more
Caused by: java.lang.IllegalArgumentException: Illegal configuration:
raft.grpc.message.size.max(= 32MB (=33554432)) must be 1m (=1048576) larger
than raft.server.log.appender.buffer.byte-limit(= 33554432).
at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:184)
at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:152)
at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:57)
at
org.apache.ratis.grpc.server.GrpcService$Builder.build(GrpcService.java:111)
at org.apache.ratis.grpc.GrpcFactory.newRaftServerRpc(GrpcFactory.java:133)
at org.apache.ratis.grpc.GrpcFactory.newRaftServerRpc(GrpcFactory.java:40)
at
org.apache.ratis.server.impl.RaftServerProxy.<init>(RaftServerProxy.java:212)
at
org.apache.ratis.server.impl.ServerImplUtils.lambda$newRaftServer$0(ServerImplUtils.java:74)
at org.apache.ratis.util.JavaUtils.lambda$attempt$7(JavaUtils.java:212)
at org.apache.ratis.util.JavaUtils.attempt(JavaUtils.java:225)
at org.apache.ratis.util.JavaUtils.attempt(JavaUtils.java:212)
at org.apache.ratis.util.JavaUtils.attemptRepeatedly(JavaUtils.java:204)
at
org.apache.ratis.server.impl.ServerImplUtils.newRaftServer(ServerImplUtils.java:73)
at
org.apache.ratis.server.impl.ServerImplUtils.newRaftServer(ServerImplUtils.java:61)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.apache.ratis.server.RaftServer$Builder.newRaftServer(RaftServer.java:191)
... 20 more
2024-08-27 15:54:46,045 INFO
[shutdown-hook-0]-org.apache.hadoop.ozone.HddsDatanodeService: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down HddsDatanodeService at host/xx.xx.xx.xx{code}
was:
All Ozone DN startup is failing with below error:
{code:java}
2024-08-27 15:54:46,040 ERROR
[main]-org.apache.hadoop.ozone.HddsDatanodeService: Exception in
HddsDatanodeService.
java.lang.RuntimeException: Can't start the HDDS datanode plugin
at
org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:336)
at
org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:209)
at
org.apache.hadoop.ozone.HddsDatanodeService.call(HddsDatanodeService.java:177)
at
org.apache.hadoop.ozone.HddsDatanodeService.call(HddsDatanodeService.java:95)
at picocli.CommandLine.executeUserObject(CommandLine.java:2045)
at picocli.CommandLine.access$1500(CommandLine.java:148)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2465)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2457)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2419)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2277)
at picocli.CommandLine$RunLast.execute(CommandLine.java:2421)
at picocli.CommandLine.execute(CommandLine.java:2174)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
at
org.apache.hadoop.ozone.HddsDatanodeService.main(HddsDatanodeService.java:159)
Caused by: java.io.IOException: java.lang.IllegalArgumentException: Illegal
configuration: raft.grpc.message.size.max(= 32MB (=33554432)) must be 1m
(=1048576) larger than raft.server.log.appender.buffer.byte-limit(= 33554432).
at org.apache.ratis.util.IOUtils.asIOException(IOUtils.java:56)
at
org.apache.ratis.server.RaftServer$Builder.newRaftServer(RaftServer.java:196)
at org.apache.ratis.server.RaftServer$Builder.build(RaftServer.java:210)
at
org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis.<init>(XceiverServerRatis.java:214)
at
org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis.newXceiverServerRatis(XceiverServerRatis.java:533)
at
org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer.<init>(OzoneContainer.java:209)
at
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.<init>(DatanodeStateMachine.java:183)
at
org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:291)
... 14 more
Caused by: java.lang.IllegalArgumentException: Illegal configuration:
raft.grpc.message.size.max(= 32MB (=33554432)) must be 1m (=1048576) larger
than raft.server.log.appender.buffer.byte-limit(= 33554432).
at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:184)
at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:152)
at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:57)
at
org.apache.ratis.grpc.server.GrpcService$Builder.build(GrpcService.java:111)
at org.apache.ratis.grpc.GrpcFactory.newRaftServerRpc(GrpcFactory.java:133)
at org.apache.ratis.grpc.GrpcFactory.newRaftServerRpc(GrpcFactory.java:40)
at
org.apache.ratis.server.impl.RaftServerProxy.<init>(RaftServerProxy.java:212)
at
org.apache.ratis.server.impl.ServerImplUtils.lambda$newRaftServer$0(ServerImplUtils.java:74)
at org.apache.ratis.util.JavaUtils.lambda$attempt$7(JavaUtils.java:212)
at org.apache.ratis.util.JavaUtils.attempt(JavaUtils.java:225)
at org.apache.ratis.util.JavaUtils.attempt(JavaUtils.java:212)
at org.apache.ratis.util.JavaUtils.attemptRepeatedly(JavaUtils.java:204)
at
org.apache.ratis.server.impl.ServerImplUtils.newRaftServer(ServerImplUtils.java:73)
at
org.apache.ratis.server.impl.ServerImplUtils.newRaftServer(ServerImplUtils.java:61)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.apache.ratis.server.RaftServer$Builder.newRaftServer(RaftServer.java:191)
... 20 more
2024-08-27 15:54:46,045 INFO
[shutdown-hook-0]-org.apache.hadoop.ozone.HddsDatanodeService: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down HddsDatanodeService at
ccycloud-1.xyz/10.111.89.111{code}
cc: [~duongnguyen] [~weichiu] [~Sammi]
Summary: DN Startup fails with Illegal configuration (was: DN Startup
fails with "RuntimeException: Can't start the HDDS datanode plugin")
> DN Startup fails with Illegal configuration
> -------------------------------------------
>
> Key: HDDS-11375
> URL: https://issues.apache.org/jira/browse/HDDS-11375
> Project: Apache Ozone
> Issue Type: Bug
> Components: Ozone Datanode
> Reporter: Pratyush Bhatt
> Assignee: Tsz-wo Sze
> Priority: Major
>
> This is a problem if Ozone is upgraded to the latest (unreleased) Ratis code
> base. Ozone currently is using Ratis 3.1.0 which does not have this problem.
> All Ozone DN startup is failing with below error:
> {code:java}
> 2024-08-27 15:54:46,040 ERROR
> [main]-org.apache.hadoop.ozone.HddsDatanodeService: Exception in
> HddsDatanodeService.
> java.lang.RuntimeException: Can't start the HDDS datanode plugin
> at
> org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:336)
> at
> org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:209)
> at
> org.apache.hadoop.ozone.HddsDatanodeService.call(HddsDatanodeService.java:177)
> at
> org.apache.hadoop.ozone.HddsDatanodeService.call(HddsDatanodeService.java:95)
> at picocli.CommandLine.executeUserObject(CommandLine.java:2045)
> at picocli.CommandLine.access$1500(CommandLine.java:148)
> at
> picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2465)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2457)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2419)
> at
> picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2277)
> at picocli.CommandLine$RunLast.execute(CommandLine.java:2421)
> at picocli.CommandLine.execute(CommandLine.java:2174)
> at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
> at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
> at
> org.apache.hadoop.ozone.HddsDatanodeService.main(HddsDatanodeService.java:159)
> Caused by: java.io.IOException: java.lang.IllegalArgumentException: Illegal
> configuration: raft.grpc.message.size.max(= 32MB (=33554432)) must be 1m
> (=1048576) larger than raft.server.log.appender.buffer.byte-limit(= 33554432).
> at org.apache.ratis.util.IOUtils.asIOException(IOUtils.java:56)
> at
> org.apache.ratis.server.RaftServer$Builder.newRaftServer(RaftServer.java:196)
> at org.apache.ratis.server.RaftServer$Builder.build(RaftServer.java:210)
> at
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis.<init>(XceiverServerRatis.java:214)
> at
> org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis.newXceiverServerRatis(XceiverServerRatis.java:533)
> at
> org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer.<init>(OzoneContainer.java:209)
> at
> org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.<init>(DatanodeStateMachine.java:183)
> at
> org.apache.hadoop.ozone.HddsDatanodeService.start(HddsDatanodeService.java:291)
> ... 14 more
> Caused by: java.lang.IllegalArgumentException: Illegal configuration:
> raft.grpc.message.size.max(= 32MB (=33554432)) must be 1m (=1048576) larger
> than raft.server.log.appender.buffer.byte-limit(= 33554432).
> at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:184)
> at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:152)
> at org.apache.ratis.grpc.server.GrpcService.<init>(GrpcService.java:57)
> at
> org.apache.ratis.grpc.server.GrpcService$Builder.build(GrpcService.java:111)
> at
> org.apache.ratis.grpc.GrpcFactory.newRaftServerRpc(GrpcFactory.java:133)
> at org.apache.ratis.grpc.GrpcFactory.newRaftServerRpc(GrpcFactory.java:40)
> at
> org.apache.ratis.server.impl.RaftServerProxy.<init>(RaftServerProxy.java:212)
> at
> org.apache.ratis.server.impl.ServerImplUtils.lambda$newRaftServer$0(ServerImplUtils.java:74)
> at org.apache.ratis.util.JavaUtils.lambda$attempt$7(JavaUtils.java:212)
> at org.apache.ratis.util.JavaUtils.attempt(JavaUtils.java:225)
> at org.apache.ratis.util.JavaUtils.attempt(JavaUtils.java:212)
> at org.apache.ratis.util.JavaUtils.attemptRepeatedly(JavaUtils.java:204)
> at
> org.apache.ratis.server.impl.ServerImplUtils.newRaftServer(ServerImplUtils.java:73)
> at
> org.apache.ratis.server.impl.ServerImplUtils.newRaftServer(ServerImplUtils.java:61)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> at
> org.apache.ratis.server.RaftServer$Builder.newRaftServer(RaftServer.java:191)
> ... 20 more
> 2024-08-27 15:54:46,045 INFO
> [shutdown-hook-0]-org.apache.hadoop.ozone.HddsDatanodeService: SHUTDOWN_MSG:
> /************************************************************
> SHUTDOWN_MSG: Shutting down HddsDatanodeService at host/xx.xx.xx.xx{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
