[ https://issues.apache.org/jira/browse/HDDS-4018?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Attila Doroszlai updated HDDS-4018: ----------------------------------- Description: {code} datanode_1 | 2020-07-22 13:11:47,845 [Datanode State Machine Thread - 0] WARN statemachine.StateContext: No available thread in pool for past 2 seconds. datanode_1 | 2020-07-22 13:11:47,846 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) datanode_1 | 2020-07-22 13:11:47,847 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) datanode_1 | 2020-07-22 13:11:47,848 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) datanode_1 | 2020-07-22 13:11:47,848 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) datanode_1 | 2020-07-22 13:11:47,851 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) ... thread in pool for past 22 seconds. datanode_1 | 2020-07-22 13:11:47,854 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) {code} This increases acceptance test logs to several hundred MBs. was: {code} datanode_1 | 2020-07-22 13:11:47,845 [Datanode State Machine Thread - 0] WARN statemachine.StateContext: No available thread in pool for past 2 seconds. datanode_1 | 2020-07-22 13:11:47,846 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) datanode_1 | 2020-07-22 13:11:47,847 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) datanode_1 | 2020-07-22 13:11:47,848 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) datanode_1 | 2020-07-22 13:11:47,848 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) datanode_1 | 2020-07-22 13:11:47,851 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) ... thread in pool for past 22 seconds. datanode_1 | 2020-07-22 13:11:47,854 [Datanode State Machine Thread - 0] ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. datanode_1 | java.lang.NullPointerException datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) datanode_1 | at org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) datanode_1 | at org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) {code} This makes increases acceptance test logs to several hundred MBs. > Datanode log spammed by NPE > --------------------------- > > Key: HDDS-4018 > URL: https://issues.apache.org/jira/browse/HDDS-4018 > Project: Hadoop Distributed Data Store > Issue Type: Bug > Components: Ozone Datanode > Affects Versions: 0.6.0 > Reporter: Attila Doroszlai > Priority: Blocker > > {code} > datanode_1 | 2020-07-22 13:11:47,845 [Datanode State Machine Thread - 0] > WARN statemachine.StateContext: No available thread in pool for past 2 > seconds. > datanode_1 | 2020-07-22 13:11:47,846 [Datanode State Machine Thread - 0] > ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. > datanode_1 | java.lang.NullPointerException > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) > datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) > datanode_1 | 2020-07-22 13:11:47,847 [Datanode State Machine Thread - 0] > ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. > datanode_1 | java.lang.NullPointerException > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) > datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) > datanode_1 | 2020-07-22 13:11:47,848 [Datanode State Machine Thread - 0] > ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. > datanode_1 | java.lang.NullPointerException > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) > datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) > datanode_1 | 2020-07-22 13:11:47,848 [Datanode State Machine Thread - 0] > ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. > datanode_1 | java.lang.NullPointerException > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) > datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) > datanode_1 | 2020-07-22 13:11:47,851 [Datanode State Machine Thread - 0] > ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. > datanode_1 | java.lang.NullPointerException > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) > datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) > ... > thread in pool for past 22 seconds. > datanode_1 | 2020-07-22 13:11:47,854 [Datanode State Machine Thread - 0] > ERROR statemachine.DatanodeStateMachine: Unable to finish the execution. > datanode_1 | java.lang.NullPointerException > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:218) > datanode_1 | at > org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState.await(RunningDatanodeState.java:50) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.StateContext.execute(StateContext.java:451) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.start(DatanodeStateMachine.java:225) > datanode_1 | at > org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.lambda$startDaemon$0(DatanodeStateMachine.java:396) > datanode_1 | at java.base/java.lang.Thread.run(Thread.java:834) > {code} > This increases acceptance test logs to several hundred MBs. -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-issues-h...@hadoop.apache.org