[
https://issues.apache.org/jira/browse/HDDS-11355?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17896874#comment-17896874
]
Attila Doroszlai commented on HDDS-11355:
-----------------------------------------
In other runs of the same test:
{code}
Nov 09, 2024 7:51:02 AM
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference
cleanQueue
SEVERE: *~*~*~ Previous channel ManagedChannelImpl{logId=297,
target=127.0.0.1:15021} was not shutdown properly!!! ~*~*~*
Make sure to call shutdown()/shutdownNow() and wait until
awaitTermination() returns true.
java.lang.RuntimeException: ManagedChannel allocation site
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper$ManagedChannelReference.<init>(ManagedChannelOrphanWrapper.java:102)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:60)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelOrphanWrapper.<init>(ManagedChannelOrphanWrapper.java:51)
at
org.apache.ratis.thirdparty.io.grpc.internal.ManagedChannelImplBuilder.build(ManagedChannelImplBuilder.java:631)
at
org.apache.ratis.thirdparty.io.grpc.internal.AbstractManagedChannelImplBuilder.build(AbstractManagedChannelImplBuilder.java:297)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient.buildChannel(GrpcClientProtocolClient.java:154)
at
org.apache.ratis.grpc.client.GrpcClientProtocolClient.<init>(GrpcClientProtocolClient.java:113)
at
org.apache.ratis.grpc.client.GrpcClientRpc.lambda$new$0(GrpcClientRpc.java:59)
at
org.apache.ratis.util.PeerProxyMap.createProxyImpl(PeerProxyMap.java:112)
at org.apache.ratis.util.PeerProxyMap.access$100(PeerProxyMap.java:42)
at
org.apache.ratis.util.PeerProxyMap$PeerAndProxy.lambda$getProxy$0(PeerProxyMap.java:70)
at
org.apache.ratis.util.LifeCycle.startAndTransition(LifeCycle.java:277)
at
org.apache.ratis.util.PeerProxyMap$PeerAndProxy.getProxy(PeerProxyMap.java:69)
at org.apache.ratis.util.PeerProxyMap.getProxy(PeerProxyMap.java:124)
at
org.apache.ratis.grpc.client.GrpcClientRpc.sendRequestAsync(GrpcClientRpc.java:69)
at
org.apache.ratis.client.impl.OrderedAsync.sendRequestWithRetry(OrderedAsync.java:206)
at
org.apache.ratis.util.SlidingWindow$Client.sendOrDelayRequest(SlidingWindow.java:315)
at
org.apache.ratis.util.SlidingWindow$Client.submitNewRequest(SlidingWindow.java:294)
at org.apache.ratis.client.impl.OrderedAsync.send(OrderedAsync.java:173)
at
org.apache.ratis.client.impl.OrderedAsync.newInstance(OrderedAsync.java:125)
at
org.apache.ratis.client.impl.RaftClientImpl.lambda$new$1(RaftClientImpl.java:211)
at org.apache.ratis.util.MemoizedSupplier.get(MemoizedSupplier.java:63)
at
org.apache.ratis.client.impl.RaftClientImpl.getOrderedAsync(RaftClientImpl.java:279)
at org.apache.ratis.client.impl.AsyncImpl.send(AsyncImpl.java:41)
at org.apache.ratis.client.impl.AsyncImpl.send(AsyncImpl.java:46)
at org.apache.ratis.client.api.AsyncApi.send(AsyncApi.java:46)
at
org.apache.hadoop.hdds.scm.XceiverClientRatis.lambda$sendRequestAsync$3(XceiverClientRatis.java:274)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:169)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:159)
at
org.apache.hadoop.hdds.scm.XceiverClientRatis.sendRequestAsync(XceiverClientRatis.java:259)
at
org.apache.hadoop.hdds.scm.XceiverClientRatis.sendCommandAsync(XceiverClientRatis.java:372)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.writeChunkAsync(ContainerProtocolCalls.java:478)
at
org.apache.hadoop.hdds.scm.storage.BlockOutputStream.writeChunkToContainer(BlockOutputStream.java:963)
at
org.apache.hadoop.hdds.scm.storage.BlockOutputStream.writeChunk(BlockOutputStream.java:665)
at
org.apache.hadoop.hdds.scm.storage.BlockOutputStream.writeChunkIfNeeded(BlockOutputStream.java:366)
at
org.apache.hadoop.hdds.scm.storage.BlockOutputStream.write(BlockOutputStream.java:392)
at
org.apache.hadoop.ozone.client.io.BlockOutputStreamEntry.write(BlockOutputStreamEntry.java:186)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.writeToOutputStream(KeyOutputStream.java:307)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:283)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.lambda$write$1(KeyOutputStream.java:249)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.doInWriteLock(KeyOutputStream.java:260)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.write(KeyOutputStream.java:248)
at
org.apache.hadoop.ozone.client.io.OzoneOutputStream.write(OzoneOutputStream.java:94)
at java.base/java.io.OutputStream.write(OutputStream.java:127)
at
org.apache.hadoop.ozone.client.rpc.TestMultiBlockWritesWithDnFailures.testMultiBlockWritesWithIntermittentDnFailures(TestMultiBlockWritesWithDnFailures.java:202)
{code}
As well as:
{code:title=https://github.com/apache/ozone/actions/runs/11754279233/job/32748235886#step:6:3711}
java.io.IOException: java.util.ConcurrentModificationException
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:293)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.lambda$write$1(KeyOutputStream.java:249)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.doInWriteLock(KeyOutputStream.java:260)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.write(KeyOutputStream.java:248)
at
org.apache.hadoop.ozone.client.io.OzoneOutputStream.write(OzoneOutputStream.java:94)
at java.base/java.io.OutputStream.write(OutputStream.java:127)
at
org.apache.hadoop.ozone.client.rpc.TestMultiBlockWritesWithDnFailures.testMultiBlockWritesWithIntermittentDnFailures(TestMultiBlockWritesWithDnFailures.java:216)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
Caused by: java.util.ConcurrentModificationException
at java.base/java.util.ArrayList.forEach(ArrayList.java:1513)
at
org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList.addDatanodes(ExcludeList.java:83)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleStreamAction(KeyOutputStream.java:621)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleFlushOrClose(KeyOutputStream.java:590)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:287)
... 9 more
{code}
> Intermittent failure in
> TestMultiBlockWritesWithDnFailures#testMultiBlockWritesWithIntermittentDnFailures
> ---------------------------------------------------------------------------------------------------------
>
> Key: HDDS-11355
> URL: https://issues.apache.org/jira/browse/HDDS-11355
> Project: Apache Ozone
> Issue Type: Sub-task
> Reporter: Ethan Rose
> Priority: Major
> Attachments: it-client.zip
>
>
> Failed in [this
> run|https://github.com/apache/ozone/actions/runs/10456779998/job/28955527873].
> {code}
> -------------------------------------------------------------------------------
> Test set:
> org.apache.hadoop.ozone.client.rpc.TestMultiBlockWritesWithDnFailures
> -------------------------------------------------------------------------------
> Tests run: 2, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 173.435 s <<<
> FAILURE! - in
> org.apache.hadoop.ozone.client.rpc.TestMultiBlockWritesWithDnFailures
> org.apache.hadoop.ozone.client.rpc.TestMultiBlockWritesWithDnFailures.testMultiBlockWritesWithIntermittentDnFailures
> Time elapsed: 112.791 s <<< ERROR!
> org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException:
> Unable to find the block with bcsID 45 .Container 1 bcsId is 40.
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.validateContainerResponse(ContainerProtocolCalls.java:790)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.lambda$createValidators$4(ContainerProtocolCalls.java:801)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithRetry(XceiverClientGrpc.java:447)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.lambda$sendCommandWithTraceIDAndRetry$1(XceiverClientGrpc.java:373)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:169)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:159)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithTraceIDAndRetry(XceiverClientGrpc.java:365)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommand(XceiverClientGrpc.java:346)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:238)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.lambda$getBlock$0(ContainerProtocolCalls.java:204)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.tryEachDatanode(ContainerProtocolCalls.java:158)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:203)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockDataUsingClient(BlockInputStream.java:287)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockData(BlockInputStream.java:260)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.initialize(BlockInputStream.java:164)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.readWithStrategy(BlockInputStream.java:375)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:56)
> at
> org.apache.hadoop.hdds.scm.storage.ByteArrayReader.readFromBlock(ByteArrayReader.java:54)
> at
> org.apache.hadoop.hdds.scm.storage.MultipartInputStream.readWithStrategy(MultipartInputStream.java:96)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:56)
> at
> org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:56)
> at java.base/java.io.InputStream.read(InputStream.java:218)
> at
> org.apache.hadoop.ozone.container.TestHelper.validateData(TestHelper.java:190)
> at
> org.apache.hadoop.ozone.client.rpc.TestMultiBlockWritesWithDnFailures.validateData(TestMultiBlockWritesWithDnFailures.java:241)
> at
> org.apache.hadoop.ozone.client.rpc.TestMultiBlockWritesWithDnFailures.testMultiBlockWritesWithIntermittentDnFailures(TestMultiBlockWritesWithDnFailures.java:229)
> at java.base/java.lang.reflect.Method.invoke(Method.java:569)
> at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
> at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
