[
https://issues.apache.org/jira/browse/ARTEMIS-5693?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Gary Tully resolved ARTEMIS-5693.
---------------------------------
Fix Version/s: 2.43.0
Assignee: Gary Tully
Resolution: Fixed
> jgroups scaledown race with shared channel
> ------------------------------------------
>
> Key: ARTEMIS-5693
> URL: https://issues.apache.org/jira/browse/ARTEMIS-5693
> Project: ActiveMQ Artemis
> Issue Type: Bug
> Components: Clustering
> Affects Versions: 2.42.0
> Reporter: Gary Tully
> Assignee: Gary Tully
> Priority: Major
> Labels: pull-request-available
> Fix For: 2.43.0
>
> Time Spent: 40m
> Remaining Estimate: 0h
>
> AMQ222181: Unable to scaleDown messages
> logs:
> {{
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=1,
> channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel),
> channelName='activemq_broadcast_channel', connected=true}::removeReceiver:
> org.apache.activemq.artemis.api.core.jgroups.JGroupsReceiver@642857e0 on
> activemq_broadcast_channel
> java.lang.Exception: Trace
> at
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.removeReceiver(JChannelWrapper.java:107)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:132)
> at
> org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
> at
> org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
> at
> org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
> at
> org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
> at
> org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
> at
> org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> at
> org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=0,
> channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel),
> channelName='activemq_broadcast_channel', connected=true}::RefCount-- 0 on
> channel activemq_broadcast_channel
> java.lang.Exception: Trace
> at
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.close(JChannelWrapper.java:87)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalCloseChannel(JGroupsBroadcastEndpoint.java:143)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:135)
> at
> org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
> at
> org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
> at
> org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
> at
> org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
> at
> org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
> at
> org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> at
> org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=0,
> channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel),
> channelName='activemq_broadcast_channel', connected=false}::Closing Channel:
> activemq_broadcast_channel
> java.lang.Exception: Trace
> at
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.closeChannel(JChannelWrapper.java:100)
> at
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.close(JChannelWrapper.java:90)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalCloseChannel(JGroupsBroadcastEndpoint.java:143)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:135)
> at
> org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
> at
> org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
> at
> org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
> at
> org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
> at
> org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
> at
> org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> at
> org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=1,
> channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel),
> channelName='activemq_broadcast_channel', connected=false}::RefCount++ = 1 on
> channel activemq_broadcast_channel
> AMQ221029: Stopped bridge
> $.artemis.internal.sf.my-cluster.c9cfdf48-9f87-11f0-bd48-e24e2a7d3973
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=1,
> channel=org.jgroups.JChannel@574a89e2,
> channelName='activemq_broadcast_channel', connected=false}:: Connecting
> activemq_broadcast_channel
> java.lang.Exception: Trace
> at
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.connect(JChannelWrapper.java:115)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalOpen(JGroupsBroadcastEndpoint.java:124)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.openClient(JGroupsBroadcastEndpoint.java:102)
> at
> org.apache.activemq.artemis.core.cluster.DiscoveryGroup.start(DiscoveryGroup.java:110)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:300)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:284)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.createSessionFactory(ServerLocatorImpl.java:677)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:570)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:549)
> at
> org.apache.activemq.artemis.core.server.NodeLocator.connectToCluster(NodeLocator.java:85)
> at
> org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.connectToScaleDownTarget(PrimaryOnlyActivation.java:148)
> at
> org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.freezeConnections(PrimaryOnlyActivation.java:116)
> at
> org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.freezeConnections(ActiveMQServerImpl.java:1595)
> at
> org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1334)
> at
> org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1251)
> at
> org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.fail(ActiveMQServerImpl.java:1230)
> at
> org.apache.activemq.artemis.core.management.impl.ActiveMQServerControlImpl.scaleDown(ActiveMQServerControlImpl.java:4209)
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> ...
> at
> org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:678)
> at
> org.eclipse.jetty.server.internal.HttpConnection.onFillable(HttpConnection.java:416)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1211)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
> at java.base/java.lang.Thread.run(Thread.java:840)
> AMQ222181: Unable to scaleDown messages
> org.apache.activemq.artemis.api.core.ActiveMQInternalErrorException:
> AMQ219004: Failed to initialise session factory
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:288)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.createSessionFactory(ServerLocatorImpl.java:677)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:570)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:549)
> at
> org.apache.activemq.artemis.core.server.NodeLocator.connectToCluster(NodeLocator.java:85)
> at
> org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.connectToScaleDownTarget(PrimaryOnlyActivation.java:148)
> at
> org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.freezeConnections(PrimaryOnlyActivation.java:116)
> at
> org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.freezeConnections(ActiveMQServerImpl.java:1595)
> at
> org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1334)
> at
> org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1251)
> at
> org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.fail(ActiveMQServerImpl.java:1230)
> at
> org.apache.activemq.artemis.core.management.impl.ActiveMQServerControlImpl.scaleDown(ActiveMQServerControlImpl.java:4209)
> ...
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
> at java.base/java.lang.Thread.run(Thread.java:840)
> Caused by:
> org.apache.activemq.artemis.api.core.ActiveMQInternalErrorException: channel
> is closed
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:302)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:284)
> ... 135 more
> Caused by: java.lang.IllegalStateException: channel is closed
> at org.jgroups.JChannel.checkClosed(JChannel.java:998)
> at org.jgroups.JChannel._preConnect(JChannel.java:809)
> at org.jgroups.JChannel.connect(JChannel.java:328)
> at org.jgroups.JChannel.connect(JChannel.java:321)
> at
> org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.connect(JChannelWrapper.java:126)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalOpen(JGroupsBroadcastEndpoint.java:124)
> at
> org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.openClient(JGroupsBroadcastEndpoint.java:102)
> at
> org.apache.activemq.artemis.core.cluster.DiscoveryGroup.start(DiscoveryGroup.java:110)
> at
> org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:300)
> ... 136 more
> }}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
For further information, visit: https://activemq.apache.org/contact
