[
https://issues.apache.org/jira/browse/CASSANDRA-3118?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13096995#comment-13096995
]
deng commented on CASSANDRA-3118:
---------------------------------
I changed the code and added some debugg infomation,the code is:
public static void calculatePendingRanges(
AbstractReplicationStrategy strategy, String table) {
logger_.debug("Calculating pending ranges for {} with {}",
table, strategy);
TokenMetadata tm = StorageService.instance.getTokenMetadata();
Multimap<Range, InetAddress> pendingRanges =
HashMultimap.create();
Map<Token, InetAddress> bootstrapTokens =
tm.getBootstrapTokens();
Set<InetAddress> leavingEndpoints = tm.getLeavingEndpoints();
for(InetAddress leave:leavingEndpoints){
System.out.println("leavingEndpoints===="+leave.getHostAddress());
}
if (bootstrapTokens.isEmpty() && leavingEndpoints.isEmpty()
&& tm.getMovingEndpoints().isEmpty()) {
if (logger_.isDebugEnabled())
logger_.debug(
"No bootstrapping, leaving or
moving nodes -> empty pending ranges for {}",
table);
tm.setPendingRanges(table, pendingRanges);
return;
}
Multimap<InetAddress, Range> addressRanges = strategy
.getAddressRanges();
// Copy of metadata reflecting the situation after all leave
operations
// are finished.
TokenMetadata allLeftMetadata = tm.cloneAfterAllLeft();
// get all ranges that will be affected by leaving nodes
Set<Range> affectedRanges = new HashSet<Range>();
for (InetAddress endpoint : leavingEndpoints)
affectedRanges.addAll(addressRanges.get(endpoint));
// for each of those ranges, find what new nodes will be
responsible for
// the range when
// all leaving nodes are gone.
for (Range range : affectedRanges) {
Collection<InetAddress> currentEndpoints = strategy
.calculateNaturalEndpoints(range.right,
tm);
Collection<InetAddress> newEndpoints = strategy
.calculateNaturalEndpoints(range.right,
allLeftMetadata);
System.out.println("olddAddressSize===="+currentEndpoints.size());
for (InetAddress olddAddress : currentEndpoints) {
System.out.println("OLDAddress+++++"
+ olddAddress.getHostAddress()
+ olddAddress.getHostName());
}
System.out.println("newEndpointsSize===="+newEndpoints.size());
for (InetAddress newAddress : newEndpoints) {
System.out.println("NEWAddress+++++"
+ newAddress.getHostAddress()
+ newAddress.getHostName());
}
newEndpoints.removeAll(currentEndpoints);
pendingRanges.putAll(range, newEndpoints);
}
// At this stage pendingRanges has been updated according to
leave
// operations. We can
// now continue the calculation by checking bootstrapping nodes.
// For each of the bootstrapping nodes, simply add and remove
them one
// by one to
// allLeftMetadata and check in between what their ranges would
be.
for (Map.Entry<Token, InetAddress> entry :
bootstrapTokens.entrySet()) {
InetAddress endpoint = entry.getValue();
allLeftMetadata.updateNormalToken(entry.getKey(),
endpoint);
for (Range range :
strategy.getAddressRanges(allLeftMetadata).get(
endpoint))
pendingRanges.put(range, endpoint);
allLeftMetadata.removeEndpoint(endpoint);
}
// At this stage pendingRanges has been updated according to
leaving and
// bootstrapping nodes.
// We can now finish the calculation by checking moving nodes.
// For each of the moving nodes, we do the same thing we did for
// bootstrapping:
// simply add and remove them one by one to allLeftMetadata and
check in
// between what their ranges would be.
for (Pair<Token, InetAddress> moving : tm.getMovingEndpoints())
{
InetAddress endpoint = moving.right; // address of the
moving node
// moving.left is a new token of the endpoint
allLeftMetadata.updateNormalToken(moving.left,
endpoint);
for (Range range :
strategy.getAddressRanges(allLeftMetadata).get(
endpoint)) {
pendingRanges.put(range, endpoint);
}
allLeftMetadata.removeEndpoint(endpoint);
}
tm.setPendingRanges(table, pendingRanges);
if (logger_.isDebugEnabled())
logger_.debug("Pending ranges:\n"
+ (pendingRanges.isEmpty() ? "<empty>"
: tm
.printPendingRanges()));
}
I run the commad decommission and the error is:
[root@devapp3 bin]# ./nodetool -h10.86.17.9 decommission
Exception in thread "main" java.lang.UnsupportedOperationException
at java.util.AbstractList.remove(AbstractList.java:144)
at java.util.AbstractList$Itr.remove(AbstractList.java:360)
at java.util.AbstractCollection.removeAll(AbstractCollection.java:337)
at
org.apache.cassandra.service.StorageService.calculatePendingRanges(StorageService.java:1065)
at
org.apache.cassandra.service.StorageService.calculatePendingRanges(StorageService.java:1006)
at
org.apache.cassandra.service.StorageService.handleStateLeaving(StorageService.java:877)
at
org.apache.cassandra.service.StorageService.onChange(StorageService.java:732)
at org.apache.cassandra.gms.Gossiper.doNotifications(Gossiper.java:839)
at
org.apache.cassandra.gms.Gossiper.addLocalApplicationState(Gossiper.java:986)
at
org.apache.cassandra.service.StorageService.startLeaving(StorageService.java:1866)
at
org.apache.cassandra.service.StorageService.decommission(StorageService.java:1885)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
at
com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
at
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
at
com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
at
javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1426)
at
javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
at
javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1264)
at
javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1359)
at
javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:305)
at sun.rmi.transport.Transport$1.run(Transport.java:159)
at java.security.AccessController.doPrivileged(Native Method)
at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
I saw the log:
DEBUG 11:38:37,513 DECOMMISSIONING
DEBUG 11:38:37,528 Node /10.86.17.9 state leaving, token 14668421489936677549167
5769537457222604
DEBUG 11:38:37,528 Calculating pending ranges for edl with org.apache.cassandra.
locator.SimpleStrategy@7f8062
leavingEndpoints====10.86.17.9
olddAddressSize====1
OLDAddress+++++10.86.17.9szpc662G
newEndpointsSize====1
INFO 11:38:38,872 Started hinted handoff for endpoint /10.86.12.224
INFO 11:38:38,872 Finished hinted handoff of 0 rows to endpoint /10.86.12.224
DEBUG 11:38:38,872 Checking remote schema before delivering hints
DEBUG 11:38:38,872 schema for /172.28.16.127 matches local schema
DEBUG 11:38:38,872 Sleeping 8676ms to stagger hint delivery
NEWAddress+++++172.28.16.126172.28.16.126
DEBUG 11:38:42,044 Pending ranges:
172.28.16.126/172.28.16.126:(138871267011481555303305975400762244971,14668421489
9366775491675769537457222604]
DEBUG 11:38:42,044 Calculating pending ranges for Staging with org.apache.cassan
dra.locator.SimpleStrategy@181b3d4
leavingEndpoints====10.86.17.9
olddAddressSize====1
OLDAddress+++++10.86.17.9szpc662G
newEndpointsSize====1
NEWAddress+++++172.28.16.126172.28.16.126
DEBUG 11:38:42,060 Pending ranges:
172.28.16.126/172.28.16.126:(138871267011481555303305975400762244971,14668421489
9366775491675769537457222604]
172.28.16.126/172.28.16.126:(138871267011481555303305975400762244971,14668421489
9366775491675769537457222604]
DEBUG 11:38:42,075 Calculating pending ranges for test1 with org.apache.cassandr
a.locator.LocalStrategy@45378f
leavingEndpoints====10.86.17.9
olddAddressSize====1
OLDAddress+++++10.86.17.9szpc662G
newEndpointsSize====1
NEWAddress+++++10.86.17.9szpc662G
please check!!!!
> nodetool can not decommission a node
> --------------------------------------
>
> Key: CASSANDRA-3118
> URL: https://issues.apache.org/jira/browse/CASSANDRA-3118
> Project: Cassandra
> Issue Type: Bug
> Components: Tools
> Affects Versions: 0.8.4
> Environment: Cassandra0.84
> Reporter: deng
> Fix For: 0.8.5
>
> Attachments: 3118-debug.txt
>
>
> when i use nodetool ring and get the result ,and than i want to decommission
> 100.86.17.90 node ,but i get the error:
> [root@ip bin]# ./nodetool -h10.86.12.225 ring
> Address DC Rack Status State Load Owns
> Token
>
> 154562542458917734942660802527609328132
> 100.86.17.90 datacenter1 rack1 Up Leaving 1.08 MB
> 11.21% 3493450320433654773610109291263389161
> 100.86.12.225 datacenter1 rack1 Up Normal 558.25 MB
> 14.25% 27742979166206700793970535921354744095
> 100.86.12.224 datacenter1 rack1 Up Normal 5.01 GB 6.58%
> 38945137636148605752956920077679425910
> ERROR:
> root@ip bin]# ./nodetool -h100.86.17.90 decommission
> Exception in thread "main" java.lang.UnsupportedOperationException
> at java.util.AbstractList.remove(AbstractList.java:144)
> at java.util.AbstractList$Itr.remove(AbstractList.java:360)
> at java.util.AbstractCollection.removeAll(AbstractCollection.java:337)
> at
> org.apache.cassandra.service.StorageService.calculatePendingRanges(StorageService.java:1041)
> at
> org.apache.cassandra.service.StorageService.calculatePendingRanges(StorageService.java:1006)
> at
> org.apache.cassandra.service.StorageService.handleStateLeaving(StorageService.java:877)
> at
> org.apache.cassandra.service.StorageService.onChange(StorageService.java:732)
> at
> org.apache.cassandra.gms.Gossiper.doNotifications(Gossiper.java:839)
> at
> org.apache.cassandra.gms.Gossiper.addLocalApplicationState(Gossiper.java:986)
> at
> org.apache.cassandra.service.StorageService.startLeaving(StorageService.java:1836)
> at
> org.apache.cassandra.service.StorageService.decommission(StorageService.java:1855)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
> at
> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
> at
> com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
> at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
> at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
> at
> com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
> at
> javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1426)
> at
> javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
> at
> javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1264)
> at
> javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1359)
> at
> javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:305)
> at sun.rmi.transport.Transport$1.run(Transport.java:159)
> at java.security.AccessController.doPrivileged(Native Method)
> at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
> at
> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:619)
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
