Print debug information on 'waitForRent' timeout. (cherry picked from commit 7cf26d9)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f3e4f78f Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f3e4f78f Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f3e4f78f Branch: refs/heads/ignite-3553 Commit: f3e4f78fc96c98f4376ff86ab3e76ba54d6b09a4 Parents: 14e3353 Author: sboikov <[email protected]> Authored: Tue Jul 26 15:48:36 2016 +0300 Committer: sboikov <[email protected]> Committed: Tue Jul 26 16:05:13 2016 +0300 ---------------------------------------------------------------------- .../distributed/dht/GridDhtLocalPartition.java | 2 +- .../dht/GridDhtPartitionTopologyImpl.java | 36 +++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/f3e4f78f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java index f587adb..5061136 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java @@ -518,7 +518,7 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>, /** * @return {@code true} If there is a group reservation. */ - private boolean groupReserved() { + boolean groupReserved() { for (GridDhtPartitionsReservation reservation : reservations) { if (!reservation.invalidate()) return true; // Failed to invalidate reservation -> we are reserved. http://git-wip-us.apache.org/repos/asf/ignite/blob/f3e4f78f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index c95817d..50f7f0f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -30,8 +30,10 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.DiscoveryEvent; +import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.affinity.GridAffinityAssignment; @@ -51,6 +53,7 @@ import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT; import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST; import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED; import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING; @@ -188,6 +191,11 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology { lock.readLock().unlock(); } + final long longOpDumpTimeout = + IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, 60_000); + + int dumpCnt = 0; + GridDhtLocalPartition part; for (int i = 0; i < locPartsCopy.length; i++) { @@ -203,7 +211,33 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology { log.debug("Waiting for renting partition: " + part); // Wait for partition to empty out. - part.rent(true).get(); + if (longOpDumpTimeout > 0) { + while (true) { + try { + part.rent(true).get(longOpDumpTimeout); + + break; + } + catch (IgniteFutureTimeoutCheckedException e) { + if (dumpCnt++ < GridDhtPartitionsExchangeFuture.DUMP_PENDING_OBJECTS_THRESHOLD) { + U.warn(log, "Failed to wait for partition eviction [" + + "topVer=" + topVer + + ", cache=" + cctx.name() + + ", part=" + part.id() + + ", partState=" + part.state() + + ", size=" + part.size() + + ", reservations=" + part.reservations() + + ", grpReservations=" + part.groupReserved() + + ", node=" + cctx.localNodeId() + "]"); + + if (IgniteSystemProperties.getBoolean(IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT, false)) + U.dumpThreads(log); + } + } + } + } + else + part.rent(true).get(); if (log.isDebugEnabled()) log.debug("Finished waiting for renting partition: " + part);
