IGNITE-3748 Data rebalancing of large cache can hang out.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f80bfbd1 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f80bfbd1 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f80bfbd1 Branch: refs/heads/ignite-4242 Commit: f80bfbd19e7870554bf3abd13bde89b0f39aaee1 Parents: d15eba4 Author: Anton Vinogradov <[email protected]> Authored: Tue Nov 22 12:02:57 2016 +0300 Committer: Anton Vinogradov <[email protected]> Committed: Tue Nov 22 12:02:57 2016 +0300 ---------------------------------------------------------------------- .../dht/preloader/GridDhtPreloader.java | 18 ++++++++++++- .../ignite/internal/util/GridLogThrottle.java | 28 +++++++++++++++----- 2 files changed, 39 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/f80bfbd1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java index 6ed58e7..0865d9f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java @@ -59,6 +59,7 @@ import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.GPC; +import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.lang.IgnitePredicate; @@ -788,7 +789,22 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter { GridDhtLocalPartition part = partsToEvict.poll(); if (part != null) - part.tryEvict(); + try { + part.tryEvict(); + } + catch (Throwable ex) { + if (cctx.kernalContext().isStopping()) { + LT.warn(log, ex, "Partition eviction failed (current node is stopping).", + false, + true); + + partsToEvict.clear(); + + return true; + } + else + LT.error(log, ex, "Partition eviction failed, this can cause grid hang."); + } } finally { if (!partsToEvict.isEmptyx()) http://git-wip-us.apache.org/repos/asf/ignite/blob/f80bfbd1/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java index ce6783a..c8ba865 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java @@ -81,12 +81,12 @@ public class GridLogThrottle { * @param log Logger. * @param e Error (optional). * @param msg Message. - * @param byMessage Errors group by message, not by tuple(error, msg). + * @param byMsg Errors group by message, not by tuple(error, msg). */ - public static void error(@Nullable IgniteLogger log, @Nullable Throwable e, String msg, boolean byMessage) { + public static void error(@Nullable IgniteLogger log, @Nullable Throwable e, String msg, boolean byMsg) { assert !F.isEmpty(msg); - log(log, e, msg, null, LogLevel.ERROR, false, byMessage); + log(log, e, msg, null, LogLevel.ERROR, false, byMsg); } /** @@ -109,6 +109,22 @@ public class GridLogThrottle { * @param e Error (optional). * @param msg Message. * @param quite Print warning anyway. + * @param byMsg Errors group by message, not by tuple(error, msg). + */ + public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, String msg, boolean quite, boolean byMsg) { + assert !F.isEmpty(msg); + + log(log, e, msg, null, LogLevel.WARN, quite, byMsg); + } + + + /** + * Logs warning if needed. + * + * @param log Logger. + * @param e Error (optional). + * @param msg Message. + * @param quite Print warning anyway. */ public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, String msg, boolean quite) { assert !F.isEmpty(msg); @@ -168,15 +184,15 @@ public class GridLogThrottle { * @param longMsg Long message (or just message). * @param shortMsg Short message for quite logging. * @param level Level where messages should appear. - * @param byMessage Errors group by message, not by tuple(error, msg). + * @param byMsg Errors group by message, not by tuple(error, msg). */ @SuppressWarnings({"RedundantTypeArguments"}) private static void log(@Nullable IgniteLogger log, @Nullable Throwable e, String longMsg, @Nullable String shortMsg, - LogLevel level, boolean quiet, boolean byMessage) { + LogLevel level, boolean quiet, boolean byMsg) { assert !F.isEmpty(longMsg); IgniteBiTuple<Class<? extends Throwable>, String> tup = - e != null && !byMessage ? F.<Class<? extends Throwable>, String>t(e.getClass(), e.getMessage()) : + e != null && !byMsg ? F.<Class<? extends Throwable>, String>t(e.getClass(), e.getMessage()) : F.<Class<? extends Throwable>, String>t(null, longMsg); while (true) {
