Repository: incubator-geode Updated Branches: refs/heads/feature/GEODE-409 20c39d7f3 -> 4a42443bd
GEODE-392: Some enhancements to GemFireDeadlockDetectorDUnitTest I originally thought this test might be the cause of this bug, so I made a few changes to make sure it no additional threads are still running when the test exits. I think that is still a good change to have, even though those threads were not the cause of the issue. Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/4a42443b Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/4a42443b Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/4a42443b Branch: refs/heads/feature/GEODE-409 Commit: 4a42443bdddda7c65bad48064f5b6b2895e48055 Parents: ca4991d Author: Dan Smith <[email protected]> Authored: Fri Oct 23 10:55:02 2015 -0700 Committer: Dan Smith <[email protected]> Committed: Fri Oct 23 11:01:51 2015 -0700 ---------------------------------------------------------------------- .../GemFireDeadlockDetectorDUnitTest.java | 49 +++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/4a42443b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java index 01af1f9..32d21e5 100644 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java @@ -22,8 +22,10 @@ import com.gemstone.gemfire.cache.execute.FunctionService; import com.gemstone.gemfire.cache.execute.ResultCollector; import com.gemstone.gemfire.cache30.CacheTestCase; import com.gemstone.gemfire.distributed.DistributedLockService; +import com.gemstone.gemfire.distributed.LockServiceDestroyedException; import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; +import dunit.AsyncInvocation; import dunit.Host; import dunit.SerializableCallable; import dunit.SerializableRunnable; @@ -41,11 +43,22 @@ public class GemFireDeadlockDetectorDUnitTest extends CacheTestCase { @Override public void tearDown2() throws Exception { + disconnectAllFromDS(); + } + + private void stopStuckThreads() { invokeInEveryVM(new SerializableRunnable() { public void run() { for(Thread thread: stuckThreads) { thread.interrupt(); + disconnectFromDS(); + try { + thread.join(30000); + assertTrue(!thread.isAlive()); + } catch (InterruptedException e) { + fail("interrupted", e); + } } } }); @@ -74,7 +87,7 @@ public class GemFireDeadlockDetectorDUnitTest extends CacheTestCase { private static final Lock lock = new ReentrantLock(); - public void testDistributedDeadlockWithFunction() throws InterruptedException { + public void testDistributedDeadlockWithFunction() throws Throwable { Host host = Host.getHost(0); VM vm0 = host.getVM(0); VM vm1 = host.getVM(1); @@ -86,21 +99,24 @@ public class GemFireDeadlockDetectorDUnitTest extends CacheTestCase { //This thread locks the lock member1 first, then member2. - lockTheLocks(vm0, member2); + AsyncInvocation async1 = lockTheLocks(vm0, member2); //This thread locks the lock member2 first, then member1. - lockTheLocks(vm1, member1); + AsyncInvocation async2 = lockTheLocks(vm1, member1); Thread.sleep(5000); GemFireDeadlockDetector detect = new GemFireDeadlockDetector(); LinkedList<Dependency> deadlock = detect.find().findCycle(); getLogWriter().info("Deadlock=" + DeadlockDetector.prettyFormat(deadlock)); assertEquals(8, deadlock.size()); + stopStuckThreads(); + async1.getResult(30000); + async2.getResult(30000); } - private void lockTheLocks(VM vm0, final InternalDistributedMember member) { - vm0.invokeAsync(new SerializableRunnable() { + private AsyncInvocation lockTheLocks(VM vm0, final InternalDistributedMember member) { + return vm0.invokeAsync(new SerializableRunnable() { public void run() { lock.lock(); @@ -117,13 +133,13 @@ public class GemFireDeadlockDetectorDUnitTest extends CacheTestCase { }); } - public void testDistributedDeadlockWithDLock() throws InterruptedException { + public void testDistributedDeadlockWithDLock() throws Throwable { Host host = Host.getHost(0); VM vm0 = host.getVM(0); VM vm1 = host.getVM(1); - lockTheDLocks(vm0, "one", "two"); - lockTheDLocks(vm1, "two", "one"); + AsyncInvocation async1 = lockTheDLocks(vm0, "one", "two"); + AsyncInvocation async2 = lockTheDLocks(vm1, "two", "one"); getSystem(); GemFireDeadlockDetector detect = new GemFireDeadlockDetector(); @@ -139,10 +155,14 @@ public class GemFireDeadlockDetectorDUnitTest extends CacheTestCase { assertTrue(deadlock != null); getLogWriter().info("Deadlock=" + DeadlockDetector.prettyFormat(deadlock)); assertEquals(4, deadlock.size()); + stopStuckThreads(); + disconnectAllFromDS(); + async1.getResult(30000); + async2.getResult(30000); } - private void lockTheDLocks(VM vm, final String first, final String second) { - vm.invokeAsync(new SerializableRunnable() { + private AsyncInvocation lockTheDLocks(VM vm, final String first, final String second) { + return vm.invokeAsync(new SerializableRunnable() { public void run() { getCache(); @@ -154,7 +174,11 @@ public class GemFireDeadlockDetectorDUnitTest extends CacheTestCase { } catch (InterruptedException e) { e.printStackTrace(); } - dls.lock(second, 10 * 1000, -1); + try { + dls.lock(second, 10 * 1000, -1); + } catch(LockServiceDestroyedException expected) { + //this is ok, the test is terminating + } } }); @@ -172,7 +196,7 @@ public class GemFireDeadlockDetectorDUnitTest extends CacheTestCase { private static class TestFunction implements Function { - private static final int LOCK_WAIT_TIME = 1000; + private static final int LOCK_WAIT_TIME = 5; public boolean hasResult() { return true; @@ -186,6 +210,7 @@ public class GemFireDeadlockDetectorDUnitTest extends CacheTestCase { } catch (InterruptedException e) { //ingore } + stuckThreads.remove(Thread.currentThread()); context.getResultSender().lastResult(null); }
