Author: kwright
Date: Thu Jan 23 20:12:20 2014
New Revision: 1560796
URL: http://svn.apache.org/r1560796
Log:
Debug zookeeper no-busy-wait locks
Modified:
manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java
Modified:
manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java?rev=1560796&r1=1560795&r2=1560796&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
(original)
+++
manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
Thu Jan 23 20:12:20 2014
@@ -212,7 +212,6 @@ public class ZooKeeperConnection
}
}
}
- System.out.println("...done");
}
catch (KeeperException e)
{
@@ -251,28 +250,55 @@ public class ZooKeeperConnection
{
// Assert that we want a write lock
lockNode = createSequentialChild(lockPath,WRITE_PREFIX);
- String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 +
WRITE_PREFIX.length());
- // See if we got it
- List<String> children = zookeeper.getChildren(lockPath,false);
- for (String x : children)
- {
- String otherLock;
- if (x.startsWith(WRITE_PREFIX))
- otherLock = x.substring(WRITE_PREFIX.length());
- else if (x.startsWith(NONEXWRITE_PREFIX))
- otherLock = x.substring(NONEXWRITE_PREFIX.length());
- else if (x.startsWith(READ_PREFIX))
- otherLock = x.substring(READ_PREFIX.length());
- else
- continue;
- if (otherLock.compareTo(lockSequenceNumber) < 0)
+ try
+ {
+ String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 +
WRITE_PREFIX.length());
+ // See if we got it
+ List<String> children = zookeeper.getChildren(lockPath,false);
+ for (String x : children)
{
- // We didn't get the lock. Clean up and exit
- zookeeper.delete(lockNode,-1);
- lockNode = null;
- return false;
+ String otherLock;
+ if (x.startsWith(WRITE_PREFIX))
+ otherLock = x.substring(WRITE_PREFIX.length());
+ else if (x.startsWith(NONEXWRITE_PREFIX))
+ otherLock = x.substring(NONEXWRITE_PREFIX.length());
+ else if (x.startsWith(READ_PREFIX))
+ otherLock = x.substring(READ_PREFIX.length());
+ else
+ continue;
+ if (otherLock.compareTo(lockSequenceNumber) < 0)
+ {
+ // We didn't get the lock. Clean up and exit
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ return false;
+ }
}
}
+ catch (KeeperException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (InterruptedException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (Error e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
// We got it!
return true;
}
@@ -288,12 +314,102 @@ public class ZooKeeperConnection
public void obtainWriteLock(String lockPath)
throws ManifoldCFException, InterruptedException
{
- // MHL to do this the proper ZooKeeper way
- while (true)
+ if (lockNode != null)
+ throw new IllegalStateException("Already have a lock in place:
'"+lockNode+"'; can't also write lock '"+lockPath+"'");
+
+ try
{
- if (obtainWriteLockNoWait(lockPath))
- return;
- ManifoldCF.sleep(10L);
+ // Assert that we want a write lock
+ lockNode = createSequentialChild(lockPath,WRITE_PREFIX);
+ try
+ {
+ long lockSequenceNumber = new
Long(lockNode.substring(lockPath.length() + 1 +
WRITE_PREFIX.length())).longValue();
+ //System.out.println("Trying to get write lock for
'"+lockSequenceNumber+"'");
+ while (true)
+ {
+ //System.out.println("Assessing whether we got lock for
'"+lockNode+"'...");
+ // See if we got it
+ List<String> children = zookeeper.getChildren(lockPath,false);
+ String previousLock = null;
+ boolean gotLock = true;
+ long highestPreviousLockIndex = -1L;
+ for (String x : children)
+ {
+ String otherLock;
+ if (x.startsWith(WRITE_PREFIX))
+ otherLock = x.substring(WRITE_PREFIX.length());
+ else if (x.startsWith(NONEXWRITE_PREFIX))
+ otherLock = x.substring(NONEXWRITE_PREFIX.length());
+ else if (x.startsWith(READ_PREFIX))
+ otherLock = x.substring(READ_PREFIX.length());
+ else
+ continue;
+ long otherLockSequenceNumber = new Long(otherLock).longValue();
+ //System.out.println("Saw other child sequence number
"+otherLockSequenceNumber);
+ if (otherLockSequenceNumber < lockSequenceNumber)
+ {
+ // We didn't get the lock. But keep going because we're looking
for the node right before the
+ // one we just asserted.
+ gotLock = false;
+ if (otherLockSequenceNumber > highestPreviousLockIndex)
+ {
+ previousLock = x;
+ highestPreviousLockIndex = otherLockSequenceNumber;
+ }
+ }
+ }
+
+ if (gotLock)
+ {
+ // We got it!
+ //System.out.println("Got write lock for
'"+lockSequenceNumber+"'");
+ return;
+ }
+
+ // There SHOULD be a previous node immediately prior to the one we
asserted. If we didn't find one, go back around;
+ // the previous lock was probably created and destroyed before we
managed to get the children.
+ if (previousLock != null)
+ {
+ //System.out.println(" Waiting on '"+previousLock+"' for write
lock '"+lockSequenceNumber+"'");
+ // Create an exists() watch on the previous node, and wait until
we are awakened by that watch firing.
+ ExistsWatcher w = new ExistsWatcher();
+ Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+ if (s != null)
+ w.waitForEvent();
+ }
+ //else
+ // System.out.println(" Retrying for write lock
'"+lockSequenceNumber+"'");
+ }
+ }
+ catch (KeeperException e)
+ {
+ //System.out.println("Unexpected keeper exception: "+e.getMessage());
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (InterruptedException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (Error e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ }
+ catch (KeeperException e)
+ {
+ throw new ManifoldCFException(e.getMessage(),e);
}
}
@@ -311,26 +427,53 @@ public class ZooKeeperConnection
{
// Assert that we want a read lock
lockNode = createSequentialChild(lockPath,NONEXWRITE_PREFIX);
- String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 +
NONEXWRITE_PREFIX.length());
- // See if we got it
- List<String> children = zookeeper.getChildren(lockPath,false);
- for (String x : children)
- {
- String otherLock;
- if (x.startsWith(WRITE_PREFIX))
- otherLock = x.substring(WRITE_PREFIX.length());
- else if (x.startsWith(READ_PREFIX))
- otherLock = x.substring(READ_PREFIX.length());
- else
- continue;
- if (otherLock.compareTo(lockSequenceNumber) < 0)
+ try
+ {
+ String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 +
NONEXWRITE_PREFIX.length());
+ // See if we got it
+ List<String> children = zookeeper.getChildren(lockPath,false);
+ for (String x : children)
{
- // We didn't get the lock. Clean up and exit
- zookeeper.delete(lockNode,-1);
- lockNode = null;
- return false;
+ String otherLock;
+ if (x.startsWith(WRITE_PREFIX))
+ otherLock = x.substring(WRITE_PREFIX.length());
+ else if (x.startsWith(READ_PREFIX))
+ otherLock = x.substring(READ_PREFIX.length());
+ else
+ continue;
+ if (otherLock.compareTo(lockSequenceNumber) < 0)
+ {
+ // We didn't get the lock. Clean up and exit
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ return false;
+ }
}
}
+ catch (KeeperException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (InterruptedException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (Error e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
// We got it!
return true;
}
@@ -346,12 +489,91 @@ public class ZooKeeperConnection
public void obtainNonExWriteLock(String lockPath)
throws ManifoldCFException, InterruptedException
{
- // MHL to do this the proper ZooKeeper way
- while (true)
+ if (lockNode != null)
+ throw new IllegalStateException("Already have a lock in place:
'"+lockNode+"'; can't also non-ex write lock '"+lockPath+"'");
+
+ try
{
- if (obtainNonExWriteLockNoWait(lockPath))
- return;
- ManifoldCF.sleep(10L);
+ // Assert that we want a read lock
+ lockNode = createSequentialChild(lockPath,NONEXWRITE_PREFIX);
+ try
+ {
+ long lockSequenceNumber = new
Long(lockNode.substring(lockPath.length() + 1 +
NONEXWRITE_PREFIX.length())).longValue();
+ while (true)
+ {
+ // See if we got it
+ List<String> children = zookeeper.getChildren(lockPath,false);
+ String previousLock = null;
+ boolean gotLock = true;
+ long highestPreviousLockIndex = -1L;
+ for (String x : children)
+ {
+ String otherLock;
+ if (x.startsWith(WRITE_PREFIX))
+ otherLock = x.substring(WRITE_PREFIX.length());
+ else if (x.startsWith(READ_PREFIX))
+ otherLock = x.substring(READ_PREFIX.length());
+ else
+ continue;
+ long otherLockSequenceNumber = new Long(otherLock).longValue();
+ //System.out.println("Saw other child sequence number
"+otherLockSequenceNumber);
+ if (otherLockSequenceNumber < lockSequenceNumber)
+ {
+ // We didn't get the lock. But keep going because we're looking
for the node right before the
+ // one we just asserted.
+ gotLock = false;
+ if (otherLockSequenceNumber > highestPreviousLockIndex)
+ {
+ previousLock = x;
+ highestPreviousLockIndex = otherLockSequenceNumber;
+ }
+ }
+ }
+
+ if (gotLock)
+ // We got it!
+ return;
+
+ // There SHOULD be a previous node immediately prior to the one we
asserted. If we didn't find one, go back around;
+ // the previous lock was probably created and destroyed before we
managed to get the children.
+ if (previousLock != null)
+ {
+ // Create an exists() watch on the previous node, and wait until
we are awakened by that watch firing.
+ ExistsWatcher w = new ExistsWatcher();
+ Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+ if (s != null)
+ w.waitForEvent();
+ }
+ }
+ }
+ catch (KeeperException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (InterruptedException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (Error e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ }
+ catch (KeeperException e)
+ {
+ throw new ManifoldCFException(e.getMessage(),e);
}
}
@@ -369,26 +591,53 @@ public class ZooKeeperConnection
{
// Assert that we want a read lock
lockNode = createSequentialChild(lockPath,READ_PREFIX);
- String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 +
READ_PREFIX.length());
- // See if we got it
- List<String> children = zookeeper.getChildren(lockPath,false);
- for (String x : children)
- {
- String otherLock;
- if (x.startsWith(WRITE_PREFIX))
- otherLock = x.substring(WRITE_PREFIX.length());
- else if (x.startsWith(NONEXWRITE_PREFIX))
- otherLock = x.substring(NONEXWRITE_PREFIX.length());
- else
- continue;
- if (otherLock.compareTo(lockSequenceNumber) < 0)
+ try
+ {
+ String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 +
READ_PREFIX.length());
+ // See if we got it
+ List<String> children = zookeeper.getChildren(lockPath,false);
+ for (String x : children)
{
- // We didn't get the lock. Clean up and exit
- zookeeper.delete(lockNode,-1);
- lockNode = null;
- return false;
+ String otherLock;
+ if (x.startsWith(WRITE_PREFIX))
+ otherLock = x.substring(WRITE_PREFIX.length());
+ else if (x.startsWith(NONEXWRITE_PREFIX))
+ otherLock = x.substring(NONEXWRITE_PREFIX.length());
+ else
+ continue;
+ if (otherLock.compareTo(lockSequenceNumber) < 0)
+ {
+ // We didn't get the lock. Clean up and exit
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ return false;
+ }
}
}
+ catch (KeeperException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (InterruptedException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (Error e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
// We got it!
return true;
}
@@ -404,12 +653,100 @@ public class ZooKeeperConnection
public void obtainReadLock(String lockPath)
throws ManifoldCFException, InterruptedException
{
- // MHL to do this the proper ZooKeeper way
- while (true)
+ if (lockNode != null)
+ throw new IllegalStateException("Already have a lock in place:
'"+lockNode+"'; can't also read lock '"+lockPath+"'");
+
+ try
+ {
+ // Assert that we want a read lock
+ lockNode = createSequentialChild(lockPath,READ_PREFIX);
+ try
+ {
+ long lockSequenceNumber = new
Long(lockNode.substring(lockPath.length() + 1 +
READ_PREFIX.length())).longValue();
+ //System.out.println("Trying to get read lock for
'"+lockSequenceNumber+"'");
+ while (true)
+ {
+ // See if we got it
+ List<String> children = zookeeper.getChildren(lockPath,false);
+ String previousLock = null;
+ boolean gotLock = true;
+ long highestPreviousLockIndex = -1L;
+ for (String x : children)
+ {
+ String otherLock;
+ if (x.startsWith(WRITE_PREFIX))
+ otherLock = x.substring(WRITE_PREFIX.length());
+ else if (x.startsWith(NONEXWRITE_PREFIX))
+ otherLock = x.substring(NONEXWRITE_PREFIX.length());
+ else
+ continue;
+ long otherLockSequenceNumber = new Long(otherLock).longValue();
+ //System.out.println("Saw other child sequence number
"+otherLockSequenceNumber);
+ if (otherLockSequenceNumber < lockSequenceNumber)
+ {
+ // We didn't get the lock. But keep going because we're looking
for the node right before the
+ // one we just asserted.
+ gotLock = false;
+ if (otherLockSequenceNumber > highestPreviousLockIndex)
+ {
+ previousLock = x;
+ highestPreviousLockIndex = otherLockSequenceNumber;
+ }
+ }
+ }
+
+ if (gotLock)
+ {
+ // We got it!
+ //System.out.println("Got read lock for '"+lockSequenceNumber+"'");
+ return;
+ }
+
+ // There SHOULD be a previous node immediately prior to the one we
asserted. If we didn't find one, go back around;
+ // the previous lock was probably created and destroyed before we
managed to get the children.
+ if (previousLock != null)
+ {
+ //System.out.println(" Waiting on '"+previousLock+"' for read lock
'"+lockSequenceNumber+"'");
+ // Create an exists() watch on the previous node, and wait until
we are awakened by that watch firing.
+ ExistsWatcher w = new ExistsWatcher();
+ Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+ if (s != null)
+ w.waitForEvent();
+ }
+ //else
+ // System.out.println(" Retrying for read lock
'"+lockSequenceNumber+"'");
+
+ }
+ }
+ catch (KeeperException e)
+ {
+ //System.out.println("Unexpected keeper exception: "+e.getMessage());
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (InterruptedException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (Error e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ zookeeper.delete(lockNode,-1);
+ lockNode = null;
+ throw e;
+ }
+ }
+ catch (KeeperException e)
{
- if (obtainReadLockNoWait(lockPath))
- return;
- ManifoldCF.sleep(10L);
+ throw new ManifoldCFException(e.getMessage(),e);
}
}
@@ -420,6 +757,7 @@ public class ZooKeeperConnection
{
if (lockNode == null)
throw new IllegalStateException("Can't release lock we don't hold");
+ //System.out.println("Releasing lock '"+lockNode+"'");
try
{
zookeeper.delete(lockNode,-1);
@@ -647,10 +985,43 @@ public class ZooKeeperConnection
{
}
+ @Override
+ public void process(WatchedEvent event)
+ {
+ }
+
+ }
+
+ /** Watcher class for exists state changes, so we get notified about
deletions of lock request nodes. */
+ protected static class ExistsWatcher implements Watcher
+ {
+ protected boolean eventTriggered = false;
+
+ public ExistsWatcher()
+ {
+ }
+
+ @Override
public void process(WatchedEvent event)
{
+ synchronized (this)
+ {
+ eventTriggered = true;
+ notifyAll();
+ }
}
+ public void waitForEvent()
+ throws InterruptedException
+ {
+ synchronized (this)
+ {
+ if (eventTriggered)
+ return;
+ wait();
+ }
+ }
+
}
}
Modified:
manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java?rev=1560796&r1=1560795&r2=1560796&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java
(original)
+++
manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java
Thu Jan 23 20:12:20 2014
@@ -287,14 +287,22 @@ public class TestZooKeeperLocks extends
// LockPool is a dummy
LockPool lp = new LockPool(factory);
LockGate lo;
- // Take write locks but free them if read is what's active
+ // The reader threads require ALL of the readers to get into the
protected area. If
+ // we try to write before that happens, ordering requirements produce
a deadlock. So wait.
+ while (ai.get() < readerThreadCount)
+ {
+ Thread.sleep(100L);
+ }
+
+ /*
+ // Take write locks but free them immediately if read is what's active
while (true)
{
lo = lp.getObject(lockKey);
enterWriteLock(threadID,lo);
- System.out.println("Made it into write lock");
try
{
+ System.out.println("Made it into read-time write lock");
// Check if we made it in during read cycle... that would be bad.
if (ai.get() > 0 && ai.get() < readerThreadCount)
throw new Exception("Was able to write even when readers were
active");
@@ -303,12 +311,15 @@ public class TestZooKeeperLocks extends
}
finally
{
+ System.out.println("Leaving read-time write lock");
leaveWriteLock(lo);
+ System.out.println("Left read-time write lock");
}
- Thread.sleep(100L);
+ Thread.sleep(1000L);
}
+ */
- // Get write lock, increment twice, and leave write lock
+ // Get write lock, increment twice, and leave write lock. Meanwhile,
reader threads will be trying to gain access.
lo = lp.getObject(lockKey);
enterWriteLock(threadID,lo);
try
