Author: kwright
Date: Fri Feb 8 13:59:16 2013
New Revision: 1444015
URL: http://svn.apache.org/r1444015
Log:
Pull up fix for CONNECTORS-638 from trunk.
Modified:
manifoldcf/branches/release-1.1-branch/ (props changed)
manifoldcf/branches/release-1.1-branch/CHANGES.txt
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
Propchange: manifoldcf/branches/release-1.1-branch/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1442101,1442171,1442313,1442578,1443521,1443655
Modified: manifoldcf/branches/release-1.1-branch/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/CHANGES.txt?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.1-branch/CHANGES.txt Fri Feb 8 13:59:16 2013
@@ -1,6 +1,12 @@
ManifoldCF Change Log
$Id$
+======================= Release 1.1.1 =====================
+
+CONNECTORS-638: Fix issue with database pool constriction due to
+connection expiration.
+(Erlend Garåsen, Maciej Li¿ewski, Karl Wright)
+
======================= Release 1.1 =====================
CONNECTORS-630: Work around SolrJ paramname encoding bug.
Modified:
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
---
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java
(original)
+++
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java
Fri Feb 8 13:59:16 2013
@@ -179,6 +179,7 @@ public class ConnectionFactory
}
public ConnectionPoolManager createPoolManager()
+ throws ManifoldCFException
{
synchronized (poolExistenceLock)
{
Modified:
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
---
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java
(original)
+++
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java
Fri Feb 8 13:59:16 2013
@@ -30,18 +30,22 @@ public class ConnectionPool
{
public static final String _rcsid = "@(#)$Id$";
- protected String dbURL;
- protected String userName;
- protected String password;
+ protected final String dbURL;
+ protected final String userName;
+ protected final String password;
protected volatile int freePointer;
protected volatile int activeConnections;
protected volatile boolean closed;
- protected Connection[] freeConnections;
- protected long[] connectionCleanupTimeouts;
- protected long expiration;
+ protected final Connection[] freeConnections;
+ protected final long[] connectionCleanupTimeouts;
+ protected final long expiration;
+
+ protected final boolean debug;
+
+ protected final Set<WrappedConnection> outstandingConnections = new
HashSet<WrappedConnection>();
/** Constructor */
- public ConnectionPool(String dbURL, String userName, String password, int
maxConnections, long expiration)
+ public ConnectionPool(String dbURL, String userName, String password, int
maxConnections, long expiration, boolean debug)
{
this.dbURL = dbURL;
this.userName = userName;
@@ -52,6 +56,7 @@ public class ConnectionPool
this.activeConnections = 0;
this.closed = false;
this.expiration = expiration;
+ this.debug = debug;
}
/** Obtain a connection from the pool.
@@ -62,6 +67,12 @@ public class ConnectionPool
public WrappedConnection getConnection()
throws SQLException, InterruptedException
{
+ Exception instantiationException;
+ if (debug)
+ instantiationException = new Exception("Possibly leaked db connection");
+ else
+ instantiationException = null;
+ Connection rval = null;
while (true)
{
synchronized (this)
@@ -70,12 +81,24 @@ public class ConnectionPool
{
if (closed)
throw new InterruptedException("Pool already closed");
- Connection rval = freeConnections[--freePointer];
+ rval = freeConnections[--freePointer];
freeConnections[freePointer] = null;
- return new WrappedConnection(this,rval);
+ break;
}
if (activeConnections == freeConnections.length)
{
+ // If properly configured, we really shouldn't be getting here.
+ if (debug)
+ {
+ synchronized (outstandingConnections)
+ {
+ Logging.db.warn("Out of db connections, list of outstanding ones
follows.");
+ for (WrappedConnection c : outstandingConnections)
+ {
+ Logging.db.warn("Found a possibly leaked db
connection",c.getInstantiationException());
+ }
+ }
+ }
// Wait until kicked; we hope something will free up...
this.wait();
continue;
@@ -85,22 +108,58 @@ public class ConnectionPool
break;
}
}
-
- // Create a new connection. If we fail at this we need to restore the
number of active connections, so catch any failures
- Connection rval2 = null;
+
+ boolean returnedValue = true;
try
{
- if (userName != null)
- rval2 = DriverManager.getConnection(dbURL, userName, password);
- else
- rval2 = DriverManager.getConnection(dbURL);
+ if (rval == null)
+ {
+ if (userName != null)
+ rval = DriverManager.getConnection(dbURL, userName, password);
+ else
+ rval = DriverManager.getConnection(dbURL);
+ }
+
+ WrappedConnection wc = new
WrappedConnection(this,rval,instantiationException);
+ if (debug)
+ {
+ synchronized (outstandingConnections)
+ {
+ outstandingConnections.add(wc);
+ }
+ }
+ return wc;
+ }
+ catch (Error e)
+ {
+ returnedValue = false;
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ returnedValue = false;
+ throw e;
+ }
+ catch (SQLException e)
+ {
+ returnedValue = false;
+ throw e;
}
finally
{
- if (rval2 == null)
- activeConnections--;
+ if (!returnedValue)
+ {
+ // We didn't finish. Restore the pool to the correct form.
+ // Note: We should always be able to just return any current
connection to the pool. This is
+ // safe because we reserved a slot when we decided to create the
connection (if that's what
+ // we did), or we just used a connection that was already allocated.
Either way, we can put
+ // it into the pool.
+ if (rval != null)
+ {
+ release(rval);
+ }
+ }
}
- return new WrappedConnection(this,rval2);
}
/** Close down the pool.
@@ -133,15 +192,10 @@ public class ConnectionPool
{
if (connectionCleanupTimeouts[i] <= currentTime)
{
- try
- {
- freeConnections[i].close();
- }
- catch (SQLException e)
- {
- Logging.db.warn("Error closing pooled connection:
"+e.getMessage(),e);
- }
+ Connection c = freeConnections[i];
+ freeConnections[i] = null;
freePointer--;
+ activeConnections--;
if (freePointer == i)
{
freeConnections[i] = null;
@@ -152,18 +206,44 @@ public class ConnectionPool
connectionCleanupTimeouts[i] =
connectionCleanupTimeouts[freePointer];
freeConnections[freePointer] = null;
}
+ try
+ {
+ c.close();
+ }
+ catch (SQLException e)
+ {
+ Logging.db.warn("Error closing pooled connection:
"+e.getMessage(),e);
+ }
}
else
i++;
}
}
- public synchronized void releaseConnection(Connection connection)
+ public void releaseConnection(WrappedConnection connection)
{
- freeConnections[freePointer] = connection;
- connectionCleanupTimeouts[freePointer] = System.currentTimeMillis() +
expiration;
- freePointer++;
- notifyAll();
+
+ if (debug)
+ {
+ synchronized (outstandingConnections)
+ {
+ outstandingConnections.remove(connection);
+ }
+ }
+
+ release(connection.getConnection());
+ }
+
+ protected void release(Connection c)
+ {
+ synchronized (this)
+ {
+ freeConnections[freePointer] = c;
+ connectionCleanupTimeouts[freePointer] = System.currentTimeMillis() +
expiration;
+ freePointer++;
+ notifyAll();
+ }
+
}
}
Modified:
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
---
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java
(original)
+++
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java
Fri Feb 8 13:59:16 2013
@@ -23,18 +23,24 @@ import javax.sql.*;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.system.ManifoldCF;
+
/** An instance of this class manages a number of (independent) connection
pools.
*/
public class ConnectionPoolManager
{
public static final String _rcsid = "@(#)$Id$";
- protected Map<String,ConnectionPool> poolMap;
- protected ConnectionCloserThread connectionCloserThread;
+ protected final Map<String,ConnectionPool> poolMap;
+ protected final ConnectionCloserThread connectionCloserThread;
protected volatile AtomicBoolean shuttingDown = new AtomicBoolean(false);
-
+ protected final boolean debug;
+
public ConnectionPoolManager(int count)
+ throws ManifoldCFException
{
+ debug =
ManifoldCF.getBooleanProperty(ManifoldCF.databaseConnectionTrackingProperty,
false);
poolMap = new HashMap<String,ConnectionPool>(count);
connectionCloserThread = new ConnectionCloserThread();
connectionCloserThread.start();
@@ -54,7 +60,7 @@ public class ConnectionPoolManager
throws ClassNotFoundException, InstantiationException,
IllegalAccessException
{
Class.forName(driverClassName).newInstance();
- ConnectionPool cp = new
ConnectionPool(dbURL,userName,password,maxSize,expiration);
+ ConnectionPool cp = new
ConnectionPool(dbURL,userName,password,maxSize,expiration,debug);
poolMap.put(poolKey,cp);
return cp;
}
Modified:
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
---
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java
(original)
+++
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java
Fri Feb 8 13:59:16 2013
@@ -19,6 +19,7 @@
package org.apache.manifoldcf.core.jdbcpool;
import java.sql.*;
+import org.apache.manifoldcf.core.system.Logging;
/** The class that represents a connection from a pool.
*/
@@ -28,12 +29,21 @@ public class WrappedConnection
protected Connection connection;
protected ConnectionPool owner;
+ /** Exception, to keep track of where the connection was allocated */
+ protected Exception instantiationException;
/** Constructor */
public WrappedConnection(ConnectionPool owner, Connection connection)
{
+ this(owner,connection,null);
+ }
+
+ /** Constructor */
+ public WrappedConnection(ConnectionPool owner, Connection connection,
Exception instantiationException)
+ {
this.owner = owner;
this.connection = connection;
+ this.instantiationException = instantiationException;
}
/** Get the JDBC connection object.
@@ -47,9 +57,17 @@ public class WrappedConnection
*/
public void release()
{
- owner.releaseConnection(this.connection);
+ owner.releaseConnection(this);
this.connection = null;
}
+
+ /** Get instantiation exception.
+ */
+ public Exception getInstantiationException()
+ {
+ return instantiationException;
+ }
+
}
Modified:
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
---
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
(original)
+++
manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
Fri Feb 8 13:59:16 2013
@@ -109,6 +109,8 @@ public class ManifoldCF
public static final String databaseHandleMaxcountProperty =
"org.apache.manifoldcf.database.maxhandles";
/** Database handle timeout property */
public static final String databaseHandleTimeoutProperty =
"org.apache.manifoldcf.database.handletimeout";
+ /** Connection tracking debug property */
+ public static final String databaseConnectionTrackingProperty =
"org.apache.manifoldcf.database.connectiontracking";
// Database performance monitoring properties
/** Elapsed time a query can take before a warning is output to the log, in
seconds */
Modified:
manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
---
manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
(original)
+++
manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
Fri Feb 8 13:59:16 2013
@@ -801,6 +801,7 @@ cd example
<tr><td>org.apache.manifoldcf.synchdirectory</td><td>Yes, if
file-based synchronization class is used</td><td>Specifies the path of a
synchronization directory. All ManifoldCF process owners <strong>must</strong>
have read/write privileges to this directory.</td></tr>
<tr><td>org.apache.manifoldcf.database.maxhandles</td><td>No</td><td>Specifies
the maximum number of database connection handles that will by pooled.
Recommended value is 200.</td></tr>
<tr><td>org.apache.manifoldcf.database.handletimeout</td><td>No</td><td>Specifies
the maximum time a handle is to live before it is presumed dead. Recommend a
value of 604800, which is the maximum allowable.</td></tr>
+
<tr><td>org.apache.manifoldcf.database.connectiontracking</td><td>No</td><td>True
or false. When "true", will track all allocated database connection handles,
and will dump an allocation stack trace when the pool is exhausted. Useful for
diagnosing connection leaks.</td></tr>
<tr><td>org.apache.manifoldcf.logconfigfile</td><td>No</td><td>Specifies
location of logging configuration file.</td></tr>
<tr><td>org.apache.manifoldcf.database.name</td><td>No</td><td>Describes
database name for ManifoldCF; defaults to "dbname" if not specified.</td></tr>
<tr><td>org.apache.manifoldcf.database.username</td><td>No</td><td>Describes
database user name for ManifoldCF; defaults to "manifoldcf" if not
specified.</td></tr>
Modified:
manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
---
manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
(original)
+++
manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
Fri Feb 8 13:59:16 2013
@@ -801,6 +801,7 @@ cd example
<tr><td>org.apache.manifoldcf.synchdirectory</td><td>Yes, if
file-based synchronization class is used</td><td>Specifies the path of a
synchronization directory. All ManifoldCF process owners <strong>must</strong>
have read/write privileges to this directory.</td></tr>
<tr><td>org.apache.manifoldcf.database.maxhandles</td><td>No</td><td>Specifies
the maximum number of database connection handles that will by pooled.
Recommended value is 200.</td></tr>
<tr><td>org.apache.manifoldcf.database.handletimeout</td><td>No</td><td>Specifies
the maximum time a handle is to live before it is presumed dead. Recommend a
value of 604800, which is the maximum allowable.</td></tr>
+
<tr><td>org.apache.manifoldcf.database.connectiontracking</td><td>No</td><td>True
or false. When "true", will track all allocated database connection handles,
and will dump an allocation stack trace when the pool is exhausted. Useful for
diagnosing connection leaks.</td></tr>
<tr><td>org.apache.manifoldcf.logconfigfile</td><td>No</td><td>Specifies
location of logging configuration file.</td></tr>
<tr><td>org.apache.manifoldcf.database.name</td><td>No</td><td>Describes
database name for ManifoldCF; defaults to "dbname" if not specified.</td></tr>
<tr><td>org.apache.manifoldcf.database.username</td><td>No</td><td>Describes
database user name for ManifoldCF; defaults to "manifoldcf" if not
specified.</td></tr>
