Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1548081&r1=1548080&r2=1548081&view=diff ============================================================================== --- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original) +++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Thu Dec 5 10:31:01 2013 @@ -105,6 +105,15 @@ public class ManifoldCF extends org.apac public static void localCleanup(IThreadContext tc) { + try + { + RepositoryConnectorPoolFactory.make(tc).closeAllConnectors(); + } + catch (ManifoldCFException e) + { + if (Logging.root != null) + Logging.root.warn("Exception tossed on repository connector pool cleanup: "+e.getMessage(),e); + } } /** Create system database using superuser properties from properties.xml. @@ -948,8 +957,7 @@ public class ManifoldCF extends org.apac break; // Calculate new priorities for all these documents - writeDocumentPriorities(threadContext,connectionManager,jobManager,docs,connectionMap,jobDescriptionMap, - rt,updateTime); + writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,rt,updateTime); Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms"); } @@ -959,12 +967,15 @@ public class ManifoldCF extends org.apac /** Write a set of document priorities, based on the current queue tracker. */ - public static void writeDocumentPriorities(IThreadContext threadContext, IRepositoryConnectionManager mgr, - IJobManager jobManager, DocumentDescription[] descs, + public static void writeDocumentPriorities(IThreadContext threadContext, DocumentDescription[] descs, Map<String,IRepositoryConnection> connectionMap, Map<Long,IJobDescription> jobDescriptionMap, ReprioritizationTracker rt, long currentTime) throws ManifoldCFException { + IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext); + IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(threadContext); + IJobManager jobManager = JobManagerFactory.make(threadContext); + if (Logging.scheduling.isDebugEnabled()) Logging.scheduling.debug("Reprioritizing "+Integer.toString(descs.length)+" documents"); @@ -992,10 +1003,7 @@ public class ManifoldCF extends org.apac String[] binNames; // Grab a connector handle - IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext, - connection.getClassName(), - connection.getConfigParams(), - connection.getMaxConnections()); + IRepositoryConnector connector = repositoryConnectorPool.grab(connection); try { if (connector == null) @@ -1006,7 +1014,7 @@ public class ManifoldCF extends org.apac } finally { - RepositoryConnectorFactory.release(connector); + repositoryConnectorPool.release(connector); } priorities[i] = new PriorityCalculator(rt,connection,binNames); @@ -1217,6 +1225,7 @@ public class ManifoldCF extends org.apac { try { + IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(tc); IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc); IOutputConnection connection = connectionManager.load(connectionName); if (connection == null) @@ -1227,7 +1236,7 @@ public class ManifoldCF extends org.apac String results; // Grab a connection handle, and call the test method - IOutputConnector connector = OutputConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections()); + IOutputConnector connector = outputConnectorPool.grab(connection); try { results = connector.check(); @@ -1238,7 +1247,7 @@ public class ManifoldCF extends org.apac } finally { - OutputConnectorFactory.release(connector); + outputConnectorPool.release(connector); } ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE); @@ -1258,6 +1267,7 @@ public class ManifoldCF extends org.apac { try { + IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(tc); IAuthorityConnectionManager connectionManager = AuthorityConnectionManagerFactory.make(tc); IAuthorityConnection connection = connectionManager.load(connectionName); if (connection == null) @@ -1268,7 +1278,7 @@ public class ManifoldCF extends org.apac String results; // Grab a connection handle, and call the test method - IAuthorityConnector connector = AuthorityConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections()); + IAuthorityConnector connector = authorityConnectorPool.grab(connection); try { results = connector.check(); @@ -1279,7 +1289,7 @@ public class ManifoldCF extends org.apac } finally { - AuthorityConnectorFactory.release(connector); + authorityConnectorPool.release(connector); } ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE); @@ -1299,6 +1309,7 @@ public class ManifoldCF extends org.apac { try { + IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(tc); IMappingConnectionManager connectionManager = MappingConnectionManagerFactory.make(tc); IMappingConnection connection = connectionManager.load(connectionName); if (connection == null) @@ -1309,7 +1320,7 @@ public class ManifoldCF extends org.apac String results; // Grab a connection handle, and call the test method - IMappingConnector connector = MappingConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections()); + IMappingConnector connector = mappingConnectorPool.grab(connection); try { results = connector.check(); @@ -1320,7 +1331,7 @@ public class ManifoldCF extends org.apac } finally { - MappingConnectorFactory.release(connector); + mappingConnectorPool.release(connector); } ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE); @@ -1340,6 +1351,7 @@ public class ManifoldCF extends org.apac { try { + IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(tc); IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc); IRepositoryConnection connection = connectionManager.load(connectionName); if (connection == null) @@ -1350,7 +1362,7 @@ public class ManifoldCF extends org.apac String results; // Grab a connection handle, and call the test method - IRepositoryConnector connector = RepositoryConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections()); + IRepositoryConnector connector = repositoryConnectorPool.grab(connection); try { results = connector.check(); @@ -1361,7 +1373,7 @@ public class ManifoldCF extends org.apac } finally { - RepositoryConnectorFactory.release(connector); + repositoryConnectorPool.release(connector); } ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE); @@ -1382,6 +1394,7 @@ public class ManifoldCF extends org.apac { try { + IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(tc); IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc); IOutputConnection connection = connectionManager.load(connectionName); if (connection == null) @@ -1391,14 +1404,14 @@ public class ManifoldCF extends org.apac } // Grab a connection handle, and call the test method - IOutputConnector connector = OutputConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections()); + IOutputConnector connector = outputConnectorPool.grab(connection); try { return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND; } finally { - OutputConnectorFactory.release(connector); + outputConnectorPool.release(connector); } } catch (ManifoldCFException e) @@ -1414,6 +1427,7 @@ public class ManifoldCF extends org.apac { try { + IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(tc); IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc); IRepositoryConnection connection = connectionManager.load(connectionName); if (connection == null) @@ -1423,14 +1437,14 @@ public class ManifoldCF extends org.apac } // Grab a connection handle, and call the test method - IRepositoryConnector connector = RepositoryConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections()); + IRepositoryConnector connector = repositoryConnectorPool.grab(connection); try { return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND; } finally { - RepositoryConnectorFactory.release(connector); + repositoryConnectorPool.release(connector); } } catch (ManifoldCFException e)
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff ============================================================================== --- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java (original) +++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java Thu Dec 5 10:31:01 2013 @@ -68,11 +68,8 @@ public class SeedingThread extends Threa IRepositoryConnectionManager connectionMgr = RepositoryConnectionManagerFactory.make(threadContext); ReprioritizationTracker rt = new ReprioritizationTracker(threadContext); - IDBInterface database = DBInterfaceFactory.make(threadContext, - ManifoldCF.getMasterDatabaseName(), - ManifoldCF.getMasterDatabaseUsername(), - ManifoldCF.getMasterDatabasePassword()); - + IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext); + String[] identifiers = new String[MAX_COUNT]; // Loop while (true) @@ -126,17 +123,12 @@ public class SeedingThread extends Threa int hopcountMethod = jobDescription.getHopcountMode(); IRepositoryConnection connection = connectionMgr.load(jobDescription.getConnectionName()); - IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext, - connection.getClassName(), - connection.getConfigParams(), - connection.getMaxConnections()); + IRepositoryConnector connector = repositoryConnectorPool.grab(connection); // Null will come back if the connector instance could not be obtained, so just skip in that case. if (connector == null) continue; try { - - // Get the number of link types. String[] legalLinkTypes = connector.getRelationshipTypes(); @@ -174,7 +166,7 @@ public class SeedingThread extends Threa } finally { - RepositoryConnectorFactory.release(connector); + repositoryConnectorPool.release(connector); } Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff ============================================================================== --- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java (original) +++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java Thu Dec 5 10:31:01 2013 @@ -126,7 +126,7 @@ public class SetPriorityThread extends T DocumentDescription desc = blockingDocuments.getBlockingDocument(); if (desc != null) { - ManifoldCF.writeDocumentPriorities(threadContext,mgr,jobManager, + ManifoldCF.writeDocumentPriorities(threadContext, new DocumentDescription[]{desc},connectionMap,jobDescriptionMap,rt,currentTime); processedCount++; continue; Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff ============================================================================== --- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java (original) +++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java Thu Dec 5 10:31:01 2013 @@ -62,11 +62,8 @@ public class StartupThread extends Threa IRepositoryConnectionManager connectionMgr = RepositoryConnectionManagerFactory.make(threadContext); ReprioritizationTracker rt = new ReprioritizationTracker(threadContext); - IDBInterface database = DBInterfaceFactory.make(threadContext, - ManifoldCF.getMasterDatabaseName(), - ManifoldCF.getMasterDatabaseUsername(), - ManifoldCF.getMasterDatabasePassword()); - + IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext); + // Loop while (true) { @@ -117,22 +114,19 @@ public class StartupThread extends Threa int hopcountMethod = jobDescription.getHopcountMode(); IRepositoryConnection connection = connectionMgr.load(jobDescription.getConnectionName()); - IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext, - connection.getClassName(), - connection.getConfigParams(), - connection.getMaxConnections()); + IRepositoryConnector connector = repositoryConnectorPool.grab(connection); // If the attempt to grab a connector instance failed, don't start the job, of course. if (connector == null) continue; - // Only now record the fact that we are trying to start the job. - connectionMgr.recordHistory(jobDescription.getConnectionName(), - null,connectionMgr.ACTIVITY_JOBSTART,null, - jobID.toString()+"("+jobDescription.getDescription()+")",null,null,null); - try { + // Only now record the fact that we are trying to start the job. + connectionMgr.recordHistory(jobDescription.getConnectionName(), + null,connectionMgr.ACTIVITY_JOBSTART,null, + jobID.toString()+"("+jobDescription.getDescription()+")",null,null,null); + int model = connector.getConnectorModel(); // Get the number of link types. String[] legalLinkTypes = connector.getRelationshipTypes(); @@ -175,7 +169,7 @@ public class StartupThread extends Threa } finally { - RepositoryConnectorFactory.release(connector); + repositoryConnectorPool.release(connector); } // Start this job! Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff ============================================================================== --- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java (original) +++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java Thu Dec 5 10:31:01 2013 @@ -89,6 +89,8 @@ public class StufferThread extends Threa IJobManager jobManager = JobManagerFactory.make(threadContext); ReprioritizationTracker rt = new ReprioritizationTracker(threadContext); + IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext); + Logging.threads.debug("Stuffer thread: Low water mark is "+Integer.toString(lowWaterMark)+"; amount per stuffing is "+Integer.toString(stuffAmt)); // This is used to adjust the number of records returned for jobs @@ -250,10 +252,7 @@ public class StufferThread extends Threa try { // Grab a connector handle - IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext, - connection.getClassName(), - connection.getConfigParams(), - connection.getMaxConnections()); + IRepositoryConnector connector = repositoryConnectorPool.grab(connection); if (connector == null) { maxDocuments = 1; @@ -270,7 +269,7 @@ public class StufferThread extends Threa } finally { - RepositoryConnectorFactory.release(connector); + repositoryConnectorPool.release(connector); } } } Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff ============================================================================== --- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java (original) +++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java Thu Dec 5 10:31:01 2013 @@ -79,6 +79,8 @@ public class WorkerThread extends Thread IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(threadContext); ReprioritizationTracker rt = new ReprioritizationTracker(threadContext); + IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext); + List<DocumentToProcess> fetchList = new ArrayList<DocumentToProcess>(); Map<String,String> versionMap = new HashMap<String,String>(); List<QueuedDocument> finishList = new ArrayList<QueuedDocument>(); @@ -257,10 +259,7 @@ public class WorkerThread extends Thread IRepositoryConnector connector = null; if (activeDocuments.size() > 0 || hopcountremoveList.size() > 0) { - connector = RepositoryConnectorFactory.grab(threadContext, - connection.getClassName(), - connection.getConfigParams(), - connection.getMaxConnections()); + connector = repositoryConnectorPool.grab(connection); // If we wind up with a null here, it means that a document got queued for a connector which is now gone. // Basically, what we want to do in that case is to treat this kind of like a service interruption - the document @@ -832,7 +831,7 @@ public class WorkerThread extends Thread } finally { - RepositoryConnectorFactory.release(connector); + repositoryConnectorPool.release(connector); } }
