Author: kwright
Date: Tue Nov 26 18:27:47 2013
New Revision: 1545764
URL: http://svn.apache.org/r1545764
Log:
Add docbins table, for tracking bin counts cross-cluster.
Added:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/BinManager.java
(with props)
Modified:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Added:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/BinManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/BinManager.java?rev=1545764&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/BinManager.java
(added)
+++
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/BinManager.java
Tue Nov 26 18:27:47 2013
@@ -0,0 +1,136 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.jobs;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.CacheKeyFactory;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import java.util.*;
+
+/** This class manages the docbins table.
+* A row in this table represents a document bin. The count that is kept is the
+* number of documents in this particular bin that have been assigned a
document priority.
+*
+* <br><br>
+* <b>docbins</b>
+* <table border="1" cellpadding="3" cellspacing="0">
+* <tr class="TableHeadingColor">
+*
<th>Field</th><th>Type</th><th>Description </th>
+* <tr><td>binname</td><td>VARCHAR(255)</td><td>Primary Key</td></tr>
+* <tr><td>bincounter</td><td>BIGINT</td><td></td></tr>
+* </table>
+* <br><br>
+*
+*/
+public class BinManager extends org.apache.manifoldcf.core.database.BaseTable
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ // Field names
+ public final static String binNameField = "binname";
+ public final static String binCounterField = "bincounter";
+
+ /** Constructor.
+ *@param database is the database handle.
+ */
+ public BinManager(IDBInterface database)
+ throws ManifoldCFException
+ {
+ super(database,"docbins");
+ }
+
+ /** Install or upgrade this table.
+ */
+ public void install()
+ throws ManifoldCFException
+ {
+ // Standard practice: outer loop for installs
+ while (true)
+ {
+ Map existing = getTableSchema(null,null);
+ if (existing == null)
+ {
+ HashMap map = new HashMap();
+ map.put(binNameField,new
ColumnDescription("VARCHAR(255)",true,true,null,null,false));
+ map.put(binCounterField,new
ColumnDescription("BIGINT",false,false,null,null,false));
+ performCreate(map,null);
+ }
+ else
+ {
+ // Upgrade goes here if needed
+ }
+
+ // Index management goes here
+
+ break;
+ }
+ }
+
+ /** Uninstall.
+ */
+ public void deinstall()
+ throws ManifoldCFException
+ {
+ performDrop(null);
+ }
+
+ /** Reset all bins */
+ public void reset()
+ throws ManifoldCFException
+ {
+ performDelete("", null, null);
+ }
+
+ /** Get a bin value (and set next one). If the record does not yet exist,
create it with a starting value.
+ * We expect this to happen within a transaction!!
+ *@param binName is the name of the bin (256 char max)
+ *@param newBinValue is the value to use if there is no such bin yet. This
is the value that will be
+ * returned; what will be stored will be that value + 1.
+ *@return the counter value.
+ */
+ public long getIncrementBinValue(String binName, long newBinValue)
+ throws ManifoldCFException
+ {
+ // SELECT FOR UPDATE/MODIFY is the most common path
+ ArrayList params = new ArrayList();
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(binNameField,binName)});
+ IResultSet result = performQuery("SELECT "+binCounterField+" FROM
"+getTableName()+" WHERE "+query+" FOR UPDATE",params,null,null);
+ if (result.getRowCount() > 0)
+ {
+ IResultRow row = result.getRow(0);
+ Long value = (Long)row.getValue(binCounterField);
+ long rval = value.longValue();
+ HashMap map = new HashMap();
+ map.put(binCounterField,new Long(rval+1L));
+ performUpdate(map," WHERE "+query,params,null);
+ return rval;
+ }
+ else
+ {
+ HashMap map = new HashMap();
+ map.put(binNameField,binName);
+ map.put(binCounterField,new Long(newBinValue+1L));
+ performInsert(map,null);
+ return newBinValue;
+ }
+ }
+
+}
Propchange:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/BinManager.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/BinManager.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1545764&r1=1545763&r2=1545764&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Tue Nov 26 18:27:47 2013
@@ -39,16 +39,17 @@ public class JobManager implements IJobM
protected static final String hopLock = "_HOPLOCK_";
// Member variables
- protected IDBInterface database;
- protected IOutputConnectionManager outputMgr;
- protected IRepositoryConnectionManager connectionMgr;
- protected ILockManager lockManager;
- protected IThreadContext threadContext;
- protected JobQueue jobQueue;
- protected Jobs jobs;
- protected HopCount hopCount;
- protected Carrydown carryDown;
- protected EventManager eventManager;
+ protected final IDBInterface database;
+ protected final IOutputConnectionManager outputMgr;
+ protected final IRepositoryConnectionManager connectionMgr;
+ protected final ILockManager lockManager;
+ protected final IThreadContext threadContext;
+ protected final BinManager binManager;
+ protected final JobQueue jobQueue;
+ protected final Jobs jobs;
+ protected final HopCount hopCount;
+ protected final Carrydown carryDown;
+ protected final EventManager eventManager;
protected static Random random = new Random();
@@ -67,10 +68,10 @@ public class JobManager implements IJobM
hopCount = new HopCount(threadContext,database);
carryDown = new Carrydown(database);
eventManager = new EventManager(database);
+ binManager = new BinManager(database);
outputMgr = OutputConnectionManagerFactory.make(threadContext);
connectionMgr = RepositoryConnectionManagerFactory.make(threadContext);
lockManager = LockManagerFactory.make(threadContext);
-
}
/** Install.
@@ -83,6 +84,7 @@ public class JobManager implements IJobM
hopCount.install(jobs.getTableName(),jobs.idField);
carryDown.install(jobs.getTableName(),jobs.idField);
eventManager.install();
+ binManager.install();
}
/** Uninstall.
@@ -90,6 +92,7 @@ public class JobManager implements IJobM
public void deinstall()
throws ManifoldCFException
{
+ binManager.deinstall();
eventManager.deinstall();
carryDown.deinstall();
hopCount.deinstall();
