Author: kwright
Date: Fri Nov 29 01:59:41 2013
New Revision: 1546493
URL: http://svn.apache.org/r1546493
Log:
Add ability to get multiple bin values at once.
Modified:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
Modified:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java?rev=1546493&r1=1546492&r2=1546493&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
(original)
+++
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
Fri Nov 29 01:59:41 2013
@@ -22,6 +22,7 @@ import org.apache.manifoldcf.core.interf
import org.apache.manifoldcf.crawler.interfaces.*;
import org.apache.manifoldcf.crawler.interfaces.CacheKeyFactory;
import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.apache.manifoldcf.crawler.system.Logging;
import java.util.*;
/** This class manages the docbins table.
@@ -101,17 +102,18 @@ public class BinManager extends org.apac
performDelete("", null, null);
}
- /** Get a bin value (and set next one). If the record does not yet exist,
create it with a starting value.
- * We expect this to happen within a transaction!!
+ /** Get N bin values (and set next one). If the record does not yet exist,
create it with a starting value.
+ * We expect this to happen within a transaction!!.
*@param binName is the name of the bin (256 char max)
*@param newBinValue is the value to use if there is no such bin yet. This
is the value that will be
* returned; what will be stored will be that value + 1.
- *@return the counter value.
+ *@param count is the number of values desired.
+ *@return the counter values.
*/
- @Override
- public double getIncrementBinValue(String binName, double newBinValue)
+ public double[] getIncrementBinValues(String binName, double newBinValue,
int count)
throws ManifoldCFException
{
+ double[] returnValues = new double[count];
// SELECT FOR UPDATE/MODIFY is the most common path
ArrayList params = new ArrayList();
String query = buildConjunctionClause(params,new ClauseDescription[]{
@@ -124,18 +126,78 @@ public class BinManager extends org.apac
double rval = value.doubleValue();
if (rval < newBinValue)
rval = newBinValue;
+ // rval is the starting value; compute the entire array based on it.
+ for (int i = 0; i < count; i++)
+ {
+ returnValues[i] = rval;
+ rval += 1.0;
+ }
HashMap map = new HashMap();
- map.put(binCounterField,new Double(rval+1.0));
+ map.put(binCounterField,new Double(rval));
performUpdate(map," WHERE "+query,params,null);
- return rval;
}
else
{
+ for (int i = 0; i < count; i++)
+ {
+ returnValues[i] = newBinValue;
+ newBinValue += 1.0;
+ }
HashMap map = new HashMap();
map.put(binNameField,binName);
- map.put(binCounterField,new Double(newBinValue+1.0));
+ map.put(binCounterField,new Double(newBinValue));
performInsert(map,null);
- return newBinValue;
+ }
+ return returnValues;
+ }
+
+ /** Get N bin values (and set next one). If the record does not yet exist,
create it with a starting value.
+ * This method invokes its own retry-able transaction.
+ *@param binName is the name of the bin (256 char max)
+ *@param newBinValue is the value to use if there is no such bin yet. This
is the value that will be
+ * returned; what will be stored will be that value + 1.
+ *@param count is the number of values desired.
+ *@return the counter values.
+ */
+ @Override
+ public double[] getIncrementBinValuesInTransaction(String binName, double
newBinValue, int count)
+ throws ManifoldCFException
+ {
+ while (true)
+ {
+ long sleepAmt = 0L;
+ beginTransaction();
+ try
+ {
+ return getIncrementBinValues(binName, newBinValue, count);
+ }
+ catch (Error e)
+ {
+ signalRollback();
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ signalRollback();
+ throw e;
+ }
+ catch (ManifoldCFException e)
+ {
+ signalRollback();
+ if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+ {
+ if (Logging.perf.isDebugEnabled())
+ Logging.perf.debug("Aborted transaction obtaining docpriorities:
"+e.getMessage());
+ sleepAmt = getSleepAmt();
+ continue;
+ }
+ throw e;
+ }
+ finally
+ {
+ endTransaction();
+ sleepFor(sleepAmt);
+ }
}
}
Modified:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java?rev=1546493&r1=1546492&r2=1546493&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
(original)
+++
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
Fri Nov 29 01:59:41 2013
@@ -41,14 +41,26 @@ public interface IBinManager
public void reset()
throws ManifoldCFException;
- /** Get a bin value (and set next one). If the record does not yet exist,
create it with a starting value.
+ /** Get N bin values (and set next one). If the record does not yet exist,
create it with a starting value.
* We expect this to happen within a transaction!!
*@param binName is the name of the bin (256 char max)
*@param newBinValue is the value to use if there is no such bin yet. This
is the value that will be
* returned; what will be stored will be that value + 1.
- *@return the counter value.
+ *@param count is the number of values desired.
+ *@return the counter values.
*/
- public double getIncrementBinValue(String binName, double newBinValue)
+ public double[] getIncrementBinValues(String binName, double newBinValue,
int count)
+ throws ManifoldCFException;
+
+ /** Get N bin values (and set next one). If the record does not yet exist,
create it with a starting value.
+ * This method invokes its own retry-able transaction.
+ *@param binName is the name of the bin (256 char max)
+ *@param newBinValue is the value to use if there is no such bin yet. This
is the value that will be
+ * returned; what will be stored will be that value + 1.
+ *@param count is the number of values desired.
+ *@return the counter values.
+ */
+ public double[] getIncrementBinValuesInTransaction(String binName, double
newBinValue, int count)
throws ManifoldCFException;
}
Modified:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1546493&r1=1546492&r2=1546493&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Fri Nov 29 01:59:41 2013
@@ -4536,8 +4536,6 @@ public class JobManager implements IJobM
IResultSet set = database.performQuery(sb.toString(),list,null,null);
- boolean priorityUsed;
-
if (set.getRowCount() > 0)
{
// Found a row, and it is now locked.
Modified:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java?rev=1546493&r1=1546492&r2=1546493&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
(original)
+++
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
Fri Nov 29 01:59:41 2013
@@ -255,7 +255,7 @@ public class ReprioritizationTracker
public double getIncrementBinValue(String binName, double
weightedMinimumDepth)
throws ManifoldCFException
{
- return binManager.getIncrementBinValue(binName, weightedMinimumDepth);
+ return binManager.getIncrementBinValues(binName,
weightedMinimumDepth,1)[0];
}
// Protected methods
