Author: kwright
Date: Wed May 24 22:22:15 2023
New Revision: 1910036
URL: http://svn.apache.org/viewvc?rev=1910036&view=rev
Log:
CONNECTORS-1747: Add global config flag to disable hopcount tracking completely
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1910036&r1=1910035&r2=1910036&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed May 24 22:22:15 2023
@@ -6,6 +6,9 @@ $Id$
CONNECTORS-1743: Retry on 502 and 503 errors in Solr connector.
(Markus Günther)
+CONNECTORS-1747: Add global property to disable hopcount for all connectors.
+(Mingchun Zhao)
+
======================= Release 2.24 =====================
CONNECTORS-1739: Reuse escaping facilities.
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp?rev=1910036&r1=1910035&r2=1910036&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp Wed May
24 22:22:15 2023
@@ -54,6 +54,13 @@ try
INotificationConnectorPool notificationConnectorPool =
NotificationConnectorPoolFactory.make(threadContext);
ITransformationConnectorPool transformationConnectorPool =
TransformationConnectorPoolFactory.make(threadContext);
+ ILockManager lockManager = LockManagerFactory.make(threadContext);
+
+ /** If the global cluster property "storehopcount" is set to false(defaults
to true), disable support for hopcount handling completely,
+ * the "Hop Filters" tab should not appear in the UI for any job.
+ */
+ Boolean storeHopCount =
lockManager.getSharedConfiguration().getBooleanProperty("org.apache.manifoldcf.crawler.jobs.storehopcount",true);
+
// Figure out tab name and sequence number
String tabName = variableContext.getParameter("tabname");
String tabSequenceNumber = variableContext.getParameter("sequencenumber");
@@ -218,7 +225,7 @@ try
{
tabsArray.add(Messages.getString(pageContext.getRequest().getLocale(),"editjob.Scheduling"));
sequenceArray.add(null);
- if (relationshipTypes != null && relationshipTypes.length > 0)
+ if (storeHopCount && relationshipTypes != null && relationshipTypes.length
> 0)
{
tabsArray.add(Messages.getString(pageContext.getRequest().getLocale(),"editjob.HopFilters"));
sequenceArray.add(null);
@@ -902,7 +909,11 @@ function isRegularExpression(value)
}
// Hop Filters tab
- if
(tabName.equals(Messages.getString(pageContext.getRequest().getLocale(),"editjob.HopFilters"))
&& tabSequenceInt == -1)
+ if (!storeHopCount)
+ {
+ // Do nothing
+ }
+ else if
(tabName.equals(Messages.getString(pageContext.getRequest().getLocale(),"editjob.HopFilters"))
&& tabSequenceInt == -1)
{
if (relationshipTypes != null)
{
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp?rev=1910036&r1=1910035&r2=1910036&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp Wed May
24 22:22:15 2023
@@ -46,6 +46,13 @@ try
INotificationConnectorPool notificationConnectorPool =
NotificationConnectorPoolFactory.make(threadContext);
ITransformationConnectorPool transformationConnectorPool =
TransformationConnectorPoolFactory.make(threadContext);
+ ILockManager lockManager = LockManagerFactory.make(threadContext);
+
+ /** If the global cluster property "storehopcount" is set to false(defaults
to true), disable support for hopcount handling completely,
+ * the hopcount information should not appear in the UI for any job.
+ */
+ Boolean storeHopCount =
lockManager.getSharedConfiguration().getBooleanProperty("org.apache.manifoldcf.crawler.jobs.storehopcount",true);
+
String jobID = variableContext.getParameter("jobid");
IJobDescription job = manager.load(new Long(jobID));
if (job == null)
@@ -595,7 +602,7 @@ try
}
}
- if (relationshipTypes != null && relationshipTypes.length > 0)
+ if (storeHopCount && relationshipTypes != null && relationshipTypes.length
> 0)
{
int k = 0;
while (k < relationshipTypes.length)
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java?rev=1910036&r1=1910035&r2=1910036&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
Wed May 24 22:22:15 2023
@@ -148,6 +148,14 @@ public class HopCount extends org.apache
/** Thread context */
protected IThreadContext threadContext;
+ /** Lock manager */
+ protected final ILockManager lockManager;
+
+ /** If the global cluster property "storehopcount" is set to false(defaults
to true), disable support for hopcount handling completely,
+ * the hopcount will never be recorded in the "intrinsiclink" or "hopcount"
tables for any job at all.
+ */
+ protected static Boolean storeHopCount = true;
+
/** Constructor.
*@param database is the database handle.
*/
@@ -158,6 +166,8 @@ public class HopCount extends org.apache
this.threadContext = tc;
intrinsicLinkManager = new IntrinsicLink(database);
deleteDepsManager = new HopDeleteDeps(database);
+ lockManager = LockManagerFactory.make(tc);
+ storeHopCount =
lockManager.getSharedConfiguration().getBooleanProperty("org.apache.manifoldcf.crawler.jobs.storehopcount",true);
}
/** Install or upgrade.
@@ -389,6 +399,12 @@ public class HopCount extends org.apache
// this method would need to be revised to not process any additions until
the finishParents() call
// is made. At the moment, revertParents() is not used by any thread.
// TBD, MHL
+ if (!storeHopCount)
+ {
+ // Do nothing
+ return null;
+ }
+
boolean[] rval = new boolean[targetDocumentIDHashes.length];
for (int i = 0; i < rval.length; i++)
{
