Author: kwright
Date: Sat Nov 9 14:48:30 2013
New Revision: 1540317
URL: http://svn.apache.org/r1540317
Log:
Fix for CONNECTORS-803.
Added:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
(with props)
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp
manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml
manifoldcf/trunk/framework/example-multiprocess/properties.xml
manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml
manifoldcf/trunk/framework/example-singleprocess/properties.xml
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sat Nov 9 14:48:30 2013
@@ -3,6 +3,10 @@ $Id$
======================= 1.5-dev =====================
+CONNECTORS-803: Add automatic and manual ability to clean out
+repohistory rows.
+(Marcello Lorenzi, Karl Wright)
+
CONNECTORS-801: Refactor JDBC authority to bring it into compliance with
current
conventions in JDBC connector. Warning: this change is not backwards
compatible! JDBC authority queries now require explicit return column
Modified:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java
(original)
+++
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java
Sat Nov 9 14:48:30 2013
@@ -91,6 +91,12 @@ public class LockManagerFactory
{
return make(tc).getSharedConfiguration().getIntProperty(s, defaultValue);
}
+
+ public static long getLongProperty(IThreadContext tc, String s, long
defaultValue)
+ throws ManifoldCFException
+ {
+ return make(tc).getSharedConfiguration().getLongProperty(s, defaultValue);
+ }
public static double getDoubleProperty(IThreadContext tc, String s, double
defaultValue)
throws ManifoldCFException
Modified:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java
(original)
+++
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java
Sat Nov 9 14:48:30 2013
@@ -104,6 +104,24 @@ public class ManifoldCFConfiguration ext
}
}
+ /** Read a long property, either from the system properties, or from the
local configuration file.
+ */
+ public long getLongProperty(String s, long defaultValue)
+ throws ManifoldCFException
+ {
+ String value = getProperty(s);
+ if (value == null)
+ return defaultValue;
+ try
+ {
+ return Long.parseLong(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new ManifoldCFException("Illegal property value for long property
'"+s+"': '"+value+"': "+e.getMessage(),e,ManifoldCFException.SETUP_ERROR);
+ }
+ }
+
/** Read a float property, either from the system properties, or from the
local configuration file.
*/
public double getDoubleProperty(String s, double defaultValue)
Modified:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
(original)
+++
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
Sat Nov 9 14:48:30 2013
@@ -393,6 +393,14 @@ public class ManifoldCF
return localConfiguration.getIntProperty(s, defaultValue);
}
+ /** Read a long property, either from the system properties, or from the
local configuration file.
+ */
+ public static long getLongProperty(String s, long defaultValue)
+ throws ManifoldCFException
+ {
+ return localConfiguration.getLongProperty(s, defaultValue);
+ }
+
/** Read a float property, either from the system properties, or from the
local configuration file.
*/
public static double getDoubleProperty(String s, double defaultValue)
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp Sat Nov
9 14:48:30 2013
@@ -203,6 +203,28 @@
<jsp:forward page="listconnections.jsp"/>
<%
}
+ else if (op.equals("ClearHistory"))
+ {
+ try
+ {
+ String connectionName =
variableContext.getParameter("connname");
+ if (connectionName == null)
+ throw new
ManifoldCFException("Missing connection parameter");
+
connManager.cleanUpHistoryData(connectionName);
+%>
+ <jsp:forward
page="listconnections.jsp"/>
+<%
+ }
+ catch (ManifoldCFException e)
+ {
+ e.printStackTrace();
+
variableContext.setParameter("text",e.getMessage());
+
variableContext.setParameter("target","listconnections.jsp");
+%>
+ <jsp:forward page="error.jsp"/>
+<%
+ }
+ }
else
{
// Error
Modified:
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp
(original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp
Sat Nov 9 14:48:30 2013
@@ -45,6 +45,16 @@
}
}
+ function ClearHistory(connectionName)
+ {
+ if
(confirm("<%=Messages.getBodyJavascriptString(pageContext.getRequest().getLocale(),"viewconnection.Thiscommandwillclearallhistoryrelatedto")%>
'"+connectionName+"'
<%=Messages.getBodyJavascriptString(pageContext.getRequest().getLocale(),"viewconnection.period")%>"))
+ {
+ document.viewconnection.op.value="ClearHistory";
+ document.viewconnection.connname.value=connectionName;
+ document.viewconnection.submit();
+ }
+ }
+
//-->
</script>
@@ -207,8 +217,12 @@
<tr>
<td class="separator" colspan="4"><hr/></td>
</tr>
- <tr><td class="message" colspan="4"><a
href='<%="viewconnection.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Refresh")%></a> <a
href='<%="editconnection.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.EditThisConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Edit")%></a> <a
href="javascript:void()"
onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.Deletethisconnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Delete"
)%></a>
- </td></tr>
+ <tr>
+ <td class="message" colspan="4">
+ <nobr><a
href='<%="viewconnection.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Refresh")%></a> <a
href='<%="editconnection.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.EditThisConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Edit")%></a> <a
href="javascript:void()"
onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.Deletethisconnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Delete")%></a></nobr>
+ <nobr><a href="javascript:void()"
onclick='<%="javascript:ClearHistory(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.ClearHistoryAssociatedWithThisConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.ClearAllRelatedHistory")%></a></nobr>
+ </td>
+ </tr>
</table>
<%
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp
(original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp Sat
Nov 9 14:48:30 2013
@@ -159,12 +159,12 @@
<tr>
<td class="separator" colspan="4"><hr/></td>
</tr>
- <tr>
- <td class="message" colspan="4">
- <nobr><a
href='<%="viewoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%></a> <a
href='<%="editoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.EditThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Edit")%></a> <a
href="javascript:void()"
onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.DeleteThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Delete")%></a></nobr><br/>
- <nobr><a href="javascript:void()"
onclick='<%="javascript:ReingestAll(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllDocumentsAssociatedWithThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllAssociatedDocuments")%></a></nobr>
- </td>
- </tr>
+ <tr>
+ <td class="message" colspan="4">
+ <nobr><a
href='<%="viewoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%></a> <a
href='<%="editoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.EditThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Edit")%></a> <a
href="javascript:void()"
onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.DeleteThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Delete")%></a></nobr><br/>
+ <nobr><a href="javascript:void()"
onclick='<%="javascript:ReingestAll(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>'
alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllDocumentsAssociatedWithThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllAssociatedDocuments")%></a></nobr>
+ </td>
+ </tr>
</table>
<%
Modified:
manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml
(original)
+++ manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml
Sat Nov 9 14:48:30 2013
@@ -34,6 +34,7 @@
<property name="org.apache.manifoldcf.dbsuperuserpassword" value=""/>
<property name="org.apache.manifoldcf.database.maxhandles" value="100"/>
<property name="org.apache.manifoldcf.crawler.threads" value="50"/>
+ <property name="org.apache.manifoldcf.crawler.historycleanupinterval"
value="2592000000"/>
<!-- Point to a specific (common) logging file -->
<property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/>
<!-- Specify the connectors to be loaded -->
Modified: manifoldcf/trunk/framework/example-multiprocess/properties.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/example-multiprocess/properties.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/example-multiprocess/properties.xml (original)
+++ manifoldcf/trunk/framework/example-multiprocess/properties.xml Sat Nov 9
14:48:30 2013
@@ -34,6 +34,7 @@
<property name="org.apache.manifoldcf.dbsuperuserpassword" value=""/>
<property name="org.apache.manifoldcf.database.maxhandles" value="100"/>
<property name="org.apache.manifoldcf.crawler.threads" value="50"/>
+ <property name="org.apache.manifoldcf.crawler.historycleanupinterval"
value="2592000000"/>
<!-- Point to a specific (common) logging file -->
<property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/>
<!-- Specify the connectors to be loaded -->
Modified:
manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml
(original)
+++ manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml
Sat Nov 9 14:48:30 2013
@@ -30,6 +30,7 @@
<property name="org.apache.manifoldcf.derbydatabasepath" value="."/>
<property name="org.apache.manifoldcf.database.maxhandles" value="100"/>
<property name="org.apache.manifoldcf.crawler.threads" value="50"/>
+ <property name="org.apache.manifoldcf.crawler.historycleanupinterval"
value="2592000000"/>
<!-- Point to a specific logging file -->
<property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/>
<!-- Specify the connectors to be loaded -->
Modified: manifoldcf/trunk/framework/example-singleprocess/properties.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/example-singleprocess/properties.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/example-singleprocess/properties.xml (original)
+++ manifoldcf/trunk/framework/example-singleprocess/properties.xml Sat Nov 9
14:48:30 2013
@@ -30,6 +30,7 @@
<property name="org.apache.manifoldcf.derbydatabasepath" value="."/>
<property name="org.apache.manifoldcf.database.maxhandles" value="100"/>
<property name="org.apache.manifoldcf.crawler.threads" value="50"/>
+ <property name="org.apache.manifoldcf.crawler.historycleanupinterval"
value="2592000000"/>
<!-- Point to a specific logging file -->
<property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/>
<!-- Specify the connectors to be loaded -->
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java
Sat Nov 9 14:48:30 2013
@@ -121,6 +121,18 @@ public interface IRepositoryConnectionMa
// Reporting and analysis related
+ /** Delete history rows related to a specific connection, upon user request.
+ *@param connectionName is the connection whose history records should be
removed.
+ */
+ public void cleanUpHistoryData(String connectionName)
+ throws ManifoldCFException;
+
+ /** Delete history rows older than a specified timestamp.
+ *@param timeCutoff is the timestamp to delete older rows before.
+ */
+ public void cleanUpHistoryData(long timeCutoff)
+ throws ManifoldCFException;
+
// Activities the Connector Framework records
/** Start a job */
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
Sat Nov 9 14:48:30 2013
@@ -528,7 +528,7 @@ public class RepositoryConnectionManager
throw new ManifoldCFException("Can't delete repository connection
'"+name+"': existing jobs refer to it");
ManifoldCF.noteConfigurationChange();
throttleSpecManager.deleteRows(name);
- historyManager.deleteOwner(name,null);
+ historyManager.deleteOwner(name);
ArrayList params = new ArrayList();
String query = buildConjunctionClause(params,new ClauseDescription[]{
new UnitaryClause(nameField,name)});
@@ -580,6 +580,7 @@ public class RepositoryConnectionManager
*@param className is the class name of the connector.
*@return the repository connections that use that connector.
*/
+ @Override
public String[] findConnectionsForConnector(String className)
throws ManifoldCFException
{
@@ -607,6 +608,7 @@ public class RepositoryConnectionManager
*@param name is the name of the connection to check.
*@return true if the underlying connector is registered.
*/
+ @Override
public boolean checkConnectorExists(String name)
throws ManifoldCFException
{
@@ -648,6 +650,7 @@ public class RepositoryConnectionManager
/** Return the name column.
*@return the name column.
*/
+ @Override
public String getConnectionNameColumn()
{
return nameField;
@@ -656,7 +659,26 @@ public class RepositoryConnectionManager
// Reporting and analysis related
-
+ /** Delete history rows related to a specific connection, upon user request.
+ *@param connectionName is the connection whose history records should be
removed.
+ */
+ @Override
+ public void cleanUpHistoryData(String connectionName)
+ throws ManifoldCFException
+ {
+ historyManager.deleteOwner(connectionName);
+ }
+
+ /** Delete history rows older than a specified timestamp.
+ *@param timeCutoff is the timestamp to delete older rows before.
+ */
+ @Override
+ public void cleanUpHistoryData(long timeCutoff)
+ throws ManifoldCFException
+ {
+ historyManager.deleteOldRows(timeCutoff);
+ }
+
/** Record time-stamped information about the activity of the connection.
This information can originate from
* either the connector or from the framework. The reason it is here is that
it is viewed as 'belonging' to an
* individual connection, and is segregated accordingly.
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java
Sat Nov 9 14:48:30 2013
@@ -161,15 +161,27 @@ public class RepositoryHistoryManager ex
*@param owner is the name of the owner.
*@param invKeys are the invalidation keys.
*/
- public void deleteOwner(String owner, StringSet invKeys)
+ public void deleteOwner(String owner)
throws ManifoldCFException
{
ArrayList params = new ArrayList();
String query = buildConjunctionClause(params,new ClauseDescription[]{
new UnitaryClause(ownerNameField,owner)});
- performDelete("WHERE "+query,params,invKeys);
+ performDelete("WHERE "+query,params,null);
}
+ /** Delete records older than a specified time.
+ *@param timeCutoff is the time, earlier than which records are removed.
+ */
+ public void deleteOldRows(long timeCutoff)
+ throws ManifoldCFException
+ {
+ ArrayList params = new ArrayList();
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(startTimeField,"<",new Long(timeCutoff))});
+ performDelete("WHERE "+query,params,null);
+ }
+
/** Add row to table, and reanalyze if necessary.
*/
public Long addRow(String connectionName, long startTime, long endTime, long
dataSize, String activityType,
Added:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java?rev=1540317&view=auto
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
(added)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
Sat Nov 9 14:48:30 2013
@@ -0,0 +1,131 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.system;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import java.util.*;
+import java.lang.reflect.*;
+
+/** This class describes the thread that cleans up history records.
+* It fires infrequently and removes history records older than a
configuration-determined cutoff.
+*/
+public class HistoryCleanupThread extends Thread
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ protected static final String historyCleanupIntervalProperty =
"org.apache.manifoldcf.crawler.historycleanupinterval";
+
+ /** Constructor.
+ */
+ public HistoryCleanupThread()
+ throws ManifoldCFException
+ {
+ super();
+ setName("History cleanup thread");
+ setDaemon(true);
+ }
+
+ public void run()
+ {
+ try
+ {
+ // Create a thread context object.
+ IThreadContext threadContext = ThreadContextFactory.make();
+ IRepositoryConnectionManager connectionManager =
RepositoryConnectionManagerFactory.make(threadContext);
+ // Default zero value means we never clean up, which is the
backwards-compatible behavior
+ long historyCleanupInterval =
LockManagerFactory.getLongProperty(threadContext,
historyCleanupIntervalProperty, 0L);
+ // Loop
+ while (true)
+ {
+ if (Thread.currentThread().isInterrupted())
+ break;
+
+ // Do another try/catch around everything in the loop
+ try
+ {
+ // Get current time
+ long currentTime = System.currentTimeMillis();
+ // Log it
+ if (Logging.threads.isDebugEnabled())
+ Logging.threads.debug("History cleanup thread - removing old
history at "+new Long(currentTime).toString());
+ if (historyCleanupInterval > 0L && historyCleanupInterval <
currentTime)
+ connectionManager.cleanUpHistoryData(currentTime -
historyCleanupInterval);
+ else
+ Logging.threads.debug(" History cleanup thread did nothing because
cleanup disabled");
+ // Loop around again, after resting a while
+ ManifoldCF.sleep(60L * 60L * 1000L);
+ }
+ catch (ManifoldCFException e)
+ {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ break;
+
+ if (e.getErrorCode() ==
ManifoldCFException.DATABASE_CONNECTION_ERROR)
+ {
+ Logging.threads.error("History thread aborting and restarting due
to database connection reset: "+e.getMessage(),e);
+ try
+ {
+ // Give the database a chance to catch up/wake up
+ ManifoldCF.sleep(10000L);
+ }
+ catch (InterruptedException se)
+ {
+ break;
+ }
+ continue;
+ }
+
+ // Log it, but keep the thread alive
+ Logging.threads.error("Exception tossed: "+e.getMessage(),e);
+
+ if (e.getErrorCode() == ManifoldCFException.SETUP_ERROR)
+ {
+ // Shut the whole system down!
+ System.exit(1);
+ }
+ }
+ catch (InterruptedException e)
+ {
+ // We're supposed to quit
+ break;
+ }
+ catch (OutOfMemoryError e)
+ {
+ System.err.println("agents process ran out of memory - shutting
down");
+ e.printStackTrace(System.err);
+ System.exit(-200);
+ }
+ catch (Throwable e)
+ {
+ // A more severe error - but stay alive
+ Logging.threads.fatal("Error tossed: "+e.getMessage(),e);
+ }
+ }
+ }
+ catch (Throwable e)
+ {
+ // Severe error on initialization
+ System.err.println("agents process could not start - shutting down");
+ Logging.threads.fatal("HistoryCleanupThread initialization error tossed:
"+e.getMessage(),e);
+ System.exit(-300);
+ }
+ }
+
+}
Propchange:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
Sat Nov 9 14:48:30 2013
@@ -73,6 +73,7 @@ public class ManifoldCF extends org.apac
protected static SeedingThread seedingThread = null;
protected static IdleCleanupThread idleCleanupThread = null;
protected static SetPriorityThread setPriorityThread = null;
+ protected static HistoryCleanupThread historyCleanupThread = null;
// Reset managers
/** Worker thread pool reset manager */
@@ -640,6 +641,7 @@ public class ManifoldCF extends org.apac
stufferThread = new
StufferThread(documentQueue,numWorkerThreads,workerResetManager,queueTracker,blockingDocuments,lowWaterFactor,stuffAmtFactor);
expireStufferThread = new
ExpireStufferThread(expireQueue,numExpireThreads,workerResetManager);
setPriorityThread = new
SetPriorityThread(queueTracker,numWorkerThreads,blockingDocuments);
+ historyCleanupThread = new HistoryCleanupThread();
workerThreads = new WorkerThread[numWorkerThreads];
int i = 0;
@@ -755,6 +757,7 @@ public class ManifoldCF extends org.apac
stufferThread.start();
expireStufferThread.start();
setPriorityThread.start();
+ historyCleanupThread.start();
i = 0;
while (i < numWorkerThreads)
@@ -820,7 +823,7 @@ public class ManifoldCF extends org.apac
finisherThread != null || notificationThread != null || workerThreads
!= null || expireStufferThread != null || expireThreads != null ||
deleteStufferThread != null || deleteThreads != null ||
cleanupStufferThread != null || cleanupThreads != null ||
- jobResetThread != null || seedingThread != null || idleCleanupThread
!= null || setPriorityThread != null)
+ jobResetThread != null || seedingThread != null || idleCleanupThread
!= null || setPriorityThread != null || historyCleanupThread != null)
{
// Send an interrupt to all threads that are still there.
// In theory, this only needs to be done once. In practice, I have
seen cases where the thread loses track of the fact that it has been
@@ -829,6 +832,10 @@ public class ManifoldCF extends org.apac
{
initializationThread.interrupt();
}
+ if (historyCleanupThread != null)
+ {
+ historyCleanupThread.interrupt();
+ }
if (setPriorityThread != null)
{
setPriorityThread.interrupt();
@@ -941,6 +948,11 @@ public class ManifoldCF extends org.apac
if (!initializationThread.isAlive())
initializationThread = null;
}
+ if (historyCleanupThread != null)
+ {
+ if (!historyCleanupThread.isAlive())
+ historyCleanupThread = null;
+ }
if (setPriorityThread != null)
{
if (!setPriorityThread.isAlive())
Modified:
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
(original)
+++
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
Sat Nov 9 14:48:30 2013
@@ -24,6 +24,10 @@ editjob.Name=Name
editjob.Connection=Connection
viewconnection.NoneGlobalAuthority=None (global authority)
+viewconnection.ClearHistoryAssociatedWithThisConnection=Clear history
associated with this connection
+viewconnection.ClearAllRelatedHistory=Clear All Related History
+viewconnection.Thiscommandwillclearallhistoryrelatedto=This command will clear
all history related to connection
+viewconnection.period=.
listconnections.GlobalAuthority=None(globalAuthority)
Modified:
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
(original)
+++
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
Sat Nov 9 14:48:30 2013
@@ -24,6 +24,10 @@ editjob.Name=åå
editjob.Connection=ã³ãã¯ã·ã§ã³
viewconnection.NoneGlobalAuthority=ãªãï¼globalAuthorityï¼
+viewconnection.ClearHistoryAssociatedWithThisConnection=Clear history
associated with this connection
+viewconnection.ClearAllRelatedHistory=Clear All Related History
+viewconnection.Thiscommandwillclearallhistoryrelatedto=This command will clear
all history related to connection
+viewconnection.period=.
listconnections.GlobalAuthority=ãªãï¼globalAuthorityï¼
Modified:
manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
(original)
+++
manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
Sat Nov 9 14:48:30 2013
@@ -836,6 +836,7 @@ cd example
<tr><td>org.apache.manifoldcf.crawler.expirethreads</td><td>No</td><td>Number
of crawler expiration threads created. Suggest a value of 10.</td></tr>
<tr><td>org.apache.manifoldcf.crawler.cleanupthreads</td><td>No</td><td>Number
of crawler cleanup threads created. Suggest a value of 10.</td></tr>
<tr><td>org.apache.manifoldcf.crawler.deletethreads</td><td>No</td><td>Number
of crawler delete threads created. Suggest a value of 10.</td></tr>
+
<tr><td>org.apache.manifoldcf.crawler.historycleanupinterval</td><td>No</td><td>Milliseconds
to retain history records. Default is 0. Zero means "forever".</td></tr>
<tr><td>org.apache.manifoldcf.misc</td><td>No</td><td>Miscellaneous debugging
output. Legal values INFO, WARN, or DEBUG.</td></tr>
<tr><td>org.apache.manifoldcf.db</td><td>No</td><td>Database
debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
<tr><td>org.apache.manifoldcf.lock</td><td>No</td><td>Lock
management debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
Modified:
manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
---
manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
(original)
+++
manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
Sat Nov 9 14:48:30 2013
@@ -766,6 +766,8 @@ cd example
when operating without any apparent stalls due to the above
issues, Derby is still only about 1/4 as fast as PostgreSQL. At the moment
this limits Derby's utility for
ManifoldCF to demonstration and testing.</p>
</section>
+
+
</section>
<section>
@@ -834,6 +836,7 @@ cd example
<tr><td>org.apache.manifoldcf.crawler.expirethreads</td><td>No</td><td>Number
of crawler expiration threads created. Suggest a value of 10.</td></tr>
<tr><td>org.apache.manifoldcf.crawler.cleanupthreads</td><td>No</td><td>Number
of crawler cleanup threads created. Suggest a value of 10.</td></tr>
<tr><td>org.apache.manifoldcf.crawler.deletethreads</td><td>No</td><td>Number
of crawler delete threads created. Suggest a value of 10.</td></tr>
+
<tr><td>org.apache.manifoldcf.crawler.historycleanupinterval</td><td>No</td><td>Milliseconds
to retain history records. Default is 0. Zero means "forever".</td></tr>
<tr><td>org.apache.manifoldcf.misc</td><td>No</td><td>Miscellaneous debugging
output. Legal values INFO, WARN, or DEBUG.</td></tr>
<tr><td>org.apache.manifoldcf.db</td><td>No</td><td>Database
debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
<tr><td>org.apache.manifoldcf.lock</td><td>No</td><td>Lock
management debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
@@ -940,7 +943,47 @@ mvn exec:exec
<p>In a multi process setup, all of the ManifoldCF processes might as
well exist on their own. You can learn how to programmatically start the
agents process by looking at the code
in the AgentRun command class, as described above. Similarly, the
command classes that register connectors are very small and should be easy to
understand.</p>
</section>
-
+
+ <section>
+ <title>Integrating ManifoldCF with a search engine</title>
+ <p></p>
+ <p>ManifoldCF's Authority Service is designed to allow maximum
flexibility in integrating ManifoldCF security with search engines. The
+ service receives a user identity (as a set of authorization
domain/user name tuples), and produces a set of tokens. It also returns a
+ summary of the status of all authorities that were involved in
the assembly of the set of tokens, as a nicety. A search engine user
+ interface could thus signal the user when the results they
might be seeing are incomplete, and why.</p>
+ <p>The Authority Service expects the following arguments, passed as
URL arguments and properly URL encoded:</p>
+ <p></p>
+ <table>
+ <caption>Authority Service URL parameters</caption>
+ <tr><th>Authority Service URL parameter</th><th>Meaning</th></tr>
+ <tr><td>username</td><td>the username, if there is only one
authorization domain</td></tr>
+ <tr><td>domain</td><td>the optional authorization domain if there
is only one authorization domain (defaults to empty string)</td></tr>
+ <tr><td>username_<em>XX</em></td><td>username number <em>XX</em>,
where <em>XX</em> is an integer starting at zero</td></tr>
+ <tr><td>domain_<em>XX</em></td><td>authorization domain
<em>XX</em>, where <em>XX</em> is an integer starting at zero</td></tr>
+ </table>
+ <p></p>
+ <p>Access tokens and authority statuses are returned in the HTTP
response separated by newline characters. Each line has a prefix
+ as follows:</p>
+ <p></p>
+ <table>
+ <caption>Authority Service response prefixes</caption>
+ <tr><th>Authority Service response prefix</th><th>Meaning</th></tr>
+ <tr><td>TOKEN:</td><td>An access token</td></tr>
+ <tr><td>AUTHORIZED:</td><td>The name of an authority that found
the user to be authorized</td></tr>
+ <tr><td>UNREACHABLEAUTHORITY:</td><td>The name of an authority
that was found to be unreachable or unusable</td></tr>
+ <tr><td>UNAUTHORIZED:</td><td>The name of an authority that found
the user to be unauthorized</td></tr>
+ <tr><td>USERNOTFOUND:</td><td>The name of an authority that could
not find the user</td></tr>
+ </table>
+ <p></p>
+ <p>It is important to remember that only the "TOKEN:" lines actually
matter for security. Even if any of the error conditions apply, the set
+ of tokens returned by the Authority Service will be correctly
supplied in order to apply appropriate security to documents being searched.</p>
+ <p>If you choose to deploy a search-engine plugin supplied by the
Apache ManifoldCF project (for example, the Solr plugin), you will not need
+ know any of the above, since part of the plugin's purpose is
to communicate with the Authority Service and apply the access tokens that are
+ returned to the search query automatically. Some plugins,
such as the ElasticSearch plugin, are more or less like toolkits, but still
hide most
+ of the above from the integrator. In a more highly customized
system, however, you may need to develop your own code which interacts
+ with the Authority Service in order to meet your goals.</p>
+ </section>
+
</section>
</body>
