Author: kwright
Date: Sun Aug 31 18:17:42 2014
New Revision: 1621613
URL: http://svn.apache.org/r1621613
Log:
More work on CONNECTORS-977.
Modified:
manifoldcf/trunk/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java
manifoldcf/trunk/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java
manifoldcf/trunk/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java
manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchSpecs.java
manifoldcf/trunk/connectors/email/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/email/EmailConnector.java
manifoldcf/trunk/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputSpecs.java
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java
manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
manifoldcf/trunk/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java
manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
manifoldcf/trunk/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java
manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/MeridioConnector.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSpecs.java
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/TestingRepositoryConnector.java
Modified:
manifoldcf/trunk/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java
(original)
+++
manifoldcf/trunk/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java
Sun Aug 31 18:17:42 2014
@@ -45,6 +45,7 @@ import org.apache.commons.lang.StringUti
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.Specification;
import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
import org.apache.manifoldcf.core.interfaces.IPasswordMapperActivity;
import org.apache.manifoldcf.core.interfaces.IPostParameters;
@@ -52,9 +53,9 @@ import org.apache.manifoldcf.core.interf
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.core.interfaces.SpecificationNode;
import org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector;
-import org.apache.manifoldcf.crawler.interfaces.DocumentSpecification;
import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
+import org.apache.manifoldcf.crawler.interfaces.DocumentSpecification;
import org.apache.manifoldcf.crawler.system.Logging;
public class AlfrescoRepositoryConnector extends BaseRepositoryConnector {
@@ -388,35 +389,36 @@ public class AlfrescoRepositoryConnector
}
/** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
- * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
- *
- * This method can choose to find repository changes that happen only during
the specified time interval.
- * The seeds recorded by this method will be viewed by the framework based
on what the
- * getConnectorModel() method returns.
- *
- * It is not a big problem if the connector chooses to create more seeds
than are
- * strictly necessary; it is merely a question of overall work required.
- *
- * The times passed to this method may be interpreted for greatest
efficiency. The time ranges
- * any given job uses with this connector will not overlap, but will proceed
starting at 0 and going
- * to the "current time", each time the job is run. For continuous crawling
jobs, this method will
- * be called once, when the job starts, and at various periodic intervals as
the job executes.
- *
- * When a job's specification is changed, the framework automatically resets
the seeding start time to 0. The
- * seeding start time may also be set to 0 on each job run, depending on the
connector model returned by
- * getConnectorModel().
- *
- * Note that it is always ok to send MORE documents rather than less to this
method.
- *@param activities is the interface this method should use to perform
whatever framework actions are desired.
- *@param spec is a document specification (that comes from the job).
- *@param startTime is the beginning of the time range to consider, inclusive.
- *@param endTime is the end of the time range to consider, exclusive.
- *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
- */
+ * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only during
the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based on
what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds than
are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals as
the job executes.
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
+ *@param activities is the interface this method should use to perform
whatever framework actions are desired.
+ *@param spec is a document specification (that comes from the job).
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
+ */
@Override
- public void addSeedDocuments(ISeedingActivity activities,
- DocumentSpecification spec, long startTime, long endTime)
- throws ManifoldCFException, ServiceInterruption {
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption {
String luceneQuery = StringUtils.EMPTY;
int i = 0;
@@ -452,6 +454,7 @@ public class AlfrescoRepositoryConnector
Logging.connectors.warn("Alfresco: IOException: " + e.getMessage(), e);
handleIOException(e);
}
+ return "";
}
/** Get the maximum number of documents to amalgamate together into one
batch, for this connector.
@@ -684,7 +687,7 @@ public class AlfrescoRepositoryConnector
/** Fill in Velocity parameters for the LuceneQuery tab.
*/
- private static void fillInLuceneQueryParameters(Map<String,String> paramMap,
DocumentSpecification ds)
+ private static void fillInLuceneQueryParameters(Map<String,String> paramMap,
Specification ds)
{
int i = 0;
String luceneQuery = "";
Modified:
manifoldcf/trunk/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java
(original)
+++
manifoldcf/trunk/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java
Sun Aug 31 18:17:42 2014
@@ -57,6 +57,7 @@ import org.apache.commons.lang.StringUti
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.Specification;
import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
import org.apache.manifoldcf.core.interfaces.IPasswordMapperActivity;
import org.apache.manifoldcf.core.interfaces.IPostParameters;
@@ -627,35 +628,36 @@ public class CmisRepositoryConnector ext
}
/** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
- * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
- *
- * This method can choose to find repository changes that happen only during
the specified time interval.
- * The seeds recorded by this method will be viewed by the framework based
on what the
- * getConnectorModel() method returns.
- *
- * It is not a big problem if the connector chooses to create more seeds
than are
- * strictly necessary; it is merely a question of overall work required.
- *
- * The times passed to this method may be interpreted for greatest
efficiency. The time ranges
- * any given job uses with this connector will not overlap, but will proceed
starting at 0 and going
- * to the "current time", each time the job is run. For continuous crawling
jobs, this method will
- * be called once, when the job starts, and at various periodic intervals as
the job executes.
- *
- * When a job's specification is changed, the framework automatically resets
the seeding start time to 0. The
- * seeding start time may also be set to 0 on each job run, depending on the
connector model returned by
- * getConnectorModel().
- *
- * Note that it is always ok to send MORE documents rather than less to this
method.
- *@param activities is the interface this method should use to perform
whatever framework actions are desired.
- *@param spec is a document specification (that comes from the job).
- *@param startTime is the beginning of the time range to consider, inclusive.
- *@param endTime is the end of the time range to consider, exclusive.
- *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
- */
+ * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only during
the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based on
what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds than
are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals as
the job executes.
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
+ *@param activities is the interface this method should use to perform
whatever framework actions are desired.
+ *@param spec is a document specification (that comes from the job).
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
+ */
@Override
- public void addSeedDocuments(ISeedingActivity activities,
- DocumentSpecification spec, long startTime, long endTime, int jobMode)
- throws ManifoldCFException, ServiceInterruption {
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption {
getSession();
@@ -685,6 +687,7 @@ public class CmisRepositoryConnector ext
}
}
+ return "";
}
Modified:
manifoldcf/trunk/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java
(original)
+++
manifoldcf/trunk/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java
Sun Aug 31 18:17:42 2014
@@ -992,16 +992,46 @@ public class DCTM extends org.apache.man
/** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
* are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only during
the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based on
what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds than
are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals as
the job executes.
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
*@param activities is the interface this method should use to perform
whatever framework actions are desired.
*@param spec is a document specification (that comes from the job).
- *@param startTime is the beginning of the time range to consider, inclusive.
- *@param endTime is the end of the time range to consider, exclusive.
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
*/
@Override
- public void addSeedDocuments(ISeedingActivity activities,
DocumentSpecification spec,
- long startTime, long endTime)
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
throws ManifoldCFException, ServiceInterruption
{
+ // Extract startTime
+ long startTime;
+ if (lastSeedVersion == null)
+ startTime = 0L;
+ else
+ {
+ // Unpack seed time from seed version string
+ startTime = new Long(lastSeedVersion).longValue();
+ }
+
// First, build the query
int i = 0;
@@ -1069,7 +1099,7 @@ public class DCTM extends org.apache.man
else
startTime = 0L;
StringBuilder strDQLend = new StringBuilder(" where r_modify_date >= " +
buildDateString(startTime) +
- " and r_modify_date<=" + buildDateString(endTime) +
+ " and r_modify_date<=" + buildDateString(seedTime) +
" AND (i_is_deleted=TRUE Or (i_is_deleted=FALSE AND a_full_text=TRUE
AND r_content_size>0");
// append maxsize if set
@@ -1218,7 +1248,7 @@ public class DCTM extends org.apache.man
}
throw new ManifoldCFException(e.getMessage(),e);
}
-
+ return new Long(seedTime).toString();
}
/** Do a query and read back the name column */
Modified:
manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
(original)
+++
manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
Sun Aug 31 18:17:42 2014
@@ -42,6 +42,7 @@ import org.apache.manifoldcf.core.interf
import org.apache.manifoldcf.core.common.XThreadInputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.core.interfaces.Specification;
import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
import org.apache.manifoldcf.core.interfaces.IPasswordMapperActivity;
import org.apache.manifoldcf.core.interfaces.IPostParameters;
@@ -696,50 +697,38 @@ public class DropboxRepositoryConnector
Messages.outputResourceWithVelocity(out,locale,EDIT_SPEC_HEADER_FORWARD,paramMap);
}
- /**
- * Queue "seed" documents. Seed documents are the starting places for
- * crawling activity. Documents are seeded when this method calls
- * appropriate methods in the passed in ISeedingActivity object.
- *
- * This method can choose to find repository changes that happen only during
- * the specified time interval. The seeds recorded by this method will be
- * viewed by the framework based on what the getConnectorModel() method
- * returns.
- *
- * It is not a big problem if the connector chooses to create more seeds
- * than are strictly necessary; it is merely a question of overall work
- * required.
- *
- * The times passed to this method may be interpreted for greatest
- * efficiency. The time ranges any given job uses with this connector will
- * not overlap, but will proceed starting at 0 and going to the "current
- * time", each time the job is run. For continuous crawling jobs, this
- * method will be called once, when the job starts, and at various periodic
- * intervals as the job executes.
- *
- * When a job's specification is changed, the framework automatically resets
- * the seeding start time to 0. The seeding start time may also be set to 0
- * on each job run, depending on the connector model returned by
- * getConnectorModel().
- *
- * Note that it is always ok to send MORE documents rather than less to this
- * method.
- *
- * @param activities is the interface this method should use to perform
- * whatever framework actions are desired.
- * @param spec is a document specification (that comes from the job).
- * @param startTime is the beginning of the time range to consider,
- * inclusive.
- * @param endTime is the end of the time range to consider, exclusive.
- * @param jobMode is an integer describing how the job is being run, whether
- * continuous or once-only.
- */
+ /** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
+ * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only during
the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based on
what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds than
are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals as
the job executes.
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
+ *@param activities is the interface this method should use to perform
whatever framework actions are desired.
+ *@param spec is a document specification (that comes from the job).
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
+ */
@Override
- public void addSeedDocuments(ISeedingActivity activities,
- DocumentSpecification spec, long startTime, long endTime, int jobMode)
- throws ManifoldCFException, ServiceInterruption {
-
-
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption {
+
String dropboxPath = StringUtils.EMPTY;
int i = 0;
while (i < spec.getChildCount()) {
@@ -787,6 +776,7 @@ public class DropboxRepositoryConnector
Logging.connectors.warn("DROPBOX: Error adding seed documents: " +
e.getMessage(), e);
handleDropboxException(e);
}
+ return "";
}
protected class GetSeedsThread extends Thread {
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Sun Aug 31 18:17:42 2014
@@ -381,18 +381,6 @@ public class ElasticSearchConnector exte
return new ElasticSearchSpecs(getSpecNode(os));
}
- final private ElasticSearchSpecs getSpecsCache(String outputDescription)
- throws ManifoldCFException
- {
- try
- {
- return new ElasticSearchSpecs(new JSONObject(outputDescription));
- } catch (JSONException e)
- {
- throw new ManifoldCFException(e);
- }
- }
-
@Override
public VersionContext getPipelineDescription(Specification os)
throws ManifoldCFException
@@ -405,22 +393,13 @@ public class ElasticSearchConnector exte
public boolean checkLengthIndexable(VersionContext outputDescription, long
length, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
- ElasticSearchSpecs specs =
getSpecsCache(outputDescription.getVersionString());
+ ElasticSearchSpecs specs = new
ElasticSearchSpecs(getSpecNode(outputDescription.getSpecification()));
long maxFileSize = specs.getMaxFileSize();
if (length > maxFileSize)
return false;
return super.checkLengthIndexable(outputDescription, length, activities);
}
- @Override
- public boolean checkDocumentIndexable(VersionContext outputDescription, File
localFile, IOutputCheckActivity activities)
- throws ManifoldCFException, ServiceInterruption
- {
- // No filtering here; we don't look inside the file and don't know its
extension. That's done via the url
- // filter
- return true;
- }
-
/** Pre-determine whether a document's URL is indexable by this connector.
This method is used by participating repository connectors
* to help filter out documents that are not worth indexing.
*@param outputDescription is the document's output version.
@@ -431,7 +410,7 @@ public class ElasticSearchConnector exte
public boolean checkURLIndexable(VersionContext outputDescription, String
url, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
- ElasticSearchSpecs specs =
getSpecsCache(outputDescription.getVersionString());
+ ElasticSearchSpecs specs = new
ElasticSearchSpecs(getSpecNode(outputDescription.getSpecification()));
return specs.checkExtension(FilenameUtils.getExtension(url));
}
@@ -439,7 +418,7 @@ public class ElasticSearchConnector exte
public boolean checkMimeTypeIndexable(VersionContext outputDescription,
String mimeType, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
- ElasticSearchSpecs specs =
getSpecsCache(outputDescription.getVersionString());
+ ElasticSearchSpecs specs = new
ElasticSearchSpecs(getSpecNode(outputDescription.getSpecification()));
return specs.checkMimeType(mimeType);
}
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchSpecs.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchSpecs.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchSpecs.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchSpecs.java
Sun Aug 31 18:17:42 2014
@@ -50,29 +50,6 @@ public class ElasticSearchSpecs extends
private Set<String> mimeTypeSet;
- /** Build a set of ElasticSearch parameters by reading an JSON object
- *
- * @param json
- * @throws JSONException
- * @throws ManifoldCFException */
- public ElasticSearchSpecs(JSONObject json) throws JSONException,
- ManifoldCFException
- {
- super(SPECIFICATIONLIST);
- extensionSet = null;
- mimeTypeSet = null;
- for (ParameterEnum param : SPECIFICATIONLIST)
- {
- String value = null;
- value = json.getString(param.name());
- if (value == null)
- value = param.defaultValue;
- put(param, value);
- }
- extensionSet = createStringSet(getExtensions());
- mimeTypeSet = createStringSet(getMimeTypes());
- }
-
/** Build a set of ElasticSearch parameters by reading an instance of
* SpecificationNode.
*
Modified:
manifoldcf/trunk/connectors/email/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/email/EmailConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/email/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/email/EmailConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/email/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/email/EmailConnector.java
(original)
+++
manifoldcf/trunk/connectors/email/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/email/EmailConnector.java
Sun Aug 31 18:17:42 2014
@@ -236,39 +236,47 @@ public class EmailConnector extends org.
return 10;
}
- /**
- * Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
+ /** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
* are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
- * <p/>
+ *
* This method can choose to find repository changes that happen only during
the specified time interval.
* The seeds recorded by this method will be viewed by the framework based on
what the
* getConnectorModel() method returns.
- * <p/>
+ *
* It is not a big problem if the connector chooses to create more seeds than
are
* strictly necessary; it is merely a question of overall work required.
- * <p/>
- * The times passed to this method may be interpreted for greatest
efficiency. The time ranges
- * any given job uses with this connector will not overlap, but will proceed
starting at 0 and going
- * to the "current time", each time the job is run. For continuous crawling
jobs, this method will
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
* be called once, when the job starts, and at various periodic intervals as
the job executes.
- * <p/>
- * When a job's specification is changed, the framework automatically resets
the seeding start time to 0. The
- * seeding start time may also be set to 0 on each job run, depending on the
connector model returned by
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
* getConnectorModel().
- * <p/>
- * Note that it is always ok to send MORE documents rather than less to this
method.
*
- * @param activities is the interface this method should use to perform
whatever framework actions are desired.
- * @param spec is a document specification (that comes from the job).
- * @param startTime is the beginning of the time range to consider, inclusive.
- * @param endTime is the end of the time range to consider, exclusive.
- * @param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
+ *@param activities is the interface this method should use to perform
whatever framework actions are desired.
+ *@param spec is a document specification (that comes from the job).
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
*/
@Override
- public void addSeedDocuments(ISeedingActivity activities,
- DocumentSpecification spec, long startTime, long endTime, int jobMode)
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
throws ManifoldCFException, ServiceInterruption {
+ long startTime;
+ if (lastSeedVersion == null)
+ startTime = 0L;
+ else
+ {
+ // Unpack seed time from seed version string
+ startTime = new Long(lastSeedVersion).longValue();
+ }
+
getSession();
int i = 0;
@@ -296,7 +304,7 @@ public class EmailConnector extends org.
Folder folder = oft.finishUp();
try
{
- Message[] messages = findMessages(folder, startTime, endTime,
findMap);
+ Message[] messages = findMessages(folder, startTime, seedTime,
findMap);
for (Message message : messages) {
String emailID = ((MimeMessage) message).getMessageID();
activities.addSeedDocument(createDocumentIdentifier(folderName,emailID));
@@ -315,6 +323,7 @@ public class EmailConnector extends org.
}
}
+ return new Long(seedTime).toString();
}
/*
Modified:
manifoldcf/trunk/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java
(original)
+++
manifoldcf/trunk/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java
Sun Aug 31 18:17:42 2014
@@ -683,18 +683,47 @@ public class FilenetConnector extends or
/** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
* are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only during
the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based on
what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds than
are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals as
the job executes.
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
*@param activities is the interface this method should use to perform
whatever framework actions are desired.
*@param spec is a document specification (that comes from the job).
- *@param startTime is the beginning of the time range to consider, inclusive.
- *@param endTime is the end of the time range to consider, exclusive.
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
*/
@Override
- public void addSeedDocuments(ISeedingActivity activities,
DocumentSpecification spec,
- long startTime, long endTime)
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
throws ManifoldCFException, ServiceInterruption
{
Logging.connectors.debug("FileNet: Inside addSeedDocuments");
+ long startTime;
+ if (lastSeedVersion == null)
+ startTime = 0L;
+ else
+ {
+ // Unpack seed time from seed version string
+ startTime = new Long(lastSeedVersion).longValue();
+ }
+
// Go through all the document classes and do a query for each one
//get all mimetypes and build a SQL "where condition"
@@ -746,7 +775,7 @@ public class FilenetConnector extends or
sqlBuffer.append(" AND [DateLastModified] >=
").append(buildTime(c,startTime));
}
- sqlBuffer.append(" AND [DateLastModified] <=
").append(buildTime(c,endTime));
+ sqlBuffer.append(" AND [DateLastModified] <=
").append(buildTime(c,seedTime));
// Folders are also based just on objectstore, so we add those here.
boolean seenAny = false;
@@ -826,6 +855,7 @@ public class FilenetConnector extends or
i++;
}
+ return new Long(seedTime).toString();
}
protected static String quoteSQLString(String value)
Modified:
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
(original)
+++
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
Sun Aug 31 18:17:42 2014
@@ -171,27 +171,26 @@ public class FileOutputConnector extends
FileOutputConfig config = getConfigParameters(null);
- FileOutputSpecs specs = null;
+ FileOutputSpecs specs = new
FileOutputSpecs(getSpecNode(outputDescription.getSpecification()));;
StringBuffer path = new StringBuffer();
- try {
- specs = new FileOutputSpecs(outputDescription.getVersionString());
+ try {
/*
- * make file path
- */
+ * make file path
+ */
if (specs.getRootPath() != null) {
path.append(specs.getRootPath());
}
-
+
// If the path does not yet exist at the root level, it is dangerous to
create it.
File currentPath = new File(path.toString());
if (!currentPath.exists())
throw new ManifoldCFException("Root path does not yet exist:
'"+currentPath+"'");
if (!currentPath.isDirectory())
throw new ManifoldCFException("Root path is not a directory:
'"+currentPath+"'");
-
+
String filePath = documentURItoFilePath(documentURI);
-
+
// Build path one level at a time. This is needed because there may be
a collision at
// every level.
int index = 0;
@@ -226,7 +225,7 @@ public class FileOutputConnector extends
currentPath = newPath;
// Go on to the next one.
}
-
+
// Path successfully created. Now create file.
FileOutputStream output = null;
String fileName = filePath.substring(index);
@@ -256,8 +255,8 @@ public class FileOutputConnector extends
try {
/*
- * lock file
- */
+ * lock file
+ */
FileChannel channel = output.getChannel();
FileLock lock = channel.tryLock();
if (lock == null)
@@ -266,8 +265,8 @@ public class FileOutputConnector extends
try {
/*
- * write file
- */
+ * write file
+ */
InputStream input = document.getBinaryStream();
byte buf[] = new byte[65536];
int len;
@@ -290,9 +289,6 @@ public class FileOutputConnector extends
} catch (IOException e) {
}
}
- } catch (JSONException e) {
- handleJSONException(e);
- return DOCUMENTSTATUS_REJECTED;
} catch (URISyntaxException e) {
handleURISyntaxException(e);
return DOCUMENTSTATUS_REJECTED;
@@ -361,10 +357,8 @@ public class FileOutputConnector extends
FileOutputConfig config = getConfigParameters(null);
- FileOutputSpecs specs = null;
StringBuffer path = new StringBuffer();
try {
- specs = new FileOutputSpecs(outputDescription);
// We cannot remove documents, because it is unsafe to do so.
// Paths that were created when the document existed will not
@@ -437,8 +431,6 @@ public class FileOutputConnector extends
}
// Just close it, to make a zero-length grave marker.
output.close();
- } catch (JSONException e) {
- handleJSONException(e);
} catch (URISyntaxException e) {
handleURISyntaxException(e);
} catch (FileNotFoundException e) {
Modified:
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputSpecs.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputSpecs.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputSpecs.java
(original)
+++
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputSpecs.java
Sun Aug 31 18:17:42 2014
@@ -44,36 +44,6 @@ public class FileOutputSpecs extends Fil
private String rootPath;
- /** Build a set of ElasticSearch parameters by reading an JSON object
- *
- * @param json
- * @throws JSONException
- * @throws ManifoldCFException
- */
- public FileOutputSpecs(String json) throws JSONException,
ManifoldCFException {
- this(new JSONObject(json));
- }
-
- /** Build a set of ElasticSearch parameters by reading an JSON object
- *
- * @param json
- * @throws JSONException
- * @throws ManifoldCFException
- */
- public FileOutputSpecs(JSONObject json) throws JSONException,
ManifoldCFException {
- super(SPECIFICATIONLIST);
- rootPath = null;
- for (ParameterEnum param : SPECIFICATIONLIST) {
- String value = null;
- value = json.getString(param.name());
- if (value == null) {
- value = param.defaultValue;
- }
- put(param, value);
- }
- rootPath = getRootPath();
- }
-
/** Build a set of ElasticSearch parameters by reading an instance of
* SpecificationNode.
*
Modified:
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
(original)
+++
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
Sun Aug 31 18:17:42 2014
@@ -166,26 +166,64 @@ public class FileConnector extends org.a
}
- /** Given a document specification, get either a list of starting document
identifiers (seeds),
- * or a list of changes (deltas), depending on whether this is a "crawled"
connector or not.
- * These document identifiers will be loaded into the job's queue at the
beginning of the
- * job's execution.
- * This method can return changes only (because it is provided a time range).
For full
- * recrawls, the start time is always zero.
- * Note that it is always ok to return MORE documents rather than less with
this method.
+ /** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
+ * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only during
the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based on
what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds than
are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals as
the job executes.
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
+ *@param activities is the interface this method should use to perform
whatever framework actions are desired.
*@param spec is a document specification (that comes from the job).
- *@param startTime is the beginning of the time range to consider, inclusive.
- *@param endTime is the end of the time range to consider, exclusive.
- *@return the stream of local document identifiers that should be added to
the queue.
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
*/
@Override
- public IDocumentIdentifierStream
getDocumentIdentifiers(DocumentSpecification spec, long startTime, long endTime)
- throws ManifoldCFException
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption
{
- return new IdentifierStream(spec);
+ try
+ {
+ // Walk the specification for the "startpoint" types. Amalgamate these
into a list of strings.
+ // Presume that all roots are startpoint nodes
+ for (int i = 0; i < spec.getChildCount(); i++)
+ {
+ SpecificationNode n = spec.getChild(i);
+ if (n.getType().equals("startpoint"))
+ {
+ // The id returned MUST be in canonical form!!!
+ String seed = new
File(n.getAttributeValue("path")).getCanonicalPath();
+ if (Logging.connectors.isDebugEnabled())
+ {
+ Logging.connectors.debug("Seed = '"+seed+"'");
+ }
+ activities.addSeedDocument(seed);
+ }
+ }
+ }
+ catch (IOException e)
+ {
+ throw new ManifoldCFException("Could not get a canonical path:
"+e.getMessage(),e);
+ }
+ return "";
}
-
-
+
/** Get document versions given an array of document identifiers.
* This method is called for EVERY document that is considered. It is
therefore important to perform
* as little work as possible here.
@@ -1309,73 +1347,4 @@ public class FileConnector extends org.a
}
}
- /** Document identifier stream.
- */
- protected static class IdentifierStream implements IDocumentIdentifierStream
- {
- protected String[] ids = null;
- protected int currentIndex = 0;
-
- public IdentifierStream(DocumentSpecification spec)
- throws ManifoldCFException
- {
- try
- {
- // Walk the specification for the "startpoint" types. Amalgamate
these into a list of strings.
- // Presume that all roots are startpoint nodes
- int i = 0;
- int j = 0;
- while (i < spec.getChildCount())
- {
- SpecificationNode n = spec.getChild(i);
- if (n.getType().equals("startpoint"))
- j++;
- i++;
- }
- ids = new String[j];
- i = 0;
- j = 0;
- while (i < ids.length)
- {
- SpecificationNode n = spec.getChild(i);
- if (n.getType().equals("startpoint"))
- {
- // The id returned MUST be in canonical form!!!
- ids[j] = new File(n.getAttributeValue("path")).getCanonicalPath();
- if (Logging.connectors.isDebugEnabled())
- {
- Logging.connectors.debug("Seed = '"+ids[j]+"'");
- }
- j++;
- }
- i++;
- }
- }
- catch (IOException e)
- {
- throw new ManifoldCFException("Could not get a canonical path",e);
- }
- }
-
- /** Get the next identifier.
- *@return the next document identifier, or null if there are no more.
- */
- public String getNextIdentifier()
- throws ManifoldCFException, ServiceInterruption
- {
- if (currentIndex == ids.length)
- return null;
- return ids[currentIndex++];
- }
-
- /** Close the stream.
- */
- public void close()
- throws ManifoldCFException
- {
- ids = null;
- }
-
- }
-
}
Modified:
manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java
(original)
+++
manifoldcf/trunk/connectors/generic/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/generic/GenericConnector.java
Sun Aug 31 18:17:42 2014
@@ -189,10 +189,19 @@ public class GenericConnector extends Ba
}
@Override
- public void addSeedDocuments(ISeedingActivity activities,
DocumentSpecification spec,
- long startTime, long endTime)
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
throws ManifoldCFException, ServiceInterruption {
+ long startTime;
+ if (lastSeedVersion == null)
+ startTime = 0L;
+ else
+ {
+ // Unpack seed time from seed version string
+ startTime = new Long(lastSeedVersion).longValue();
+ }
+
HttpClient client = getClient();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
@@ -201,7 +210,7 @@ public class GenericConnector extends Ba
if (startTime > 0) {
url.append("&startTime=").append(sdf.format(new Date(startTime)));
}
- url.append("&endTime=").append(sdf.format(new Date(endTime)));
+ url.append("&endTime=").append(sdf.format(new Date(seedTime)));
for (int i = 0; i < spec.getChildCount(); i++) {
SpecificationNode sn = spec.getChild(i);
if (sn.getType().equals("param")) {
@@ -245,6 +254,7 @@ public class GenericConnector extends Ba
throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
ManifoldCFException.INTERRUPTED);
}
+ return new Long(seedTime).toString();
}
@Override
Modified:
manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
(original)
+++
manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
Sun Aug 31 18:17:42 2014
@@ -38,6 +38,7 @@ import org.apache.manifoldcf.crawler.sys
import org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.Specification;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.commons.lang.StringUtils;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
@@ -747,48 +748,37 @@ public class GoogleDriveRepositoryConnec
Messages.outputResourceWithVelocity(out,locale,EDIT_SPEC_HEADER_FORWARD,paramMap);
}
- /**
- * Queue "seed" documents. Seed documents are the starting places for
- * crawling activity. Documents are seeded when this method calls
- * appropriate methods in the passed in ISeedingActivity object.
- *
- * This method can choose to find repository changes that happen only during
- * the specified time interval. The seeds recorded by this method will be
- * viewed by the framework based on what the getConnectorModel() method
- * returns.
- *
- * It is not a big problem if the connector chooses to create more seeds
- * than are strictly necessary; it is merely a question of overall work
- * required.
- *
- * The times passed to this method may be interpreted for greatest
- * efficiency. The time ranges any given job uses with this connector will
- * not overlap, but will proceed starting at 0 and going to the "current
- * time", each time the job is run. For continuous crawling jobs, this
- * method will be called once, when the job starts, and at various periodic
- * intervals as the job executes.
- *
- * When a job's specification is changed, the framework automatically resets
- * the seeding start time to 0. The seeding start time may also be set to 0
- * on each job run, depending on the connector model returned by
- * getConnectorModel().
- *
- * Note that it is always ok to send MORE documents rather than less to this
- * method.
- *
- * @param activities is the interface this method should use to perform
- * whatever framework actions are desired.
- * @param spec is a document specification (that comes from the job).
- * @param startTime is the beginning of the time range to consider,
- * inclusive.
- * @param endTime is the end of the time range to consider, exclusive.
- * @param jobMode is an integer describing how the job is being run, whether
- * continuous or once-only.
- */
+ /** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
+ * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only during
the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based on
what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds than
are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals as
the job executes.
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
+ *@param activities is the interface this method should use to perform
whatever framework actions are desired.
+ *@param spec is a document specification (that comes from the job).
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
+ */
@Override
- public void addSeedDocuments(ISeedingActivity activities,
- DocumentSpecification spec, long startTime, long endTime, int jobMode)
- throws ManifoldCFException, ServiceInterruption {
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption {
String googleDriveQuery = GoogleDriveConfig.GOOGLEDRIVE_QUERY_DEFAULT;
int i = 0;
@@ -843,6 +833,7 @@ public class GoogleDriveRepositoryConnec
Logging.connectors.warn("GOOGLEDRIVE: Error adding seed documents: " +
e.getMessage(), e);
handleIOException(e);
}
+ return "";
}
protected class GetSeedsThread extends Thread {
Modified:
manifoldcf/trunk/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
(original)
+++
manifoldcf/trunk/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
Sun Aug 31 18:17:42 2014
@@ -39,6 +39,7 @@ import org.apache.commons.lang.StringUti
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.Specification;
import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
import org.apache.manifoldcf.core.interfaces.IPasswordMapperActivity;
import org.apache.manifoldcf.core.interfaces.IPostParameters;
@@ -323,47 +324,37 @@ public class GridFSRepositoryConnector e
return super.getRelationshipTypes();
}
- /**
- * Queue "seed" documents. Seed documents are the starting places for
- * crawling activity. Documents are seeded when this method calls
- * appropriate methods in the passed in ISeedingActivity object.
- *
- * This method can choose to find repository changes that happen only
during
- * the specified time interval. The seeds recorded by this method will be
- * viewed by the framework based on what the getConnectorModel() method
- * returns.
- *
- * It is not a big problem if the connector chooses to create more seeds
- * than are strictly necessary; it is merely a question of overall work
- * required.
- *
- * The times passed to this method may be interpreted for greatest
- * efficiency. The time ranges any given job uses with this connector will
- * not overlap, but will proceed starting at 0 and going to the "current
- * time", each time the job is run. For continuous crawling jobs, this
- * method will be called once, when the job starts, and at various periodic
- * intervals as the job executes.
- *
- * When a job's specification is changed, the framework automatically
resets
- * the seeding start time to 0. The seeding start time may also be set to 0
- * on each job run, depending on the connector model returned by
- * getConnectorModel().
- *
- * Note that it is always ok to send MORE documents rather than less to
this
- * method. The connector will be connected before this method can be
called.
- *
- * @param activities is the interface this method should use to perform
- * whatever framework actions are desired.
- * @param spec is a document specification (that comes from the job).
- * @param startTime is the beginning of the time range to consider,
- * inclusive.
- * @param endTime is the end of the time range to consider, exclusive.
- * @throws org.apache.manifoldcf.core.interfaces.ManifoldCFException
- * @throws org.apache.manifoldcf.agents.interfaces.ServiceInterruption
- */
- @Override
- public void addSeedDocuments(ISeedingActivity activities,
DocumentSpecification spec,
- long startTime, long endTime) throws ManifoldCFException,
ServiceInterruption {
+ /** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
+ * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only
during the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based
on what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds
than are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals
as the job executes.
+ *
+ * When a job's specification is changed, the framework automatically
resets the seeding version string to null. The
+ * seeding version string may also be set to null on each job run,
depending on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to
this method.
+ * The connector will be connected before this method can be called.
+ *@param activities is the interface this method should use to perform
whatever framework actions are desired.
+ *@param spec is a document specification (that comes from the job).
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
+ */
+ @Override
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption {
getSession();
DBCollection fsFiles = session.getCollection(
bucket + GridFSConstants.COLLECTION_SEPERATOR +
GridFSConstants.FILES_COLLECTION_NAME
@@ -377,6 +368,7 @@ public class GridFSRepositoryConnector e
Logging.connectors.debug("GridFS: Document _id = " + _id + "
added to queue");
}
}
+ return "";
}
/**
Modified:
manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java
(original)
+++
manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java
Sun Aug 31 18:17:42 2014
@@ -239,17 +239,11 @@ public class GTSConnector extends org.ap
public VersionContext getPipelineDescription(Specification spec)
throws ManifoldCFException, ServiceInterruption
{
- // The information we want in this string is:
- // (1) the collection name(s), in sorted order.
- // (2) the document template
- // (3) the ingest URI
-
- ArrayList collectionList = new ArrayList();
+ List<String> collectionList = new ArrayList<String>();
String documentTemplate = "";
- int i = 0;
- while (i < spec.getChildCount())
+ for (int i = 0; i < spec.getChildCount(); i++)
{
- SpecificationNode sn = spec.getChild(i++);
+ SpecificationNode sn = spec.getChild(i);
if (sn.getType().equals(GTSConfig.NODE_COLLECTION))
{
collectionList.add(sn.getAttributeValue(GTSConfig.ATTRIBUTE_VALUE));
@@ -260,20 +254,24 @@ public class GTSConnector extends org.ap
}
}
- String[] sortArray = new String[collectionList.size()];
- i = 0;
- while (i < sortArray.length)
- {
- sortArray[i] = (String)collectionList.get(i);
- i++;
- }
- java.util.Arrays.sort(sortArray);
-
// Get the config info too. This will be constant for any given connector
instance, so we don't have to worry about it changing
// out from under us.
String ingestURI = params.getParameter(GTSConfig.PARAM_INGESTURI);
// Now, construct the appropriate string
+ // The information we want in this string is:
+ // (1) the collection name(s), in sorted order.
+ // (2) the document template
+ // (3) the ingest URI
+
+ String[] sortArray = new String[collectionList.size()];
+ int j = 0;
+ for (String collection : collectionList)
+ {
+ sortArray[j++] = collection;
+ }
+ java.util.Arrays.sort(sortArray);
+
StringBuilder sb = new StringBuilder();
packList(sb,sortArray,'+');
pack(sb,documentTemplate,'+');
@@ -300,27 +298,28 @@ public class GTSConnector extends org.ap
public int addOrReplaceDocumentWithException(String documentURI,
VersionContext pipelineDescription, RepositoryDocument document, String
authorityNameString, IOutputAddActivity activities)
throws ManifoldCFException, ServiceInterruption, IOException
{
- String outputDescription = pipelineDescription.getVersionString();
-
- // Establish a session
- getSession();
-
- // Unpack what we need from the output description. This consists of the
collection names, plus the document template.
- ArrayList collections = new ArrayList();
- StringBuilder documentTemplateBuffer = new StringBuilder();
- int startPosition = unpackList(collections,outputDescription,0,'+');
- startPosition =
unpack(documentTemplateBuffer,outputDescription,startPosition,'+');
-
- String[] collectionArray = new String[collections.size()];
- int i = 0;
- while (i < collectionArray.length)
+ // Grab the information we need to index
+ Specification spec = pipelineDescription.getSpecification();
+ List<String> collectionList = new ArrayList<String>();
+ String documentTemplate = "";
+ for (int i = 0; i < spec.getChildCount(); i++)
{
- collectionArray[i] = (String)collections.get(i);
- i++;
+ SpecificationNode sn = spec.getChild(i);
+ if (sn.getType().equals(GTSConfig.NODE_COLLECTION))
+ {
+ collectionList.add(sn.getAttributeValue(GTSConfig.ATTRIBUTE_VALUE));
+ }
+ else if (sn.getType().equals(GTSConfig.NODE_DOCUMENTTEMPLATE))
+ {
+ documentTemplate = sn.getAttributeValue(GTSConfig.ATTRIBUTE_VALUE);
+ }
}
+ // Establish a session
+ getSession();
+
// Now, go off and call the ingest API.
- if
(poster.indexPost(documentURI,collectionArray,documentTemplateBuffer.toString(),authorityNameString,document,activities))
+ if
(poster.indexPost(documentURI,collectionList,documentTemplate,authorityNameString,document,activities))
return DOCUMENTSTATUS_ACCEPTED;
return DOCUMENTSTATUS_REJECTED;
}
Modified:
manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java
(original)
+++
manifoldcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java
Sun Aug 31 18:17:42 2014
@@ -146,7 +146,7 @@ public class HttpPoster
* @throws ManifoldCFException, ServiceInterruption
*/
public boolean indexPost(String documentURI,
- String[] collections, String documentTemplate, String authorityNameString,
+ List<String> collections, String documentTemplate, String
authorityNameString,
RepositoryDocument document, IOutputAddActivity activities)
throws ManifoldCFException, ServiceInterruption
{
@@ -671,7 +671,7 @@ public class HttpPoster
{
protected String documentURI;
protected String aclXmlString;
- protected String[] collections;
+ protected List<String> collections;
protected String documentTemplate;
protected RepositoryDocument document;
@@ -683,7 +683,7 @@ public class HttpPoster
protected boolean readFromDocumentStreamYet = false;
protected boolean rval = false;
- public IngestThread(String documentURI, String aclXmlString, String[]
collections, String documentTemplate, RepositoryDocument document)
+ public IngestThread(String documentURI, String aclXmlString, List<String>
collections, String documentTemplate, RepositoryDocument document)
{
super();
setDaemon(true);
@@ -761,10 +761,8 @@ public class HttpPoster
// Do the collections
if (collections != null)
{
- int index = 0;
- while (index < collections.length)
+ for (String collectionName : collections)
{
- String collectionName = collections[index++];
String encodedValue = metadataEncode(collectionName);
//System.out.println("collection metadata:
collection_name = '"+encodedValue+"'");
tmp = ("Document-Metadata:
collection_name="+encodedValue+"\r\n").getBytes(StandardCharsets.UTF_8);
Modified:
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
(original)
+++
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
Sun Aug 31 18:17:42 2014
@@ -271,9 +271,8 @@ public class HDFSOutputConnector extends
@Override
public int addOrReplaceDocumentWithException(String documentURI,
VersionContext pipelineDescription, RepositoryDocument document, String
authorityNameString, IOutputAddActivity activities)
throws ManifoldCFException, ServiceInterruption, IOException {
- String outputDescription = pipelineDescription.getVersionString();
+ HDFSOutputSpecs specs = new
HDFSOutputSpecs(getSpecNode(pipelineDescription.getSpecification()));
try {
- HDFSOutputSpecs specs = new HDFSOutputSpecs(outputDescription);
/*
* make file path
@@ -290,9 +289,6 @@ public class HDFSOutputConnector extends
createFile(path, document.getBinaryStream());
activities.recordActivity(startTime, INGEST_ACTIVITY, new
Long(document.getBinaryLength()), documentURI, "OK", null);
return DOCUMENTSTATUS_ACCEPTED;
- } catch (JSONException e) {
- handleJSONException(e);
- return DOCUMENTSTATUS_REJECTED;
} catch (URISyntaxException e) {
handleURISyntaxException(e);
return DOCUMENTSTATUS_REJECTED;
Modified:
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1621613&r1=1621612&r2=1621613&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
(original)
+++
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
Sun Aug 31 18:17:42 2014
@@ -262,48 +262,37 @@ public class HDFSRepositoryConnector ext
}
}
- /**
- * Queue "seed" documents. Seed documents are the starting places for
- * crawling activity. Documents are seeded when this method calls
- * appropriate methods in the passed in ISeedingActivity object.
- *
- * This method can choose to find repository changes that happen only during
- * the specified time interval. The seeds recorded by this method will be
- * viewed by the framework based on what the getConnectorModel() method
- * returns.
- *
- * It is not a big problem if the connector chooses to create more seeds
- * than are strictly necessary; it is merely a question of overall work
- * required.
- *
- * The times passed to this method may be interpreted for greatest
- * efficiency. The time ranges any given job uses with this connector will
- * not overlap, but will proceed starting at 0 and going to the "current
- * time", each time the job is run. For continuous crawling jobs, this
- * method will be called once, when the job starts, and at various periodic
- * intervals as the job executes.
- *
- * When a job's specification is changed, the framework automatically resets
- * the seeding start time to 0. The seeding start time may also be set to 0
- * on each job run, depending on the connector model returned by
- * getConnectorModel().
- *
- * Note that it is always ok to send MORE documents rather than less to this
- * method.
- *
- * @param activities is the interface this method should use to perform
- * whatever framework actions are desired.
- * @param spec is a document specification (that comes from the job).
- * @param startTime is the beginning of the time range to consider,
- * inclusive.
- * @param endTime is the end of the time range to consider, exclusive.
- * @param jobMode is an integer describing how the job is being run, whether
- * continuous or once-only.
- */
+ /** Queue "seed" documents. Seed documents are the starting places for
crawling activity. Documents
+ * are seeded when this method calls appropriate methods in the passed in
ISeedingActivity object.
+ *
+ * This method can choose to find repository changes that happen only during
the specified time interval.
+ * The seeds recorded by this method will be viewed by the framework based on
what the
+ * getConnectorModel() method returns.
+ *
+ * It is not a big problem if the connector chooses to create more seeds than
are
+ * strictly necessary; it is merely a question of overall work required.
+ *
+ * The end time and seeding version string passed to this method may be
interpreted for greatest efficiency.
+ * For continuous crawling jobs, this method will
+ * be called once, when the job starts, and at various periodic intervals as
the job executes.
+ *
+ * When a job's specification is changed, the framework automatically resets
the seeding version string to null. The
+ * seeding version string may also be set to null on each job run, depending
on the connector model returned by
+ * getConnectorModel().
+ *
+ * Note that it is always ok to send MORE documents rather than less to this
method.
+ * The connector will be connected before this method can be called.
+ *@param activities is the interface this method should use to perform
whatever framework actions are desired.
+ *@param spec is a document specification (that comes from the job).
+ *@param seedTime is the end of the time range of documents to consider,
exclusive.
+ *@param lastSeedVersionString is the last seeding version string for this
job, or null if the job has no previous seeding version string.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@return an updated seeding version string, to be stored with the job.
+ */
@Override
- public void addSeedDocuments(ISeedingActivity activities,
- DocumentSpecification spec, long startTime, long endTime, int jobMode)
- throws ManifoldCFException, ServiceInterruption {
+ public String addSeedDocuments(ISeedingActivity activities, Specification
spec,
+ String lastSeedVersion, long seedTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption {
String path = StringUtils.EMPTY;
int i = 0;
@@ -319,6 +308,7 @@ public class HDFSRepositoryConnector ext
}
i++;
}
+ return "";
}
/** Get document versions given an array of document identifiers.
