Author: kwright
Date: Wed Sep 10 09:26:55 2014
New Revision: 1623956
URL: http://svn.apache.org/r1623956
Log:
Rearrange test connectors to be use only primary IRepositoryConnector methods.
Part of CONNECTORS-977.
Modified:
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/TestingRepositoryConnector.java
Modified:
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java?rev=1623956&r1=1623955&r2=1623956&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
Wed Sep 10 09:26:55 2014
@@ -55,48 +55,44 @@ public class InterruptionRepositoryConne
return "";
}
+ /** Process a set of documents.
+ * This is the method that should cause each document to be fetched,
processed, and the results either added
+ * to the queue of documents for the current job, and/or entered into the
incremental ingestion manager.
+ * The document specification allows this class to filter what is done based
on the job.
+ * The connector will be connected before this method can be called.
+ *@param documentIdentifiers is the set of document identifiers to process.
+ *@param statuses are the currently-stored document versions for each
document in the set of document identifiers
+ * passed in above.
+ *@param activities is the interface this method should use to queue up new
document references
+ * and ingest documents.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@param usesDefaultAuthority will be true only if the authority in use for
these documents is the default one.
+ */
@Override
- public String[] getDocumentVersions(String[] documentIdentifiers, String[]
oldVersions, IVersionActivity activities,
- DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
- throws ManifoldCFException, ServiceInterruption
- {
- String[] rval = new String[documentIdentifiers.length];
- for (int i = 0; i < rval.length; i++)
- {
- rval[i] = "";
- }
- return rval;
- }
-
- @Override
- public void processDocuments(String[] documentIdentifiers, String[]
versions, IProcessActivity activities,
- DocumentSpecification spec, boolean[] scanOnly, int jobMode)
+ public void processDocuments(String[] documentIdentifiers, IExistingVersions
statuses, Specification spec,
+ IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
throws ManifoldCFException, ServiceInterruption
{
for (int i = 0; i < documentIdentifiers.length; i++)
{
String documentIdentifier = documentIdentifiers[i];
- String version = versions[i];
- if (!scanOnly[i])
+ if (documentIdentifier.equals("test0.txt"))
+ {
+ // This will emulate one particular document failing (and being
skipped)
+ long currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("Pretending there's a service
interruption",
+ null,currentTime+1000L,currentTime+5000L,10,false);
+ }
+ RepositoryDocument rd = new RepositoryDocument();
+ byte[] bytes = documentIdentifier.getBytes(StandardCharsets.UTF_8);
+ rd.setBinary(new ByteArrayInputStream(bytes),bytes.length);
+ try
+ {
+
activities.ingestDocumentWithException(documentIdentifier,"","http://"+documentIdentifier,rd);
+ }
+ catch (IOException e)
{
- if (documentIdentifier.equals("test0.txt"))
- {
- // This will emulate one particular document failing (and being
skipped)
- long currentTime = System.currentTimeMillis();
- throw new ServiceInterruption("Pretending there's a service
interruption",
- null,currentTime+1000L,currentTime+5000L,10,false);
- }
- RepositoryDocument rd = new RepositoryDocument();
- byte[] bytes = documentIdentifier.getBytes(StandardCharsets.UTF_8);
- rd.setBinary(new ByteArrayInputStream(bytes),bytes.length);
- try
- {
-
activities.ingestDocumentWithException(documentIdentifier,version,"http://"+documentIdentifier,rd);
- }
- catch (IOException e)
- {
- throw new RuntimeException("Shouldn't be seeing IOException from
binary array input stream: "+e.getMessage(),e);
- }
+ throw new RuntimeException("Shouldn't be seeing IOException from
binary array input stream: "+e.getMessage(),e);
}
}
}
Modified:
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java?rev=1623956&r1=1623955&r2=1623956&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java
Wed Sep 10 09:26:55 2014
@@ -61,22 +61,22 @@ public class SchedulingRepositoryConnect
return "";
}
+ /** Process a set of documents.
+ * This is the method that should cause each document to be fetched,
processed, and the results either added
+ * to the queue of documents for the current job, and/or entered into the
incremental ingestion manager.
+ * The document specification allows this class to filter what is done based
on the job.
+ * The connector will be connected before this method can be called.
+ *@param documentIdentifiers is the set of document identifiers to process.
+ *@param statuses are the currently-stored document versions for each
document in the set of document identifiers
+ * passed in above.
+ *@param activities is the interface this method should use to queue up new
document references
+ * and ingest documents.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@param usesDefaultAuthority will be true only if the authority in use for
these documents is the default one.
+ */
@Override
- public String[] getDocumentVersions(String[] documentIdentifiers, String[]
oldVersions, IVersionActivity activities,
- DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
- throws ManifoldCFException, ServiceInterruption
- {
- String[] rval = new String[documentIdentifiers.length];
- for (int i = 0; i < rval.length; i++)
- {
- rval[i] = "";
- }
- return rval;
- }
-
- @Override
- public void processDocuments(String[] documentIdentifiers, String[]
versions, IProcessActivity activities,
- DocumentSpecification spec, boolean[] scanOnly, int jobMode)
+ public void processDocuments(String[] documentIdentifiers, IExistingVersions
statuses, Specification spec,
+ IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
throws ManifoldCFException, ServiceInterruption
{
String documentsPerSeedString = params.getParameter("documentsperseed");
@@ -105,37 +105,34 @@ public class SchedulingRepositoryConnect
}
else
{
- if (!scanOnly[i])
+ System.out.println("Fetching "+documentIdentifier);
+ // Find the bin
+ String bin =
documentIdentifier.substring(0,documentIdentifier.indexOf("/"));
+ // For now they are all the same
+ long binTimePerDocument = timePerDocument;
+ long now = System.currentTimeMillis();
+ long whenFetch;
+ synchronized (nextFetchTime)
+ {
+ Long time = nextFetchTime.get(bin);
+ if (time == null)
+ whenFetch = now;
+ else
+ whenFetch = time.longValue();
+ nextFetchTime.put(bin,new Long(whenFetch + binTimePerDocument));
+ }
+ if (whenFetch > now)
{
- System.out.println("Fetching "+documentIdentifier);
- // Find the bin
- String bin =
documentIdentifier.substring(0,documentIdentifier.indexOf("/"));
- // For now they are all the same
- long binTimePerDocument = timePerDocument;
- long now = System.currentTimeMillis();
- long whenFetch;
- synchronized (nextFetchTime)
+ System.out.println("Waiting "+(whenFetch-now)+" to fetch
"+documentIdentifier);
+ try
{
- Long time = nextFetchTime.get(bin);
- if (time == null)
- whenFetch = now;
- else
- whenFetch = time.longValue();
- nextFetchTime.put(bin,new Long(whenFetch + binTimePerDocument));
+ ManifoldCF.sleep(whenFetch-now);
}
- if (whenFetch > now)
+ catch (InterruptedException e)
{
- System.out.println("Waiting "+(whenFetch-now)+" to fetch
"+documentIdentifier);
- try
- {
- ManifoldCF.sleep(whenFetch-now);
- }
- catch (InterruptedException e)
- {
- throw new
ManifoldCFException(e.getMessage(),ManifoldCFException.INTERRUPTED);
- }
- System.out.println("Wait complete for "+documentIdentifier);
+ throw new
ManifoldCFException(e.getMessage(),ManifoldCFException.INTERRUPTED);
}
+ System.out.println("Wait complete for "+documentIdentifier);
}
}
}
Modified:
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/TestingRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/TestingRepositoryConnector.java?rev=1623956&r1=1623955&r2=1623956&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/TestingRepositoryConnector.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/TestingRepositoryConnector.java
Wed Sep 10 09:26:55 2014
@@ -55,41 +55,37 @@ public class TestingRepositoryConnector
return "";
}
+ /** Process a set of documents.
+ * This is the method that should cause each document to be fetched,
processed, and the results either added
+ * to the queue of documents for the current job, and/or entered into the
incremental ingestion manager.
+ * The document specification allows this class to filter what is done based
on the job.
+ * The connector will be connected before this method can be called.
+ *@param documentIdentifiers is the set of document identifiers to process.
+ *@param statuses are the currently-stored document versions for each
document in the set of document identifiers
+ * passed in above.
+ *@param activities is the interface this method should use to queue up new
document references
+ * and ingest documents.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@param usesDefaultAuthority will be true only if the authority in use for
these documents is the default one.
+ */
@Override
- public String[] getDocumentVersions(String[] documentIdentifiers, String[]
oldVersions, IVersionActivity activities,
- DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
- throws ManifoldCFException, ServiceInterruption
- {
- String[] rval = new String[documentIdentifiers.length];
- for (int i = 0; i < rval.length; i++)
- {
- rval[i] = "";
- }
- return rval;
- }
-
- @Override
- public void processDocuments(String[] documentIdentifiers, String[]
versions, IProcessActivity activities,
- DocumentSpecification spec, boolean[] scanOnly, int jobMode)
+ public void processDocuments(String[] documentIdentifiers, IExistingVersions
statuses, Specification spec,
+ IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
throws ManifoldCFException, ServiceInterruption
{
for (int i = 0; i < documentIdentifiers.length; i++)
{
String documentIdentifier = documentIdentifiers[i];
- String version = versions[i];
- if (!scanOnly[i])
+ RepositoryDocument rd = new RepositoryDocument();
+ byte[] bytes = documentIdentifier.getBytes(StandardCharsets.UTF_8);
+ rd.setBinary(new ByteArrayInputStream(bytes),bytes.length);
+ try
+ {
+
activities.ingestDocumentWithException(documentIdentifier,"","http://"+documentIdentifier,rd);
+ }
+ catch (IOException e)
{
- RepositoryDocument rd = new RepositoryDocument();
- byte[] bytes = documentIdentifier.getBytes(StandardCharsets.UTF_8);
- rd.setBinary(new ByteArrayInputStream(bytes),bytes.length);
- try
- {
-
activities.ingestDocumentWithException(documentIdentifier,version,"http://"+documentIdentifier,rd);
- }
- catch (IOException e)
- {
- throw new RuntimeException("Shouldn't be seeing IOException from
binary array input stream: "+e.getMessage(),e);
- }
+ throw new RuntimeException("Shouldn't be seeing IOException from
binary array input stream: "+e.getMessage(),e);
}
}
}
