Author: kwright
Date: Sun Sep 14 21:21:45 2014
New Revision: 1624906
URL: http://svn.apache.org/r1624906
Log:
Upgrade RSS connector. Part of CONNECTORS-977
Modified:
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
Modified:
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java?rev=1624906&r1=1624905&r2=1624906&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
(original)
+++
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
Sun Sep 14 21:21:45 2014
@@ -702,27 +702,22 @@ public class RSSConnector extends org.ap
}
-
- /** Get document versions given an array of document identifiers.
- * This method is called for EVERY document that is considered. It is
- * therefore important to perform as little work as possible here.
- *@param documentIdentifiers is the array of local document identifiers, as
understood by this connector.
- *@param oldVersions is the corresponding array of version strings that have
been saved for the document identifiers.
- * A null value indicates that this is a first-time fetch, while an empty
string indicates that the previous document
- * had an empty version string.
- *@param activities is the interface this method should use to perform
whatever framework actions are desired.
- *@param spec is the current document specification for the current job. If
there is a dependency on this
- * specification, then the version string should include the pertinent data,
so that reingestion will occur
- * when the specification changes. This is primarily useful for metadata.
- *@param jobType is an integer describing how the job is being run, whether
continuous or once-only.
+ /** Process a set of documents.
+ * This is the method that should cause each document to be fetched,
processed, and the results either added
+ * to the queue of documents for the current job, and/or entered into the
incremental ingestion manager.
+ * The document specification allows this class to filter what is done based
on the job.
+ * The connector will be connected before this method can be called.
+ *@param documentIdentifiers is the set of document identifiers to process.
+ *@param statuses are the currently-stored document versions for each
document in the set of document identifiers
+ * passed in above.
+ *@param activities is the interface this method should use to queue up new
document references
+ * and ingest documents.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
*@param usesDefaultAuthority will be true only if the authority in use for
these documents is the default one.
- *@return the corresponding version strings, with null in the places where
the document no longer exists.
- * Empty version strings indicate that there is no versioning ability for the
corresponding document, and the document
- * will always be processed.
*/
@Override
- public String[] getDocumentVersions(String[] documentIdentifiers, String[]
oldVersions, IVersionActivity activities,
- DocumentSpecification spec, int jobType, boolean usesDefaultAuthority)
+ public void processDocuments(String[] documentIdentifiers, IExistingVersions
statuses, Specification spec,
+ IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
throws ManifoldCFException, ServiceInterruption
{
getSession();
@@ -732,6 +727,8 @@ public class RSSConnector extends org.ap
// thought to set up properly.
int connectionLimit = 200;
+ String[] fixedList = new String[2];
+
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("RSS: In getDocumentVersions for
"+Integer.toString(documentIdentifiers.length)+" documents");
@@ -757,183 +754,185 @@ public class RSSConnector extends org.ap
int feedTimeout = f.getFeedTimeoutValue();
- // A preliminary fetch must be done in either case, because otherwise we
cannot detect a deletion.
- // But, since we don't want to fetch twice, write these urls to temporary
files. The
- // files will be cleaned up as soon as possible.
-
- String[] rval = new String[documentIdentifiers.length];
-
- // HttpClient client = new HttpClient(httpConMan.getManager());
-
// The document specification has already been used to trim out documents
that are not
// allowed from appearing in the queue. So, even that has already been
done.
- int i = 0;
- while (i < documentIdentifiers.length)
+ for (String documentIdentifier : documentIdentifiers)
{
// If it is in this list, we presume that it has been vetted against the
map etc., so we don't do that again. We just fetch it.
// And, if the content type is xml, we calculate the version as if it is
a feed rather than a document.
// Get the url
- String urlValue = documentIdentifiers[i];
+ String urlValue = documentIdentifier;
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("RSS: Getting version string for
'"+urlValue+"'");
- // If there's a carrydown "data" value for this url, we use that value
rather than actually fetching the document. This also means we don't need to
- // do a robots check, because we aren't actually crawling anything. So,
ALWAYS do this first...
- CharacterInput[] dechromedData =
activities.retrieveParentDataAsFiles(urlValue,"data");
+ String versionString;
+ String ingestURL = null;
+ String[] pubDates = null;
+ String[] sources = null;
+ String[] titles = null;
+ String[] authorNames = null;
+ String[] authorEmails = null;
+ String[] categories = null;
+ String[] descriptions = null;
+
try
{
- if (dechromedData.length > 0)
+ // If there's a carrydown "data" value for this url, we use that value
rather than actually fetching the document. This also means we don't need to
+ // do a robots check, because we aren't actually crawling anything.
So, ALWAYS do this first...
+ CharacterInput[] dechromedData =
activities.retrieveParentDataAsFiles(urlValue,"data");
+ try
{
- // Data already available. The fetch cycle can be entirely avoided,
as can the robots check.
- String ingestURL = f.mapDocumentURL(urlValue);
- if (ingestURL != null)
+ if (dechromedData.length > 0)
{
- // Open up an input stream corresponding to the carrydown data.
The stream will be encoded as utf-8.
- try
+ // Data already available. The fetch cycle can be entirely
avoided, as can the robots check.
+ ingestURL = f.mapDocumentURL(urlValue);
+ if (ingestURL != null)
{
- InputStream is = dechromedData[0].getUtf8Stream();
+ // Open up an input stream corresponding to the carrydown data.
The stream will be encoded as utf-8.
try
{
- StringBuilder sb = new StringBuilder();
- long checkSum =
cache.addData(activities,urlValue,"text/html",is);
- // Grab what we need from the passed-down data for the
document. These will all become part
- // of the version string.
- String[] pubDates =
activities.retrieveParentData(urlValue,"pubdate");
- String[] sources =
activities.retrieveParentData(urlValue,"source");
- String[] titles =
activities.retrieveParentData(urlValue,"title");
- String[] authorNames =
activities.retrieveParentData(urlValue,"authorname");
- String[] authorEmails =
activities.retrieveParentData(urlValue,"authoremail");
- String[] categories =
activities.retrieveParentData(urlValue,"category");
- String[] descriptions =
activities.retrieveParentData(urlValue,"description");
- java.util.Arrays.sort(pubDates);
- java.util.Arrays.sort(sources);
- java.util.Arrays.sort(titles);
- java.util.Arrays.sort(authorNames);
- java.util.Arrays.sort(authorEmails);
- java.util.Arrays.sort(categories);
- java.util.Arrays.sort(descriptions);
-
- if (sources.length == 0)
+ InputStream is = dechromedData[0].getUtf8Stream();
+ try
{
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Warning; URL
'"+ingestURL+"' doesn't seem to have any RSS feed source!");
- }
+ StringBuilder sb = new StringBuilder();
+ long checkSum =
cache.addData(activities,urlValue,"text/html",is);
+ // Grab what we need from the passed-down data for the
document. These will all become part
+ // of the version string.
+ pubDates = activities.retrieveParentData(urlValue,"pubdate");
+ sources = activities.retrieveParentData(urlValue,"source");
+ titles = activities.retrieveParentData(urlValue,"title");
+ authorNames =
activities.retrieveParentData(urlValue,"authorname");
+ authorEmails =
activities.retrieveParentData(urlValue,"authoremail");
+ categories =
activities.retrieveParentData(urlValue,"category");
+ descriptions =
activities.retrieveParentData(urlValue,"description");
+ java.util.Arrays.sort(pubDates);
+ java.util.Arrays.sort(sources);
+ java.util.Arrays.sort(titles);
+ java.util.Arrays.sort(authorNames);
+ java.util.Arrays.sort(authorEmails);
+ java.util.Arrays.sort(categories);
+ java.util.Arrays.sort(descriptions);
+
+ if (sources.length == 0)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Warning; URL
'"+ingestURL+"' doesn't seem to have any RSS feed source!");
+ }
- sb.append('+');
- packList(sb,acls,'+');
- if (acls.length > 0)
- {
sb.append('+');
- pack(sb,defaultAuthorityDenyToken,'+');
- }
- else
- sb.append('-');
- // The ingestion URL
- pack(sb,ingestURL,'+');
- // The pub dates
- packList(sb,pubDates,'+');
- // The titles
- packList(sb,titles,'+');
- // The sources
- packList(sb,sources,'+');
- // The categories
- packList(sb,categories,'+');
- // The descriptions
- packList(sb,descriptions,'+');
- // The author names
- packList(sb,authorNames,'+');
- // The author emails
- packList(sb,authorEmails,'+');
+ packList(sb,acls,'+');
+ if (acls.length > 0)
+ {
+ sb.append('+');
+ pack(sb,defaultAuthorityDenyToken,'+');
+ }
+ else
+ sb.append('-');
+ // The ingestion URL
+ pack(sb,ingestURL,'+');
+ // The pub dates
+ packList(sb,pubDates,'+');
+ // The titles
+ packList(sb,titles,'+');
+ // The sources
+ packList(sb,sources,'+');
+ // The categories
+ packList(sb,categories,'+');
+ // The descriptions
+ packList(sb,descriptions,'+');
+ // The author names
+ packList(sb,authorNames,'+');
+ // The author emails
+ packList(sb,authorEmails,'+');
- // Do the checksum part, which does not need to be parseable.
- sb.append(new Long(checkSum).toString());
+ // Do the checksum part, which does not need to be parseable.
+ sb.append(new Long(checkSum).toString());
- rval[i] = sb.toString();
+ versionString = sb.toString();
+ }
+ finally
+ {
+ is.close();
+ }
}
- finally
+ catch (java.net.SocketTimeoutException e)
{
- is.close();
+ throw new ManifoldCFException("IO exception reading data from
string: "+e.getMessage(),e);
+ }
+ catch (InterruptedIOException e)
+ {
+ throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (IOException e)
+ {
+ throw new ManifoldCFException("IO exception reading data from
string: "+e.getMessage(),e);
}
}
- catch (java.net.SocketTimeoutException e)
- {
- throw new ManifoldCFException("IO exception reading data from
string: "+e.getMessage(),e);
- }
- catch (InterruptedIOException e)
- {
- throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (IOException e)
+ else
{
- throw new ManifoldCFException("IO exception reading data from
string: "+e.getMessage(),e);
+ // Document a seed or unmappable; just skip
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Skipping carry-down document
'"+urlValue+"' because it is unmappable or is a seed.");
}
}
else
{
- // Document a seed or unmappable; just skip
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Skipping carry-down document
'"+urlValue+"' because it is unmappable or is a seed.");
- }
- }
- else
- {
- // Get the old version string
- String oldVersionString = oldVersions[i];
+ // Get the old version string
+ String oldVersionString =
statuses.getIndexedVersionString(documentIdentifier);
- // Unpack the old version as much as possible.
- // We are interested in what the ETag and Last-Modified headers were
last time.
- String lastETagValue = null;
- String lastModifiedValue = null;
- // Note well: Non-continuous jobs cannot use etag because the rss
document MUST be fetched each time for such jobs,
- // or the documents it points at would get deleted.
- //
- // NOTE: I disabled this code because we really need the feed's TTL
value in order to reschedule properly. I can't get the
- // TTL value without refetching the document - therefore ETag and
Last-Modified cannot be used :-(
- if (false && jobType == JOBMODE_CONTINUOUS && oldVersionString !=
null && oldVersionString.startsWith("-"))
- {
- // It's a feed, so the last etag and last-modified fields should
be encoded in this version string.
- StringBuilder lastETagBuffer = new StringBuilder();
- int unpackPos = unpack(lastETagBuffer,oldVersionString,1,'+');
- StringBuilder lastModifiedBuffer = new StringBuilder();
- unpackPos =
unpack(lastModifiedBuffer,oldVersionString,unpackPos,'+');
- if (lastETagBuffer.length() > 0)
- lastETagValue = lastETagBuffer.toString();
- if (lastModifiedBuffer.length() > 0)
- lastModifiedValue = lastModifiedBuffer.toString();
- }
-
- if (Logging.connectors.isDebugEnabled() && (lastETagValue != null ||
lastModifiedValue != null))
- Logging.connectors.debug("RSS: Document '"+urlValue+"' was found
to have a previous ETag value of
'"+((lastETagValue==null)?"null":lastETagValue)+
- "' and a previous Last-Modified value of
'"+((lastModifiedValue==null)?"null":lastModifiedValue)+"'");
+ // Unpack the old version as much as possible.
+ // We are interested in what the ETag and Last-Modified headers
were last time.
+ String lastETagValue = null;
+ String lastModifiedValue = null;
+ // Note well: Non-continuous jobs cannot use etag because the rss
document MUST be fetched each time for such jobs,
+ // or the documents it points at would get deleted.
+ //
+ // NOTE: I disabled this code because we really need the feed's
TTL value in order to reschedule properly. I can't get the
+ // TTL value without refetching the document - therefore ETag and
Last-Modified cannot be used :-(
+ if (false && jobMode == JOBMODE_CONTINUOUS && oldVersionString !=
null && oldVersionString.startsWith("-"))
+ {
+ // It's a feed, so the last etag and last-modified fields should
be encoded in this version string.
+ StringBuilder lastETagBuffer = new StringBuilder();
+ int unpackPos = unpack(lastETagBuffer,oldVersionString,1,'+');
+ StringBuilder lastModifiedBuffer = new StringBuilder();
+ unpackPos =
unpack(lastModifiedBuffer,oldVersionString,unpackPos,'+');
+ if (lastETagBuffer.length() > 0)
+ lastETagValue = lastETagBuffer.toString();
+ if (lastModifiedBuffer.length() > 0)
+ lastModifiedValue = lastModifiedBuffer.toString();
+ }
+
+ if (Logging.connectors.isDebugEnabled() && (lastETagValue != null
|| lastModifiedValue != null))
+ Logging.connectors.debug("RSS: Document '"+urlValue+"' was found
to have a previous ETag value of
'"+((lastETagValue==null)?"null":lastETagValue)+
+ "' and a previous Last-Modified value of
'"+((lastModifiedValue==null)?"null":lastModifiedValue)+"'");
- // Robots check. First, we need to separate the url into its
components
- try
- {
- URL url = new URL(urlValue);
- String protocol = url.getProtocol();
- int port = url.getPort();
- String hostName = url.getHost();
- String pathPart = url.getFile();
-
- // Check with robots to see if it's allowed
- if (robotsUsage >= ROBOTS_DATA &&
!robots.isFetchAllowed(currentContext,throttleGroupName,
- protocol,port,hostName,url.getPath(),
- userAgent,from,
- proxyHost, proxyPort, proxyAuthDomain, proxyAuthUsername,
proxyAuthPassword,
- activities, connectionLimit))
- {
- activities.recordActivity(null,ACTIVITY_FETCH,
- null,urlValue,Integer.toString(-2),"Robots exclusion",null);
-
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Skipping url '"+urlValue+"'
because robots.txt says to");
- rval[i] = null;
- }
- else
+ // Robots check. First, we need to separate the url into its
components
+ try
{
+ URL url = new URL(urlValue);
+ String protocol = url.getProtocol();
+ int port = url.getPort();
+ String hostName = url.getHost();
+ String pathPart = url.getFile();
+
+ // Check with robots to see if it's allowed
+ if (robotsUsage >= ROBOTS_DATA &&
!robots.isFetchAllowed(currentContext,throttleGroupName,
+ protocol,port,hostName,url.getPath(),
+ userAgent,from,
+ proxyHost, proxyPort, proxyAuthDomain, proxyAuthUsername,
proxyAuthPassword,
+ activities, connectionLimit))
+ {
+ activities.recordActivity(null,ACTIVITY_FETCH,
+ null,urlValue,Integer.toString(-2),"Robots exclusion",null);
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Skipping url '"+urlValue+"'
because robots.txt says to");
+ activities.deleteDocument(documentIdentifier);
+ continue;
+ }
+
// Now, use the fetcher, and get the file.
IThrottledConnection connection =
fetcher.createConnection(currentContext,
throttleGroupName,
@@ -958,7 +957,7 @@ public class RSSConnector extends org.ap
switch (status)
{
case IThrottledConnection.STATUS_NOCHANGE:
- rval[i] = oldVersionString;
+ versionString = oldVersionString;
break;
case IThrottledConnection.STATUS_OK:
try
@@ -987,7 +986,7 @@ public class RSSConnector extends org.ap
contentType.startsWith("application/rdf+xml") ||
contentType.startsWith("text/application") ||
contentType.startsWith("XML") ));
- String ingestURL = null;
+ ingestURL = null;
if (!isXML)
{
// If the chromed content mode is set to "skip", and
we got here, it means
@@ -996,7 +995,7 @@ public class RSSConnector extends org.ap
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("RSS: Removing url
'"+urlValue+"' because it no longer has dechromed content available");
- rval[i] = null;
+ versionString = null;
break;
}
@@ -1007,7 +1006,7 @@ public class RSSConnector extends org.ap
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("RSS: Removing url
'"+urlValue+"' because it had the wrong content type:
"+((contentType==null)?"null":"'"+contentType+"'"));
- rval[i] = null;
+ versionString = null;
break;
}
@@ -1023,7 +1022,7 @@ public class RSSConnector extends org.ap
// Remove the feed from consideration, since it has
left the list of seeds
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("RSS: Removing feed url
'"+urlValue+"' because it is not a seed.");
- rval[i] = null;
+ versionString = null;
break;
}
}
@@ -1039,13 +1038,13 @@ public class RSSConnector extends org.ap
// Grab what we need from the passed-down data for
the document. These will all become part
// of the version string.
- String[] pubDates =
activities.retrieveParentData(urlValue,"pubdate");
- String[] sources =
activities.retrieveParentData(urlValue,"source");
- String[] titles =
activities.retrieveParentData(urlValue,"title");
- String[] authorNames =
activities.retrieveParentData(urlValue,"authorname");
- String[] authorEmails =
activities.retrieveParentData(urlValue,"authoremail");
- String[] categories =
activities.retrieveParentData(urlValue,"category");
- String[] descriptions =
activities.retrieveParentData(urlValue,"description");
+ pubDates =
activities.retrieveParentData(urlValue,"pubdate");
+ sources =
activities.retrieveParentData(urlValue,"source");
+ titles =
activities.retrieveParentData(urlValue,"title");
+ authorNames =
activities.retrieveParentData(urlValue,"authorname");
+ authorEmails =
activities.retrieveParentData(urlValue,"authoremail");
+ categories =
activities.retrieveParentData(urlValue,"category");
+ descriptions =
activities.retrieveParentData(urlValue,"description");
java.util.Arrays.sort(pubDates);
java.util.Arrays.sort(sources);
java.util.Arrays.sort(titles);
@@ -1105,7 +1104,7 @@ public class RSSConnector extends org.ap
// Do the checksum part, which does not need to be
parseable.
sb.append(new Long(checkSum).toString());
- rval[i] = sb.toString();
+ versionString = sb.toString();
}
finally
{
@@ -1115,12 +1114,12 @@ public class RSSConnector extends org.ap
catch (java.net.SocketTimeoutException e)
{
Logging.connectors.warn("RSS: Socket timeout exception
fetching document contents '"+urlValue+"' - skipping: "+e.getMessage(), e);
- rval[i] = null;
+ versionString = null;
}
catch (ConnectTimeoutException e)
{
Logging.connectors.warn("RSS: Connecto timeout exception
fetching document contents '"+urlValue+"' - skipping: "+e.getMessage(), e);
- rval[i] = null;
+ versionString = null;
}
catch (InterruptedIOException e)
{
@@ -1129,7 +1128,7 @@ public class RSSConnector extends org.ap
catch (IOException e)
{
Logging.connectors.warn("RSS: IO exception fetching
document contents '"+urlValue+"' - skipping: "+e.getMessage(), e);
- rval[i] = null;
+ versionString = null;
}
break;
@@ -1140,7 +1139,7 @@ public class RSSConnector extends org.ap
// Record an *empty* version.
// This signals the processDocuments() method that we
really don't want to ingest this document, but we also don't
// want to blow the document out of the queue, since then
we'd wind up perhaps fetching it multiple times.
- rval[i] = "";
+ versionString = "";
break;
}
}
@@ -1153,342 +1152,205 @@ public class RSSConnector extends org.ap
{
connection.close();
}
- }
- }
- catch (MalformedURLException e)
- {
- Logging.connectors.debug("RSS: URL '"+urlValue+"' is malformed;
skipping",e);
- rval[i] = null;
- }
- }
- }
- finally
- {
- int j = 0;
- while (j < dechromedData.length)
- {
- CharacterInput ci = (CharacterInput)dechromedData[j++];
- if (ci != null)
- ci.discard();
- }
-
- }
-
- i++;
- }
-
- return rval;
- }
-
- /** Process a set of documents.
- * This is the method that should cause each document to be fetched,
processed, and the results either added
- * to the queue of documents for the current job, and/or entered into the
incremental ingestion manager.
- * The document specification allows this class to filter what is done based
on the job.
- *@param documentIdentifiers is the set of document identifiers to process.
- *@param activities is the interface this method should use to queue up new
document references
- * and ingest documents.
- *@param spec is the document specification.
- *@param scanOnly is an array corresponding to the document identifiers. It
is set to true to indicate when the processing
- * should only find other references, and should not actually call the
ingestion methods.
- */
- @Override
- public void processDocuments(String[] documentIdentifiers, String[]
versions, IProcessActivity activities,
- DocumentSpecification spec, boolean[] scanOnly, int jobType)
- throws ManifoldCFException, ServiceInterruption
- {
- getSession();
-
- // The version strings contain the ingest url as well as everything we
need to determine if this url is a feed or just a document.
- // So, there is no need to reparse the specification.
- Filter f = null;
-
- String[] fixedList = new String[2];
-
- int i = 0;
- while (i < documentIdentifiers.length)
- {
- String urlValue = documentIdentifiers[i];
- String version = versions[i];
-
- if (version.length() == 0)
- {
- // This document had an http response incompatible with its existence
on the web.
- i++;
- // Leave document in jobqueue, but do NOT get rid of it, or we will
wind up seeing it queued again by
- // somebody else. We *do* have to signal the document to be removed
from the index, however, or it will
- // stick around until the job is deleted.
- activities.noDocument(urlValue,version);
- continue;
- }
-
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Processing '"+urlValue+"'");
+
+ if (versionString == null)
+ {
+ activities.deleteDocument(documentIdentifier);
+ continue;
+ }
+
+ if (!(versionString.length() == 0 ||
activities.checkDocumentNeedsReindexing(documentIdentifier,versionString)))
+ continue;
+
+ // Process document!
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Processing '"+urlValue+"'");
- // The only links we extract come from documents that we think are RSS
feeds.
- // When we think that's the case, we attempt to parse it as RSS XML.
- if (version.startsWith("-"))
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Interpreting document '"+urlValue+"'
as a feed");
+ // The only links we extract come from documents that we think
are RSS feeds.
+ // When we think that's the case, we attempt to parse it as RSS
XML.
+ if (ingestURL == null)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Interpreting document
'"+urlValue+"' as a feed");
- // We think it is a feed.
- // The version string does not have anything useful in it.
- if (f == null)
- f = new Filter(spec,false);
-
- // If this is a continuous job, AND scanonly is true, it means that
the document was either identical to the
- // previous fetch, or was not fetched at all. In that case, it may
not even be there, and we *certainly* don't
- // want to attempt to process it in any case.
- //
-
- // NOTE: I re-enabled the scan permanently because we need the TTL
value to be set whatever the cost. If the
- // TTL value is not set, we default to the specified job's feed-rescan
time, which is not going to be current enough for some feeds.
- if (true || scanOnly[i] == false || jobType != JOBMODE_CONTINUOUS)
- {
- handleRSSFeedSAX(urlValue,activities,f);
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Extraction of feed '"+urlValue+"'
complete");
+ // We think it is a feed.
+ // If this is a continuous job, AND scanonly is true, it means
that the document was either identical to the
+ // previous fetch, or was not fetched at all. In that case,
it may not even be there, and we *certainly* don't
+ // want to attempt to process it in any case.
+ //
+
+ // NOTE: I re-enabled the scan permanently because we need the
TTL value to be set whatever the cost. If the
+ // TTL value is not set, we default to the specified job's
feed-rescan time, which is not going to be current enough for some feeds.
+ if (true || jobMode != JOBMODE_CONTINUOUS)
+ {
+ handleRSSFeedSAX(urlValue,activities,f);
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Extraction of feed
'"+urlValue+"' complete");
- // Record the feed's version string, so we won't refetch unless
needed.
- // This functionality is required for the last ETag and
Last-Modified fields to be sent to the rss server, and to
- // keep track of the adaptive parameters.
- activities.recordDocument(urlValue,version);
- }
- else
- {
- // The problem here is that we really do need to set the rescan time
to something reasonable.
- // But we might not even have read the feed! So what to do??
- // One answer is to build a connector-specific table that carries
the last value of every feed around.
- // Another answer is to change the version code to always read the
feed (and the heck with ETag and Last-Modified).
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Feed '"+urlValue+"' does not appear
to differ from previous fetch for a continuous job; not extracting!");
+ // Record the feed's version string, so we won't refetch
unless needed.
+ // This functionality is required for the last ETag and
Last-Modified fields to be sent to the rss server, and to
+ // keep track of the adaptive parameters.
+ activities.recordDocument(documentIdentifier,versionString);
+ }
+ else
+ {
+ // The problem here is that we really do need to set the
rescan time to something reasonable.
+ // But we might not even have read the feed! So what to do??
+ // One answer is to build a connector-specific table that
carries the last value of every feed around.
+ // Another answer is to change the version code to always
read the feed (and the heck with ETag and Last-Modified).
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Feed '"+urlValue+"' does
not appear to differ from previous fetch for a continuous job; not
extracting!");
- long currentTime = System.currentTimeMillis();
+ long currentTime = System.currentTimeMillis();
- Long defaultRescanTime = f.getDefaultRescanTime(currentTime);
+ Long defaultRescanTime = f.getDefaultRescanTime(currentTime);
- if (defaultRescanTime != null)
- {
- Long minimumTime = f.getMinimumRescanTime(currentTime);
- if (minimumTime != null)
- {
- if (defaultRescanTime.longValue() < minimumTime.longValue())
- defaultRescanTime = minimumTime;
- }
- }
+ if (defaultRescanTime != null)
+ {
+ Long minimumTime = f.getMinimumRescanTime(currentTime);
+ if (minimumTime != null)
+ {
+ if (defaultRescanTime.longValue() <
minimumTime.longValue())
+ defaultRescanTime = minimumTime;
+ }
+ }
-
activities.setDocumentScheduleBounds(urlValue,defaultRescanTime,defaultRescanTime,null,null);
+
activities.setDocumentScheduleBounds(urlValue,defaultRescanTime,defaultRescanTime,null,null);
- }
- }
- else if (scanOnly[i] == false && version.startsWith("+"))
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Interpreting '"+urlValue+"' as a
document");
+ }
+ }
+ else
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Interpreting '"+urlValue+"'
as a document");
+ if (isDataIngestable(activities,urlValue))
+ {
+ // Treat it as an ingestable document.
+
+ long dataSize = cache.getDataLength(urlValue);
+ RepositoryDocument rd = new RepositoryDocument();
+ // Set content type
+ rd.setMimeType(cache.getContentType(urlValue));
- if (isDataIngestable(activities,urlValue))
- {
- // Treat it as an ingestable document.
- // Version *should* start with a "+".
- ArrayList acls = new ArrayList();
- StringBuilder denyAclBuffer = new StringBuilder();
- int startPos = unpackList(acls,version,1,'+');
- if (startPos < version.length() && version.charAt(startPos++) == '+')
- {
- startPos = unpack(denyAclBuffer,version,startPos,'+');
- }
- StringBuilder ingestUrlBuffer = new StringBuilder();
- startPos = unpack(ingestUrlBuffer,version,startPos,'+');
- String ingestURL = ingestUrlBuffer.toString();
- ArrayList pubDates = new ArrayList();
- startPos = unpackList(pubDates,version,startPos,'+');
- ArrayList titles = new ArrayList();
- startPos = unpackList(titles,version,startPos,'+');
- ArrayList sources = new ArrayList();
- startPos = unpackList(sources,version,startPos,'+');
- ArrayList categories = new ArrayList();
- startPos = unpackList(categories,version,startPos,'+');
- ArrayList descriptions = new ArrayList();
- startPos = unpackList(descriptions,version,startPos,'+');
- ArrayList authorNames = new ArrayList();
- startPos = unpackList(authorNames,version,startPos,'+');
- ArrayList authorEmails = new ArrayList();
- startPos = unpackList(authorEmails,version,startPos,'+');
-
- if (ingestURL.length() > 0)
- {
- long dataSize = cache.getDataLength(urlValue);
- RepositoryDocument rd = new RepositoryDocument();
-
- // Set content type
- rd.setMimeType(cache.getContentType(urlValue));
-
- // Turn into acls and add into description
- String[] aclArray = new String[acls.size()];
- int j = 0;
- while (j < aclArray.length)
- {
- aclArray[j] = (String)acls.get(j);
- j++;
- }
-
rd.setSecurityACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,aclArray);
-
- // Deny acl too
- if (denyAclBuffer.length() > 0)
- {
- String[] denyAclArray = new String[]{denyAclBuffer.toString()};
-
rd.setSecurityDenyACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,denyAclArray);
- }
-
- int k;
-
- // Loop through the titles to add those to the metadata
- String[] titleValues = new String[titles.size()];
- k = 0;
- while (k < titleValues.length)
- {
- titleValues[k] = (String)titles.get(k);
- k++;
- }
- if (k > 0)
- rd.addField("title",titleValues);
-
- // Loop through the author names to add those to the metadata
- String[] authorNameValues = new String[authorNames.size()];
- k = 0;
- while (k < authorNameValues.length)
- {
- authorNameValues[k] = (String)authorNames.get(k);
- k++;
- }
- if (k > 0)
- rd.addField("authorname",authorNameValues);
-
- // Loop through the author emails to add those to the metadata
- String[] authorEmailValues = new String[authorEmails.size()];
- k = 0;
- while (k < authorEmailValues.length)
- {
- authorEmailValues[k] = (String)authorEmails.get(k);
- k++;
- }
- if (k > 0)
- rd.addField("authoremail",authorEmailValues);
-
- // Loop through the descriptions to add those to the metadata
- String[] descriptionValues = new String[descriptions.size()];
- k = 0;
- while (k < descriptionValues.length)
- {
- descriptionValues[k] = (String)descriptions.get(k);
- k++;
- }
- if (k > 0)
- rd.addField("summary",descriptionValues);
-
- // Loop through the sources to add those to the metadata
- String[] sourceValues = new String[sources.size()];
- k = 0;
- while (k < sourceValues.length)
- {
- sourceValues[k] = (String)sources.get(k);
- k++;
- }
- if (k > 0)
- rd.addField("source",sourceValues);
-
- // Add the categories now
- String[] categoryValues = new String[categories.size()];
- k = 0;
- while (k < categoryValues.length)
- {
- categoryValues[k] = (String)categories.get(k);
- k++;
- }
- if (k > 0)
- rd.addField("category",categoryValues);
-
- // The pubdates are a ms since epoch value; we want the minimum
one for the origination time.
- Long minimumOrigTime = null;
- String[] pubDateValues = new String[pubDates.size()];
- String[] pubDateValuesISO = new String[pubDates.size()];
- TimeZone tz = TimeZone.getTimeZone("UTC");
- DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
- df.setTimeZone(tz);
- k = 0;
- while (k < pubDates.size())
- {
- String pubDate = (String)pubDates.get(k);
- pubDateValues[k] = pubDate;
- try
- {
- Long pubDateLong = new Long(pubDate);
- if (minimumOrigTime == null || pubDateLong.longValue() <
minimumOrigTime.longValue())
- minimumOrigTime = pubDateLong;
- pubDateValuesISO[k] = df.format(new
Date(pubDateLong.longValue()));
- }
- catch (NumberFormatException e)
- {
- // Do nothing; the version string seems to not mean anything
- }
- k++;
- }
- if (k > 0)
- {
- rd.addField("pubdate",pubDateValues);
- rd.addField("pubdateiso",pubDateValuesISO);
- }
+ // Turn into acls and add into description
+ String[] denyAcls;
+ if (acls == null)
+ denyAcls = null;
+ else if (acls.length == 0)
+ denyAcls = new String[0];
+ else
+ denyAcls = new String[]{defaultAuthorityDenyToken};
+
+ if (acls != null && denyAcls != null)
+
rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,acls,denyAcls);
+
+ if (titles != null && titles.length > 0)
+ rd.addField("title",titles);
+ if (authorNames != null && authorNames.length > 0)
+ rd.addField("authorname",authorNames);
+ if (authorEmails != null && authorEmails.length > 0)
+ rd.addField("authoremail",authorEmails);
+ if (descriptions != null && descriptions.length > 0)
+ rd.addField("summary",descriptions);
+ if (sources != null && sources.length > 0)
+ rd.addField("source",sources);
+ if (categories != null && categories.length > 0)
+ rd.addField("category",categories);
+
+ // The pubdates are a ms since epoch value; we want the
minimum one for the origination time.
+ Long minimumOrigTime = null;
+ if (pubDates != null && pubDates.length > 0)
+ {
+ String[] pubDateValuesISO = new String[pubDates.length];
+ TimeZone tz = TimeZone.getTimeZone("UTC");
+ DateFormat df = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
+ df.setTimeZone(tz);
+ for (int k = 0; k < pubDates.length; k++)
+ {
+ String pubDate = pubDates[k];
+ try
+ {
+ Long pubDateLong = new Long(pubDate);
+ if (minimumOrigTime == null || pubDateLong.longValue()
< minimumOrigTime.longValue())
+ minimumOrigTime = pubDateLong;
+ pubDateValuesISO[k] = df.format(new
Date(pubDateLong.longValue()));
+ }
+ catch (NumberFormatException e)
+ {
+ // Do nothing; the version string seems to not mean
anything
+ pubDateValuesISO[k] = "";
+ }
+ }
+ rd.addField("pubdate",pubDates);
+ rd.addField("pubdateiso",pubDateValuesISO);
+ }
- if (minimumOrigTime != null)
- activities.setDocumentOriginationTime(urlValue,minimumOrigTime);
+ if (minimumOrigTime != null)
+
activities.setDocumentOriginationTime(urlValue,minimumOrigTime);
- InputStream is = cache.getData(urlValue);
- if (is != null)
- {
- try
- {
- rd.setBinary(is,dataSize);
- try
- {
-
activities.ingestDocumentWithException(urlValue,version,ingestURL,rd);
- }
- catch (IOException e)
- {
- handleIOException(e,"reading data");
- }
- }
- finally
- {
- try
- {
- is.close();
+ InputStream is = cache.getData(urlValue);
+ if (is != null)
+ {
+ try
+ {
+ rd.setBinary(is,dataSize);
+ try
+ {
+
activities.ingestDocumentWithException(documentIdentifier,versionString,ingestURL,rd);
+ }
+ catch (IOException e)
+ {
+ handleIOException(e,"reading data");
+ }
+ }
+ finally
+ {
+ try
+ {
+ is.close();
+ }
+ catch (IOException e)
+ {
+ handleIOException(e,"closing stream");
+ }
+ }
+ }
}
- catch (IOException e)
+ else
{
- handleIOException(e,"closing stream");
+ activities.noDocument(documentIdentifier,versionString);
+
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("RSS: Skipping document
'"+urlValue+"' because it cannot be indexed");
}
}
}
+ catch (MalformedURLException e)
+ {
+ Logging.connectors.debug("RSS: URL '"+urlValue+"' is malformed;
skipping",e);
+ activities.deleteDocument(documentIdentifier);
+ continue;
+ }
}
}
- else
+ finally
{
- activities.noDocument(urlValue,version);
+ for (CharacterInput ci : dechromedData)
+ {
+ if (ci != null)
+ ci.discard();
+ }
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Skipping document '"+urlValue+"'
because it cannot be indexed");
}
}
- else
+ finally
{
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("RSS: Skipping document '"+urlValue+"'
because it cannot have interesting links");
+ // Remove any fetched documents.
+ cache.deleteData(documentIdentifier);
}
- i++;
}
-
}
protected static void handleIOException(IOException e, String context)
@@ -1502,31 +1364,6 @@ public class RSSConnector extends org.ap
throw new ManifoldCFException("IO error "+context+": "+e.getMessage(),e);
}
- /** Free a set of documents. This method is called for all documents whose
versions have been fetched using
- * the getDocumentVersions() method, including those that returned null
versions. It may be used to free resources
- * committed during the getDocumentVersions() method. It is guaranteed to be
called AFTER any calls to
- * processDocuments() for the documents in question.
- *@param documentIdentifiers is the set of document identifiers.
- *@param versions is the corresponding set of version identifiers (individual
identifiers may be null).
- */
- @Override
- public void releaseDocumentVersions(String[] documentIdentifiers, String[]
versions)
- throws ManifoldCFException
- {
- int i = 0;
- while (i < documentIdentifiers.length)
- {
- String version = versions[i];
- if (version != null)
-
- {
- String urlValue = documentIdentifiers[i];
- cache.deleteData(urlValue);
- }
- i++;
- }
- }
-
// UI support methods.
//
// These support methods come in two varieties. The first bunch is involved
in setting up connection configuration information. The second bunch
