Author: kwright
Date: Thu Nov 14 12:38:31 2013
New Revision: 1541890
URL: http://svn.apache.org/r1541890
Log:
Add author support at the item level to RSS Connector. Part of CONNECTORS-805.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1541890&r1=1541889&r2=1541890&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Thu Nov 14 12:38:31 2013
@@ -3,6 +3,9 @@ $Id$
======================= 1.5-dev =====================
+CONNECTORS-805: Add author name/email support to RSS connector.
+(Benjamin Brandmeier, Karl Wright)
+
CONNECTORS-13: Provide a ZooKeeper method of coordinating
various ManifoldCF processes, along with an example.
(Karl Wright)
Modified:
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java?rev=1541890&r1=1541889&r2=1541890&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
(original)
+++
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
Thu Nov 14 12:38:31 2013
@@ -815,11 +815,15 @@ public class RSSConnector extends org.ap
String[] pubDates =
activities.retrieveParentData(urlValue,"pubdate");
String[] sources =
activities.retrieveParentData(urlValue,"source");
String[] titles =
activities.retrieveParentData(urlValue,"title");
+ String[] authorNames =
activities.retrieveParentData(urlValue,"authorname");
+ String[] authorEmails =
activities.retrieveParentData(urlValue,"authoremail");
String[] categories =
activities.retrieveParentData(urlValue,"category");
String[] descriptions =
activities.retrieveParentData(urlValue,"description");
java.util.Arrays.sort(pubDates);
java.util.Arrays.sort(sources);
java.util.Arrays.sort(titles);
+ java.util.Arrays.sort(authorNames);
+ java.util.Arrays.sort(authorEmails);
java.util.Arrays.sort(categories);
java.util.Arrays.sort(descriptions);
@@ -852,6 +856,10 @@ public class RSSConnector extends org.ap
packList(sb,categories,'+');
// The descriptions
packList(sb,descriptions,'+');
+ // The author names
+ packList(sb,authorNames,'+');
+ // The author emails
+ packList(sb,authorEmails,'+');
// Do the checksum part, which does not need to be parseable.
sb.append(new Long(checkSum).toString());
@@ -1055,11 +1063,15 @@ public class RSSConnector extends org.ap
String[] pubDates =
activities.retrieveParentData(urlValue,"pubdate");
String[] sources =
activities.retrieveParentData(urlValue,"source");
String[] titles =
activities.retrieveParentData(urlValue,"title");
+ String[] authorNames =
activities.retrieveParentData(urlValue,"authorname");
+ String[] authorEmails =
activities.retrieveParentData(urlValue,"authoremail");
String[] categories =
activities.retrieveParentData(urlValue,"category");
String[] descriptions =
activities.retrieveParentData(urlValue,"description");
java.util.Arrays.sort(pubDates);
java.util.Arrays.sort(sources);
java.util.Arrays.sort(titles);
+ java.util.Arrays.sort(authorNames);
+ java.util.Arrays.sort(authorEmails);
java.util.Arrays.sort(categories);
java.util.Arrays.sort(descriptions);
@@ -1092,7 +1104,10 @@ public class RSSConnector extends org.ap
packList(sb,categories,'+');
// The descriptions
packList(sb,descriptions,'+');
-
+ // The author names
+ packList(sb,authorNames,'+');
+ // The author emails
+ packList(sb,authorEmails,'+');
}
else
{
@@ -1322,6 +1337,10 @@ public class RSSConnector extends org.ap
startPos = unpackList(categories,version,startPos,'+');
ArrayList descriptions = new ArrayList();
startPos = unpackList(descriptions,version,startPos,'+');
+ ArrayList authorNames = new ArrayList();
+ startPos = unpackList(authorNames,version,startPos,'+');
+ ArrayList authorEmails = new ArrayList();
+ startPos = unpackList(authorEmails,version,startPos,'+');
if (ingestURL.length() > 0)
{
@@ -1389,6 +1408,28 @@ public class RSSConnector extends org.ap
}
if (k > 0)
rd.addField("title",titleValues);
+
+ // Loop through the author names to add those to the metadata
+ String[] authorNameValues = new String[authorNames.size()];
+ k = 0;
+ while (k < authorNameValues.length)
+ {
+ authorNameValues[k] = (String)authorNames.get(k);
+ k++;
+ }
+ if (k > 0)
+ rd.addField("authorname",authorNameValues);
+
+ // Loop through the author emails to add those to the metadata
+ String[] authorEmailValues = new String[authorEmails.size()];
+ k = 0;
+ while (k < authorEmailValues.length)
+ {
+ authorEmailValues[k] = (String)authorEmails.get(k);
+ k++;
+ }
+ if (k > 0)
+ rd.addField("authoremail",authorEmailValues);
// Loop through the descriptions to add those to the metadata
String[] descriptionValues = new String[descriptions.size()];
@@ -3808,6 +3849,7 @@ public class RSSConnector extends org.ap
protected String pubDateField = null;
protected String titleField = null;
protected String descriptionField = null;
+ protected String authorEmailField = null;
protected ArrayList categoryField = new ArrayList();
protected File contentsFile = null;
@@ -3847,6 +3889,11 @@ public class RSSConnector extends org.ap
// "category" tag
return new
XMLStringParsingContext(theStream,namespace,localName,qName,atts);
}
+ else if (localName.equals("author"))
+ {
+ // "author" tag, which contains email
+ return new
XMLStringParsingContext(theStream,namespace,localName,qName,atts);
+ }
else
{
// Handle potentially longer fields. Both "description" and "content"
fields can potentially be large; they are thus
@@ -3943,6 +3990,10 @@ public class RSSConnector extends org.ap
{
categoryField.add(((XMLStringParsingContext)theContext).getValue());
}
+ else if (theTag.equals("author"))
+ {
+ authorEmailField = ((XMLStringParsingContext)theContext).getValue();
+ }
else
{
// What we want is: (a) if dechromed mode is NONE, just put the
description file in the description field; (b)
@@ -4040,22 +4091,24 @@ public class RSSConnector extends org.ap
if (contentsFile == null && filter.getChromedContentMode() !=
CHROMED_METADATA_ONLY)
{
// It's a reference! Add it.
- String[] dataNames = new
String[]{"pubdate","title","source","category","description"};
+ String[] dataNames = new
String[]{"pubdate","title","source","authoremail","category","description"};
String[][] dataValues = new String[dataNames.length][];
if (origDate != null)
dataValues[0] = new String[]{origDate.toString()};
if (titleField != null)
dataValues[1] = new String[]{titleField};
dataValues[2] = new String[]{documentIdentifier};
- dataValues[3] = new String[categoryField.size()];
+ if (authorEmailField != null)
+ dataValues[3] = new String[]{authorEmailField};
+ dataValues[4] = new String[categoryField.size()];
int q = 0;
while (q < categoryField.size())
{
- (dataValues[3])[q] = (String)categoryField.get(q);
+ (dataValues[4])[q] = (String)categoryField.get(q);
q++;
}
if (descriptionField != null)
- dataValues[4] = new String[]{descriptionField};
+ dataValues[5] = new String[]{descriptionField};
// Add document reference, not including the data to pass
down, but including a description
activities.addDocumentReference(newIdentifier,documentIdentifier,null,dataNames,dataValues,origDate);
}
@@ -4069,30 +4122,32 @@ public class RSSConnector extends org.ap
// Since the dechromed data is available from the feed, the
possibility remains of passing the document
// Now, set up the carrydown info
- String[] dataNames = new
String[]{"pubdate","title","source","category","data","description"};
+ String[] dataNames = new
String[]{"pubdate","title","source","authoremail","category","data","description"};
Object[][] dataValues = new Object[dataNames.length][];
if (origDate != null)
dataValues[0] = new String[]{origDate.toString()};
if (titleField != null)
dataValues[1] = new String[]{titleField};
dataValues[2] = new String[]{documentIdentifier};
- dataValues[3] = new String[categoryField.size()];
+ if (authorEmailField != null)
+ dataValues[3] = new String[]{authorEmailField};
+ dataValues[4] = new String[categoryField.size()];
int q = 0;
while (q < categoryField.size())
{
- (dataValues[3])[q] = (String)categoryField.get(q);
+ (dataValues[4])[q] = (String)categoryField.get(q);
q++;
}
if (descriptionField != null)
- dataValues[5] = new String[]{descriptionField};
+ dataValues[6] = new String[]{descriptionField};
if (contentsFile == null)
{
CharacterInput ci = new NullCharacterInput();
try
{
- dataValues[4] = new Object[]{ci};
+ dataValues[5] = new Object[]{ci};
// Add document reference, including the data to pass
down, and the dechromed content too
activities.addDocumentReference(newIdentifier,documentIdentifier,null,dataNames,dataValues,origDate);
@@ -4108,7 +4163,7 @@ public class RSSConnector extends org.ap
try
{
contentsFile = null;
- dataValues[4] = new Object[]{ci};
+ dataValues[5] = new Object[]{ci};
// Add document reference, including the data to pass
down, and the dechromed content too
activities.addDocumentReference(newIdentifier,documentIdentifier,null,dataNames,dataValues,origDate);
@@ -4254,6 +4309,7 @@ public class RSSConnector extends org.ap
protected String linkField = null;
protected String pubDateField = null;
protected String titleField = null;
+ protected String authorNameField = null;
protected String descriptionField = null;
protected File contentsFile = null;
@@ -4283,6 +4339,11 @@ public class RSSConnector extends org.ap
// "title" tag
return new
XMLStringParsingContext(theStream,namespace,localName,qName,atts);
}
+ else if (localName.equals("creator"))
+ {
+ // "creator" tag (e.g. "dc:creator")
+ return new
XMLStringParsingContext(theStream,namespace,localName,qName,atts);
+ }
else
{
switch (dechromedContentMode)
@@ -4368,6 +4429,10 @@ public class RSSConnector extends org.ap
{
titleField = ((XMLStringParsingContext)theContext).getValue();
}
+ else if (theTag.equals("creator"))
+ {
+ authorNameField = ((XMLStringParsingContext)theContext).getValue();
+ }
else
{
switch (dechromedContentMode)
@@ -4452,15 +4517,17 @@ public class RSSConnector extends org.ap
if (contentsFile == null && filter.getChromedContentMode() !=
CHROMED_METADATA_ONLY)
{
// It's a reference! Add it.
- String[] dataNames = new
String[]{"pubdate","title","source","description"};
+ String[] dataNames = new
String[]{"pubdate","title","source","authorname","description"};
String[][] dataValues = new String[dataNames.length][];
if (origDate != null)
dataValues[0] = new String[]{origDate.toString()};
if (titleField != null)
dataValues[1] = new String[]{titleField};
dataValues[2] = new String[]{documentIdentifier};
+ if (authorNameField != null)
+ dataValues[3] = new String[]{authorNameField};
if (descriptionField != null)
- dataValues[3] = new String[]{descriptionField};
+ dataValues[4] = new String[]{descriptionField};
// Add document reference, including the data to pass down
activities.addDocumentReference(newIdentifier,documentIdentifier,null,dataNames,dataValues,origDate);
@@ -4473,22 +4540,24 @@ public class RSSConnector extends org.ap
// right here.
// Now, set up the carrydown info
- String[] dataNames = new
String[]{"pubdate","title","source","data","description"};
+ String[] dataNames = new
String[]{"pubdate","title","source","authorname","data","description"};
Object[][] dataValues = new Object[dataNames.length][];
if (origDate != null)
dataValues[0] = new String[]{origDate.toString()};
if (titleField != null)
dataValues[1] = new String[]{titleField};
dataValues[2] = new String[]{documentIdentifier};
+ if (authorNameField != null)
+ dataValues[3] = new String[]{authorNameField};
if (descriptionField != null)
- dataValues[4] = new String[]{descriptionField};
+ dataValues[5] = new String[]{descriptionField};
if (contentsFile == null)
{
CharacterInput ci = new NullCharacterInput();
try
{
- dataValues[3] = new Object[]{ci};
+ dataValues[4] = new Object[]{ci};
// Add document reference, including the data to pass
down, and the dechromed content too
activities.addDocumentReference(newIdentifier,documentIdentifier,null,dataNames,dataValues,origDate);
@@ -4504,7 +4573,7 @@ public class RSSConnector extends org.ap
try
{
contentsFile = null;
- dataValues[3] = new Object[]{ci};
+ dataValues[4] = new Object[]{ci};
// Add document reference, including the data to pass
down, and the dechromed content too
activities.addDocumentReference(newIdentifier,documentIdentifier,null,dataNames,dataValues,origDate);
@@ -4650,6 +4719,8 @@ public class RSSConnector extends org.ap
protected List<String> linkField = new ArrayList<String>();
protected String pubDateField = null;
protected String titleField = null;
+ protected String authorNameField = null;
+ protected String authorEmailField = null;
protected ArrayList categoryField = new ArrayList();
protected File contentsFile = null;
protected String descriptionField = null;
@@ -4683,6 +4754,10 @@ public class RSSConnector extends org.ap
// "title" tag
return new
XMLStringParsingContext(theStream,namespace,localName,qName,atts);
}
+ else if (localName.equals("author"))
+ {
+ return new
FeedAuthorContextClass(theStream,namespace,localName,qName,atts);
+ }
else if (localName.equals("category"))
{
String category = atts.get("term");
@@ -4771,6 +4846,12 @@ public class RSSConnector extends org.ap
{
titleField = ((XMLStringParsingContext)theContext).getValue();
}
+ else if (theTag.equals("author"))
+ {
+ FeedAuthorContextClass authorContext =
(FeedAuthorContextClass)theContext;
+ authorEmailField = authorContext.getAuthorEmail();
+ authorNameField = authorContext.getAuthorName();
+ }
else
{
switch (dechromedContentMode)
@@ -4953,6 +5034,69 @@ public class RSSConnector extends org.ap
}
}
+ protected class FeedAuthorContextClass extends XMLParsingContext
+ {
+ protected String authorNameField = null;
+ protected String authorEmailField = null;
+
+ public FeedAuthorContextClass(XMLFuzzyHierarchicalParseState theStream,
String namespace, String localName, String qName, Map<String,String> atts)
+ {
+ super(theStream,namespace,localName,qName,atts);
+ }
+
+ @Override
+ protected XMLParsingContext beginTag(String namespace, String localName,
String qName, Map<String,String> atts)
+ throws ManifoldCFException
+ {
+ if (localName.equals("name"))
+ {
+ // "name" tag
+ return new
XMLStringParsingContext(theStream,namespace,localName,qName,atts);
+ }
+ else if (localName.equals("email"))
+ {
+ // "email" tag
+ return new
XMLStringParsingContext(theStream,namespace,localName,qName,atts);
+ }
+ else
+ {
+ // Skip everything else.
+ return super.beginTag(namespace,localName,qName,atts);
+ }
+ }
+
+ /** Convert the individual sub-fields of the item context into their final
forms */
+ @Override
+ protected void endTag()
+ throws ManifoldCFException
+ {
+ XMLParsingContext theContext = theStream.getContext();
+ String theTag = theContext.getLocalname();
+ if (theTag.equals("name"))
+ {
+ authorNameField = ((XMLStringParsingContext)theContext).getValue();
+ }
+ else if (theTag.equals("email"))
+ {
+ authorEmailField = ((XMLStringParsingContext)theContext).getValue();
+ }
+ else
+ {
+ super.endTag();
+ }
+ }
+
+ public String getAuthorName()
+ {
+ return authorNameField;
+ }
+
+ public String getAuthorEmail()
+ {
+ return authorEmailField;
+ }
+ }
+
protected class UrlsetContextClass extends XMLParsingContext
{
/** The document identifier */
