Author: kwright
Date: Fri Jun 28 00:19:33 2013
New Revision: 1497617
URL: http://svn.apache.org/r1497617
Log:
Changes for CONNECTORS-735. Add crawl data in the standard RepositoryDocument
data set, and modify incremental indexer and Solr output connector to transmit
it to Solr.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1497617&r1=1497616&r2=1497617&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Fri Jun 28 00:19:33 2013
@@ -3,11 +3,15 @@ $Id$
======================= 1.3-dev =====================
+CONNECTORS-735: Include crawl date in output. Modified Solr connector
+to allow you to specify indexeddate attribute name.
+(Stephane Gamard, Karl Wright)
+
CONNECTORS-728: Add HDFS connector
-{Minoru Osuka, Karl Wright)
+(Minoru Osuka, Karl Wright)
CONNECTORS-727: Implemented generic API connector
-{Maciej Lizewski, Karl Wright)
+(Maciej Lizewski, Karl Wright)
CONNECTORS-734: Catch deadlock error with EXPLAIN ANALYZE in
postgresql, and ignore it.
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1497617&r1=1497616&r2=1497617&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Fri Jun 28 00:19:33 2013
@@ -107,6 +107,7 @@ public class HttpPoster
private String idAttributeName;
private String modifiedDateAttributeName;
private String createdDateAttributeName;
+ private String indexedDateAttributeName;
private String fileNameAttributeName;
private String mimeTypeAttributeName;
@@ -131,7 +132,7 @@ public class HttpPoster
int zkClientTimeout, int zkConnectTimeout,
String updatePath, String removePath, String statusPath,
String allowAttributeName, String denyAttributeName, String
idAttributeName,
- String modifiedDateAttributeName, String createdDateAttributeName,
+ String modifiedDateAttributeName, String createdDateAttributeName, String
indexedDateAttributeName,
String fileNameAttributeName, String mimeTypeAttributeName,
Long maxDocumentLength,
String commitWithin)
@@ -149,6 +150,7 @@ public class HttpPoster
this.idAttributeName = idAttributeName;
this.modifiedDateAttributeName = modifiedDateAttributeName;
this.createdDateAttributeName = createdDateAttributeName;
+ this.indexedDateAttributeName = indexedDateAttributeName;
this.fileNameAttributeName = fileNameAttributeName;
this.mimeTypeAttributeName = mimeTypeAttributeName;
@@ -176,7 +178,7 @@ public class HttpPoster
String updatePath, String removePath, String statusPath,
String realm, String userID, String password,
String allowAttributeName, String denyAttributeName, String
idAttributeName,
- String modifiedDateAttributeName, String createdDateAttributeName,
+ String modifiedDateAttributeName, String createdDateAttributeName, String
indexedDateAttributeName,
String fileNameAttributeName, String mimeTypeAttributeName,
IKeystoreManager keystoreManager, Long maxDocumentLength,
String commitWithin)
@@ -194,6 +196,7 @@ public class HttpPoster
this.idAttributeName = idAttributeName;
this.modifiedDateAttributeName = modifiedDateAttributeName;
this.createdDateAttributeName = createdDateAttributeName;
+ this.indexedDateAttributeName = indexedDateAttributeName;
this.fileNameAttributeName = fileNameAttributeName;
this.mimeTypeAttributeName = mimeTypeAttributeName;
@@ -832,6 +835,13 @@ public class HttpPoster
// Write value
writeField(out,LITERAL+createdDateAttributeName,DateParser.formatISO8601Date(date));
}
+ if (indexedDateAttributeName != null)
+ {
+ Date date = document.getIndexingDate();
+ if (date != null)
+ // Write value
+
writeField(out,LITERAL+indexedDateAttributeName,DateParser.formatISO8601Date(date));
+ }
if (fileNameAttributeName != null)
{
String fileName = document.getFileName();
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1497617&r1=1497616&r2=1497617&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
Fri Jun 28 00:19:33 2013
@@ -87,6 +87,8 @@ public class SolrConfig
public static final String PARAM_MODIFIEDDATEFIELD = "Solr modified date
field name";
/** Optional created date field */
public static final String PARAM_CREATEDDATEFIELD = "Solr created date field
name";
+ /** Optional indexed date field */
+ public static final String PARAM_INDEXEDDATEFIELD = "Solr indexed date field
name";
/** Optional file name field */
public static final String PARAM_FILENAMEFIELD = "Solr filename field name";
/** Optional mime type field */
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1497617&r1=1497616&r2=1497617&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Fri Jun 28 00:19:33 2013
@@ -160,6 +160,10 @@ public class SolrConnector extends org.a
if (createdDateAttributeName == null ||
createdDateAttributeName.length() == 0)
createdDateAttributeName = null;
+ String indexedDateAttributeName =
params.getParameter(SolrConfig.PARAM_INDEXEDDATEFIELD);
+ if (indexedDateAttributeName == null ||
indexedDateAttributeName.length() == 0)
+ indexedDateAttributeName = null;
+
String fileNameAttributeName =
params.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
if (fileNameAttributeName == null || fileNameAttributeName.length() == 0)
fileNameAttributeName = null;
@@ -273,7 +277,7 @@ public class SolrConnector extends org.a
connectTimeout,socketTimeout,
updatePath,removePath,statusPath,realm,userID,password,
allowAttributeName,denyAttributeName,idAttributeName,
- modifiedDateAttributeName,createdDateAttributeName,
+
modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
fileNameAttributeName,mimeTypeAttributeName,
keystoreManager,maxDocumentLength,commitWithin);
@@ -328,7 +332,7 @@ public class SolrConnector extends org.a
zkClientTimeout,zkConnectTimeout,
updatePath,removePath,statusPath,
allowAttributeName,denyAttributeName,idAttributeName,
- modifiedDateAttributeName,createdDateAttributeName,
+
modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
fileNameAttributeName,mimeTypeAttributeName,
maxDocumentLength,commitWithin);
@@ -1059,6 +1063,10 @@ public class SolrConnector extends org.a
String createdDateField =
parameters.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
if (createdDateField == null)
createdDateField = "";
+
+ String indexedDateField =
parameters.getParameter(SolrConfig.PARAM_INDEXEDDATEFIELD);
+ if (indexedDateField == null)
+ indexedDateField = "";
String fileNameField =
parameters.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
if (fileNameField == null)
@@ -1503,6 +1511,12 @@ public class SolrConnector extends org.a
" </td>\n"+
" </tr>\n"+
" <tr>\n"+
+" <td class=\"description\"><nobr>" +
Messages.getBodyString(locale,"SolrConnector.IndexedDateFieldName") +
"</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"indexeddatefield\" type=\"text\" size=\"32\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(indexedDateField)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
+" <tr>\n"+
" <td class=\"description\"><nobr>" +
Messages.getBodyString(locale,"SolrConnector.FileNameFieldName") +
"</nobr></td>\n"+
" <td class=\"value\">\n"+
" <input name=\"filenamefield\" type=\"text\" size=\"32\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(fileNameField)+"\"/>\n"+
@@ -1523,6 +1537,7 @@ public class SolrConnector extends org.a
"<input type=\"hidden\" name=\"idfield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(idField)+"\"/>\n"+
"<input type=\"hidden\" name=\"modifieddatefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(modifiedDateField)+"\"/>\n"+
"<input type=\"hidden\" name=\"createddatefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(createdDateField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"indexeddatefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(indexedDateField)+"\"/>\n"+
"<input type=\"hidden\" name=\"filenamefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(fileNameField)+"\"/>\n"+
"<input type=\"hidden\" name=\"mimetypefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(mimeTypeField)+"\"/>\n"
);
@@ -1812,6 +1827,10 @@ public class SolrConnector extends org.a
if (createdDateField != null)
parameters.setParameter(SolrConfig.PARAM_CREATEDDATEFIELD,createdDateField);
+ String indexedDateField = variableContext.getParameter("indexeddatefield");
+ if (indexedDateField != null)
+
parameters.setParameter(SolrConfig.PARAM_INDEXEDDATEFIELD,indexedDateField);
+
String fileNameField = variableContext.getParameter("filenamefield");
if (fileNameField != null)
parameters.setParameter(SolrConfig.PARAM_FILENAMEFIELD,fileNameField);
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties?rev=1497617&r1=1497616&r2=1497617&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
Fri Jun 28 00:19:33 2013
@@ -54,6 +54,7 @@ SolrConnector.Certificate=Certificate:
SolrConnector.IDFieldName=ID field name:
SolrConnector.ModifiedDateFieldName=Modified date field name:
SolrConnector.CreatedDateFieldName=Created date field name:
+SolrConnector.IndexedDateFieldName=Indexed date field name:
SolrConnector.FileNameFieldName=File name field name:
SolrConnector.MimeTypeFieldName=Mime type field name:
SolrConnector.MaximumDocumentLength=Maximum document length:
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties?rev=1497617&r1=1497616&r2=1497617&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
Fri Jun 28 00:19:33 2013
@@ -54,6 +54,7 @@ SolrConnector.Certificate=証æè
SolrConnector.IDFieldName=IDãã£ã¼ã«ãåï¼
SolrConnector.ModifiedDateFieldName=æ´æ°æ¥ä»ãã£ã¼ã«ãåï¼
SolrConnector.CreatedDateFieldName=ä½ææ¥ä»ãã£ã¼ã«ãåï¼
+SolrConnector.IndexedDateFieldName=Indexed date field name:
SolrConnector.FileNameFieldName=ãã¡ã¤ã«å称ãã£ã¼ã«ãåï¼
SolrConnector.MimeTypeFieldName=MIMEã¿ã¤ããã£ã¼ã«ãåï¼
SolrConnector.MaximumDocumentLength=æ大ã³ã³ãã³ãé·ï¼
Modified:
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1497617&r1=1497616&r2=1497617&view=diff
==============================================================================
---
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Fri Jun 28 00:19:33 2013
@@ -1647,6 +1647,8 @@ public class IncrementalIngester extends
IOutputAddActivity activities)
throws ManifoldCFException, ServiceInterruption
{
+ // Set indexing date
+ document.setIndexingDate(new Date());
IOutputConnector connector =
OutputConnectorFactory.grab(threadContext,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
if (connector == null)
// The connector is not installed; treat this as a service interruption.
Modified:
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java?rev=1497617&r1=1497616&r2=1497617&view=diff
==============================================================================
---
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
(original)
+++
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Fri Jun 28 00:19:33 2013
@@ -50,6 +50,7 @@ public class RepositoryDocument
protected String contentMimeType = "application/octet-stream";
protected Date createdDate = null;
protected Date modifiedDate = null;
+ protected Date indexingDate = null;
/** Constructor.
*/
@@ -88,6 +89,22 @@ public class RepositoryDocument
{
return modifiedDate;
}
+
+ /** Set the document's indexing date. Use null to indicate that the date is
unknown.
+ *@param date is the date.
+ */
+ public void setIndexingDate(Date date)
+ {
+ indexingDate = date;
+ }
+
+ /** Get the document's indexing date. Returns null of the date is unknown.
+ *@return the date.
+ */
+ public Date getIndexingDate()
+ {
+ return indexingDate;
+ }
/** Set the document's mime type.
*@param mimeType is the mime type.
