svn commit: r1097391 - in /nutch/trunk: CHANGES.txt src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java

[markus]

Author: markus
Date: Thu Apr 28 09:58:26 2011
New Revision: 1097391

URL: http://svn.apache.org/viewvc?rev=1097391&view=rev
Log:
NUTCH-986 Dedup fails due to incorrect date format

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1097391&r1=1097390&r2=1097391&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Apr 28 09:58:26 2011
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Release 2.0 - Current Development
 
+* NUTCH 986 SolrDedup fails due to date incorrect format (markus)
+
 * NUTCH-977 SolrMappingReader uses hardcoded configuration parameter name for 
mapping file (markus)
 
 * NUTCH-976 Rename properties solrindex.* to solr.* (markus)

Modified: 
nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java?rev=1097391&r1=1097390&r2=1097391&view=diff
==============================================================================
--- 
nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java 
(original)
+++ 
nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java 
Thu Apr 28 09:58:26 2011
@@ -20,7 +20,10 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 
@@ -118,7 +121,15 @@ implements Tool {
     public void readSolrDocument(SolrDocument doc) {
       id = (String)doc.getFieldValue(SolrConstants.ID_FIELD);
       boost = (Float)doc.getFieldValue(SolrConstants.BOOST_FIELD);
-      tstamp = (Long)doc.getFieldValue(SolrConstants.TIMESTAMP_FIELD);
+
+      // Attempt to convert Solr formatted date to internally used long
+      try {
+        DateFormat formatter = new 
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+        Date date = 
(Date)formatter.parse((String)doc.getFieldValue(SolrConstants.TIMESTAMP_FIELD));
+        tstamp = (Long)date.getTime();
+      } catch (Exception e) {
+        LOG.error("Could not convert date to long: " + e);
+      }
     }
 
     @Override