svn commit: r725627 - in /lucene/solr/trunk/contrib/dataimporthandler: CHANGES.txt src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java

[shalin]

Author: shalin
Date: Thu Dec 11 01:05:39 2008
New Revision: 725627

URL: http://svn.apache.org/viewvc?rev=725627&view=rev
Log:
SOLR-846 -- Reduce memory consumption during delta import by removing keys when 
used

Modified:
    lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
    
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java

Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=725627&r1=725626&r2=725627&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Thu Dec 11 01:05:39 
2008
@@ -35,11 +35,13 @@
 5. SOLR-887:  A Transformer to strip HTML tags.
               (Ahmed Hammad via shalin)
 
-6. SOLR-886: DataImportHandler should rollback when an import fails or it is 
aborted (shalin)
+6. SOLR-886:  DataImportHandler should rollback when an import fails or it is 
aborted
+              (shalin)
 
 Optimizations
 ----------------------
-
+1. SOLR-846:  Reduce memory consumption during delta import by removing keys 
when used
+              (Ricky Leung, Noble Paul via shalin)
 
 Bug Fixes
 ----------------------

Modified: 
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: 
http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=725627&r1=725626&r2=725627&view=diff
==============================================================================
--- 
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
 (original)
+++ 
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
 Thu Dec 11 01:05:39 2008
@@ -207,12 +207,16 @@
       // Make sure that documents are not re-created
       allPks.removeAll(deletedKeys);
     }
+    deletedKeys = null;
 
     statusMessages.put("Total Changed Documents", allPks.size());
-    for (Map<String, Object> pk : allPks) {
-      VariableResolverImpl vri = getVariableResolver(dataImporter);
-      vri.addNamespace(DataConfig.IMPORTER_NS + ".delta", pk);
-      buildDocument(vri, null, pk, root, true, null);
+    VariableResolverImpl vri = getVariableResolver(dataImporter);
+    Iterator<Map<String, Object>> pkIter = allPks.iterator();
+    while (pkIter.hasNext()) {
+      Map<String, Object> map = pkIter.next();
+      vri.addNamespace(DataConfig.IMPORTER_NS + ".delta", map);
+      buildDocument(vri, null, map, root, true, null);
+      pkIter.remove();
     }
 
     if (!stop.get()) {
@@ -223,8 +227,11 @@
 
   private void deleteAll(Set<Map<String, Object>> deletedKeys) {
     LOG.info("Deleting stale documents ");
-    for (Map<String, Object> deletedKey : deletedKeys) {
-      writer.deleteDoc(deletedKey.get(root.pk));
+    Iterator<Map<String, Object>> iter = deletedKeys.iterator();
+    while (iter.hasNext()) {
+      Map<String, Object> map = iter.next();
+      writer.deleteDoc(map.get(root.pk));
+      iter.remove();
     }
   }