svn commit: r770467 - in /lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model: GenericDataModel.java file/FileDataModel.java

[srowen]

Author: srowen
Date: Thu Apr 30 22:09:22 2009
New Revision: 770467

URL: http://svn.apache.org/viewvc?rev=770467&view=rev
Log:
Traded a bit of processing time for better memory usage in FileDataModel

Modified:
    
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java
    
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java

Modified: 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java
URL: 
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java?rev=770467&r1=770466&r2=770467&view=diff
==============================================================================
--- 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java
 (original)
+++ 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java
 Thu Apr 30 22:09:22 2009
@@ -29,6 +29,8 @@
 import org.apache.mahout.cf.taste.model.Item;
 import org.apache.mahout.cf.taste.model.Preference;
 import org.apache.mahout.cf.taste.model.User;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -46,6 +48,8 @@
  */
 public final class GenericDataModel implements DataModel, Serializable {
 
+  private static final Logger log = 
LoggerFactory.getLogger(GenericDataModel.class);
+
   private static final Preference[] NO_PREFS_ARRAY = new Preference[0];
   private static final Iterable<Preference> NO_PREFS_ITERABLE = new 
EmptyIterable<Preference>();
 
@@ -72,6 +76,7 @@
     // I'm abusing generics a little here since I want to use this (huge) map 
to hold Lists,
     // then arrays, and don't want to allocate two Maps at once here.
     Map<Object, Object> prefsForItems = new FastMap<Object, Object>();
+    int currentCount = 0;
     for (User user : users) {
       userMap.put(user.getID(), user);
       Preference[] prefsArray = user.getPreferencesAsArray();
@@ -86,6 +91,10 @@
         }
         prefsForItem.add(preference);
       }
+      currentCount++;
+      if (currentCount % 10000 == 0) {
+        log.info("Processed {} users", currentCount);
+      }
     }
 
     List<User> usersCopy = new ArrayList<User>(userMap.values());

Modified: 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
URL: 
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java?rev=770467&r1=770466&r2=770467&view=diff
==============================================================================
--- 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
 (original)
+++ 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
 Thu Apr 30 22:09:22 2009
@@ -228,7 +228,7 @@
     }
     List<Preference> prefs = data.get(userID);
     if (prefs == null) {
-      prefs = new ArrayList<Preference>();
+      prefs = new ArrayList<Preference>(2);
       data.put(userID, prefs);
     }
 
@@ -433,7 +433,10 @@
     @Override
     public User next() {
       Map.Entry<String, List<Preference>> datum = dataIterator.next();
-      return buildUser(datum.getKey(), datum.getValue());
+      String key = datum.getKey();
+      List<Preference> value = datum.getValue();
+      dataIterator.remove();
+      return buildUser(key, value);
     }
     @Override
     public void remove() {