Author: srowen
Date: Wed Aug 6 20:23:39 2008
New Revision: 683487
URL: http://svn.apache.org/viewvc?rev=683487&view=rev
Log:
PearsonCorrelation now caches num items/users for efficiency, at some tradeoff
for accuracy; introduced Weighting enum to clarify API calls that use a
'weighted' param. Other minor cleanup.
Added:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/common/Weighting.java
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/AbstractCorrelation.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelation.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelation.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelation.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/SlopeOneRecommender.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelationTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelationTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelationTest.java
Added:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/common/Weighting.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/common/Weighting.java?rev=683487&view=auto
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/common/Weighting.java
(added)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/common/Weighting.java
Wed Aug 6 20:23:39 2008
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.cf.taste.common;
+
+/**
+ * <p>A simple enum which gives symbolic names to the ideas of "weighted" and
"unweighted",
+ * to make various API calls which take a weighting parameter more
readable.</p>
+ */
+public enum Weighting {
+
+ WEIGHTED,
+ UNWEIGHTED
+
+}
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/AbstractCorrelation.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/AbstractCorrelation.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/AbstractCorrelation.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/AbstractCorrelation.java
Wed Aug 6 20:23:39 2008
@@ -27,6 +27,7 @@
import org.apache.mahout.cf.taste.transforms.CorrelationTransform;
import org.apache.mahout.cf.taste.transforms.PreferenceTransform;
import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.common.Weighting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,28 +45,27 @@
private PreferenceTransform prefTransform;
private CorrelationTransform<Object> correlationTransform;
private boolean weighted;
+ private int cachedNumItems;
+ private int cachedNumUsers;
/**
- * <p>Creates a normal (unweighted) [EMAIL PROTECTED]
org.apache.mahout.cf.taste.impl.correlation.PearsonCorrelation}.</p>
- *
- * @param dataModel
+ * <p>Creates a normal (unweighted) [EMAIL PROTECTED]
AbstractCorrelation}.</p>
*/
- AbstractCorrelation(DataModel dataModel) {
- this(dataModel, false);
+ AbstractCorrelation(DataModel dataModel) throws TasteException {
+ this(dataModel, Weighting.UNWEIGHTED);
}
/**
- * <p>Creates a weighted [EMAIL PROTECTED]
org.apache.mahout.cf.taste.impl.correlation.PearsonCorrelation}.</p>
- *
- * @param dataModel
- * @param weighted
+ * <p>Creates a possibly weighted [EMAIL PROTECTED] AbstractCorrelation}.</p>
*/
- AbstractCorrelation(DataModel dataModel, boolean weighted) {
+ AbstractCorrelation(DataModel dataModel, Weighting weighting) throws
TasteException {
if (dataModel == null) {
throw new IllegalArgumentException("dataModel is null");
}
this.dataModel = dataModel;
- this.weighted = weighted;
+ this.weighted = weighting == Weighting.WEIGHTED;
+ this.cachedNumItems = dataModel.getNumItems();
+ this.cachedNumUsers = dataModel.getNumUsers();
}
final DataModel getDataModel() {
@@ -232,7 +232,7 @@
}
if (!Double.isNaN(result)) {
- result = normalizeWeightResult(result, count, dataModel.getNumItems());
+ result = normalizeWeightResult(result, count, cachedNumItems);
}
if (log.isTraceEnabled()) {
@@ -320,7 +320,7 @@
}
if (!Double.isNaN(result)) {
- result = normalizeWeightResult(result, count, dataModel.getNumUsers());
+ result = normalizeWeightResult(result, count, cachedNumUsers);
}
if (log.isTraceEnabled()) {
@@ -349,6 +349,13 @@
public final void refresh() {
dataModel.refresh();
+ try {
+ cachedNumItems = dataModel.getNumItems();
+ cachedNumUsers = dataModel.getNumUsers();
+ } catch (TasteException te) {
+ // hmm, continue?
+ log.warn("Unable to refresh number of users and items", te);
+ }
if (inferrer != null) {
inferrer.refresh();
}
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelation.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelation.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelation.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelation.java
Wed Aug 6 20:23:39 2008
@@ -18,6 +18,8 @@
package org.apache.mahout.cf.taste.impl.correlation;
import org.apache.mahout.cf.taste.model.DataModel;
+import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.common.Weighting;
/**
* <p>An implementation of a "correlation" based on the Euclidean "distance"
between two
@@ -30,12 +32,12 @@
*/
public final class EuclideanDistanceCorrelation extends AbstractCorrelation {
- public EuclideanDistanceCorrelation(DataModel dataModel) {
+ public EuclideanDistanceCorrelation(DataModel dataModel) throws
TasteException {
super(dataModel);
}
- public EuclideanDistanceCorrelation(DataModel dataModel, boolean weighted) {
- super(dataModel, weighted);
+ public EuclideanDistanceCorrelation(DataModel dataModel, Weighting
weighting) throws TasteException {
+ super(dataModel, weighting);
}
double computeResult(int n, double sumXY, double sumX2, double sumY2, double
sumXYdiff2) {
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelation.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelation.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelation.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelation.java
Wed Aug 6 20:23:39 2008
@@ -20,6 +20,8 @@
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Item;
import org.apache.mahout.cf.taste.model.User;
+import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.common.Weighting;
/**
* <p>An implementation of the Pearson correlation. For [EMAIL PROTECTED]
User}s X and Y, the following values
@@ -47,12 +49,12 @@
*/
public final class PearsonCorrelation extends AbstractCorrelation {
- public PearsonCorrelation(DataModel dataModel) {
+ public PearsonCorrelation(DataModel dataModel) throws TasteException {
super(dataModel);
}
- public PearsonCorrelation(DataModel dataModel, boolean weighted) {
- super(dataModel, weighted);
+ public PearsonCorrelation(DataModel dataModel, Weighting weighting) throws
TasteException {
+ super(dataModel, weighting);
}
double computeResult(int n, double sumXY, double sumX2, double sumY2, double
sumXYdiff2) {
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelation.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelation.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelation.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelation.java
Wed Aug 6 20:23:39 2008
@@ -41,7 +41,7 @@
private final UserCorrelation rankingUserCorrelation;
private final ReentrantLock refreshLock;
- public SpearmanCorrelation(DataModel dataModel) {
+ public SpearmanCorrelation(DataModel dataModel) throws TasteException {
if (dataModel == null) {
throw new IllegalArgumentException("dataModel is null");
}
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
Wed Aug 6 20:23:39 2008
@@ -33,7 +33,6 @@
import java.io.File;
import java.io.FileNotFoundException;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -128,7 +127,7 @@
}
String userID = line.substring(0, commaOne);
String itemID = line.substring(commaOne + 1, commaTwo);
- double preferenceValue = Double.valueOf(line.substring(commaTwo + 1));
+ double preferenceValue = Double.parseDouble(line.substring(commaTwo + 1));
List<Preference> prefs = data.get(userID);
if (prefs == null) {
prefs = new ArrayList<Preference>();
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/SlopeOneRecommender.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/SlopeOneRecommender.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/SlopeOneRecommender.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/SlopeOneRecommender.java
Wed Aug 6 20:23:39 2008
@@ -18,6 +18,7 @@
package org.apache.mahout.cf.taste.impl.recommender.slopeone;
import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.common.Weighting;
import org.apache.mahout.cf.taste.impl.common.RunningAverage;
import org.apache.mahout.cf.taste.impl.common.RunningAverageAndStdDev;
import org.apache.mahout.cf.taste.impl.recommender.AbstractRecommender;
@@ -58,7 +59,10 @@
* @param dataModel data model
*/
public SlopeOneRecommender(DataModel dataModel) throws TasteException {
- this(dataModel, true, true, new MemoryDiffStorage(dataModel, true, false,
Long.MAX_VALUE));
+ this(dataModel,
+ Weighting.WEIGHTED,
+ Weighting.WEIGHTED,
+ new MemoryDiffStorage(dataModel, true, false, Long.MAX_VALUE));
}
/**
@@ -69,25 +73,24 @@
* item-item ratings diffs with lower standard deviation more highly, on the
theory that they are more
* reliable.</p>
*
- * @param dataModel
- * @param weighted if <code>true</code>, acts as a weighted slope one
recommender
- * @param stdDevWeighted use optional standard deviation weighting of diffs
+ * @param weighting if [EMAIL PROTECTED] Weighting#WEIGHTED}, acts as a
weighted slope one recommender
+ * @param stdDevWeighting use optional standard deviation weighting of diffs
* @throws IllegalArgumentException if <code>diffStorage</code> is null, or
stdDevWeighted is set
* when weighted is not set
*/
public SlopeOneRecommender(DataModel dataModel,
- boolean weighted,
- boolean stdDevWeighted,
+ Weighting weighting,
+ Weighting stdDevWeighting,
DiffStorage diffStorage) {
super(dataModel);
- if (stdDevWeighted && !weighted) {
+ if (stdDevWeighting == Weighting.WEIGHTED && weighting ==
Weighting.UNWEIGHTED) {
throw new IllegalArgumentException("weighted required when
stdDevWeighted is set");
}
if (diffStorage == null) {
throw new IllegalArgumentException("diffStorage is null");
}
- this.weighted = weighted;
- this.stdDevWeighted = stdDevWeighted;
+ this.weighted = weighting == Weighting.WEIGHTED;
+ this.stdDevWeighted = stdDevWeighting == Weighting.WEIGHTED;
this.diffStorage = diffStorage;
}
Modified:
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelationTest.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelationTest.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelationTest.java
(original)
+++
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/EuclideanDistanceCorrelationTest.java
Wed Aug 6 20:23:39 2008
@@ -17,6 +17,7 @@
package org.apache.mahout.cf.taste.impl.correlation;
+import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.correlation.ItemCorrelation;
import org.apache.mahout.cf.taste.impl.model.GenericItem;
import org.apache.mahout.cf.taste.impl.model.GenericPreference;
@@ -24,6 +25,7 @@
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Preference;
import org.apache.mahout.cf.taste.model.User;
+import org.apache.mahout.cf.taste.common.Weighting;
import java.util.Collections;
@@ -44,7 +46,7 @@
User user1 = getUser("test1", 3.0, -2.0);
User user2 = getUser("test2", 3.0, -2.0);
DataModel dataModel = getDataModel(user1, user2);
- double correlation = new EuclideanDistanceCorrelation(dataModel,
true).userCorrelation(user1, user2);
+ double correlation = new EuclideanDistanceCorrelation(dataModel,
Weighting.WEIGHTED).userCorrelation(user1, user2);
assertCorrelationEquals(1.0, correlation);
}
@@ -68,7 +70,7 @@
User user1 = getUser("test1", 3.0, -2.0);
User user2 = getUser("test2", -3.0, 2.0);
DataModel dataModel = getDataModel(user1, user2);
- double correlation = new EuclideanDistanceCorrelation(dataModel,
true).userCorrelation(user1, user2);
+ double correlation = new EuclideanDistanceCorrelation(dataModel,
Weighting.WEIGHTED).userCorrelation(user1, user2);
assertCorrelationEquals(0.8081551272944483, correlation);
}
@@ -102,7 +104,7 @@
User user1 = getUser("test1", 1.0, 2.0, 3.0);
User user2 = getUser("test2", 2.0, 5.0, 6.0);
DataModel dataModel = getDataModel(user1, user2);
- double correlation = new EuclideanDistanceCorrelation(dataModel,
true).userCorrelation(user1, user2);
+ double correlation = new EuclideanDistanceCorrelation(dataModel,
Weighting.WEIGHTED).userCorrelation(user1, user2);
assertCorrelationEquals(0.8974062142054332, correlation);
}
@@ -170,12 +172,12 @@
User user2 = getUser("test2", 2.0, 5.0);
User user3 = getUser("test3", 3.0, 6.0);
DataModel dataModel = getDataModel(user1, user2, user3);
- ItemCorrelation itemCorrelation = new
EuclideanDistanceCorrelation(dataModel, true);
+ ItemCorrelation itemCorrelation = new
EuclideanDistanceCorrelation(dataModel, Weighting.WEIGHTED);
double correlation =
itemCorrelation.itemCorrelation(dataModel.getItem("0"), dataModel.getItem("1"));
assertCorrelationEquals(0.8974062142054332, correlation);
}
- public void testRefresh() {
+ public void testRefresh() throws TasteException {
// Make sure this doesn't throw an exception
new EuclideanDistanceCorrelation(getDataModel()).refresh();
}
Modified:
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelationTest.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelationTest.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelationTest.java
(original)
+++
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/PearsonCorrelationTest.java
Wed Aug 6 20:23:39 2008
@@ -17,6 +17,7 @@
package org.apache.mahout.cf.taste.impl.correlation;
+import org.apache.mahout.cf.taste.common.Weighting;
import org.apache.mahout.cf.taste.correlation.ItemCorrelation;
import org.apache.mahout.cf.taste.impl.model.GenericItem;
import org.apache.mahout.cf.taste.impl.model.GenericPreference;
@@ -44,7 +45,7 @@
User user1 = getUser("test1", 3.0, -2.0);
User user2 = getUser("test2", 3.0, -2.0);
DataModel dataModel = getDataModel(user1, user2);
- double correlation = new PearsonCorrelation(dataModel,
true).userCorrelation(user1, user2);
+ double correlation = new PearsonCorrelation(dataModel,
Weighting.WEIGHTED).userCorrelation(user1, user2);
assertCorrelationEquals(1.0, correlation);
}
@@ -69,7 +70,7 @@
User user1 = getUser("test1", 3.0, -2.0);
User user2 = getUser("test2", -3.0, 2.0);
DataModel dataModel = getDataModel(user1, user2);
- double correlation = new PearsonCorrelation(dataModel,
true).userCorrelation(user1, user2);
+ double correlation = new PearsonCorrelation(dataModel,
Weighting.WEIGHTED).userCorrelation(user1, user2);
assertCorrelationEquals(-1.0, correlation);
}
@@ -103,7 +104,7 @@
User user1 = getUser("test1", 1.0, 2.0, 3.0);
User user2 = getUser("test2", 2.0, 5.0, 6.0);
DataModel dataModel = getDataModel(user1, user2);
- double correlation = new PearsonCorrelation(dataModel,
true).userCorrelation(user1, user2);
+ double correlation = new PearsonCorrelation(dataModel,
Weighting.WEIGHTED).userCorrelation(user1, user2);
assertCorrelationEquals(0.9901922307076306, correlation);
}
@@ -171,12 +172,12 @@
User user2 = getUser("test2", 2.0, 5.0);
User user3 = getUser("test3", 3.0, 6.0);
DataModel dataModel = getDataModel(user1, user2, user3);
- ItemCorrelation itemCorrelation = new PearsonCorrelation(dataModel, true);
+ ItemCorrelation itemCorrelation = new PearsonCorrelation(dataModel,
Weighting.WEIGHTED);
double correlation =
itemCorrelation.itemCorrelation(dataModel.getItem("0"), dataModel.getItem("1"));
assertCorrelationEquals(0.9901922307076306, correlation);
}
- public void testRefresh() {
+ public void testRefresh() throws Exception {
// Make sure this doesn't throw an exception
new PearsonCorrelation(getDataModel()).refresh();
}
Modified:
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelationTest.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelationTest.java?rev=683487&r1=683486&r2=683487&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelationTest.java
(original)
+++
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/correlation/SpearmanCorrelationTest.java
Wed Aug 6 20:23:39 2008
@@ -19,6 +19,7 @@
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.User;
+import org.apache.mahout.cf.taste.common.TasteException;
/**
* <p>Tests [EMAIL PROTECTED] SpearmanCorrelation}.</p>
@@ -57,7 +58,7 @@
assertCorrelationEquals(-0.5, correlation);
}
- public void testRefresh() {
+ public void testRefresh() throws TasteException {
// Make sure this doesn't throw an exception
new SpearmanCorrelation(getDataModel()).refresh();
}
