Author: ssc
Date: Tue Jul 10 15:16:47 2012
New Revision: 1359722
URL: http://svn.apache.org/viewvc?rev=1359722&view=rev
Log:
MAHOUT-1042 Hotspot in RecommenderJob-PartialMultiplyMapper-Reducer
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/AggregateAndRecommendReducer.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java
mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/AggregateAndRecommendReducer.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/AggregateAndRecommendReducer.java?rev=1359722&r1=1359721&r2=1359722&view=diff
==============================================================================
---
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/AggregateAndRecommendReducer.java
(original)
+++
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/AggregateAndRecommendReducer.java
Tue Jul 10 15:16:47 2012
@@ -33,6 +33,7 @@ import org.apache.mahout.math.RandomAcce
import org.apache.mahout.math.VarLongWritable;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.function.DoubleFunction;
+import org.apache.mahout.math.function.Functions;
import org.apache.mahout.math.map.OpenIntLongHashMap;
import java.io.IOException;
@@ -122,13 +123,12 @@ public final class AggregateAndRecommend
/* having boolean data, each estimated preference can only be 1,
* however we can't use this to rank the recommended items,
* so we use the sum of similarities for that. */
- Vector predictionVector = null;
- for (PrefAndSimilarityColumnWritable prefAndSimilarityColumn : values) {
- predictionVector = predictionVector == null
- ? prefAndSimilarityColumn.getSimilarityColumn()
- :
predictionVector.plus(prefAndSimilarityColumn.getSimilarityColumn());
+ Iterator<PrefAndSimilarityColumnWritable> columns = values.iterator();
+ Vector predictions = columns.next().getSimilarityColumn();
+ while (columns.hasNext()) {
+ predictions.assign(columns.next().getSimilarityColumn(), Functions.PLUS);
}
- writeRecommendedItems(userID, predictionVector, context);
+ writeRecommendedItems(userID, predictions, context);
}
private void reduceNonBooleanData(VarLongWritable userID,
@@ -151,12 +151,25 @@ public final class AggregateAndRecommend
numberOfSimilarItemsUsed.setQuick(itemIDIndex,
numberOfSimilarItemsUsed.getQuick(itemIDIndex) + 1);
}
- numerators = numerators == null
- ? prefValue == BOOLEAN_PREF_VALUE ? simColumn.clone() :
simColumn.times(prefValue)
- : numerators.plus(prefValue == BOOLEAN_PREF_VALUE ? simColumn :
simColumn.times(prefValue));
+ if (denominators == null) {
+ denominators = simColumn.clone();
+ } else {
+ denominators.assign(simColumn, Functions.PLUS_ABS);
+ }
+
+ if (numerators == null) {
+ numerators = simColumn.clone();
+ if (prefValue != BOOLEAN_PREF_VALUE) {
+ numerators.assign(Functions.MULT, prefValue);
+ }
+ } else {
+ Vector toAdd = simColumn;
+ if (prefValue != BOOLEAN_PREF_VALUE) {
+ toAdd.assign(Functions.MULT, prefValue);
+ }
+ numerators.assign(toAdd, Functions.PLUS);
+ }
- simColumn.assign(ABSOLUTE_VALUES);
- denominators = denominators == null ? simColumn :
denominators.plus(simColumn);
}
if (numerators == null) {
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java?rev=1359722&r1=1359721&r2=1359722&view=diff
==============================================================================
---
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
(original)
+++
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
Tue Jul 10 15:16:47 2012
@@ -194,7 +194,6 @@ public final class RecommenderJob extend
Job prePartialMultiply1 = prepareJob(
similarityMatrixPath, prePartialMultiplyPath1,
SequenceFileInputFormat.class,
SimilarityMatrixRowWrapperMapper.class, VarIntWritable.class,
VectorOrPrefWritable.class,
- Reducer.class, VarIntWritable.class, VectorOrPrefWritable.class,
SequenceFileOutputFormat.class);
boolean succeeded = prePartialMultiply1.waitForCompletion(true);
if (!succeeded) {
@@ -207,9 +206,6 @@ public final class RecommenderJob extend
UserVectorSplitterMapper.class,
VarIntWritable.class,
VectorOrPrefWritable.class,
- Reducer.class,
- VarIntWritable.class,
- VectorOrPrefWritable.class,
SequenceFileOutputFormat.class);
if (usersFile != null) {
prePartialMultiply2.getConfiguration().set(UserVectorSplitterMapper.USERS_FILE,
usersFile);
Modified:
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java?rev=1359722&r1=1359721&r2=1359722&view=diff
==============================================================================
---
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java
(original)
+++
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java
Tue Jul 10 15:16:47 2012
@@ -829,7 +829,7 @@ public class RecommenderJobTest extends
* check whether the explicit user/item filter works
*/
@Test
- public void testCompleteJobWithFiltering() throws Exception {
+ public void testCompleteJobWithFiltering() throws Exception {
File inputFile = getTestTempFile("prefs.txt");
File userFile = getTestTempFile("users.txt");
Modified:
mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java?rev=1359722&r1=1359721&r2=1359722&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java
(original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java
Tue Jul 10 15:16:47 2012
@@ -553,7 +553,7 @@ public abstract class AbstractVector imp
}
/* special case: we only need to iterate over the non-zero elements of the
vector to add */
- if (Functions.PLUS.equals(function)) {
+ if (Functions.PLUS.equals(function) ||
Functions.PLUS_ABS.equals(function)) {
Iterator<Vector.Element> nonZeroElements = other.iterateNonZero();
while (nonZeroElements.hasNext()) {
Vector.Element e = nonZeroElements.next();
