Github user debasish83 commented on a diff in the pull request:
https://github.com/apache/spark/pull/5829#discussion_r29493623
--- Diff:
mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
---
@@ -137,20 +141,113 @@ class MatrixFactorizationModel(
MatrixFactorizationModel.SaveLoadV1_0.save(this, path)
}
+ /**
+ * Recommends topK products for all users.
+ *
+ * @param num how many products to return for every user.
+ * @return [(Int, Array[Rating])] objects, where every tuple contains a
userID and an array of
+ * rating objects which contains the same userId, recommended productID
and a "score" in the
+ * rating field. Semantics of score is same as recommendProducts API
+ */
+ def recommendProductsForUsers(num: Int): RDD[(Int, Array[Rating])] = {
+ MatrixFactorizationModel.recommendForAll(rank, userFeatures,
productFeatures, num).map {
+ case (user, top) =>
+ val ratings = top.map { case (product, rating) => Rating(user,
product, rating) }
+ (user, ratings)
+ }
+ }
+
+
+ /**
+ * Recommends topK users for all products.
+ *
+ * @param num how many users to return for every product.
+ * @return [(Int, Array[Rating])] objects, where every tuple contains a
productID and an array
+ * of rating objects which contains the recommended userId, same
productID and a "score" in the
+ * rating field. Semantics of score is same as recommendUsers API
+ */
+ def recommendUsersForProducts(num: Int): RDD[(Int, Array[Rating])] = {
+ MatrixFactorizationModel.recommendForAll(rank, productFeatures,
userFeatures, num).map {
+ case (product, top) =>
+ val ratings = top.map { case (user, rating) => Rating(user,
product, rating) }
+ (product, ratings)
+ }
+ }
+}
+
+object MatrixFactorizationModel extends Loader[MatrixFactorizationModel] {
+
+ import org.apache.spark.mllib.util.Loader._
+
+ /**
+ * Makes recommendations for a single user (or product).
+ */
private def recommend(
recommendToFeatures: Array[Double],
recommendableFeatures: RDD[(Int, Array[Double])],
num: Int): Array[(Int, Double)] = {
- val scored = recommendableFeatures.map { case (id,features) =>
+ val scored = recommendableFeatures.map { case (id, features) =>
(id, blas.ddot(features.length, recommendToFeatures, 1, features, 1))
}
scored.top(num)(Ordering.by(_._2))
}
-}
-object MatrixFactorizationModel extends Loader[MatrixFactorizationModel] {
+ /**
+ * Makes recommendations for all users (or products).
+ * @param rank rank
+ * @param srcFeatures src features to receive recommendations
+ * @param dstFeatures dst features used to make recommendations
+ * @param num number of recommendations for each record
+ * @return an RDD of (srcId: Int, recommendations), where
recommendations are stored as an array
+ * of (dstId, rating) pairs.
+ */
+ private def recommendForAll(
+ rank: Int,
+ srcFeatures: RDD[(Int, Array[Double])],
+ dstFeatures: RDD[(Int, Array[Double])],
+ num: Int): RDD[(Int, Array[(Int, Double)])] = {
+ val srcBlocks = blockify(rank, srcFeatures)
+ val dstBlocks = blockify(rank, dstFeatures)
+ val ratings = srcBlocks.cartesian(dstBlocks).flatMap {
--- End diff --
Normally items are skinny ~ 1M...and ranks are low...50...so 1Mx50 bytes ~
50 MB...with 8M products, its 400 MB...I still think that cartesian will be
slower than the version I added in terms of runtime....did you run any
benchmark with the old code ?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
