Github user MLnick commented on a diff in the pull request:
https://github.com/apache/spark/pull/12574#discussion_r60825392
--- Diff:
mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
---
@@ -261,58 +261,93 @@ object MatrixFactorizationModel extends
Loader[MatrixFactorizationModel] {
}
/**
- * Makes recommendations for all users (or products).
- * @param rank rank
- * @param srcFeatures src features to receive recommendations
- * @param dstFeatures dst features used to make recommendations
- * @param num number of recommendations for each record
- * @return an RDD of (srcId: Int, recommendations), where
recommendations are stored as an array
- * of (dstId, rating) pairs.
+ * Makes recommendations for all users (or items).
+ *
+ * @param rank rank the dimension of the factor vectors
+ * @param srcFactors src factor to receive recommendations
+ * @param dstFactors dst factor used to make recommendations
+ * @param num number of recommendations for each user (or item)
+ * @return an RDD of (srcId, recommendations) pairs, where
recommendations are stored as an array
+ * of (dstId, score) pairs.
*/
- private def recommendForAll(
+ private[spark] def recommendForAll[T: ClassTag : Fractional](
rank: Int,
- srcFeatures: RDD[(Int, Array[Double])],
- dstFeatures: RDD[(Int, Array[Double])],
- num: Int): RDD[(Int, Array[(Int, Double)])] = {
- val srcBlocks = blockify(rank, srcFeatures)
- val dstBlocks = blockify(rank, dstFeatures)
+ srcFactors: RDD[(Int, Array[T])],
+ dstFactors: RDD[(Int, Array[T])],
+ num: Int): RDD[(Int, Array[(Int, T)])] = {
+ val srcBlocks = blockify(rank, srcFactors)
+ val dstBlocks = blockify(rank, dstFactors)
+ val tag = implicitly[ClassTag[T]].runtimeClass
val ratings = srcBlocks.cartesian(dstBlocks).flatMap {
case ((srcIds, srcFactors), (dstIds, dstFactors)) =>
val m = srcIds.length
val n = dstIds.length
- val ratings = srcFactors.transpose.multiply(dstFactors)
- val output = new Array[(Int, (Int, Double))](m * n)
- var k = 0
- ratings.foreachActive { (i, j, r) =>
- output(k) = (srcIds(i), (dstIds(j), r))
- k += 1
+ val targetMatrix = new Array[T](m * n)
+ tag match {
+ case java.lang.Float.TYPE =>
+ val A = srcFactors.asInstanceOf[Array[Float]]
+ val B = dstFactors.asInstanceOf[Array[Float]]
+ val C = targetMatrix.asInstanceOf[Array[Float]]
+ blas.sgemm("T", "N", m, n, rank, 1f, A, rank, B, rank, 0f, C,
m)
+ case java.lang.Double.TYPE =>
+ val A = srcFactors.asInstanceOf[Array[Double]]
+ val B = dstFactors.asInstanceOf[Array[Double]]
+ val C = targetMatrix.asInstanceOf[Array[Double]]
+ blas.dgemm("T", "N", m, n, rank, 1f, A, rank, B, rank, 0f, C,
m)
}
- output.toSeq
+ fillPredictions(srcIds, dstIds, targetMatrix, m, n).toSeq
}
ratings.topByKey(num)(Ordering.by(_._2))
}
/**
+ * Fills array of (srcId, Array[(targetId, score)] from the result of
multiplying the
+ * user and item factor matrices.
+ */
+ private def fillPredictions[T : ClassTag](
+ srcIds: Array[Int],
+ dstIds: Array[Int],
+ predictions: Array[T],
+ m: Int,
+ n: Int): Array[(Int, (Int, T))] = {
+ val output = new Array[(Int, (Int, T))](m * n)
+ // outer loop over columns
+ var k = 0
+ var j = 0
+ while (j < n) {
+ var i = 0
+ val indStart = j * m
+ while (i < m) {
+ output(k) = (srcIds(i), (dstIds(j), predictions(indStart + i)))
+ i += 1
+ k += 1
+ }
+ j += 1
+ }
+ output
+ }
+
+ /**
* Blockifies features to use Level-3 BLAS.
*/
- private def blockify(
+ private def blockify[T: ClassTag](
rank: Int,
- features: RDD[(Int, Array[Double])]): RDD[(Array[Int], DenseMatrix)]
= {
- val blockSize = 4096 // TODO: tune the block size
+ features: RDD[(Int, Array[T])],
+ /* TODO make blockSize a param */blockSize: Int = 4096):
RDD[(Array[Int], Array[T])] = {
--- End diff --
Thanks, will update and add a comment. Also when I get to performance
testing I'll check out a few different settings to see what the impact is.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
