Repository: mahout Updated Branches: refs/heads/master 272b16153 -> dec441fb8
MAHOUT-1529(e): Move dense/sparse matrix test in mapBlock into spark (Anand Avati via dlyubimov) All math-scala, spark tests are passing. this closes apache/mahout#29 Squashed commit of the following: commit 1e3cdb68198636c9f38f2d41d782d12edba7a2f7 Author: Anand Avati <[email protected]> Date: Mon Jul 14 17:20:09 2014 -0700 MAHOUT-1529: Move dense/sparse matrix test in mapBlock into spark/ In h2o engine, the Matrix provided to mapBlock() is an instance of "H2OBlockMatrix extends AbstractMatrix", and neither a DenseMatrix nor SparseMatrix. H2OBlockMatrix is a 0-copy virtual Matrix exposing just the partition's data (created at almost no expense), and creates a copy-on-write Matrix only if modified by the blockmapfunction. So these two tests are failing with h2obindings. Hence moving these two tests into spark module. Signed-off-by: Anand Avati <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/mahout/repo Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/dec441fb Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/dec441fb Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/dec441fb Branch: refs/heads/master Commit: dec441fb895c96d1e756619d15d75bba00b10fa3 Parents: 272b161 Author: Dmitriy Lyubimov <[email protected]> Authored: Mon Jul 21 14:54:57 2014 -0700 Committer: Dmitriy Lyubimov <[email protected]> Committed: Mon Jul 21 14:58:10 2014 -0700 ---------------------------------------------------------------------- CHANGELOG | 4 ++- .../mahout/math/drm/DrmLikeSuiteBase.scala | 29 ------------------ .../mahout/sparkbindings/drm/DrmLikeSuite.scala | 31 +++++++++++++++++++- 3 files changed, 33 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mahout/blob/dec441fb/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index 7ea84c5..f34ee13 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,7 +2,9 @@ Mahout Change Log Release 1.0 - unreleased - MAHOUT-1583: cbind() operator for Scala DRMs + MAHOUT-1529(e): Move dense/sparse matrix test in mapBlock into spark (Anand Avati via dlyubimov) + + MAHOUT-1583: cbind() operator for Scala DRMs (dlyubimov) MAHOUT-1541, MAHOUT-1568, MAHOUT-1569: Created text-delimited file I/O traits and classes on spark, a MahoutDriver for a CLI and a ItemSimilairtyDriver using the CLI http://git-wip-us.apache.org/repos/asf/mahout/blob/dec441fb/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala b/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala index 651c611..eaca53f 100644 --- a/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala +++ b/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala @@ -50,35 +50,6 @@ trait DrmLikeSuiteBase extends DistributedMahoutSuite with Matchers { } - test("DRM blockify dense") { - - val inCoreA = dense((1, 2, 3), (3, 4, 5)) - val drmA = drmParallelize(inCoreA, numPartitions = 2) - - (inCoreA - drmA.mapBlock() { - case (keys, block) => - if (!block.isInstanceOf[DenseMatrix]) - throw new AssertionError("Block must be dense.") - keys -> block - }).norm should be < 1e-4 - } - - test("DRM blockify sparse -> SRM") { - - val inCoreA = sparse( - (1, 2, 3), - 0 -> 3 :: 2 -> 5 :: Nil - ) - val drmA = drmParallelize(inCoreA, numPartitions = 2) - - (inCoreA - drmA.mapBlock() { - case (keys, block) => - if (!block.isInstanceOf[SparseRowMatrix]) - throw new AssertionError("Block must be dense.") - keys -> block - }).norm should be < 1e-4 - } - test("DRM parallelizeEmpty") { val drmEmpty = drmParallelizeEmpty(100, 50) http://git-wip-us.apache.org/repos/asf/mahout/blob/dec441fb/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala ---------------------------------------------------------------------- diff --git a/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala b/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala index e6a9055..bf635dc 100644 --- a/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala +++ b/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala @@ -27,4 +27,33 @@ import org.apache.mahout.sparkbindings.test.DistributedSparkSuite /** DRMLike tests -- just run common DRM tests in Spark. */ -class DrmLikeSuite extends FunSuite with DistributedSparkSuite with DrmLikeSuiteBase +class DrmLikeSuite extends FunSuite with DistributedSparkSuite with DrmLikeSuiteBase { + test("DRM blockify dense") { + + val inCoreA = dense((1, 2, 3), (3, 4, 5)) + val drmA = drmParallelize(inCoreA, numPartitions = 2) + + (inCoreA - drmA.mapBlock() { + case (keys, block) => + if (!block.isInstanceOf[DenseMatrix]) + throw new AssertionError("Block must be dense.") + keys -> block + }).norm should be < 1e-4 + } + + test("DRM blockify sparse -> SRM") { + + val inCoreA = sparse( + (1, 2, 3), + 0 -> 3 :: 2 -> 5 :: Nil + ) + val drmA = drmParallelize(inCoreA, numPartitions = 2) + + (inCoreA - drmA.mapBlock() { + case (keys, block) => + if (!block.isInstanceOf[SparseRowMatrix]) + throw new AssertionError("Block must be dense.") + keys -> block + }).norm should be < 1e-4 + } +}
