[systemds] branch master updated: [SYSTEMDS-3064] Fix invalid shuffle-free spark reblock

[mboehm7]

This is an automated email from the ASF dual-hosted git repository.

mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new 9fcc6d3  [SYSTEMDS-3064] Fix invalid shuffle-free spark reblock
9fcc6d3 is described below

commit 9fcc6d3b38c372539f468e9d347e8b12ebdc5110
Author: Matthias Boehm <mboe...@gmail.com>
AuthorDate: Sat Jul 17 00:17:16 2021 +0200

    [SYSTEMDS-3064] Fix invalid shuffle-free spark reblock
    
    This patch fixes a case of invalid binary reblock for cases where the
    output block size is larger than the number of rows and columns.
---
 .../instructions/spark/utils/RDDConverterUtils.java     |  4 ++--
 .../sysds/test/functions/io/binary/BlocksizeTest.java   | 17 ++++++++++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git 
a/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDConverterUtils.java
 
b/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDConverterUtils.java
index 40d3b53..7874e90 100644
--- 
a/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDConverterUtils.java
+++ 
b/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDConverterUtils.java
@@ -188,8 +188,8 @@ public class RDDConverterUtils {
                JavaPairRDD<MatrixIndexes, MatrixBlock> in, DataCharacteristics 
mcIn, DataCharacteristics mcOut)
        {
                boolean shuffleFreeReblock = mcIn.dimsKnown() && 
mcOut.dimsKnown()
-                       && (mcIn.getRows() < mcOut.getBlocksize() || 
mcIn.getBlocksize()%mcOut.getBlocksize() == 0)
-                       && (mcIn.getCols() < mcOut.getBlocksize() || 
mcIn.getBlocksize()%mcOut.getBlocksize() == 0);
+                       && (mcIn.getRows() < mcIn.getBlocksize() || 
mcIn.getBlocksize()%mcOut.getBlocksize() == 0)
+                       && (mcIn.getCols() < mcIn.getBlocksize() || 
mcIn.getBlocksize()%mcOut.getBlocksize() == 0);
 
                JavaPairRDD<MatrixIndexes, MatrixBlock> out = in
                        .flatMapToPair(new ExtractBlockForBinaryReblock(mcIn, 
mcOut));
diff --git 
a/src/test/java/org/apache/sysds/test/functions/io/binary/BlocksizeTest.java 
b/src/test/java/org/apache/sysds/test/functions/io/binary/BlocksizeTest.java
index 8ba7e48..225df91 100644
--- a/src/test/java/org/apache/sysds/test/functions/io/binary/BlocksizeTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/io/binary/BlocksizeTest.java
@@ -111,7 +111,22 @@ public class BlocksizeTest extends AutomatedTestBase
        public void testSpark2000() {
                runBlocksizeTest(1000, 2000, ExecMode.SPARK);
        }
-
+       
+       @Test
+       public void testSingleNode2xRowsCols() {
+               runBlocksizeTest(1000, 7000, ExecMode.SINGLE_NODE);
+       }
+       
+       @Test
+       public void testHybrid2xRowsCols() {
+               runBlocksizeTest(1000, 7000, ExecMode.HYBRID);
+       }
+       
+       @Test
+       public void testSpark2xRowsCols() {
+               //test for invalid shuffle-free reblock
+               runBlocksizeTest(1000, 7000, ExecMode.SPARK);
+       }
        
        private void runBlocksizeTest(int inBlksize, int outBlksize, ExecMode 
mode)
        {