(systemds) branch main updated: [MINOR] Filter pre-aggregate warning

[baunsgaard]

This is an automated email from the ASF dual-hosted git repository.

baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/main by this push:
     new 7f04d1642c [MINOR] Filter pre-aggregate warning
7f04d1642c is described below

commit 7f04d1642c1a679457c8dc4d6f9003e5e2fc4bf3
Author: Sebastian Baunsgaard <baunsga...@apache.org>
AuthorDate: Mon Oct 30 12:24:06 2023 +0100

    [MINOR] Filter pre-aggregate warning
    
    In compressed linear algebra, we print a warning in case of
    uncompressed matrix multiplication.
    This commit filters that error out if the input is one column.
    The one-column case is special since transposition is a no-op that
    only touches metadata. Therefore, we filter this error.
    
    Also introduced in this commit is an error where we try to allocate a
    pre-aggregate output larger than Integer.MAX_VALUE. This happens
    in cases where the number of columns in a single-column group is
    large, such as in a recode-bin encoding scenario of transform encoding.
---
 .../org/apache/sysds/runtime/compress/colgroup/APreAgg.java    | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git 
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java 
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java
index 17f210865b..8b8a7b7df0 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java
@@ -19,6 +19,7 @@
 
 package org.apache.sysds.runtime.compress.colgroup;
 
+import org.apache.commons.lang.NotImplementedException;
 import org.apache.sysds.runtime.DMLRuntimeException;
 import org.apache.sysds.runtime.compress.DMLCompressionException;
 import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary;
@@ -84,9 +85,11 @@ public abstract class APreAgg extends AColGroupValue {
         * @return A aggregate dictionary
         */
        public final IDictionary preAggregateThatIndexStructure(APreAgg that) {
-               int outputLength = that._colIndexes.size() * 
this.getNumValues();
+               long outputLength = (long)that._colIndexes.size() * 
this.getNumValues();
+               if(outputLength > Integer.MAX_VALUE)
+                       throw new NotImplementedException("Not supported pre 
aggregate of above integer length");
                // create empty Dictionary that we slowly fill, hence the 
dictionary is empty and no check
-               final Dictionary ret = Dictionary.createNoCheck(new 
double[outputLength]);
+               final Dictionary ret = Dictionary.createNoCheck(new 
double[(int)outputLength]);
 
                if(that instanceof ColGroupDDC)
                        preAggregateThatDDCStructure((ColGroupDDC) that, ret);
@@ -224,7 +227,8 @@ public abstract class APreAgg extends AColGroupValue {
        }
 
        private void leftMultByUncompressedColGroup(ColGroupUncompressed lhs, 
MatrixBlock result) {
-               LOG.warn("Transpose of uncompressed to fit to template need 
t(a) %*% b");
+               if(lhs.getNumCols() != 1)
+                       LOG.warn("Transpose of uncompressed to fit to template 
need t(a) %*% b");
                final MatrixBlock tmp = LibMatrixReorg.transpose(lhs.getData(), 
InfrastructureAnalyzer.getLocalParallelism());
                final int numVals = getNumValues();
                final MatrixBlock preAgg = new MatrixBlock(tmp.getNumRows(), 
numVals, false);