This is an automated email from the ASF dual-hosted git repository.
janniklinde pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 88300144ab [MINOR] Extend OOC Random Instruction
88300144ab is described below
commit 88300144abf948e6234d6974321102a2b7eabf77
Author: jessicapriebe <[email protected]>
AuthorDate: Thu Feb 19 14:54:08 2026 +0100
[MINOR] Extend OOC Random Instruction
This extends OOC rand to support non-uniform distributions and non-constant
values.
Closes #2429
---
.../instructions/ooc/DataGenOOCInstruction.java | 36 ++++++++++++++++++----
.../runtime/matrix/data/LibMatrixDatagen.java | 2 +-
.../apache/sysds/test/functions/ooc/RandTest.java | 5 ++-
src/test/scripts/functions/ooc/Rand1.dml | 2 +-
4 files changed, 34 insertions(+), 11 deletions(-)
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java
index 81b3bb7b38..1b74a3ae56 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java
@@ -22,6 +22,7 @@ package org.apache.sysds.runtime.instructions.ooc;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math3.random.Well1024a;
import org.apache.sysds.common.Opcodes;
import org.apache.sysds.common.Types;
import org.apache.sysds.hops.DataGenOp;
@@ -177,6 +178,7 @@ public class DataGenOOCInstruction extends
UnaryOOCInstruction {
@Override
public void processInstruction(ExecutionContext ec) {
final OOCStream<IndexedMatrixValue> qOut =
createWritableStream();
+ ec.getMatrixObject(output).setStreamHandle(qOut);
// process specific datagen operator
if (method == Types.OpOpDG.RAND) {
@@ -188,9 +190,6 @@ public class DataGenOOCInstruction extends
UnaryOOCInstruction {
long lcols = ec.getScalarInput(cols).getLongValue();
checkValidDimensions(lrows, lcols);
- if (!pdf.equalsIgnoreCase("uniform") || minValue !=
maxValue)
- throw new NotImplementedException(); // TODO
modified version of rng as in LibMatrixDatagen to handle blocks independently
-
OOCStream<MatrixIndexes> qIn = createWritableStream();
int nrb = (int)((lrows-1) / blen)+1;
int ncb = (int)((lcols-1) / blen)+1;
@@ -210,10 +209,37 @@ public class DataGenOOCInstruction extends
UnaryOOCInstruction {
return;
}
+ if(sparsity == 1.0 && minValue == maxValue) {
+ mapOOC(qIn, qOut, idx -> {
+ long rlen = Math.min(blen, lrows -
(idx.getRowIndex()-1) * blen);
+ long clen = Math.min(blen, lcols -
(idx.getColumnIndex()-1) * blen);
+ return new IndexedMatrixValue(idx, new
MatrixBlock((int)rlen, (int)clen, minValue));
+ });
+ return;
+ }
+
+ Well1024a bigrand =
LibMatrixDatagen.setupSeedsForRand(lSeed);
+ int nb = nrb * ncb;
+ long[] seeds = new long[nb];
+ for(int i = 0; i < nb; i++) seeds[i] =
bigrand.nextLong();
+
mapOOC(qIn, qOut, idx -> {
long rlen = Math.min(blen, lrows -
(idx.getRowIndex()-1) * blen);
long clen = Math.min(blen, lcols -
(idx.getColumnIndex()-1) * blen);
- MatrixBlock mout =
MatrixBlock.randOperations(getGenerator(rlen, clen), lSeed);
+
+ int r = (int) idx.getRowIndex()-1;
+ int c = (int) idx.getColumnIndex()-1;
+ long bSeed = seeds[r*ncb+c];
+
+ final long estnnz = ((minValue==0.0 &&
maxValue==0.0) ? 0 : (long)(sparsity * rlen * clen));
+ boolean lsparse =
MatrixBlock.evalSparseFormatInMemory(rlen, clen, estnnz);
+
+ MatrixBlock mout = new MatrixBlock();
+ mout.reset((int) rlen, (int) clen, lsparse,
estnnz);
+ mout.allocateBlock();
+
+ LibMatrixDatagen.genRandomNumbers(false, 0, 1,
0, 1, mout, getGenerator(rlen, clen), bSeed, null);
+ mout.recomputeNonZeros();
return new IndexedMatrixValue(idx, mout);
});
}
@@ -263,8 +289,6 @@ public class DataGenOOCInstruction extends
UnaryOOCInstruction {
}
else
throw new NotImplementedException();
-
- ec.getMatrixObject(output).setStreamHandle(qOut);
}
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java
index 58fb8e4fa8..1edd439e40 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java
@@ -460,7 +460,7 @@ public class LibMatrixDatagen
return lseeds;
}
- private static void genRandomNumbers(boolean invokedFromCP, int rl, int
ru, int cl, int cu, MatrixBlock out, RandomMatrixGenerator rgen, long bSeed,
long[] seeds) {
+ public static void genRandomNumbers(boolean invokedFromCP, int rl, int
ru, int cl, int cu, MatrixBlock out, RandomMatrixGenerator rgen, long bSeed,
long[] seeds) {
int rows = rgen._rows;
int cols = rgen._cols;
int blen = rgen._blocksize;
diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/RandTest.java
b/src/test/java/org/apache/sysds/test/functions/ooc/RandTest.java
index 40430aa49f..398e8d105b 100644
--- a/src/test/java/org/apache/sysds/test/functions/ooc/RandTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/ooc/RandTest.java
@@ -53,11 +53,10 @@ public class RandTest extends AutomatedTestBase {
addTestConfiguration(TEST_NAME_2, config2);
}
- // Actual rand operation not yet supported
- /*@Test
+ @Test
public void testRand() {
runRandTest(TEST_NAME_1);
- }*/
+ }
@Test
public void testConstInit() {
diff --git a/src/test/scripts/functions/ooc/Rand1.dml
b/src/test/scripts/functions/ooc/Rand1.dml
index 2861f29462..6b08a84d0e 100644
--- a/src/test/scripts/functions/ooc/Rand1.dml
+++ b/src/test/scripts/functions/ooc/Rand1.dml
@@ -19,6 +19,6 @@
#
#-------------------------------------------------------------
-res = rand(rows=1500, cols=1200, min=-1, max=1);
+res = rand(rows=1500, cols=1200, min=-1, max=1, seed=42);
write(res, $2, format="binary");
