This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 798a0df3fc [MINOR] Performance tests for compressed behavior
798a0df3fc is described below
commit 798a0df3fc179b3a4d7a903fd3755b23f52828c2
Author: Sebastian Baunsgaard <[email protected]>
AuthorDate: Fri Oct 20 17:38:34 2023 +0200
[MINOR] Performance tests for compressed behavior
Closes #1928
---
.../java/org/apache/sysds/performance/Main.java | 8 ++-
.../org/apache/sysds/performance/PerfUtil.java | 12 ++--
.../org/apache/sysds/performance/TimingUtils.java | 2 +
.../sysds/performance/compression/Serialize.java | 79 ++++++++++++++++++----
.../performance/compression/TransformPerf.java | 14 ++--
.../apache/sysds/performance/generators/Const.java | 2 +-
.../sysds/performance/generators/ConstFrame.java | 64 +++++++++---------
.../sysds/performance/generators/FrameFile.java | 76 ++++++++++-----------
.../performance/generators/FrameTransformFile.java | 78 +++++++++------------
.../sysds/performance/generators/MatrixFile.java | 46 ++++++-------
.../sysds/performance/simple/DetectTypeArray.java | 38 +++++------
.../org/apache/sysds/performance/simple/NNZ.java | 48 ++++++-------
12 files changed, 253 insertions(+), 214 deletions(-)
diff --git a/src/test/java/org/apache/sysds/performance/Main.java
b/src/test/java/org/apache/sysds/performance/Main.java
index 4e8f566a30..185a43e2c3 100644
--- a/src/test/java/org/apache/sysds/performance/Main.java
+++ b/src/test/java/org/apache/sysds/performance/Main.java
@@ -132,8 +132,10 @@ public class Main {
double sparsity = Double.parseDouble(args[4]);
int k = Integer.parseInt(args[5]);
int n = Integer.parseInt(args[6]);
-
- Serialize s = new Serialize(n, new ConstMatrix(rows, cols,
unique, sparsity), k);
+ //args[7] is id
+ Serialize s = (args.length == 9) ? //
+ new Serialize(n, new ConstMatrix(rows, cols, unique,
sparsity), k) : //
+ new Serialize(n, new ConstMatrix(rows, cols, unique,
sparsity), k, args[7], args[8]);
if(id == -1)
s.run();
@@ -179,7 +181,7 @@ public class Main {
private static void run16(String[] args) {
int len = Integer.parseInt(args[1]);
- MatrixBlock mb =
TestUtils.ceil(TestUtils.generateTestMatrixBlock(len, len, 0, 100, 0.01, len
+1));
+ MatrixBlock mb =
TestUtils.ceil(TestUtils.generateTestMatrixBlock(len, len, 0, 100, 0.01, len +
1));
System.out.println(mb);
}
diff --git a/src/test/java/org/apache/sysds/performance/PerfUtil.java
b/src/test/java/org/apache/sysds/performance/PerfUtil.java
index f93b03bdb3..9115bf5878 100644
--- a/src/test/java/org/apache/sysds/performance/PerfUtil.java
+++ b/src/test/java/org/apache/sysds/performance/PerfUtil.java
@@ -25,10 +25,10 @@ import java.io.InputStream;
public interface PerfUtil {
- public static String readSpec(String path) throws IOException {
- InputStream in = new FileInputStream(path);
- String spec = new String(in.readAllBytes());
- in.close();
- return spec;
- }
+ public static String readSpec(String path) throws IOException {
+ InputStream in = new FileInputStream(path);
+ String spec = new String(in.readAllBytes());
+ in.close();
+ return spec;
+ }
}
diff --git a/src/test/java/org/apache/sysds/performance/TimingUtils.java
b/src/test/java/org/apache/sysds/performance/TimingUtils.java
index 11e2c1dca5..0faf01c9b0 100644
--- a/src/test/java/org/apache/sysds/performance/TimingUtils.java
+++ b/src/test/java/org/apache/sysds/performance/TimingUtils.java
@@ -21,6 +21,7 @@ package org.apache.sysds.performance;
import java.util.Arrays;
+import org.apache.sysds.api.DMLScript;
import org.apache.sysds.performance.generators.IGenerate;
import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing;
@@ -93,6 +94,7 @@ public interface TimingUtils {
b.run();
while(bq.isEmpty())
Thread.sleep(bq.defaultWaitTime());
+ DMLScript.SEED = i + 1000;
time(f, times, i);
c.run();
}
diff --git
a/src/test/java/org/apache/sysds/performance/compression/Serialize.java
b/src/test/java/org/apache/sysds/performance/compression/Serialize.java
index 12316874c1..802e7f3a7b 100644
--- a/src/test/java/org/apache/sysds/performance/compression/Serialize.java
+++ b/src/test/java/org/apache/sysds/performance/compression/Serialize.java
@@ -38,9 +38,13 @@ import
org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
import org.apache.sysds.runtime.compress.CompressionStatistics;
import org.apache.sysds.runtime.compress.colgroup.scheme.CompressionScheme;
+import org.apache.sysds.runtime.compress.io.ReaderCompressed;
import org.apache.sysds.runtime.compress.io.WriterCompressed;
import org.apache.sysds.runtime.compress.lib.CLALibScheme;
+import org.apache.sysds.runtime.io.MatrixReader;
import org.apache.sysds.runtime.io.MatrixWriter;
+import org.apache.sysds.runtime.io.ReaderBinaryBlock;
+import org.apache.sysds.runtime.io.ReaderBinaryBlockParallel;
import org.apache.sysds.runtime.io.WriterBinaryBlock;
import org.apache.sysds.runtime.io.WriterBinaryBlockParallel;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
@@ -49,26 +53,37 @@ public class Serialize extends APerfTest<Serialize.InOut,
MatrixBlock> {
private final String file;
private final int k;
+ private final String codec;
public Serialize(int N, IGenerate<MatrixBlock> gen) {
super(N, gen);
- file = "tmp/perf-tmp.bin";
+ file = "./tmp/perf-tmp.bin";
k = 1;
+ codec = "none";
}
public Serialize(int N, IGenerate<MatrixBlock> gen, int k) {
super(N, gen);
- file = "tmp/perf-tmp.bin";
+ file = "./tmp/perf-tmp.bin";
this.k = k;
+ codec = "none";
}
public Serialize(int N, IGenerate<MatrixBlock> gen, int k, String file)
{
super(N, gen);
this.file = file;
this.k = k;
+ codec = "none";
}
+ public Serialize(int N, IGenerate<MatrixBlock> gen, int k, String file,
String codec) {
+ super(N, gen);
+ this.file = file == null ? "tmp/perf-tmp.bin" : file;
+ this.k = k;
+ this.codec = codec;
+ }
+
public void run() throws Exception, InterruptedException {
CompressedMatrixBlock.debug = true;
CompressedMatrixBlock.debug = false;
@@ -81,27 +96,35 @@ public class Serialize extends APerfTest<Serialize.InOut,
MatrixBlock> {
if(k == 1) {
ConfigurationManager.getCompilerConfig().set(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS,
false);
}
-
+
+
ConfigurationManager.getDMLConfig().setTextValue(DMLConfig.IO_COMPRESSION_CODEC,
codec);
+
System.out.println(ConfigurationManager.getDMLConfig().getTextValue(DMLConfig.IO_COMPRESSION_CODEC));
warmup(() -> sumTask(k), N);
+
// execute(() -> writeUncompressed(k), "Serialize");
// execute(() -> diskUncompressed(k), "CustomDisk");
-
- execute(() -> standardIO(k), () -> setFileSize(), () ->
cleanup(), "StandardDisk");
-
// execute(() -> compressTask(k), "Compress Normal");
// execute(() -> writeCompressTask(k), "Compress Normal
Serialize");
// execute(() -> diskCompressTask(k), "Compress Normal
CustomDisk");
-
- execute(() -> standardCompressedIO(k), () -> setFileSize(), ()
-> cleanup(), "Compress StandardIO");
-
- final CompressionScheme sch2 = CLALibScheme.getScheme(getC());
// execute(() -> updateAndApplySchemeFused(sch2, k),
"Update&Apply Scheme Fused");
// execute(() -> writeUpdateAndApplySchemeFused(sch2, k),
"Update&Apply Scheme Fused Serialize");
// execute(() -> diskUpdateAndApplySchemeFused(sch2, k),
"Update&Apply Scheme Fused Disk");
-
+
+ execute(() -> standardIO(k), () -> setFileSize(), () ->
cleanup(), "StandardDisk");
+ execute(() -> standardCompressedIO(k), () -> setFileSize(), ()
-> cleanup(), "Compress StandardIO");
+ final CompressionScheme sch2 = CLALibScheme.getScheme(getC());
execute(() -> standardCompressedIOUpdateAndApply(sch2, k), ()
-> setFileSize(), () -> cleanup(),
"Update&Apply Standard IO");
+
+ // write the input file to disk.
+ standardIO(k);
+ execute(() -> standardIORead(k), "StandardRead");
+ cleanup();
+ // write compressed input file to disk
+ standardCompressedIOUpdateAndApply(sch2, k);
+ // standardCompressedIO( k);
+ execute(() -> standardCompressedRead(k),
"StandardCompressedRead");
}
public void run(int i) throws Exception, InterruptedException {
@@ -177,6 +200,19 @@ public class Serialize extends APerfTest<Serialize.InOut,
MatrixBlock> {
}
}
+ private void standardIORead(int k) {
+ try {
+ MatrixBlock mb = gen.take();
+ MatrixReader r = (k == 1) ? new
ReaderBinaryBlock(false) : new ReaderBinaryBlockParallel(false);
+ MatrixBlock mbr = r.readMatrixFromHDFS(file,
mb.getNumRows(), mb.getNumColumns(), ConfigurationManager.getBlocksize(),
mb.getNonZeros());
+
+ ret.add(new
InOut(mb.getInMemorySize(),mbr.getInMemorySize()));
+ }
+ catch(Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private void compressTask(int k) {
MatrixBlock mb = gen.take();
long in = mb.getInMemorySize();
@@ -213,6 +249,20 @@ public class Serialize extends APerfTest<Serialize.InOut,
MatrixBlock> {
}
}
+ private void standardCompressedRead(int k) {
+ try {
+ MatrixBlock mb = gen.take();
+ ReaderCompressed r = new ReaderCompressed(k);
+ MatrixBlock mbr = r.readMatrixFromHDFS(file,
mb.getNumRows(), mb.getNumColumns(), ConfigurationManager.getBlocksize(),
mb.getNonZeros());
+
+ ret.add(new
InOut(mb.getInMemorySize(),mbr.getInMemorySize()));
+ }
+ catch(Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
// private void standardCompressedIOPipelined(int k) {
// try {
// // MatrixWriter w = new WriterBinaryBlockParallel(1);
@@ -443,15 +493,14 @@ public class Serialize extends APerfTest<Serialize.InOut,
MatrixBlock> {
else
fd.delete();
}
+
}
private boolean deleteDirectory(File directoryToBeDeleted) {
File[] allContents = directoryToBeDeleted.listFiles();
- if(allContents != null) {
- for(File file : allContents) {
+ if(allContents != null)
+ for(File file : allContents)
deleteDirectory(file);
- }
- }
return directoryToBeDeleted.delete();
}
diff --git
a/src/test/java/org/apache/sysds/performance/compression/TransformPerf.java
b/src/test/java/org/apache/sysds/performance/compression/TransformPerf.java
index 454d7c7cfe..7d85a30b79 100644
--- a/src/test/java/org/apache/sysds/performance/compression/TransformPerf.java
+++ b/src/test/java/org/apache/sysds/performance/compression/TransformPerf.java
@@ -50,14 +50,18 @@ public class TransformPerf extends
APerfTest<Serialize.InOut, FrameBlock> {
System.out.println(this);
CompressedMatrixBlock.debug = true;
- // execute(() -> detectSchema(k), "Detect Schema");
- // execute(() -> detectAndApply(k), "Detect&Apply Frame
Schema");
+ System.out.println(String.format("Unknown mem size: %30d",
gen.take().getInMemorySize()));
- updateGen();
+ execute(() -> detectSchema(k), "Detect Schema");
+ execute(() -> detectAndApply(k), "Detect&Apply Frame Schema");
+ execute(() -> transformEncode(k), "TransformEncode Def");
+ execute(() -> transformEncodeCompressed(k), "TransformEncode
Comp");
- // execute(() -> detectAndApply(k), "Detect&Apply Frame Schema
Known");
+ updateGen();
- // execute(() -> transformEncode(k), "TransformEncode Def");
+ System.out.println(String.format("Known mem size: %30d",
gen.take().getInMemorySize()));
+ System.out.println(gen.take().slice(0, 10));
+ execute(() -> transformEncode(k), "TransformEncode Def");
execute(() -> transformEncodeCompressed(k), "TransformEncode
Comp");
}
diff --git a/src/test/java/org/apache/sysds/performance/generators/Const.java
b/src/test/java/org/apache/sysds/performance/generators/Const.java
index 2d3adc1ace..7cb4abd40d 100644
--- a/src/test/java/org/apache/sysds/performance/generators/Const.java
+++ b/src/test/java/org/apache/sysds/performance/generators/Const.java
@@ -20,5 +20,5 @@
package org.apache.sysds.performance.generators;
public interface Const<T> extends IGenerate<T> {
- public void change(T t);
+ public void change(T t);
}
diff --git
a/src/test/java/org/apache/sysds/performance/generators/ConstFrame.java
b/src/test/java/org/apache/sysds/performance/generators/ConstFrame.java
index 13f7392380..075fd3efc9 100644
--- a/src/test/java/org/apache/sysds/performance/generators/ConstFrame.java
+++ b/src/test/java/org/apache/sysds/performance/generators/ConstFrame.java
@@ -25,43 +25,43 @@ import org.apache.sysds.runtime.frame.data.FrameBlock;
public class ConstFrame implements Const<FrameBlock> {
- protected FrameBlock fb;
+ protected FrameBlock fb;
- public ConstFrame(FrameBlock fb) {
- this.fb = fb;
- }
+ public ConstFrame(FrameBlock fb) {
+ this.fb = fb;
+ }
- @Override
- public FrameBlock take() {
- return fb;
- }
+ @Override
+ public FrameBlock take() {
+ return fb;
+ }
- @Override
- public void generate(int N) throws InterruptedException {
- // do nothing
- }
+ @Override
+ public void generate(int N) throws InterruptedException {
+ // do nothing
+ }
- @Override
- public final boolean isEmpty() {
- return false;
- }
+ @Override
+ public final boolean isEmpty() {
+ return false;
+ }
- @Override
- public final int defaultWaitTime() {
- return 0;
- }
+ @Override
+ public final int defaultWaitTime() {
+ return 0;
+ }
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(this.getClass().getSimpleName());
- sb.append(" Schema:");
- sb.append(Arrays.toString(fb.getSchema()));
- return sb.toString();
- }
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(this.getClass().getSimpleName());
+ sb.append(" Schema:");
+ sb.append(Arrays.toString(fb.getSchema()));
+ return sb.toString();
+ }
- @Override
- public void change(FrameBlock t) {
- fb = t;
- }
+ @Override
+ public void change(FrameBlock t) {
+ fb = t;
+ }
}
diff --git
a/src/test/java/org/apache/sysds/performance/generators/FrameFile.java
b/src/test/java/org/apache/sysds/performance/generators/FrameFile.java
index d89a2589d7..e106e7c063 100644
--- a/src/test/java/org/apache/sysds/performance/generators/FrameFile.java
+++ b/src/test/java/org/apache/sysds/performance/generators/FrameFile.java
@@ -31,50 +31,46 @@ import org.apache.sysds.runtime.meta.MetaDataAll;
public class FrameFile extends ConstFrame {
- final private String path;
+ final private String path;
- private FrameFile(String path, FrameBlock fb) {
- super(fb);
- this.path = path;
- System.out.println("First 10 rows:");
- System.out.println(fb.slice(0, 10));
- }
+ private FrameFile(String path, FrameBlock fb) {
+ super(fb);
+ this.path = path;
+ }
- public static FrameFile create(String path) throws Exception {
+ public static FrameFile create(String path) throws Exception {
- MetaDataAll mba = new MetaDataAll(path + ".mtd", false, true);
- if(mba.mtdExists()) {
- LOG.error(mba);
+ MetaDataAll mba = new MetaDataAll(path + ".mtd", false, true);
+ if(mba.mtdExists()) {
+ FileFormat f =
FileFormat.valueOf(mba.getFormatTypeString().toUpperCase());
+ ValueType[] schema =
FrameObject.parseSchema(mba.getSchema());
+ FileFormatProperties p = null;
+ if(f.equals(FileFormat.CSV)) {
+ p = new FileFormatPropertiesCSV();
+ ((FileFormatPropertiesCSV)
p).setHeader(mba.getHasHeader());
+ ((FileFormatPropertiesCSV)
p).setDelim(mba.getDelim());
+ }
+ FrameReader r = FrameReaderFactory.createFrameReader(f,
p);
+ FrameBlock fb = r.readFrameFromHDFS(path, schema,
mba.getDim1(), mba.getDim2());
+ return new FrameFile(path, fb);
+ }
+ else {
+ LOG.error("No Mtd file found.. please add one. Fallback
to CSV reading with header");
+ // we assume csv
+ FrameReader r =
FrameReaderFactory.createFrameReader(FileFormat.CSV);
+ FrameBlock fb = r.readFrameFromHDFS(path, -1, -1);
+ return new FrameFile(path, fb);
+ }
- // DataCharacteristics ds = mba.getDataCharacteristics();
- FileFormat f =
FileFormat.valueOf(mba.getFormatTypeString().toUpperCase());
- ValueType[] schema = FrameObject.parseSchema(mba.getSchema());
- FileFormatProperties p = null;
- if(f.equals(FileFormat.CSV)){
- p = new FileFormatPropertiesCSV();
- ((FileFormatPropertiesCSV)p).setHeader(mba.getHasHeader());
- }
- FrameReader r = FrameReaderFactory.createFrameReader(f, p);
- FrameBlock fb = r.readFrameFromHDFS(path, schema, mba.getDim1(),
mba.getDim2());
- return new FrameFile(path, fb);
- }
- else {
- LOG.error("No Mtd file found.. please add one. Fallback to CSV
reading with header");
- // we assume csv
- FrameReader r =
FrameReaderFactory.createFrameReader(FileFormat.CSV);
- FrameBlock fb = r.readFrameFromHDFS(path, -1, -1);
- return new FrameFile(path, fb);
- }
+ }
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(super.toString());
- sb.append(" From file: ");
- sb.append(path);
- return sb.toString();
- }
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(super.toString());
+ sb.append(" From file: ");
+ sb.append(path);
+ return sb.toString();
+ }
}
diff --git
a/src/test/java/org/apache/sysds/performance/generators/FrameTransformFile.java
b/src/test/java/org/apache/sysds/performance/generators/FrameTransformFile.java
index 359cbd2381..8f8130f81c 100644
---
a/src/test/java/org/apache/sysds/performance/generators/FrameTransformFile.java
+++
b/src/test/java/org/apache/sysds/performance/generators/FrameTransformFile.java
@@ -19,65 +19,51 @@
package org.apache.sysds.performance.generators;
-import java.io.IOException;
-
-import org.apache.sysds.common.Types.FileFormat;
-import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.performance.PerfUtil;
import
org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysds.runtime.frame.data.FrameBlock;
-import org.apache.sysds.runtime.io.FileFormatPropertiesCSV;
-import org.apache.sysds.runtime.io.FrameReader;
-import org.apache.sysds.runtime.io.FrameReaderFactory;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.transform.encode.EncoderFactory;
import org.apache.sysds.runtime.transform.encode.MultiColumnEncoder;
public class FrameTransformFile extends ConstMatrix {
- final private String path;
- final private String specPath;
-
- private FrameTransformFile(String path, String specPath, MatrixBlock mb)
throws IOException {
- super(mb);
- this.path = path;
- this.specPath = specPath;
- }
-
- // example:
- // src/test/resources/datasets/titanic/tfspec.json
- // src/test/resources/datasets/titanic/titanic.csv
- public static FrameTransformFile create(String path, String specPath)
throws IOException {
- // read spec
- final String spec = PerfUtil.readSpec(specPath);
+ final private String path;
+ final private String specPath;
- // MetaDataAll mba = new MetaDataAll(path + ".mtd", false, true);
- // DataCharacteristics ds = mba.getDataCharacteristics();
- // FileFormat f =
FileFormat.valueOf(mba.getFormatTypeString().toUpperCase());
+ private FrameTransformFile(String path, String specPath, MatrixBlock
mb) throws Exception {
+ super(mb);
+ this.path = path;
+ this.specPath = specPath;
+ }
- FileFormatPropertiesCSV csvP = new FileFormatPropertiesCSV();
- csvP.setHeader(true);
- FrameReader r = FrameReaderFactory.createFrameReader(FileFormat.CSV,
csvP);
- FrameBlock fb = r.readFrameFromHDFS(path, new ValueType[]
{ValueType.STRING}, -1, -1);
+ // example:
+ // src/test/resources/datasets/titanic/tfspec.json
+ // src/test/resources/datasets/titanic/titanic.csv
+ public static FrameTransformFile create(String path, String specPath)
throws Exception {
+ // read spec
+ final String spec = PerfUtil.readSpec(specPath);
+ final FrameFile fg = FrameFile.create(path);
- int k = InfrastructureAnalyzer.getLocalParallelism();
- FrameBlock sc = fb.detectSchema(k);
- fb = fb.applySchema(sc, k);
- MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec,
fb.getColumnNames(), fb.getNumColumns(), null);
- MatrixBlock mb = encoder.encode(fb, k);
+ FrameBlock fb = fg.take();
+ int k = InfrastructureAnalyzer.getLocalParallelism();
+ FrameBlock sc = fb.detectSchema(k);
+ fb = fb.applySchema(sc, k);
+ MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec,
fb.getColumnNames(), fb.getNumColumns(), null);
+ MatrixBlock mb = encoder.encode(fb, k);
- return new FrameTransformFile(path, specPath, mb);
- }
+ return new FrameTransformFile(path, specPath, mb);
+ }
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(this.getClass().getSimpleName());
- sb.append(" From file: ");
- sb.append(path);
- sb.append(" -- Transformed with: ");
- sb.append(specPath);
- return sb.toString();
- }
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(this.getClass().getSimpleName());
+ sb.append(" From file: ");
+ sb.append(path);
+ sb.append(" -- Transformed with: ");
+ sb.append(specPath);
+ return sb.toString();
+ }
}
diff --git
a/src/test/java/org/apache/sysds/performance/generators/MatrixFile.java
b/src/test/java/org/apache/sysds/performance/generators/MatrixFile.java
index 0f85528ad2..c23b5f472a 100644
--- a/src/test/java/org/apache/sysds/performance/generators/MatrixFile.java
+++ b/src/test/java/org/apache/sysds/performance/generators/MatrixFile.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -28,31 +28,31 @@ import org.apache.sysds.runtime.meta.MetaDataAll;
public class MatrixFile extends ConstMatrix {
- final private String path;
+ final private String path;
- private MatrixFile(String path, MatrixBlock mb) {
- super(mb);
- this.path = path;
- }
+ private MatrixFile(String path, MatrixBlock mb) {
+ super(mb);
+ this.path = path;
+ }
- public static MatrixFile create(String path) throws Exception {
+ public static MatrixFile create(String path) throws Exception {
- MetaDataAll mba = new MetaDataAll(path + ".mtd", false, true);
- DataCharacteristics ds = mba.getDataCharacteristics();
- FileFormat f =
FileFormat.valueOf(mba.getFormatTypeString().toUpperCase());
+ MetaDataAll mba = new MetaDataAll(path + ".mtd", false, true);
+ DataCharacteristics ds = mba.getDataCharacteristics();
+ FileFormat f =
FileFormat.valueOf(mba.getFormatTypeString().toUpperCase());
- MatrixReader r = MatrixReaderFactory.createMatrixReader(f);
- MatrixBlock mb = r.readMatrixFromHDFS(path, ds.getRows(),
ds.getCols(), ds.getBlocksize(), ds.getNonZeros());
- return new MatrixFile(path, mb);
- }
+ MatrixReader r = MatrixReaderFactory.createMatrixReader(f);
+ MatrixBlock mb = r.readMatrixFromHDFS(path, ds.getRows(),
ds.getCols(), ds.getBlocksize(), ds.getNonZeros());
+ return new MatrixFile(path, mb);
+ }
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(this.getClass().getSimpleName());
- sb.append(" From file: ");
- sb.append(path);
- return sb.toString();
- }
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(this.getClass().getSimpleName());
+ sb.append(" From file: ");
+ sb.append(path);
+ return sb.toString();
+ }
}
diff --git
a/src/test/java/org/apache/sysds/performance/simple/DetectTypeArray.java
b/src/test/java/org/apache/sysds/performance/simple/DetectTypeArray.java
index f9fdf1b954..6acf92da24 100644
--- a/src/test/java/org/apache/sysds/performance/simple/DetectTypeArray.java
+++ b/src/test/java/org/apache/sysds/performance/simple/DetectTypeArray.java
@@ -27,30 +27,30 @@ import
org.apache.sysds.runtime.frame.data.columns.ArrayFactory;
public class DetectTypeArray {
- public static void main(String[] args) {
- Array<?> a = ArrayFactory.create(generateRandomFloatString(1000, 134));
+ public static void main(String[] args) {
+ Array<?> a =
ArrayFactory.create(generateRandomFloatString(1000, 134));
- Timing t = new Timing();
- t.start();
- int N = 10000;
- for(int i = 0; i < N; i++)
- a.analyzeValueType();
+ Timing t = new Timing();
+ t.start();
+ int N = 10000;
+ for(int i = 0; i < N; i++)
+ a.analyzeValueType();
- System.out.println(t.stop() / N);
+ System.out.println(t.stop() / N);
- }
+ }
- public static String[] generateRandomFloatString(int size, int seed) {
- Random r = new Random(seed);
- String[] ret = new String[size];
- for(int i = 0; i < size; i++) {
- int e = r.nextInt(999);
- int a = r.nextInt(999);
+ public static String[] generateRandomFloatString(int size, int seed) {
+ Random r = new Random(seed);
+ String[] ret = new String[size];
+ for(int i = 0; i < size; i++) {
+ int e = r.nextInt(999);
+ int a = r.nextInt(999);
- ret[i] = String.format("%d.%03d", e, a);
- }
+ ret[i] = String.format("%d.%03d", e, a);
+ }
- return ret;
- }
+ return ret;
+ }
}
diff --git a/src/test/java/org/apache/sysds/performance/simple/NNZ.java
b/src/test/java/org/apache/sysds/performance/simple/NNZ.java
index 8ed77aea97..57407080c3 100644
--- a/src/test/java/org/apache/sysds/performance/simple/NNZ.java
+++ b/src/test/java/org/apache/sysds/performance/simple/NNZ.java
@@ -24,32 +24,32 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.test.TestUtils;
public class NNZ {
- public static void main(String[] args) {
- MatrixBlock mb = TestUtils.generateTestMatrixBlock(10000, 1000, 0,
103, 0.7, 421);
- Timing t = new Timing();
- t.start();
- for(int i = 0; i < 1000; i++) {
- mb.recomputeNonZeros();
- }
- System.out.println("single: " + t.stop()/ 1000);
- t.start();
- for(int i = 0; i < 1000; i++) {
+ public static void main(String[] args) {
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(10000, 1000,
0, 103, 0.7, 421);
+ Timing t = new Timing();
+ t.start();
+ for(int i = 0; i < 1000; i++) {
+ mb.recomputeNonZeros();
+ }
+ System.out.println("single: " + t.stop() / 1000);
+ t.start();
+ for(int i = 0; i < 1000; i++) {
- mb.recomputeNonZeros(16);
- }
+ mb.recomputeNonZeros(16);
+ }
- System.out.println("par: " + t.stop()/ 1000);
- t.start();
- for(int i = 0; i < 1000; i++) {
- mb.recomputeNonZeros();
- }
- System.out.println("single: " + t.stop()/ 1000);
- t.start();
- for(int i = 0; i < 1000; i++) {
+ System.out.println("par: " + t.stop() / 1000);
+ t.start();
+ for(int i = 0; i < 1000; i++) {
+ mb.recomputeNonZeros();
+ }
+ System.out.println("single: " + t.stop() / 1000);
+ t.start();
+ for(int i = 0; i < 1000; i++) {
- mb.recomputeNonZeros(16);
- }
+ mb.recomputeNonZeros(16);
+ }
- System.out.println("par: " + t.stop()/ 1000);
- }
+ System.out.println("par: " + t.stop() / 1000);
+ }
}
