janniklinde commented on code in PR #2560: URL: https://github.com/apache/systemds/pull/2560#discussion_r3795176763
########## src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java: ########## @@ -0,0 +1,1390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.runtime.compress.colgroup; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.sysds.runtime.compress.DMLCompressionException; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; +import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; +import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; +import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; +import org.apache.sysds.runtime.compress.utils.IntArrayList; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.data.SparseBlock; +import org.apache.sysds.runtime.data.SparseBlockMCSR; +import org.apache.sysds.runtime.functionobjects.Builtin; +import org.apache.sysds.runtime.functionobjects.Divide; +import org.apache.sysds.runtime.functionobjects.Minus; +import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.functionobjects.Plus; +import org.apache.sysds.runtime.instructions.cp.CmCovObject; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.BinaryOperator; +import org.apache.sysds.runtime.matrix.operators.CMOperator; +import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.runtime.matrix.operators.UnaryOperator; + +/** + * This class represents a new ColGroup which is compresses column into segments (piecewise linear) to represent the + * original Data each column is approximate by a set of linear segments defined by breakpoints, slopes and intercepts + */ + +public class ColGroupPiecewiseLinearCompressed extends AColGroupCompressed { + /** + * breakpoints indices per column to define the segment boundaries slopes of the regression line per segment per + * column intercepts of the regression line per segment per column + */ + int[][] breakpointsPerCol; + double[][] slopesPerCol; + double[][] interceptsPerCol; + int numRows; Review Comment: Why not private? ########## src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java: ########## @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.performance; + +import org.apache.sysds.runtime.compress.CompressionSettings; +import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ColGroupFactory; +import org.apache.sysds.runtime.compress.colgroup.ColGroupPiecewiseLinearCompressed; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; +import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.RightScalarOperator; +import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.utils.stats.Timing; +import org.apache.sysds.test.TestUtils; + +/** + * Performance benchmark for piecewise linear compression. Successive is benchmarked across large matrices to show + * scalability. DP is only used as a quality reference on small matrices due to quadratic complexity + * + */ Review Comment: Outdated comment ########## src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java: ########## @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.performance; + +import org.apache.sysds.runtime.compress.CompressionSettings; +import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ColGroupFactory; +import org.apache.sysds.runtime.compress.colgroup.ColGroupPiecewiseLinearCompressed; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; +import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.RightScalarOperator; +import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.utils.stats.Timing; +import org.apache.sysds.test.TestUtils; + +/** + * Performance benchmark for piecewise linear compression. Successive is benchmarked across large matrices to show + * scalability. DP is only used as a quality reference on small matrices due to quadratic complexity + * + */ +public class PiecewiseLinearCompressionPerformanceTest { + + // different target losses : loose, avg, strict + private static final double[] LOSSES = {1e-1, 1e-2, 1e-4}; + // how often compressed + private static final int REPS = 3; + + /** + * generate of a perfectly linear matrix to have a realistic test set up + * + * @param nr number of rows + * @param nc number of columns + * @return matrix with random generated data + */ + private static MatrixBlock generateLinearMatrix(int nr, int nc) { + MatrixBlock mb = new MatrixBlock(nr, nc, false); + mb.allocateDenseBlock(); + for(int c = 0; c < nc; c++) { + double slope = 0.5 * (c + 1); + double intercept = 10.0 - c; + for(int r = 0; r < nr; r++) { + mb.set(r, c, slope * r + intercept); + } + } + return mb; + } + + /// returns a average number of segments per column Review Comment: Unnecessary comment ########## src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java: ########## @@ -0,0 +1,1150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.test.component.compress.colgroup; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUtils; +import org.apache.sysds.runtime.compress.CompressionSettings; +import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; +import org.apache.sysds.runtime.compress.DMLCompressionException; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty; +import org.apache.sysds.runtime.compress.colgroup.ColGroupFactory; +import org.apache.sysds.runtime.compress.colgroup.ColGroupIO; +import org.apache.sysds.runtime.compress.colgroup.ColGroupPiecewiseLinearCompressed; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; +import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.data.SparseBlockMCSR; +import org.apache.sysds.runtime.functionobjects.Builtin; +import org.apache.sysds.runtime.functionobjects.Divide; +import org.apache.sysds.runtime.functionobjects.KahanPlusSq; +import org.apache.sysds.runtime.functionobjects.Minus; +import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.functionobjects.Multiply2; +import org.apache.sysds.runtime.functionobjects.Plus; +import org.apache.sysds.runtime.functionobjects.Power2; +import org.apache.sysds.runtime.functionobjects.ReduceAll; +import org.apache.sysds.runtime.functionobjects.ReduceCol; +import org.apache.sysds.runtime.functionobjects.ReduceRow; +import org.apache.sysds.runtime.functionobjects.ValueFunction; +import org.apache.sysds.runtime.functionobjects.CM; +import org.apache.sysds.runtime.instructions.cp.CmCovObject; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.AggregateOperator; +import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator; +import org.apache.sysds.runtime.matrix.operators.BinaryOperator; +import org.apache.sysds.runtime.matrix.operators.CMOperator; +import org.apache.sysds.runtime.matrix.operators.RightScalarOperator; +import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.runtime.matrix.operators.UnaryOperator; +import org.apache.sysds.runtime.util.DataConverter; +import org.apache.sysds.test.AutomatedTestBase; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Random; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests for ColGroupPiecewiseLinearCompressed operations. + */ Review Comment: Unnecessary ########## src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java: ########## @@ -0,0 +1,1390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.runtime.compress.colgroup; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.sysds.runtime.compress.DMLCompressionException; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; +import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; +import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; +import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; +import org.apache.sysds.runtime.compress.utils.IntArrayList; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.data.SparseBlock; +import org.apache.sysds.runtime.data.SparseBlockMCSR; +import org.apache.sysds.runtime.functionobjects.Builtin; +import org.apache.sysds.runtime.functionobjects.Divide; +import org.apache.sysds.runtime.functionobjects.Minus; +import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.functionobjects.Plus; +import org.apache.sysds.runtime.instructions.cp.CmCovObject; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.BinaryOperator; +import org.apache.sysds.runtime.matrix.operators.CMOperator; +import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.runtime.matrix.operators.UnaryOperator; + +/** + * This class represents a new ColGroup which is compresses column into segments (piecewise linear) to represent the + * original Data each column is approximate by a set of linear segments defined by breakpoints, slopes and intercepts + */ + +public class ColGroupPiecewiseLinearCompressed extends AColGroupCompressed { + /** + * breakpoints indices per column to define the segment boundaries slopes of the regression line per segment per + * column intercepts of the regression line per segment per column + */ Review Comment: Weird comment, maybe one above class ########## src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java: ########## @@ -0,0 +1,1150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.test.component.compress.colgroup; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUtils; +import org.apache.sysds.runtime.compress.CompressionSettings; +import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; +import org.apache.sysds.runtime.compress.DMLCompressionException; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty; +import org.apache.sysds.runtime.compress.colgroup.ColGroupFactory; +import org.apache.sysds.runtime.compress.colgroup.ColGroupIO; +import org.apache.sysds.runtime.compress.colgroup.ColGroupPiecewiseLinearCompressed; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; +import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.data.SparseBlockMCSR; +import org.apache.sysds.runtime.functionobjects.Builtin; +import org.apache.sysds.runtime.functionobjects.Divide; +import org.apache.sysds.runtime.functionobjects.KahanPlusSq; +import org.apache.sysds.runtime.functionobjects.Minus; +import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.functionobjects.Multiply2; +import org.apache.sysds.runtime.functionobjects.Plus; +import org.apache.sysds.runtime.functionobjects.Power2; +import org.apache.sysds.runtime.functionobjects.ReduceAll; +import org.apache.sysds.runtime.functionobjects.ReduceCol; +import org.apache.sysds.runtime.functionobjects.ReduceRow; +import org.apache.sysds.runtime.functionobjects.ValueFunction; +import org.apache.sysds.runtime.functionobjects.CM; +import org.apache.sysds.runtime.instructions.cp.CmCovObject; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.AggregateOperator; +import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator; +import org.apache.sysds.runtime.matrix.operators.BinaryOperator; +import org.apache.sysds.runtime.matrix.operators.CMOperator; +import org.apache.sysds.runtime.matrix.operators.RightScalarOperator; +import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.runtime.matrix.operators.UnaryOperator; +import org.apache.sysds.runtime.util.DataConverter; +import org.apache.sysds.test.AutomatedTestBase; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Random; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests for ColGroupPiecewiseLinearCompressed operations. + */ +public class ColGroupPiecewiseLinearCompressedOperationsTest extends AutomatedTestBase { + + private static final long SEED = 42L; + private static final int NROWS = 50; + private static final int NCOLS = 3; + private static final double TARGET_LOSS = 50; + private static final double DELTA = 1e-9; + + private ColGroupPiecewiseLinearCompressed piecewiseLinearColGroup; + private MatrixBlock originalMB; + private MatrixBlock decompressedMB; + private IColIndex colIndexes; + private int numRows; + private int numCols; + + @Before + public void setUp() { + numRows = NROWS; + numCols = NCOLS; + + double[][] data = getRandomMatrix(numRows, numCols, -30, 30, 1.0, SEED); + originalMB = DataConverter.convertToMatrixBlock(data); + originalMB.allocateDenseBlock(); + + colIndexes = ColIndexFactory.create(buildColArray(numCols)); + + CompressionSettings cs = new CompressionSettingsBuilder().create(); + cs.setPiecewiseTargetLoss(TARGET_LOSS); + + AColGroup result = ColGroupFactory.compressPiecewiseLinearFunctionalSuccessive(colIndexes, originalMB, cs); + assertTrue(result instanceof ColGroupPiecewiseLinearCompressed); + piecewiseLinearColGroup = (ColGroupPiecewiseLinearCompressed) result; + + decompressedMB = decompress(piecewiseLinearColGroup); + } + + private MatrixBlock decompress(AColGroup cg) { + MatrixBlock mb = new MatrixBlock(numRows, numCols, false); + mb.allocateDenseBlock(); + cg.decompressToDenseBlock(mb.getDenseBlock(), 0, numRows, 0, 0); + return mb; + } + + /// check elementwise to compare results from compressed and decompressed matrixblock + private void checkMatrixEquals(String msg, MatrixBlock mb1, MatrixBlock mb2) { + if(mb1.getNumRows() != mb2.getNumRows() || mb1.getNumColumns() != mb2.getNumColumns()) + fail(msg + " dimension mismatch"); + for(int r = 0; r < numRows; r++) + for(int c = 0; c < numCols; c++) + assertEquals(msg + "[" + r + "," + c + "]", mb1.get(r, c), mb2.get(r, c), DELTA); + } + + /// compute column sum to validate + private double[] computeSums(MatrixBlock mb) { + double[] sums = new double[numCols]; + for(int c = 0; c < numCols; c++) + for(int r = 0; r < numRows; r++) + sums[c] += mb.get(r, c); + return sums; + } + + /// create row vector + private double[] buildRowVector() { + double[] v = new double[numCols]; + for(int i = 0; i < numCols; i++) + v[i] = 0.5 * (i + 1); + return v; + } + + private int[] buildColArray(int n) { + int[] cols = new int[n]; + for(int i = 0; i < n; i++) + cols[i] = i; + return cols; + } + + private MatrixBlock applyBinaryRowOpLeft(MatrixBlock mb, BinaryOperator op, double[] v) { + MatrixBlock result = new MatrixBlock(numRows, numCols, false); + result.allocateDenseBlock(); + for(int r = 0; r < numRows; r++) + for(int c = 0; c < numCols; c++) + result.getDenseBlock().set(r, c, op.fn.execute(v[c], mb.get(r, c))); + return result; + } + + private MatrixBlock applyBinaryRowOpRight(MatrixBlock mb, BinaryOperator op, double[] v) { + MatrixBlock result = new MatrixBlock(numRows, numCols, false); + result.allocateDenseBlock(); + for(int r = 0; r < numRows; r++) + for(int c = 0; c < numCols; c++) + result.getDenseBlock().set(r, c, op.fn.execute(mb.get(r, c), v[c])); + return result; + } + + @Test + public void testComputeSum() { + double[] sumsComp = new double[1]; + piecewiseLinearColGroup.computeSum(sumsComp, numRows); + double expectedTotal = 0; + for(double s : computeSums(decompressedMB)) + expectedTotal += s; + assertEquals(expectedTotal, sumsComp[0], DELTA); + } + + @Test + public void testComputeColSums() { + double[] sumsComp = new double[numCols]; + piecewiseLinearColGroup.computeColSums(sumsComp, numRows); + assertArrayEquals(sumsComp, computeSums(decompressedMB), DELTA); + } + + @Test + public void testGetCompType() { + assertEquals(AColGroup.CompressionType.PiecewiseLinearCompressed, piecewiseLinearColGroup.getCompType()); + } + + private void testScalarOp(ScalarOperator op, double scalar) { + MatrixBlock expected = new MatrixBlock(numRows, numCols, false); + expected.allocateDenseBlock(); + for(int r = 0; r < numRows; r++) + for(int c = 0; c < numCols; c++) + expected.getDenseBlock().set(r, c, op.fn.execute(decompressedMB.get(r, c), scalar)); + + checkMatrixEquals("scalarOp " + op.fn.getClass().getSimpleName(), expected, + decompress(piecewiseLinearColGroup.scalarOperation(op))); + } + + @Test + public void testScalarPlus() { + testScalarOp(new RightScalarOperator(Plus.getPlusFnObject(), 3.7), 3.7); + } + + @Test + public void testScalarMinus() { + testScalarOp(new RightScalarOperator(Minus.getMinusFnObject(), 1.5), 1.5); + } + + @Test + public void testScalarMultiply() { + testScalarOp(new RightScalarOperator(Multiply.getMultiplyFnObject(), 2.0), 2.0); + } + + @Test + public void testScalarDivide() { + testScalarOp(new RightScalarOperator(Divide.getDivideFnObject(), 4.0), 4.0); + } + + @Test + public void testBinaryRowOpLeftPlus() { + BinaryOperator op = new BinaryOperator(Plus.getPlusFnObject()); + double[] v = buildRowVector(); + checkMatrixEquals("binaryRowOpLeft Plus", applyBinaryRowOpLeft(decompressedMB, op, v), + decompress(piecewiseLinearColGroup.binaryRowOpLeft(op, v, false))); + } + + @Test + public void testBinaryRowOpLeftMultiply() { + BinaryOperator op = new BinaryOperator(Multiply.getMultiplyFnObject()); + double[] v = buildRowVector(); + checkMatrixEquals("binaryRowOpLeft Multiply", applyBinaryRowOpLeft(decompressedMB, op, v), + decompress(piecewiseLinearColGroup.binaryRowOpLeft(op, v, false))); + } + + @Test + public void testBinaryRowOpLeftMinus() { + BinaryOperator op = new BinaryOperator(Minus.getMinusFnObject()); + double[] v = buildRowVector(); + checkMatrixEquals("binaryRowOpLeft Minus", applyBinaryRowOpLeft(decompressedMB, op, v), + decompress(piecewiseLinearColGroup.binaryRowOpLeft(op, v, false))); + } + + @Test + public void testBinaryRowOpLeftDivideThrows() { + BinaryOperator op = new BinaryOperator(Divide.getDivideFnObject()); + double[] v = buildRowVector(); + assertThrows(NotImplementedException.class, () -> piecewiseLinearColGroup.binaryRowOpLeft(op, v, false)); + } + + @Test + public void testBinaryRowOpRightMinus() { + BinaryOperator op = new BinaryOperator(Minus.getMinusFnObject()); + double[] v = buildRowVector(); + checkMatrixEquals("binaryRowOpRight Minus", applyBinaryRowOpRight(decompressedMB, op, v), + decompress(piecewiseLinearColGroup.binaryRowOpRight(op, v, false))); + } + + @Test + public void testBinaryRowOpRightDivide() { + BinaryOperator op = new BinaryOperator(Divide.getDivideFnObject()); + double[] v = buildRowVector(); + checkMatrixEquals("binaryRowOpRight Divide", applyBinaryRowOpRight(decompressedMB, op, v), + decompress(piecewiseLinearColGroup.binaryRowOpRight(op, v, false))); + } + + @Test + public void testContainsValueIntercept() { + double pattern = piecewiseLinearColGroup.getInterceptsPerCol()[0][0]; + assertTrue("intercept of col 0 seg 0 should exist", piecewiseLinearColGroup.containsValue(pattern)); + } + + @Test + public void testContainsValueEndpoint() { + double valStart = decompressedMB.get(0, 0); + double valMid = decompressedMB.get(numRows / 2, 0); + double valEnd = decompressedMB.get(numRows - 1, 0); + assertTrue("value at start of col 0 should exist", piecewiseLinearColGroup.containsValue(valStart)); + assertTrue("value at mid of col 0 should exist", piecewiseLinearColGroup.containsValue(valMid)); + assertTrue("value at end of col 0 should exist", piecewiseLinearColGroup.containsValue(valEnd)); + } + + @Test + public void testContainsValueConstantSegment() { + ColGroupPiecewiseLinearCompressed cg = (ColGroupPiecewiseLinearCompressed) ColGroupPiecewiseLinearCompressed + .create(ColIndexFactory.create(new int[] {0}), new int[][] {{0, numRows}}, new double[][] {{0.0}}, + new double[][] {{1.23}}, numRows); + + assertTrue("constant value 1.23 should exist", cg.containsValue(1.23)); + assertFalse("value 2.0 should not exist", cg.containsValue(2.0)); + } + + @Test + public void testContainsValueOutsideRange() { + assertFalse("value -10 outside data range", piecewiseLinearColGroup.containsValue(-10.0)); + assertFalse("value +10 outside data range", piecewiseLinearColGroup.containsValue(10.0)); + } + + @Test + public void testGetIdxMatchesDecompress() { + for(int c = 0; c < numCols; c++) + for(int r = 0; r < numRows; r++) + assertEquals("getIdx(" + r + "," + c + ")", decompressedMB.get(r, c), + piecewiseLinearColGroup.getIdx(r, c), 1e-10); + } + + @Test + public void testGetNumValues() { + int expected = 21; + int[][] testbp = new int[][] {{0, 10, 20}, {0, 12, 20}, {0, 5, 20}}; + double[][] testslope = new double[][] {{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}}; + double[][] testintercept = new double[][] {{0.0, 1.0}, {2.0, 3.0}, {4.0, 5.0}}; + AColGroup testColGroup = ColGroupPiecewiseLinearCompressed.create(ColIndexFactory.create(new int[] {0, 1, 2}), + testbp, testslope, testintercept, 20); + assertEquals("getNumValues() mismatch", expected, testColGroup.getNumValues()); + } + + @Test + public void testGetExactSizeOnDisk() { + Random rng = new Random(SEED); + int rows = 80 + rng.nextInt(40); + int numSegs = 1 + rng.nextInt(3); + + int[] breakpoints = new int[numSegs + 1]; + breakpoints[0] = 0; + breakpoints[numSegs] = rows; + for(int s = 1; s < numSegs; s++) + breakpoints[s] = rng.nextInt(rows * 2 / 3) + rows / 10; + + double[] slopes = new double[numSegs]; + double[] intercepts = new double[numSegs]; + for(int s = 0; s < numSegs; s++) { + slopes[s] = rng.nextDouble() * 4 - 2; + intercepts[s] = rng.nextDouble() * 4 - 2; + } + /// PLC Piecewise Linear Compressed + AColGroup colGroupPLC = ColGroupPiecewiseLinearCompressed.create( + ColIndexFactory.create(new int[] {rng.nextInt(20)}), new int[][] {breakpoints}, new double[][] {slopes}, + new double[][] {intercepts}, rows); + + assertTrue("disk size should be positive", colGroupPLC.getExactSizeOnDisk() > 0); + assertTrue("num values should be positive", colGroupPLC.getNumValues() > 0); + } Review Comment: Weak assert -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
