Github user jkbradley commented on a diff in the pull request:
https://github.com/apache/spark/pull/2451#discussion_r17803482
--- Diff: mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
---
@@ -57,13 +250,709 @@ trait Matrix extends Serializable {
* @param numCols number of columns
* @param values matrix entries in column major
*/
-class DenseMatrix(val numRows: Int, val numCols: Int, val values:
Array[Double]) extends Matrix {
+class DenseMatrix(val numRows: Int, val numCols: Int, val values:
Array[Double]) extends Matrix with Serializable {
- require(values.length == numRows * numCols)
+ require(values.length == numRows * numCols, "The number of values
supplied doesn't match the " +
+ s"size of the matrix! values.length: ${values.length}, numRows *
numCols: ${numRows * numCols}")
override def toArray: Array[Double] = values
- private[mllib] override def toBreeze: BM[Double] = new
BDM[Double](numRows, numCols, values)
+ private[mllib] def toBreeze: BM[Double] = new BDM[Double](numRows,
numCols, values)
+
+ private[mllib] def apply(i: Int): Double = values(i)
+
+ private[mllib] def apply(i: Int, j: Int): Double = values(index(i, j))
+
+ private[mllib] def index(i: Int, j: Int): Int = i + numRows * j
+
+ private[mllib] def update(i: Int, j: Int, v: Double): Unit = {
+ values(index(i, j)) = v
+ }
+
+ override def copy = new DenseMatrix(numRows, numCols, values.clone())
+
+ private[mllib] def elementWiseOperateOnColumnsInPlace(
+ f: (Double, Double) => Double,
+ y: Matrix): DenseMatrix = {
+ val y_vals = y.toArray
+ val len = y_vals.length
+ require(y_vals.length == numRows)
+ var j = 0
+ while (j < numCols){
+ var i = 0
+ while (i < len){
+ val idx = index(i, j)
+ values(idx) = f(values(idx), y_vals(i))
+ i += 1
+ }
+ j += 1
+ }
+ this
+ }
+
+ private[mllib] def elementWiseOperateOnRowsInPlace(
+ f: (Double, Double) => Double,
+ y: Matrix): DenseMatrix = {
+ val y_vals = y.toArray
+ require(y_vals.length == numCols)
+ var j = 0
+ while (j < numCols){
+ var i = 0
+ while (i < numRows){
+ val idx = index(i, j)
+ values(idx) = f(values(idx), y_vals(j))
+ i += 1
+ }
+ j += 1
+ }
+ this
+ }
+
+ private[mllib] def elementWiseOperateInPlace(f: (Double, Double) =>
Double, y: Matrix): DenseMatrix = {
+ val y_val = y.toArray
+ val len = values.length
+ require(y_val.length == values.length)
+ var j = 0
+ while (j < len){
+ values(j) = f(values(j), y_val(j))
+ j += 1
+ }
+ this
+ }
+
+ private[mllib] def elementWiseOperateScalarInPlace(f: (Double, Double)
=> Double, y: Double): DenseMatrix = {
+ var j = 0
+ val len = values.length
+ while (j < len){
+ values(j) = f(values(j), y)
+ j += 1
+ }
+ this
+ }
+
+ private[mllib] def operateInPlace(f: (Double, Double) => Double, y:
Matrix): DenseMatrix = {
+ if (y.numCols==1 || y.numRows == 1){
+ require(numCols != numRows, "Operation is ambiguous. Please use
elementWiseOperateOnRows " +
+ "or elementWiseOperateOnColumns instead")
+ }
+ if (y.numCols == 1 && y.numRows == 1){
+ elementWiseOperateScalarInPlace(f, y.toArray(0))
+ } else {
+ if (y.numCols==1) {
+ elementWiseOperateOnColumnsInPlace(f, y)
+ }else if (y.numRows==1){
+ elementWiseOperateOnRowsInPlace(f, y)
+ }else{
+ elementWiseOperateInPlace(f, y)
+ }
+ }
+ }
+
+ private[mllib] def elementWiseOperateOnColumns(f: (Double, Double) =>
Double, y: Matrix): DenseMatrix = {
+ val dup = this.copy
+ dup.elementWiseOperateOnColumnsInPlace(f, y)
+ }
+
+ private[mllib] def elementWiseOperateOnRows(f: (Double, Double) =>
Double, y: Matrix): DenseMatrix = {
+ val dup = this.copy
+ dup.elementWiseOperateOnRowsInPlace(f, y)
+ }
+
+ private[mllib] def elementWiseOperate(f: (Double, Double) => Double, y:
Matrix): DenseMatrix = {
+ val dup = this.copy
+ dup.elementWiseOperateInPlace(f, y)
+ }
+
+ private[mllib] def elementWiseOperateScalar(f: (Double, Double) =>
Double, y: Double): DenseMatrix = {
+ val dup = this.copy
+ dup.elementWiseOperateScalarInPlace(f, y)
+ }
+
+ private[mllib] def operate(f: (Double, Double) => Double, y: Matrix):
DenseMatrix = {
+ val dup = this.copy
+ dup.operateInPlace(f, y)
+ }
+
+ def map(f: Double => Double) = new DenseMatrix(numRows, numCols,
values.map(f))
+
+ def update(f: Double => Double): DenseMatrix = {
+ val len = values.length
+ var i = 0
+ while (i < len) {
+ values(i) = f(values(i))
+ i += 1
+ }
+ this
+ }
+
+ def colNorms(p: Double): DenseMatrix = {
+ if (p==1.0) return colSums(true)
+ val sums = new DenseMatrix(1, numCols, Array.fill(numCols)(0.0))
+ var j = 0
+ while (j < numCols){
+ var i = 0
+ while (i < numRows){
+ val idx = index(i, j)
+ sums.update(0,j, sums(j) + math.pow(values(idx),p))
+ i += 1
+ }
+ j += 1
+ }
+ j = 0
+ while (j < numCols){
+ sums.update(0, j, math.pow(sums(j), 1/p))
+ j += 1
+ }
+ sums
+ }
+
+ private[mllib] def negInPlace: DenseMatrix = {
+ var j = 0
+ val len = values.length
+ while (j < len){
+ values(j) *= -1
+ j += 1
+ }
+ this
+ }
+
+ private[mllib] def neg: DenseMatrix = {
+ val copy = new DenseMatrix(numRows, numCols, values.clone())
+ copy.negInPlace
+ }
+
+ private[mllib] def compareInPlace(v: Double, f: (Double, Double) =>
Boolean): DenseMatrix = {
+ var j = 0
+ val len = values.length
+ while (j < len){
+ values(j) = if (f(values(j), v)) 1.0 else 0.0
+ j += 1
+ }
+ this
+ }
+
+ private[mllib] def compare(v: Double, f: (Double, Double) => Boolean):
DenseMatrix = {
+ val copy = new DenseMatrix(numRows, numCols, values.clone())
+ copy.compareInPlace(v, f)
+ }
+
+ private[mllib] def multiplyInPlace(y: Matrix): DenseMatrix = {
+ val copy = this multiply y
+ BLAS.copy(Vectors.dense(copy.values), Vectors.dense(values))
+ this
+ }
+}
+
+object DenseMatrix {
+
+ /**
+ * Generate a `DenseMatrix` consisting of zeros.
+ * @param numRows number of rows of the matrix
+ * @param numCols number of columns of the matrix
+ * @return `DenseMatrix` with size `numRows` x `numCols` and values of
zeros
+ */
+ def zeros(numRows: Int, numCols: Int): DenseMatrix =
+ new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(0.0))
+
+ /**
+ * Generate a `DenseMatrix` consisting of ones.
+ * @param numRows number of rows of the matrix
+ * @param numCols number of columns of the matrix
+ * @return `DenseMatrix` with size `numRows` x `numCols` and values of
ones
+ */
+ def ones(numRows: Int, numCols: Int): DenseMatrix =
+ new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(1.0))
+
+ /**
+ * Generate an Identity Matrix in `DenseMatrix` format.
+ * @param n number of rows and columns of the matrix
+ * @return `DenseMatrix` with size `n` x `n` and values of ones on the
diagonal
+ */
+ def eye(n: Int): DenseMatrix = {
+ val identity = DenseMatrix.zeros(n, n)
+ var i = 0
+ while (i < n){
+ identity.update(i, i, 1.0)
+ i += 1
+ }
+ identity
+ }
+
+ /**
+ * Generate a `DenseMatrix` consisting of i.i.d. uniform random numbers.
+ * @param numRows number of rows of the matrix
+ * @param numCols number of columns of the matrix
+ * @return `DenseMatrix` with size `numRows` x `numCols` and values in
U(0, 1)
+ */
+ def rand(numRows: Int, numCols: Int): DenseMatrix = {
+ val rand = new XORShiftRandom
+ new DenseMatrix(numRows, numCols, Array.fill(numRows *
numCols)(rand.nextDouble()))
+ }
+
+ /**
+ * Generate a `DenseMatrix` consisting of i.i.d. gaussian random numbers.
+ * @param numRows number of rows of the matrix
+ * @param numCols number of columns of the matrix
+ * @return `DenseMatrix` with size `numRows` x `numCols` and values in
N(0, 1)
+ */
+ def randn(numRows: Int, numCols: Int): DenseMatrix = {
+ val rand = new XORShiftRandom
+ new DenseMatrix(numRows, numCols, Array.fill(numRows *
numCols)(rand.nextGaussian()))
+ }
+
+ /**
+ * Generate a diagonal matrix in `DenseMatrix` format from the supplied
values.
+ * @param vector a `Vector` that will form the values on the diagonal of
the matrix
+ * @return Square `DenseMatrix` with size `values.length` x
`values.length` and `values`
+ * on the diagonal
+ */
+ def diag(vector: Vector): DenseMatrix = {
+ val n = vector.size
+ val matrix = DenseMatrix.eye(n)
+ val values = vector.toArray
+ var i = 0
+ while (i < n) {
+ matrix.update(i, i, values(i))
+ i += 1
+ }
+ matrix
+ }
+}
+
+/**
+ * Column-majored sparse matrix.
+ * The entry values are stored in Compressed Sparse Column (CSC) format.
+ * For example, the following matrix
+ * {{{
+ * 1.0 0.0 4.0
+ * 0.0 3.0 5.0
+ * 2.0 0.0 6.0
+ * }}}
+ * is stored as `values: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]`,
+ * `rowIndices=[0, 2, 1, 0, 1, 2]`, `colPointers=[0, 2, 3, 6]`.
+ *
+ * @param numRows number of rows
+ * @param numCols number of columns
+ * @param colPtrs the index corresponding to the start of a new column
+ * @param rowIndices the row index of the entry. They must be in strictly
increasing order for each
+ * column
+ * @param values non-zero matrix entries in column major
+ */
+class SparseMatrix(
+ val numRows: Int,
+ val numCols: Int,
+ val colPtrs: Array[Int],
+ val rowIndices: Array[Int],
+ val values: Array[Double]) extends Matrix with Serializable {
+
+ require(values.length == rowIndices.length, "The number of row indices
and values don't match! " +
+ s"values.length: ${values.length}, rowIndices.length:
${rowIndices.length}")
+ require(colPtrs.length == numCols + 1, "The length of the column indices
should be the " +
+ s"number of columns + 1. Currently, colPointers.length:
${colPtrs.length}, " +
+ s"numCols: $numCols")
+
+ override def toArray: Array[Double] = {
+ val arr = new Array[Double](numRows * numCols)
+ var j = 0
+ while (j < numCols) {
+ var i = colPtrs(j)
+ val indEnd = colPtrs(j + 1)
+ val offset = j * numRows
+ while (i < indEnd) {
+ val rowIndex = rowIndices(i)
+ arr(offset + rowIndex) = values(i)
+ i += 1
+ }
+ j += 1
+ }
+ arr
+ }
+
+ private[mllib] def toBreeze: BM[Double] =
+ new BSM[Double](values, numRows, numCols, colPtrs, rowIndices)
+
+ private[mllib] def apply(i: Int, j: Int): Double = {
+ val ind = index(i, j)
+ if (ind < 0) 0.0 else values(ind)
+ }
+
+ private[mllib] def index(i: Int, j: Int): Int = {
+ Arrays.binarySearch(rowIndices, colPtrs(j), colPtrs(j + 1), i)
+ }
+
+ private[mllib] def update(i: Int, j: Int, v: Double): Unit = {
+ val ind = index(i, j)
+ if (ind == -1){
+ throw new NoSuchElementException("The given row and column indices
correspond to a zero " +
+ "value. Only non-zero elements in Sparse Matrices can be updated.")
+ } else {
+ values(index(i, j)) = v
+ }
+ }
+
+ override def copy = new SparseMatrix(numRows, numCols, colPtrs,
rowIndices, values.clone())
+
+ private[mllib] def elementWiseOperateOnColumnsInPlace(f: (Double,
Double) => Double, y: Matrix): Matrix = {
+ if (isMultiplication(f) || isDivision(f)) {
+ val y_vals = y.toArray
+ require(y_vals.length == numRows)
+ var j = 0
+ while (j < numCols){
+ var i = colPtrs(j)
+ val indEnd = colPtrs(j + 1)
+ while (i < indEnd){
+ values(i) = f(values(i), y_vals(rowIndices(i)))
+ i += 1
+ }
+ j += 1
+ }
+ this
+ } else {
+ val dup = this.toDense
+ dup.elementWiseOperateOnColumnsInPlace(f, y)
+ }
+ }
+
+ private[mllib] def elementWiseOperateOnRowsInPlace(
+ f: (Double, Double) => Double,
+ y: Matrix): Matrix = {
+ if (isMultiplication(f) || isDivision(f)) {
+ val y_vals = y.toArray
+ require(y_vals.length == numCols)
+ var j = 0
+ while (j < numCols){
+ var i = colPtrs(j)
+ val indEnd = colPtrs(j + 1)
+ while (i < indEnd){
+ values(i) = f(values(i), y_vals(j))
+ i += 1
+ }
+ j += 1
+ }
+ this
+ } else {
+ val dup = this.toDense
+ dup.elementWiseOperateOnRowsInPlace(f, y)
+ }
+ }
+
+ private[mllib] def elementWiseOperateInPlace(
+ f: (Double, Double) => Double,
+ y: Matrix): Matrix = {
+ require(y.numCols == numCols)
+ require(y.numRows == numRows)
+ if (isMultiplication(f) || isDivision(f)) {
+ var j = 0
+ while (j < numCols){
+ var i = colPtrs(j)
+ val indEnd = colPtrs(j + 1)
+ while (i < indEnd) {
+ values(i) = f(values(i), y(rowIndices(i), j))
+ i += 1
+ }
+ j += 1
+ }
+ this
+ } else {
+ val dup = this.toDense
+ dup.elementWiseOperateInPlace(f, y)
+ }
+ }
+
+ private[mllib] def elementWiseOperateScalarInPlace(
+ f: (Double, Double) => Double,
+ y: Double): Matrix = {
+ if (isMultiplication(f) || isDivision(f)) {
+ var j = 0
+ val len = values.length
+ while (j < len){
+ values(j) = f(values(j), y)
+ j += 1
+ }
+ this
+ } else {
+ val dup = this.toDense
+ dup.elementWiseOperateScalarInPlace(f, y)
+ }
+ }
+
+ private def isMultiplication(f: (Double, Double) => Double): Boolean = {
+ if (f(2, 9) != 18) return false
+ if (f(3, 7) != 21) return false
+ if (f(8, 9) != 72) return false
+ true
+ }
+
+ private def isDivision(f: (Double, Double) => Double): Boolean = {
+ if (f(12, 3) != 4) return false
+ if (f(72, 4) != 18) return false
+ if (f(72, 9) != 8) return false
+ true
+ }
+
+ private[mllib] def operateInPlace(f: (Double, Double) => Double, y:
Matrix): Matrix = {
+ if (y.numCols==1 || y.numRows == 1) {
+ require(numCols != numRows, "Operation is ambiguous. Please use
elementWiseMultiplyRows " +
+ "or elementWiseMultiplyColumns instead")
+ }
+ if (y.numCols == 1 && y.numRows == 1) {
+ elementWiseOperateScalarInPlace(f, y.toArray(0))
+ } else {
+ if (y.numCols == 1) {
+ elementWiseOperateOnColumnsInPlace(f, y)
+ }else if (y.numRows == 1){
+ elementWiseOperateOnRowsInPlace(f, y)
+ }else{
+ elementWiseOperateInPlace(f, y)
+ }
+ }
+ }
+
+ private[mllib] def elementWiseOperateOnColumns(
+ f: (Double, Double) => Double,
+ y: Matrix): Matrix = {
+ val dup = y match {
+ case sy: SparseMatrix => this.copy
+ case dy: DenseMatrix => this.toDense
+ }
+ dup.elementWiseOperateOnColumnsInPlace(f, y)
+ }
+ private[mllib] def elementWiseOperateOnRows(
+ f: (Double, Double) => Double,
+ y: Matrix): Matrix = {
+ val dup = y match {
+ case sy: SparseMatrix => this.copy
+ case dy: DenseMatrix => this.toDense
+ }
+ dup.elementWiseOperateOnRowsInPlace(f, y)
+ }
+ private[mllib] def elementWiseOperate(f: (Double, Double) => Double, y:
Matrix): Matrix = {
+ val dup = y match {
+ case sy: SparseMatrix => this.copy
+ case dy: DenseMatrix => this.toDense
+ }
+ dup.elementWiseOperateInPlace(f, y)
+ }
+ private[mllib] def elementWiseOperateScalar(f: (Double, Double) =>
Double, y: Double): Matrix = {
+ if (isMultiplication(f) || isDivision(f)) {
+ val dup = this.copy
+ dup.elementWiseOperateScalarInPlace(f, y)
+ } else {
+ val dup = this.toDense
+ dup.elementWiseOperateScalarInPlace(f, y)
+ }
+ }
+
+ private[mllib] def operate(f: (Double, Double) => Double, y: Matrix):
Matrix = {
+ val dup = y match {
+ case sy: SparseMatrix => this.copy
+ case dy: DenseMatrix => this.toDense
+ }
+ dup.operateInPlace(f, y)
+ }
+
+ def map(f: Double => Double) =
+ new SparseMatrix(numRows, numCols, colPtrs, rowIndices, values.map(f))
+
+ def update(f: Double => Double): SparseMatrix = {
+ val len = values.length
+ var i = 0
+ while (i < len) {
+ values(i) = f(values(i))
+ i += 1
+ }
+ this
+ }
+
+ def colNorms(p: Double): DenseMatrix = {
+ if (p==1.0) return colSums(true)
+ val sums = new DenseMatrix(1, numCols, Array.fill(numCols)(0.0))
+ var j = 0
+ while (j < numCols){
+ var i = colPtrs(j)
+ val indEnd = colPtrs(j + 1)
+ while (i < indEnd){
+ sums.values(j) += math.pow(values(i),p)
+ i += 1
+ }
+ j += 1
+ }
+ sums.update(math.pow(_, 1/p))
+ sums
+ }
+
+ private[mllib] def negInPlace: SparseMatrix = {
+ var j = 0
+ val len = values.length
+ while (j < len){
+ values(j) *= -1
+ j += 1
+ }
+ this
+ }
+
+ private[mllib] def neg: SparseMatrix = {
+ val copy = this.copy
+ copy.negInPlace
+ }
+
+ private[mllib] def compare(v: Double, f: (Double, Double) => Boolean):
DenseMatrix = {
+ val copy = new DenseMatrix(numRows, numCols, this.toArray)
+ copy.compareInPlace(v, f)
+ }
+
+ def toDense: DenseMatrix = new DenseMatrix(numRows, numCols,
this.toArray)
+}
+
+object SparseMatrix {
+
+ /**
+ * Generate an Identity Matrix in `SparseMatrix` format.
+ * @param n number of rows and columns of the matrix
+ * @return `SparseMatrix` with size `n` x `n` and values of ones on the
diagonal
+ */
+ def speye(n: Int): SparseMatrix = {
+ new SparseMatrix(n, n, (0 to n).toArray, (0 until n).toArray,
Array.fill(n)(1.0))
+ }
+
+ private def genRand(numRows: Int, numCols: Int, raw: Array[Double],
nonZero: Int): SparseMatrix = {
--- End diff --
I feel like "nonZeros" (plural) is more common than "nonZero"
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
