Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/955#discussion_r14749276
--- Diff:
mllib/src/test/scala/org/apache/spark/mllib/stat/OnlineSummarizerSuite.scala ---
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.mllib.stat
+
+import org.scalatest.FunSuite
+
+import org.apache.spark.mllib.linalg.Vectors
+
+class OnlineSummarizerSuite extends FunSuite {
+
+ def compareDouble(x: Double, y: Double, tol: Double = 1E-3): Boolean = {
+ math.abs(x - y) / (math.abs(y) + 1e-15) < tol
+ }
+
+ test("basic error handing") {
+ val summarizer = new OnlineSummarizer
+
+ assert(summarizer.count === 0, "should be zero since nothing is
added.")
+
+ assert(intercept[IllegalArgumentException] {
+ summarizer.numNonzeros
+ }.getMessage.contains(s"Nothing has been added to this summarizer."),
+ "Getting numNonzeros from empty summarizer should throw exception.")
+
+ assert(intercept[IllegalArgumentException] {
+ summarizer.variance
+ }.getMessage.contains(s"Nothing has been added to this summarizer."),
+ "Getting variance from empty summarizer should throw exception.")
+
+ assert(intercept[IllegalArgumentException] {
+ summarizer.mean
+ }.getMessage.contains(s"Nothing has been added to this summarizer."),
+ "Getting mean from empty summarizer should throw exception.")
+
+ assert(intercept[IllegalArgumentException] {
+ summarizer.max
+ }.getMessage.contains(s"Nothing has been added to this summarizer."),
+ "Getting max from empty summarizer should throw exception.")
+
+ assert(intercept[IllegalArgumentException] {
+ summarizer.min
+ }.getMessage.contains(s"Nothing has been added to this summarizer."),
+ "Getting min from empty summarizer should throw exception.")
+
+ summarizer.add(Vectors.dense(-1.0, 2.0, 6.0))
+ summarizer.add(Vectors.sparse(3, Seq((0, -2.0), (1, 6.0))))
+
+ assert(intercept[IllegalArgumentException] {
+ summarizer.add(Vectors.dense(3.0, 1.0))
+ }.getMessage.contains(s"Dimensions mismatch"),
+ "Adding a new dense sample with different array size should throw
exception.")
+
+ assert(intercept[IllegalArgumentException] {
+ summarizer.add(Vectors.sparse(5, Seq((0, -2.0), (1, 6.0))))
+ }.getMessage.contains(s"Dimensions mismatch"),
+ "Adding a new sparse sample with different array size should throw
exception.")
+
+ val summarizer2 = new OnlineSummarizer
+ summarizer2.add(Vectors.dense(1.0, -2.0, 0.0, 4.0))
+ assert(intercept[IllegalArgumentException] {
+ summarizer.add(summarizer2)
+ }.getMessage.contains(s"Dimensions mismatch"),
+ "Merging a new summarizer with different dimensions should throw
exception.")
+ }
+
+ test("dense vector input") {
+ val summarizer = new OnlineSummarizer
+
+ // For column 2, the maximum will be 0.0, and it's not explicitly
added since we ignore all
+ // the zeros; it's a case we need to test. For column 3, the minimum
will be 0.0 which we
+ // need to test as well.
+ summarizer.add(Vectors.dense(-1.0, 0.0, 6.0))
+ summarizer.add(Vectors.dense(3.0, -3.0, 0.0))
+
+ assert(summarizer.mean.toArray.corresponds(Vectors.dense(1.0, -1.5,
3.0).toArray) {
+ compareDouble(_, _)
+ }, "mean mismatch")
+
+ assert(summarizer.min.toArray.corresponds(Vectors.dense(-1.0, -3,
0.0).toArray) {
+ compareDouble(_, _)
+ }, "min mismatch")
+
+ assert(summarizer.max.toArray.corresponds(Vectors.dense(3.0, 0.0,
6.0).toArray) {
+ compareDouble(_, _)
+ }, "max mismatch")
+
+ assert(summarizer.numNonzeros.toArray.corresponds(Vectors.dense(2, 1,
1).toArray) {
+ _.toLong == _.toLong
+ }, "numNonzeros mismatch")
+
+ assert(summarizer.variance.toArray.corresponds(Vectors.dense(8.0, 4.5,
18.0).toArray) {
+ compareDouble(_, _)
+ }, "variance mismatch")
+
+ assert(summarizer.count === 2)
+ }
+
+ test("sparse vector input") {
+ val summarizer = new OnlineSummarizer
+
+ summarizer.add(Vectors.sparse(3, Seq((0, -1.0), (2, 6.0))))
+ summarizer.add(Vectors.sparse(3, Seq((0, 3.0), (1, -3.0))))
+
+ assert(summarizer.mean.toArray.corresponds(Vectors.dense(1.0, -1.5,
3.0).toArray) {
+ compareDouble(_, _)
+ }, "mean mismatch")
+
+ assert(summarizer.min.toArray.corresponds(Vectors.dense(-1.0, -3,
0.0).toArray) {
+ compareDouble(_, _)
+ }, "min mismatch")
+
+ assert(summarizer.max.toArray.corresponds(Vectors.dense(3.0, 0.0,
6.0).toArray) {
+ compareDouble(_, _)
+ }, "max mismatch")
+
+ assert(summarizer.numNonzeros.toArray.corresponds(Vectors.dense(2, 1,
1).toArray) {
+ _.toLong == _.toLong
+ }, "numNonzeros mismatch")
+
+ assert(summarizer.variance.toArray.corresponds(Vectors.dense(8.0, 4.5,
18.0).toArray) {
+ compareDouble(_, _)
+ }, "variance mismatch")
+
+ assert(summarizer.count === 2)
+ }
+
+ test("mixing dense and sparse vector input") {
+ val summarizer = new OnlineSummarizer
+
+ summarizer.add(Vectors.sparse(3, Seq((0, -2.0), (1, 2.3))))
+ summarizer.add(Vectors.dense(0.0, -1.0, -3.0))
+ summarizer.add(Vectors.sparse(3, Seq((1, -5.1))))
+ summarizer.add(Vectors.dense(3.8, 0.0, 1.9))
+ summarizer.add(Vectors.dense(1.7, -0.6, 0.0))
+ summarizer.add(Vectors.sparse(3, Seq((1, 1.9), (2, 0.0))))
+
+ assert(summarizer.mean.toArray.corresponds(Vectors.dense(0.5833,
-0.4166, -0.1833).toArray) {
+ compareDouble(_, _)
+ }, "mean mismatch")
+
+ assert(summarizer.min.toArray.corresponds(Vectors.dense(-2.0, -5.1,
-3).toArray) {
+ compareDouble(_, _)
+ }, "min mismatch")
+
+ assert(summarizer.max.toArray.corresponds(Vectors.dense(3.8, 2.3,
1.9).toArray) {
+ compareDouble(_, _)
+ }, "max mismatch")
+
+ assert(summarizer.numNonzeros.toArray.corresponds(Vectors.dense(3, 5,
2).toArray) {
+ _.toLong == _.toLong
+ }, "numNonzeros mismatch")
+
+ assert(summarizer.variance.toArray.corresponds(Vectors.dense(3.858,
7.0457, 2.4817).toArray) {
+ compareDouble(_, _)
+ }, "variance mismatch")
+
+ assert(summarizer.count === 6)
+ }
+
+ test("merging two summarizers") {
+ val summarizer1 = new OnlineSummarizer
+ val summarizer2 = new OnlineSummarizer
+
+ summarizer1.add(Vectors.sparse(3, Seq((0, -2.0), (1, 2.3))))
+ summarizer1.add(Vectors.dense(0.0, -1.0, -3.0))
+
+ summarizer2.add(Vectors.sparse(3, Seq((1, -5.1))))
+ summarizer2.add(Vectors.dense(3.8, 0.0, 1.9))
+ summarizer2.add(Vectors.dense(1.7, -0.6, 0.0))
+ summarizer2.add(Vectors.sparse(3, Seq((1, 1.9), (2, 0.0))))
+
+ val summarizer = summarizer1.add(summarizer2)
+
+ assert(summarizer.mean.toArray.corresponds(Vectors.dense(0.5833,
-0.4166, -0.1833).toArray) {
+ compareDouble(_, _)
+ }, "mean mismatch")
+
+ assert(summarizer.min.toArray.corresponds(Vectors.dense(-2.0, -5.1,
-3).toArray) {
+ compareDouble(_, _)
+ }, "min mismatch")
+
+ assert(summarizer.max.toArray.corresponds(Vectors.dense(3.8, 2.3,
1.9).toArray) {
+ compareDouble(_, _)
+ }, "max mismatch")
+
+ assert(summarizer.numNonzeros.toArray.corresponds(Vectors.dense(3, 5,
2).toArray) {
+ _.toLong == _.toLong
+ }, "numNonzeros mismatch")
+
+ assert(summarizer.variance.toArray.corresponds(Vectors.dense(3.858,
7.0457, 2.4817).toArray) {
+ compareDouble(_, _)
+ }, "variance mismatch")
+
+ assert(summarizer.count === 6)
+ }
+
+ test("merging summarizer with empty summarizer") {
+ // If one of two is non-empty, this should return the non-empty
summarizer.
+ // If both of them are empty, then just return the empty summarizer.
+ val summarizer1 = (new OnlineSummarizer).add(
+ Vectors.dense(0.0, -1.0, -3.0)).add(new OnlineSummarizer)
+ val summarizer2 = (new OnlineSummarizer).add(
+ (new OnlineSummarizer).add(Vectors.dense(0.0, -1.0, -3.0)))
+ val summarizer3 = (new OnlineSummarizer).add(new OnlineSummarizer)
--- End diff --
There is only one test for `summarizer3`. Shall we move it closer to the
test?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
