Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/19479#discussion_r150391245
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala
---
@@ -275,6 +313,127 @@ object ColumnStat extends Logging {
avgLen = row.getLong(4),
maxLen = row.getLong(5)
)
+ if (row.isNullAt(6)) {
+ cs
+ } else {
+ val ndvs = row.getArray(6).toLongArray()
+ assert(percentiles.get.numElements() == ndvs.length + 1)
+ val endpoints =
percentiles.get.toArray[Any](attr.dataType).map(_.toString.toDouble)
+ // Construct equi-height histogram
+ val bins = ndvs.zipWithIndex.map { case (ndv, i) =>
+ HistogramBin(endpoints(i), endpoints(i + 1), ndv)
+ }
+ val nonNullRows = rowCount - cs.nullCount
+ val histogram = Histogram(nonNullRows.toDouble / ndvs.length, bins)
+ cs.copy(histogram = Some(histogram))
+ }
+ }
+
+}
+
+/**
+ * This class is an implementation of equi-height histogram.
+ * Equi-height histogram represents the distribution of a column's values
by a sequence of bins.
+ * Each bin has a value range and contains approximately the same number
of rows.
+ * @param height number of rows in each bin
+ * @param bins equi-height histogram bins
+ */
+case class Histogram(height: Double, bins: Array[HistogramBin]) {
+
+ // Only for histogram equality test.
+ override def equals(other: Any): Boolean = other match {
+ case otherHgm: Histogram =>
+ height == otherHgm.height && bins.sameElements(otherHgm.bins)
+ case _ => false
+ }
+
+ override def hashCode(): Int = {
+ val temp = java.lang.Double.doubleToLongBits(height)
+ var result = (temp ^ (temp >>> 32)).toInt
+ result = 31 * result +
java.util.Arrays.hashCode(bins.asInstanceOf[Array[AnyRef]])
+ result
+ }
+}
+
+/**
+ * A bin in an equi-height histogram. We use double type for lower/higher
bound for simplicity.
+ * @param lo lower bound of the value range in this bin
--- End diff --
ditto
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
