Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/19783#discussion_r156281583
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/EstimationUtils.scala
---
@@ -114,4 +114,99 @@ object EstimationUtils {
}
}
+ /**
+ * Returns the number of the first bin into which a column value falls
for a specified
+ * numeric equi-height histogram.
+ *
+ * @param value a literal value of a column
+ * @param bins an array of bins for a given numeric equi-height histogram
+ * @return the id of the first bin into which a column value falls.
+ */
+ def findFirstBinForValue(value: Double, bins: Array[HistogramBin]): Int
= {
+ var i = 0
+ while ((i < bins.length) && (value > bins(i).hi)) {
+ i += 1
+ }
+ i
+ }
+
+ /**
+ * Returns the number of the last bin into which a column value falls
for a specified
+ * numeric equi-height histogram.
+ *
+ * @param value a literal value of a column
+ * @param bins an array of bins for a given numeric equi-height histogram
+ * @return the id of the last bin into which a column value falls.
+ */
+ def findLastBinForValue(value: Double, bins: Array[HistogramBin]): Int =
{
+ var i = bins.length - 1
+ while ((i >= 0) && (value < bins(i).lo)) {
+ i -= 1
+ }
+ i
+ }
+
+ /**
+ * Returns a percentage of a bin holding values for column value in the
range of
+ * [lowerValue, higherValue]
+ *
+ * @param higherValue a given upper bound value of a specified column
value range
+ * @param lowerValue a given lower bound value of a specified column
value range
+ * @param bin a single histogram bin
+ * @return the percentage of a single bin holding values in [lowerValue,
higherValue].
+ */
+ private def getOccupation(
+ higherValue: Double,
+ lowerValue: Double,
+ bin: HistogramBin): Double = {
+ assert(bin.lo <= lowerValue && lowerValue <= higherValue &&
higherValue <= bin.hi)
+ if (bin.hi == bin.lo) {
+ // the entire bin is covered in the range
+ 1.0
+ } else if (higherValue == lowerValue) {
+ // set percentage to 1/NDV
+ 1.0 / bin.ndv.toDouble
+ } else {
+ // Use proration since the range falls inside this bin.
+ math.min((higherValue - lowerValue) / (bin.hi - bin.lo), 1.0)
+ }
+ }
+
+ /**
+ * Returns the number of bins for column values in [lowerValue,
higherValue].
+ * The column value distribution is saved in an equi-height histogram.
The return values is a
+ * double value is because we may return a portion of a bin. For
example, a predicate
+ * "column = 8" may return the number of bins 0.2 if the holding bin has
5 distinct values.
+ *
+ * @param higherId id of the high end bin holding the high end value of
a column range
--- End diff --
nit: `higherIndex`
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
