Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/16344#discussion_r93777486
--- Diff:
mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
---
@@ -397,46 +434,118 @@ object GeneralizedLinearRegression extends
DefaultParamsReadable[GeneralizedLine
/** Trim the fitted value so that it will be in valid range. */
def project(mu: Double): Double = mu
+
}
private[regression] object Family {
/**
- * Gets the [[Family]] object from its name.
+ * Gets the [[Family]] object based on family and variancePower.
+ * 1) retrieve object based on family name
+ * 2) if family name is tweedie, retrieve object based on variancePower
*
- * @param name family name: "gaussian", "binomial", "poisson" or
"gamma".
+ * @param model a GenerealizedLinearRegressionBase object
*/
- def fromName(name: String): Family = {
- name match {
- case Gaussian.name => Gaussian
- case Binomial.name => Binomial
- case Poisson.name => Poisson
- case Gamma.name => Gamma
+ def fromModel(model: GeneralizedLinearRegressionBase): Family = {
+ model.getFamily match {
+ case "gaussian" => Gaussian
+ case "binomial" => Binomial
+ case "poisson" => Poisson
+ case "gamma" => Gamma
+ case "tweedie" =>
+ model.getVariancePower match {
+ case 0.0 => Gaussian
+ case 1.0 => Poisson
+ case 2.0 => Gamma
+ case default => new TweedieFamily(default)
+ }
}
}
}
/**
- * Gaussian exponential family distribution.
- * The default link for the Gaussian family is the identity link.
- */
- private[regression] object Gaussian extends Family("gaussian") {
+ * Tweedie exponential family distribution.
+ * This includes the special cases of Gaussian, Poisson and Gamma.
+ */
+ private[regression] class TweedieFamily(private val variancePower:
Double)
+ extends Family("tweedie") {
+
+ /*
+ The canonical link is 1 - variancePower. Except for the special
cases of Gaussian,
+ Poisson and Gamma, the canonical link is rarely used. Set Log as the
default link.
+ */
+ override val defaultLink: Link = Log
- val defaultLink: Link = Identity
+ override def initialize(y: Double, weight: Double): Double = {
+ if (variancePower >= 1.0 && variancePower < 2.0) {
+ require(y >= 0.0, s"The response variable of the specified
distribution " +
+ s"should be non-negative, but got $y")
+ } else if (variancePower >= 2.0) {
+ require(y > 0.0, s"The response variable of the specified
distribution " +
+ s"should be non-negative, but got $y")
+ }
+ if (y == 0) delta else y
+ }
- override def initialize(y: Double, weight: Double): Double = y
+ override def variance(mu: Double): Double = math.pow(mu, variancePower)
- override def variance(mu: Double): Double = 1.0
+ private def yp(y: Double, mu: Double, p: Double): Double = {
+ if (p == 0) {
+ math.log(y / mu)
+ } else {
+ (math.pow(y, p) - math.pow(mu, p)) / p
+ }
+ }
override def deviance(y: Double, mu: Double, weight: Double): Double =
{
- weight * (y - mu) * (y - mu)
+ // Force y >= delta for Poisson or compound Poisson
+ val y1 = if (variancePower >= 1.0 && variancePower < 2.0) {
+ math.max(y, delta)
+ } else {
+ y
+ }
+ 2.0 * weight *
+ (y * yp(y1, mu, 1.0 - variancePower) - yp(y, mu, 2.0 -
variancePower))
}
override def aic(
predictions: RDD[(Double, Double, Double)],
deviance: Double,
numInstances: Double,
weightSum: Double): Double = {
+ /*
+ This depends on the density of the Tweedie distribution.
+ Only implemented for Gaussian, Poisson and Gamma at this point.
+ */
+ throw new UnsupportedOperationException("No AIC available for the
tweedie family")
+ }
+
+ override def project(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu.isInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+ }
+
+ /**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ */
+ private[regression] object Gaussian extends TweedieFamily(0.0) {
--- End diff --
We should keep the concrete implementation of ```variance, deviance and
aic``` for Gaussian, Poisson and Gamma, the main reasons are:
* These functions were called very frequently, the concrete implementation
in subclasses should be more efficient.
* It's helpful to locate errors or bugs.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
