Repository: spark Updated Branches: refs/heads/master db7317ac3 -> 252d4f27f
[SPARK-16500][ML][MLLIB][OPTIMIZER] add LBFGS convergence warning for all used place in MLLib ## What changes were proposed in this pull request? Add warning_for the following case when LBFGS training not actually convergence: 1) LogisticRegression 2) AFTSurvivalRegression 3) LBFGS algorithm wrapper in mllib package ## How was this patch tested? N/A Author: WeichenXu <[email protected]> Closes #14157 from WeichenXu123/add_lbfgs_convergence_warning_for_all_used_place. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/252d4f27 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/252d4f27 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/252d4f27 Branch: refs/heads/master Commit: 252d4f27f23b547777892bcea25a2cea62d8cbab Parents: db7317a Author: WeichenXu <[email protected]> Authored: Thu Jul 14 09:11:04 2016 +0100 Committer: Sean Owen <[email protected]> Committed: Thu Jul 14 09:11:04 2016 +0100 ---------------------------------------------------------------------- .../apache/spark/ml/classification/LogisticRegression.scala | 5 +++++ .../org/apache/spark/ml/regression/AFTSurvivalRegression.scala | 5 +++++ .../main/scala/org/apache/spark/mllib/optimization/LBFGS.scala | 6 ++++++ 3 files changed, 16 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/252d4f27/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala ---------------------------------------------------------------------- diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala index e157bde..4bab801 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala @@ -424,6 +424,11 @@ class LogisticRegression @Since("1.2.0") ( throw new SparkException(msg) } + if (!state.actuallyConverged) { + logWarning("LogisticRegression training fininshed but the result " + + s"is not converged because: ${state.convergedReason.get.reason}") + } + /* The coefficients are trained in the scaled space; we're converting them back to the original space. http://git-wip-us.apache.org/repos/asf/spark/blob/252d4f27/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala ---------------------------------------------------------------------- diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala index 7c51845..366448f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala @@ -245,6 +245,11 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S throw new SparkException(msg) } + if (!state.actuallyConverged) { + logWarning("AFTSurvivalRegression training fininshed but the result " + + s"is not converged because: ${state.convergedReason.get.reason}") + } + state.x.toArray.clone() } http://git-wip-us.apache.org/repos/asf/spark/blob/252d4f27/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala ---------------------------------------------------------------------- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala index ec6ffe6..c61b2db 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala @@ -212,6 +212,12 @@ object LBFGS extends Logging { state = states.next() } lossHistory += state.value + + if (!state.actuallyConverged) { + logWarning("LBFGS training fininshed but the result " + + s"is not converged because: ${state.convergedReason.get.reason}") + } + val weights = Vectors.fromBreeze(state.x) val lossHistoryArray = lossHistory.result() --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
