Github user WeichenXu123 commented on a diff in the pull request:
https://github.com/apache/spark/pull/19020#discussion_r143726258
--- Diff:
mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala ---
@@ -744,11 +754,20 @@ object LinearRegressionModel extends
MLReadable[LinearRegressionModel] {
val dataPath = new Path(path, "data").toString
val data = sparkSession.read.format("parquet").load(dataPath)
- val Row(intercept: Double, coefficients: Vector) =
- MLUtils.convertVectorColumnsToML(data, "coefficients")
- .select("intercept", "coefficients")
- .head()
- val model = new LinearRegressionModel(metadata.uid, coefficients,
intercept)
+ val (majorVersion, minorVersion) =
majorMinorVersion(metadata.sparkVersion)
+ val model = if (majorVersion < 2 || (majorVersion == 2 &&
minorVersion <= 2)) {
+ // Spark 2.2 and before
+ val Row(intercept: Double, coefficients: Vector) =
+ MLUtils.convertVectorColumnsToML(data, "coefficients")
+ .select("intercept", "coefficients")
+ .head()
+ new LinearRegressionModel(metadata.uid, coefficients, intercept)
+ } else {
+ // Spark 2.3 and later
+ val Row(intercept: Double, coefficients: Vector, scale: Double) =
+ data.select("intercept", "coefficients", "scale").head()
+ new LinearRegressionModel(metadata.uid, coefficients, intercept,
scale)
+ }
--- End diff --
Have you test manually, saving model by spark 2.2, and then loading model
by this PR code, to check backwards compatibility ?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
