Github user jkbradley commented on a diff in the pull request:
https://github.com/apache/spark/pull/9971#discussion_r57489818
--- Diff: mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala ---
@@ -352,3 +359,128 @@ private[ml] object DefaultParamsReader {
cls.getMethod("read").invoke(null).asInstanceOf[MLReader[T]].load(path)
}
}
+
+/**
+ * Default Meta-Pipeline read and write implementation.
+ */
+private[ml] trait MetaPipelineReadWrite {
+ /**
+ * Examine the given estimator (which may be a compound estimator) and
extract a mapping
+ * from UIDs to corresponding [[Params]] instances.
+ */
+ def getUidMap(instance: Params): Map[String, Params] = {
+ val uidList = getUidMapImpl(instance)
+ val uidMap = uidList.toMap
+ if (uidList.size != uidMap.size) {
+ throw new RuntimeException("CrossValidator.load found a compound
estimator with stages" +
+ s" with duplicate UIDs. List of UIDs:
${uidList.map(_._1).mkString(", ")}")
+ }
+ uidMap
+ }
+
+ private def getUidMapImpl(instance: Params): List[(String, Params)] = {
+ val subStages: Array[Params] = instance match {
+ case p: Pipeline => p.getStages.asInstanceOf[Array[Params]]
+ case pm: PipelineModel => pm.stages.asInstanceOf[Array[Params]]
+ case v: ValidatorParams => Array(v.getEstimator, v.getEvaluator)
+ case ovr: OneVsRestParams =>
+ // TODO: SPARK-11892: This case may require special handling.
+ throw new UnsupportedOperationException("CrossValidator write will
fail because it" +
+ " cannot yet handle an estimator containing type:
${ovr.getClass.getName}")
+ case rformModel: RFormulaModel => Array(rformModel.pipelineModel)
+ case _: Params => Array()
+ }
+ val subStageMaps =
subStages.map(getUidMapImpl).foldLeft(List.empty[(String, Params)])(_ ++ _)
+ List((instance.uid, instance)) ++ subStageMaps
+ }
+
+ /**
+ * Check that [[ValidatorParams.evaluator]] and
[[ValidatorParams.estimator]] are Writable.
+ * This does not check [[ValidatorParams.estimatorParamMaps]].
+ */
+ def validateParams(instance: ValidatorParams): Unit = {
--- End diff --
This is specific to ValidatorParams, so I would put it in a ValidatorParams
companion object.
Same for save & load
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
