WeichenXu123 commented on a change in pull request #28279:
URL: https://github.com/apache/spark/pull/28279#discussion_r412699397
##########
File path: python/pyspark/ml/pipeline.py
##########
@@ -174,6 +174,55 @@ def _to_java(self):
return _java_obj
+ def _make_java_param_pair(self, param, value):
+ """
+ Makes a Java param pair.
+ """
+ sc = SparkContext._active_spark_context
+ param = self._resolveParam(param)
+ java_param = sc._jvm.org.apache.spark.ml.param.Param(param.parent,
param.name, param.doc)
+ if isinstance(value, Params) and hasattr(value, "_to_java"):
+ # Convert JavaEstimator/JavaTransformer object or
Estimator/Transformer object which implements
+ # `_to_java` method (such as OneVsRest, Pipeline object) to java
object
+ # used in the case of an estimator having another estimator as a
parameter
+ # the reason why this is not in _py2java in common.py is that
importing
+ # Estimator and Model in common.py results in a circular import
with inherit_doc
+ java_value = value._to_java()
+ else:
+ java_value = _py2java(sc, value)
+ return java_param.w(java_value)
+
+ def _transfer_param_map_to_java(self, pyParamMap):
+ """
+ Transforms a Python ParamMap into a Java ParamMap.
+ """
+ paramMap =
JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
+ for param in self.params:
+ if param in pyParamMap:
+ pair = self._make_java_param_pair(param, pyParamMap[param])
+ paramMap.put([pair])
+ return paramMap
+
+ def _transfer_param_map_from_java(self, javaParamMap):
+ """
+ Transforms a Java ParamMap into a Python ParamMap.
+ """
+ sc = SparkContext._active_spark_context
+ paramMap = dict()
+ for pair in javaParamMap.toList():
+ param = pair.param()
+ if self.hasParam(str(param.name())):
+ java_obj = pair.value()
+ if
sc._jvm.Class.forName("org.apache.spark.ml.PipelineStage").isInstance(java_obj):
+ # Note: JavaParams._from_java support both
JavaEstimator/JavaTransformer class
+ # and Estimator/Transformer class which implements
`_from_java` static method
+ # (such as OneVsRest, Pipeline class).
+ py_obj = JavaParams._from_java(java_obj)
Review comment:
Similar with above, this is why nested pipeline and nested oneVsRest
works.
Btw, in future refactoring, we can create a new mix-in called JavaParamsOps,
and move not only `_transfer_param_map_from_java`,
`_transfer_param_map_to_java` but also `_from_java`(staticmethod), `_to_java`
methods into it. Because class Pipeline & OneVsRest which do not inherit
`JavaParams` also need them.
Both JavaEstimator/Java
##########
File path: python/pyspark/ml/pipeline.py
##########
@@ -174,6 +174,55 @@ def _to_java(self):
return _java_obj
+ def _make_java_param_pair(self, param, value):
+ """
+ Makes a Java param pair.
+ """
+ sc = SparkContext._active_spark_context
+ param = self._resolveParam(param)
+ java_param = sc._jvm.org.apache.spark.ml.param.Param(param.parent,
param.name, param.doc)
+ if isinstance(value, Params) and hasattr(value, "_to_java"):
+ # Convert JavaEstimator/JavaTransformer object or
Estimator/Transformer object which implements
+ # `_to_java` method (such as OneVsRest, Pipeline object) to java
object
+ # used in the case of an estimator having another estimator as a
parameter
+ # the reason why this is not in _py2java in common.py is that
importing
+ # Estimator and Model in common.py results in a circular import
with inherit_doc
+ java_value = value._to_java()
+ else:
+ java_value = _py2java(sc, value)
+ return java_param.w(java_value)
+
+ def _transfer_param_map_to_java(self, pyParamMap):
+ """
+ Transforms a Python ParamMap into a Java ParamMap.
+ """
+ paramMap =
JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
+ for param in self.params:
+ if param in pyParamMap:
+ pair = self._make_java_param_pair(param, pyParamMap[param])
+ paramMap.put([pair])
+ return paramMap
+
+ def _transfer_param_map_from_java(self, javaParamMap):
+ """
+ Transforms a Java ParamMap into a Python ParamMap.
+ """
+ sc = SparkContext._active_spark_context
+ paramMap = dict()
+ for pair in javaParamMap.toList():
+ param = pair.param()
+ if self.hasParam(str(param.name())):
+ java_obj = pair.value()
+ if
sc._jvm.Class.forName("org.apache.spark.ml.PipelineStage").isInstance(java_obj):
+ # Note: JavaParams._from_java support both
JavaEstimator/JavaTransformer class
+ # and Estimator/Transformer class which implements
`_from_java` static method
+ # (such as OneVsRest, Pipeline class).
+ py_obj = JavaParams._from_java(java_obj)
Review comment:
Similar with above, this is why nested pipeline and nested oneVsRest
works.
Btw, in future refactoring, we can create a new mix-in called JavaParamsOps,
and move not only `_transfer_param_map_from_java`,
`_transfer_param_map_to_java` but also `_from_java`(staticmethod), `_to_java`
methods into it. Because class Pipeline & OneVsRest which do not inherit
`JavaParams` also need them.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
