WeichenXu123 commented on a change in pull request #30471:
URL: https://github.com/apache/spark/pull/30471#discussion_r529100588
##########
File path: python/pyspark/ml/tuning.py
##########
@@ -207,6 +210,205 @@ def _to_java_impl(self):
return java_estimator, java_epms, java_evaluator
+class _ValidatorSharedReadWrite:
+
+ @staticmethod
+ def saveImpl(path, instance, sc, extraMetadata=None):
+ from pyspark.ml.classification import OneVsRest
+ numParamsNotJson = 0
+ jsonEstimatorParamMaps = []
+ for paramMap in instance.getEstimatorParamMaps():
+ jsonParamMap = []
+ for p, v in paramMap.items():
+ jsonParam = {'parent': p.parent, 'name': p.name}
+ if (isinstance(v, Estimator) and not (
+ isinstance(v, _ValidatorParams) or
+ isinstance(v, OneVsRest))
+ ) or isinstance(v, Transformer) or \
+ isinstance(Evaluator):
+ relative_path = f'epm_{p.name}{numParamsNotJson}'
+ param_path = os.path.join(path, relative_path)
+ numParamsNotJson += 1
+ v.save(param_path)
+ jsonParam['value'] = relative_path
+ jsonParam['isJson'] = False
+ elif isinstance(v, MLWritable):
+ raise RuntimeError(
+ "ValidatorSharedReadWrite.saveImpl does not handle
parameters of type: "
+ "MLWritable that are not
Estimaor/Evaluator/Transformer, and if parameter is estimator,"
+ "it cannot be Validator or OneVsRest")
+ else:
+ jsonParam['value'] = v
+ jsonParam['isJson'] = True
+ jsonParamMap.append(jsonParam)
+ jsonEstimatorParamMaps.append(jsonParamMap)
+
+ skipParams = ['estimator', 'evaluator', 'estimatorParamMaps']
+
+ jsonParams = {}
+ for p, v in instance._paramMap.items():
+ if p.name not in skipParams:
+ jsonParams[p.name] = v
+
+ jsonParams['estimatorParamMaps'] = jsonEstimatorParamMaps
+
+ DefaultParamsWriter.saveMetadata(instance, path, sc, extraMetadata,
jsonParams)
+ evaluatorPath = os.path.join(path, 'evaluator')
+ instance.getEvaluator().save(evaluatorPath)
+ estimatorPath = os.path.join(path, 'estimator')
+ instance.getEstimator().save(estimatorPath)
+
+ @staticmethod
+ def load(path, sc, metadata):
+ evaluatorPath = os.path.join(path, 'evaluator')
+ evaluator = DefaultParamsReader.loadParamsInstance(evaluatorPath, sc)
+ estimatorPath = os.path.join(path, 'estimator')
+ estimator = DefaultParamsReader.loadParamsInstance(estimatorPath, sc)
+
+ uidToParams = MetaAlgorithmReadWrite.getUidMap(estimator)
Review comment:
This map: `MetaAlgorithmReadWrite.getUidMap` used to find the parent of
each param in `estimatorParamMaps`
e.g.
CrossValidator on pipeline, and pipeline include [transformer1,
transformer2, estimator1], and CrossValidator want to tune on params :
[transformer1.params1, transformer2.params1, estimator1]
Then the getUidMap is used for: providing uid, get the corresponding
transformer / estimator which is the parent of the param.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
