[
https://issues.apache.org/jira/browse/SPARK-37435?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Hyukjin Kwon updated SPARK-37435:
---------------------------------
Description:
Got this following error when loading the saved model.
{code}
ERROR:ADS Exception Traceback (most recent call last): File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/IPython/core/interactiveshell.py",
line 3441, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File
"/tmp/ipykernel_12307/1140552986.py", line 15, in <module>
LogisticRegressionModel.load(spark, "./lrmodelv2") File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/pyspark/mllib/classification.py",
line 249, in load sc._jsc.sc(), path) File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/py4j/java_gateway.py",
line 1305, in __call__ answer, self.gateway_client, self.target_id, self.name)
File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/pyspark/sql/utils.py",
line 128, in deco return f(*a, **kw) File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/py4j/protocol.py",
line 328, in get_return_value format(target_id, ".", name), value)
py4j.protocol.Py4JJavaError: An error occurred while calling
z:org.apache.spark.mllib.classification.LogisticRegressionModel.load. :
org.json4s.package$MappingException: Did not find value which can be converted
into java.lang.String at org.json4s.reflect.package$.fail(package.scala:95) at
org.json4s.Extraction$.$anonfun$convert$2(Extraction.scala:756) at
scala.Option.getOrElse(Option.scala:189) at
org.json4s.Extraction$.convert(Extraction.scala:756) at
org.json4s.Extraction$.$anonfun$extract$10(Extraction.scala:404) at
org.json4s.Extraction$.$anonfun$customOrElse$1(Extraction.scala:658) at
scala.PartialFunction.applyOrElse(PartialFunction.scala:127) at
scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) at
scala.PartialFunction$$anon$1.applyOrElse(PartialFunction.scala:257) at
org.json4s.Extraction$.customOrElse(Extraction.scala:658) at
org.json4s.Extraction$.extract(Extraction.scala:402) at
org.json4s.Extraction$.extract(Extraction.scala:40) at
org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:21) at
org.apache.spark.mllib.util.Loader$.loadMetadata(modelSaveLoad.scala:122) at
org.apache.spark.mllib.classification.LogisticRegressionModel$.load(LogisticRegression.scala:176)
at
org.apache.spark.mllib.classification.LogisticRegressionModel.load(LogisticRegression.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at
py4j.Gateway.invoke(Gateway.java:282) at
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at
py4j.commands.CallCommand.execute(CallCommand.java:79) at
py4j.GatewayConnection.run(GatewayConnection.java:238) at
java.lang.Thread.run(Thread.java:748) Py4JJavaError: An error occurred while
calling z:org.apache.spark.mllib.classification.LogisticRegressionModel.load. :
org.json4s.package$MappingException: Did not find value which can be converted
into java.lang.String at org.json4s.reflect.package$.fail(package.scala:95) at
org.json4s.Extraction$.$anonfun$convert$2(Extraction.scala:756) at
scala.Option.getOrElse(Option.scala:189) at
org.json4s.Extraction$.convert(Extraction.scala:756) at
org.json4s.Extraction$.$anonfun$extract$10(Extraction.scala:404) at
org.json4s.Extraction$.$anonfun$customOrElse$1(Extraction.scala:658) at
scala.PartialFunction.applyOrElse(PartialFunction.scala:127) at
scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) at
scala.PartialFunction$$anon$1.applyOrElse(PartialFunction.scala:257) at
org.json4s.Extraction$.customOrElse(Extraction.scala:658) at
org.json4s.Extraction$.extract(Extraction.scala:402) at
org.json4s.Extraction$.extract(Extraction.scala:40) at
org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:21) at
org.apache.spark.mllib.util.Loader$.loadMetadata(modelSaveLoad.scala:122) at
org.apache.spark.mllib.classification.LogisticRegressionModel$.load(LogisticRegression.scala:176)
at
org.apache.spark.mllib.classification.LogisticRegressionModel.load(LogisticRegression.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at
py4j.Gateway.invoke(Gateway.java:282) at
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at
py4j.commands.CallCommand.execute(CallCommand.java:79) at
py4j.GatewayConnection.run(GatewayConnection.java:238) at
java.lang.Thread.run(Thread.java:748)
{code}
Sample code snippet
{code}
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.feature import VectorAssembler
from pyspark.sql import SparkSession
from sklearn.datasets import load_iris
from pyspark.mllib.classification import LogisticRegressionModel
spark = SparkSession.builder.getOrCreate()
df = load_iris(as_frame=True).frame.rename(columns=\{"target": "label"})
df = spark.createDataFrame(df)
df = VectorAssembler(inputCols=df.columns[:-1],
outputCol="features").transform(df)
train, test = df.randomSplit([0.8, 0.2])
lor = LogisticRegression(maxIter=5)
lorModel = lor.fit(train)
lorModel.write().overwrite().save("./lrmodelv2")
LogisticRegressionModel.load(spark, "./lrmodelv2")
{code}
was:
Got this following error when loading the saved model.
```
ERROR:ADS Exception Traceback (most recent call last): File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/IPython/core/interactiveshell.py",
line 3441, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File
"/tmp/ipykernel_12307/1140552986.py", line 15, in <module>
LogisticRegressionModel.load(spark, "./lrmodelv2") File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/pyspark/mllib/classification.py",
line 249, in load sc._jsc.sc(), path) File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/py4j/java_gateway.py",
line 1305, in __call__ answer, self.gateway_client, self.target_id, self.name)
File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/pyspark/sql/utils.py",
line 128, in deco return f(*a, **kw) File
"/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/py4j/protocol.py",
line 328, in get_return_value format(target_id, ".", name), value)
py4j.protocol.Py4JJavaError: An error occurred while calling
z:org.apache.spark.mllib.classification.LogisticRegressionModel.load. :
org.json4s.package$MappingException: Did not find value which can be converted
into java.lang.String at org.json4s.reflect.package$.fail(package.scala:95) at
org.json4s.Extraction$.$anonfun$convert$2(Extraction.scala:756) at
scala.Option.getOrElse(Option.scala:189) at
org.json4s.Extraction$.convert(Extraction.scala:756) at
org.json4s.Extraction$.$anonfun$extract$10(Extraction.scala:404) at
org.json4s.Extraction$.$anonfun$customOrElse$1(Extraction.scala:658) at
scala.PartialFunction.applyOrElse(PartialFunction.scala:127) at
scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) at
scala.PartialFunction$$anon$1.applyOrElse(PartialFunction.scala:257) at
org.json4s.Extraction$.customOrElse(Extraction.scala:658) at
org.json4s.Extraction$.extract(Extraction.scala:402) at
org.json4s.Extraction$.extract(Extraction.scala:40) at
org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:21) at
org.apache.spark.mllib.util.Loader$.loadMetadata(modelSaveLoad.scala:122) at
org.apache.spark.mllib.classification.LogisticRegressionModel$.load(LogisticRegression.scala:176)
at
org.apache.spark.mllib.classification.LogisticRegressionModel.load(LogisticRegression.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at
py4j.Gateway.invoke(Gateway.java:282) at
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at
py4j.commands.CallCommand.execute(CallCommand.java:79) at
py4j.GatewayConnection.run(GatewayConnection.java:238) at
java.lang.Thread.run(Thread.java:748) Py4JJavaError: An error occurred while
calling z:org.apache.spark.mllib.classification.LogisticRegressionModel.load. :
org.json4s.package$MappingException: Did not find value which can be converted
into java.lang.String at org.json4s.reflect.package$.fail(package.scala:95) at
org.json4s.Extraction$.$anonfun$convert$2(Extraction.scala:756) at
scala.Option.getOrElse(Option.scala:189) at
org.json4s.Extraction$.convert(Extraction.scala:756) at
org.json4s.Extraction$.$anonfun$extract$10(Extraction.scala:404) at
org.json4s.Extraction$.$anonfun$customOrElse$1(Extraction.scala:658) at
scala.PartialFunction.applyOrElse(PartialFunction.scala:127) at
scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) at
scala.PartialFunction$$anon$1.applyOrElse(PartialFunction.scala:257) at
org.json4s.Extraction$.customOrElse(Extraction.scala:658) at
org.json4s.Extraction$.extract(Extraction.scala:402) at
org.json4s.Extraction$.extract(Extraction.scala:40) at
org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:21) at
org.apache.spark.mllib.util.Loader$.loadMetadata(modelSaveLoad.scala:122) at
org.apache.spark.mllib.classification.LogisticRegressionModel$.load(LogisticRegression.scala:176)
at
org.apache.spark.mllib.classification.LogisticRegressionModel.load(LogisticRegression.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at
py4j.Gateway.invoke(Gateway.java:282) at
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at
py4j.commands.CallCommand.execute(CallCommand.java:79) at
py4j.GatewayConnection.run(GatewayConnection.java:238) at
java.lang.Thread.run(Thread.java:748)
```
Sample code snippet
```
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.feature import VectorAssembler
from pyspark.sql import SparkSession
from sklearn.datasets import load_iris
from pyspark.mllib.classification import LogisticRegressionModel
spark = SparkSession.builder.getOrCreate()
df = load_iris(as_frame=True).frame.rename(columns=\{"target": "label"})
df = spark.createDataFrame(df)
df = VectorAssembler(inputCols=df.columns[:-1],
outputCol="features").transform(df)
train, test = df.randomSplit([0.8, 0.2])
lor = LogisticRegression(maxIter=5)
lorModel = lor.fit(train)
lorModel.write().overwrite().save("./lrmodelv2")
LogisticRegressionModel.load(spark, "./lrmodelv2")
```
> Did not find value which can be converted into java.lang.String
> ---------------------------------------------------------------
>
> Key: SPARK-37435
> URL: https://issues.apache.org/jira/browse/SPARK-37435
> Project: Spark
> Issue Type: Bug
> Components: PySpark
> Affects Versions: 2.4.4, 3.0.2
> Reporter: Ziqun Ye
> Priority: Major
>
> Got this following error when loading the saved model.
> {code}
> ERROR:ADS Exception Traceback (most recent call last): File
> "/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/IPython/core/interactiveshell.py",
> line 3441, in run_code exec(code_obj, self.user_global_ns, self.user_ns)
> File "/tmp/ipykernel_12307/1140552986.py", line 15, in <module>
> LogisticRegressionModel.load(spark, "./lrmodelv2") File
> "/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/pyspark/mllib/classification.py",
> line 249, in load sc._jsc.sc(), path) File
> "/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/py4j/java_gateway.py",
> line 1305, in __call__ answer, self.gateway_client, self.target_id,
> self.name) File
> "/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/pyspark/sql/utils.py",
> line 128, in deco return f(*a, **kw) File
> "/home/datascience/conda/pyspark30_p37_cpu_v2/lib/python3.7/site-packages/py4j/protocol.py",
> line 328, in get_return_value format(target_id, ".", name), value)
> py4j.protocol.Py4JJavaError: An error occurred while calling
> z:org.apache.spark.mllib.classification.LogisticRegressionModel.load. :
> org.json4s.package$MappingException: Did not find value which can be
> converted into java.lang.String at
> org.json4s.reflect.package$.fail(package.scala:95) at
> org.json4s.Extraction$.$anonfun$convert$2(Extraction.scala:756) at
> scala.Option.getOrElse(Option.scala:189) at
> org.json4s.Extraction$.convert(Extraction.scala:756) at
> org.json4s.Extraction$.$anonfun$extract$10(Extraction.scala:404) at
> org.json4s.Extraction$.$anonfun$customOrElse$1(Extraction.scala:658) at
> scala.PartialFunction.applyOrElse(PartialFunction.scala:127) at
> scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) at
> scala.PartialFunction$$anon$1.applyOrElse(PartialFunction.scala:257) at
> org.json4s.Extraction$.customOrElse(Extraction.scala:658) at
> org.json4s.Extraction$.extract(Extraction.scala:402) at
> org.json4s.Extraction$.extract(Extraction.scala:40) at
> org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:21) at
> org.apache.spark.mllib.util.Loader$.loadMetadata(modelSaveLoad.scala:122) at
> org.apache.spark.mllib.classification.LogisticRegressionModel$.load(LogisticRegression.scala:176)
> at
> org.apache.spark.mllib.classification.LogisticRegressionModel.load(LogisticRegression.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at
> py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at
> py4j.Gateway.invoke(Gateway.java:282) at
> py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at
> py4j.commands.CallCommand.execute(CallCommand.java:79) at
> py4j.GatewayConnection.run(GatewayConnection.java:238) at
> java.lang.Thread.run(Thread.java:748) Py4JJavaError: An error occurred while
> calling z:org.apache.spark.mllib.classification.LogisticRegressionModel.load.
> : org.json4s.package$MappingException: Did not find value which can be
> converted into java.lang.String at
> org.json4s.reflect.package$.fail(package.scala:95) at
> org.json4s.Extraction$.$anonfun$convert$2(Extraction.scala:756) at
> scala.Option.getOrElse(Option.scala:189) at
> org.json4s.Extraction$.convert(Extraction.scala:756) at
> org.json4s.Extraction$.$anonfun$extract$10(Extraction.scala:404) at
> org.json4s.Extraction$.$anonfun$customOrElse$1(Extraction.scala:658) at
> scala.PartialFunction.applyOrElse(PartialFunction.scala:127) at
> scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) at
> scala.PartialFunction$$anon$1.applyOrElse(PartialFunction.scala:257) at
> org.json4s.Extraction$.customOrElse(Extraction.scala:658) at
> org.json4s.Extraction$.extract(Extraction.scala:402) at
> org.json4s.Extraction$.extract(Extraction.scala:40) at
> org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:21) at
> org.apache.spark.mllib.util.Loader$.loadMetadata(modelSaveLoad.scala:122) at
> org.apache.spark.mllib.classification.LogisticRegressionModel$.load(LogisticRegression.scala:176)
> at
> org.apache.spark.mllib.classification.LogisticRegressionModel.load(LogisticRegression.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at
> py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at
> py4j.Gateway.invoke(Gateway.java:282) at
> py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at
> py4j.commands.CallCommand.execute(CallCommand.java:79) at
> py4j.GatewayConnection.run(GatewayConnection.java:238) at
> java.lang.Thread.run(Thread.java:748)
> {code}
> Sample code snippet
> {code}
> from pyspark.ml.classification import LogisticRegression
> from pyspark.ml.feature import VectorAssembler
> from pyspark.sql import SparkSession
> from sklearn.datasets import load_iris
> from pyspark.mllib.classification import LogisticRegressionModel
> spark = SparkSession.builder.getOrCreate()
> df = load_iris(as_frame=True).frame.rename(columns=\{"target": "label"})
> df = spark.createDataFrame(df)
> df = VectorAssembler(inputCols=df.columns[:-1],
> outputCol="features").transform(df)
> train, test = df.randomSplit([0.8, 0.2])
> lor = LogisticRegression(maxIter=5)
> lorModel = lor.fit(train)
> lorModel.write().overwrite().save("./lrmodelv2")
> LogisticRegressionModel.load(spark, "./lrmodelv2")
> {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
