Github user jiangxb1987 commented on a diff in the pull request:
https://github.com/apache/spark/pull/19992#discussion_r158648471
--- Diff: core/src/main/scala/org/apache/spark/util/JsonProtocol.scala ---
@@ -988,12 +991,20 @@ private[spark] object JsonProtocol {
rddInfo
}
- def storageLevelFromJson(json: JValue): StorageLevel = {
- val useDisk = (json \ "Use Disk").extract[Boolean]
- val useMemory = (json \ "Use Memory").extract[Boolean]
- val deserialized = (json \ "Deserialized").extract[Boolean]
- val replication = (json \ "Replication").extract[Int]
- StorageLevel(useDisk, useMemory, deserialized, replication)
+ def storageLevelFromJson(json: JValue): StorageLevel = json match {
+ case _: JString =>
+ // One of the predefined storage levels, e.g. "DISK_ONLY".
+ StorageLevel.fromString(json.extract[String])
+ case _: JObject =>
+ // Generic case for compatibility with older event logs and for
+ // user-defined storage levels.
+ val useDisk = (json \ "Use Disk").extract[Boolean]
+ val useMemory = (json \ "Use Memory").extract[Boolean]
+ val deserialized = (json \ "Deserialized").extract[Boolean]
+ val replication = (json \ "Replication").extract[Int]
+ StorageLevel(useDisk, useMemory, deserialized, replication)
+ case _ =>
+ throw new IllegalArgumentException(json.toString)
--- End diff --
nit:
```
throw new IllegalArgumentException(s"Invalid storage level from json:
$json.")
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
