dongjoon-hyun commented on code in PR #36885:
URL: https://github.com/apache/spark/pull/36885#discussion_r1103463061
##########
core/src/main/scala/org/apache/spark/util/JsonProtocol.scala:
##########
@@ -1218,104 +1416,146 @@ private[spark] object JsonProtocol {
replication = replication)
}
- def blockStatusFromJson(json: JValue): BlockStatus = {
- val storageLevel = storageLevelFromJson(json \ "Storage Level")
- val memorySize = (json \ "Memory Size").extract[Long]
- val diskSize = (json \ "Disk Size").extract[Long]
+ def blockStatusFromJson(json: JsonNode): BlockStatus = {
+ val storageLevel = storageLevelFromJson(json.get("Storage Level"))
+ val memorySize = json.get("Memory Size").extractLong
+ val diskSize = json.get("Disk Size").extractLong
BlockStatus(storageLevel, memorySize, diskSize)
}
- def executorInfoFromJson(json: JValue): ExecutorInfo = {
- val executorHost = (json \ "Host").extract[String]
- val totalCores = (json \ "Total Cores").extract[Int]
- val logUrls = mapFromJson(json \ "Log Urls").toMap
- val attributes = jsonOption(json \ "Attributes") match {
+ def executorInfoFromJson(json: JsonNode): ExecutorInfo = {
+ val executorHost = json.get("Host").extractString
+ val totalCores = json.get("Total Cores").extractInt
+ val logUrls = mapFromJson(json.get("Log Urls")).toMap
+ // The "Attributes" field was added in Spark 3.0.0:
+ val attributes = jsonOption(json.get("Attributes")) match {
case Some(attr) => mapFromJson(attr).toMap
case None => Map.empty[String, String]
}
- val resources = jsonOption(json \ "Resources") match {
+ // The "Resources" field was added in Spark 3.0.0:
+ val resources = jsonOption(json.get("Resources")) match {
case Some(resources) => resourcesMapFromJson(resources).toMap
case None => Map.empty[String, ResourceInformation]
}
- val resourceProfileId = jsonOption(json \ "Resource Profile Id") match {
- case Some(id) => id.extract[Int]
+ // The "Resource Profile Id" field was added in Spark 3.4.0
+ val resourceProfileId = jsonOption(json.get("Resource Profile Id")) match {
+ case Some(id) => id.extractInt
case None => ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
}
- val registrationTs = jsonOption(json \ "Registration Time") map { ts =>
- ts.extract[Long]
+ // The "Registration Time" field was added in Spark 3.4.0
+ val registrationTs = jsonOption(json.get("Registration Time")) map { ts =>
+ ts.extractLong
}
- val requestTs = jsonOption(json \ "Request Time") map { ts =>
- ts.extract[Long]
+ // The "Request Time" field was added in Spark 3.4.0
+ val requestTs = jsonOption(json.get("Request Time")) map { ts =>
+ ts.extractLong
}
new ExecutorInfo(executorHost, totalCores, logUrls, attributes.toMap,
resources.toMap,
resourceProfileId, registrationTs, requestTs)
}
- def blockUpdatedInfoFromJson(json: JValue): BlockUpdatedInfo = {
- val blockManagerId = blockManagerIdFromJson(json \ "Block Manager ID")
- val blockId = BlockId((json \ "Block ID").extract[String])
- val storageLevel = storageLevelFromJson(json \ "Storage Level")
- val memorySize = (json \ "Memory Size").extract[Long]
- val diskSize = (json \ "Disk Size").extract[Long]
+ def blockUpdatedInfoFromJson(json: JsonNode): BlockUpdatedInfo = {
+ val blockManagerId = blockManagerIdFromJson(json.get("Block Manager ID"))
+ val blockId = BlockId(json.get("Block ID").extractString)
+ val storageLevel = storageLevelFromJson(json.get("Storage Level"))
+ val memorySize = json.get("Memory Size").extractLong
+ val diskSize = json.get("Disk Size").extractLong
BlockUpdatedInfo(blockManagerId, blockId, storageLevel, memorySize,
diskSize)
}
- def resourcesMapFromJson(json: JValue): Map[String, ResourceInformation] = {
- val jsonFields = json.asInstanceOf[JObject].obj
- jsonFields.collect { case JField(k, v) =>
- val resourceInfo = ResourceInformation.parseJson(v)
- (k, resourceInfo)
+ def resourcesMapFromJson(json: JsonNode): Map[String, ResourceInformation] =
{
+ assert(json.isObject, s"expected object, got ${json.getNodeType}")
+ json.fields.asScala.map { field =>
+ val resourceInfo = ResourceInformation.parseJson(field.getValue.toString)
+ (field.getKey, resourceInfo)
}.toMap
}
/** -------------------------------- *
* Util JSON deserialization methods |
* --------------------------------- */
- def mapFromJson(json: JValue): Map[String, String] = {
- val jsonFields = json.asInstanceOf[JObject].obj
- jsonFields.collect { case JField(k, JString(v)) => (k, v) }.toMap
+ def mapFromJson(json: JsonNode): Map[String, String] = {
+ assert(json.isObject, s"expected object, got ${json.getNodeType}")
+ json.fields.asScala.map { field =>
+ (field.getKey, field.getValue.extractString)
+ }.toMap
}
- def propertiesFromJson(json: JValue): Properties = {
+ def propertiesFromJson(json: JsonNode): Properties = {
jsonOption(json).map { value =>
val properties = new Properties
- mapFromJson(json).foreach { case (k, v) => properties.setProperty(k, v) }
+ mapFromJson(value).foreach { case (k, v) => properties.setProperty(k, v)
}
properties
}.orNull
}
- def UUIDFromJson(json: JValue): UUID = {
- val leastSignificantBits = (json \ "Least Significant Bits").extract[Long]
- val mostSignificantBits = (json \ "Most Significant Bits").extract[Long]
+ def UUIDFromJson(json: JsonNode): UUID = {
+ val leastSignificantBits = json.get("Least Significant Bits").extractLong
+ val mostSignificantBits = json.get("Most Significant Bits").extractLong
new UUID(leastSignificantBits, mostSignificantBits)
}
- def stackTraceFromJson(json: JValue): Array[StackTraceElement] = {
- json.extract[List[JValue]].map { line =>
- val declaringClass = (line \ "Declaring Class").extract[String]
- val methodName = (line \ "Method Name").extract[String]
- val fileName = (line \ "File Name").extract[String]
- val lineNumber = (line \ "Line Number").extract[Int]
+ def stackTraceFromJson(json: JsonNode): Array[StackTraceElement] = {
+ json.extractElements.map { line =>
+ val declaringClass = line.get("Declaring Class").extractString
+ val methodName = line.get("Method Name").extractString
+ val fileName = line.get("File Name").extractString
Review Comment:
This change could be the usual suspect.
```scala
- val fileName = (line \ "File Name").extract[String]
+ val fileName = line.get("File Name").extractString
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
