Github user JoshRosen commented on a diff in the pull request:
https://github.com/apache/spark/pull/6672#discussion_r33068267
--- Diff: core/src/main/scala/org/apache/spark/util/JsonProtocol.scala ---
@@ -407,6 +414,52 @@ private[spark] object JsonProtocol {
("Log Urls" -> mapToJson(executorInfo.logUrlMap))
}
+ def blockUpdatedInfoToJson(blockUpdatedInfo: BlockUpdatedInfo): JValue =
{
+ ("Block Manager ID" ->
blockManagerIdToJson(blockUpdatedInfo.blockManagerId)) ~
+ ("Block ID" -> blockIdToJson(blockUpdatedInfo.blockId)) ~
+ ("Storage Level" -> storageLevelToJson(blockUpdatedInfo.storageLevel))
~
+ ("Memory Size" -> blockUpdatedInfo.memSize) ~
+ ("ExternalBlockStore Size" -> blockUpdatedInfo.externalBlockStoreSize)
~
+ ("Disk Size" -> blockUpdatedInfo.diskSize)
+ }
+
+ def blockIdToJson(blockId: BlockId): JValue = blockId match {
+ case RDDBlockId(rddId, splitIndex) =>
+ ("Class Name" -> Utils.getFormattedClassName(blockId)) ~
+ ("RDD ID" -> rddId) ~
+ ("Split Index" -> splitIndex)
+ case ShuffleBlockId(shuffleId, mapId, reduceId) =>
+ ("Class Name" -> Utils.getFormattedClassName(blockId)) ~
+ ("Shuffle ID" -> shuffleId) ~
+ ("Map ID" -> mapId) ~
+ ("Reduce ID" -> reduceId)
+ case ShuffleDataBlockId(shuffleId, mapId, reduceId) =>
+ ("Class Name" -> Utils.getFormattedClassName(blockId)) ~
+ ("Shuffle ID" -> shuffleId) ~
+ ("Map ID" -> mapId) ~
+ ("Reduce ID" -> reduceId)
+ case ShuffleIndexBlockId(shuffleId, mapId, reduceId) =>
+ ("Class Name" -> Utils.getFormattedClassName(blockId)) ~
+ ("Shuffle ID" -> shuffleId) ~
+ ("Map ID" -> mapId) ~
+ ("Reduce ID" -> reduceId)
+ case BroadcastBlockId(broadcastId, field) =>
+ ("Class Name" -> Utils.getFormattedClassName(blockId)) ~
+ ("Broadcast ID" -> broadcastId) ~
+ ("Field" -> field)
+ case TaskResultBlockId(taskId) =>
+ ("Class Name" -> Utils.getFormattedClassName(blockId)) ~
+ ("Task ID" -> taskId)
+ case StreamBlockId(streamId, uniqueId) =>
+ ("Class Name" -> Utils.getFormattedClassName(blockId)) ~
+ ("Stream ID" -> streamId) ~
+ ("Unique ID" -> uniqueId)
+ // These aren't used, but keeps compiler happy
+ case TempLocalBlockId(_) => JNothing
+ case TempShuffleBlockId(_) => JNothing
+ case TestBlockId(_) => JNothing
--- End diff --
Should we maybe add a catch-call case here to map unknown block id types to
JNothing?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
