srowen commented on a change in pull request #26397: [SPARK-29755][CORE]
Provide @JsonDeserialize for Option[Long] in LogInfo & AttemptInfoWrapper
URL: https://github.com/apache/spark/pull/26397#discussion_r343285798
##########
File path:
core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
##########
@@ -1283,6 +1285,56 @@ class FsHistoryProviderSuite extends SparkFunSuite with
Matchers with Logging {
assert(deserializedOldObj.isComplete === false)
}
+ test("SPARK-29755 LogInfo should be serialized/deserialized by jackson
properly") {
+ def assertSerDe(serializer: KVStoreScalaSerializer, info: LogInfo): Unit =
{
+ val infoAfterSerDe = serializer.deserialize(
+ serializer.serialize(info), classOf[LogInfo])
+ assert(infoAfterSerDe === info)
+ assertOptionAfterSerde(infoAfterSerDe.lastIndex, info.lastIndex)
+ }
+
+ val serializer = new KVStoreScalaSerializer()
+ val logInfoWithIndexAsNone = LogInfo("dummy", 0, LogType.EventLogs,
Some("appId"),
+ Some("attemptId"), 100, None, false)
+ assertSerDe(serializer, logInfoWithIndexAsNone)
+
+ val logInfoWithIndex = LogInfo("dummy", 0, LogType.EventLogs,
Some("appId"),
+ Some("attemptId"), 100, Some(3), false)
+ assertSerDe(serializer, logInfoWithIndex)
+ }
+
+ test("SPARK-29755 AttemptInfoWrapper should be serialized/deserialized by
jackson properly") {
+ def assertSerDe(serializer: KVStoreScalaSerializer, attempt:
AttemptInfoWrapper): Unit = {
+ val attemptAfterSerDe =
serializer.deserialize(serializer.serialize(attempt),
+ classOf[AttemptInfoWrapper])
+ assert(attemptAfterSerDe.info === attempt.info)
+ // skip comparing some fields, as they've not triggered SPARK-29755
+ assertOptionAfterSerde(attemptAfterSerDe.lastIndex, attempt.lastIndex)
+ }
+
+ val serializer = new KVStoreScalaSerializer()
+ val appInfo = new ApplicationAttemptInfo(None, new Date(1), new Date(1),
new Date(1),
+ 10, "spark", false, "dummy")
+ val attemptInfoWithIndexAsNone = new AttemptInfoWrapper(appInfo,
"dummyPath", 10, None,
+ None, None, None, None)
+ assertSerDe(serializer, attemptInfoWithIndexAsNone)
+
+ val attemptInfoWithIndex = new AttemptInfoWrapper(appInfo, "dummyPath",
10, Some(1),
+ None, None, None, None)
+ assertSerDe(serializer, attemptInfoWithIndex)
+ }
+
+ private def assertOptionAfterSerde(opt: Option[Long], expected:
Option[Long]): Unit = {
+ if (expected.isEmpty) {
+ assert(opt.isEmpty)
+ } else {
+ // The issue happens only the value in Option is being unboxed. Simple
comparison sometimes
+ // doesn't go though unboxing the value, hence ClassCastException is not
occurred.
+ // Here we ensure unboxing to Long succeeds. Please refer SPARK-29755
for more details.
+ assert(BoxesRunTime.unboxToLong(opt.get) === expected.get)
Review comment:
So, before the problem is roughly that you get an `Option[Int]` where an
`Option[Long]` should be. This code seems to _accommodate_ that possibility in
the test, allowing it work, when we want a test that would fail if it found an
`Option[Int]` here. Right? what am I missing. If it's really an `Option[Long]`
as expected (post fix), then there isn't a need to accommodate an `Option[Int]`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
