Github user rezasafi commented on a diff in the pull request:
https://github.com/apache/spark/pull/19848#discussion_r157068238
--- Diff:
core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala ---
@@ -908,6 +918,40 @@ class NewFakeFormatWithCallback() extends
NewFakeFormat {
}
}
+class YetAnotherFakeCommitter extends NewOutputCommitter with Assertions {
+ def setupJob(j: NewJobContext): Unit = {
+ JobID.jobid = j.getJobID().getId
+ }
+
+ def needsTaskCommit(t: NewTaskAttempContext): Boolean = false
+
+ def setupTask(t: NewTaskAttempContext): Unit = {
+ val jobId = t.getTaskAttemptID().getJobID().getId
+ assert(jobId === JobID.jobid)
+ }
+
+ def commitTask(t: NewTaskAttempContext): Unit = {}
+
+ def abortTask(t: NewTaskAttempContext): Unit = {}
+}
+
+class YetAnotherFakeFormat() extends NewOutputFormat[Integer, Integer]() {
+
+ def checkOutputSpecs(j: NewJobContext): Unit = {}
+
+ def getRecordWriter(t: NewTaskAttempContext): NewRecordWriter[Integer,
Integer] = {
+ new NewFakeWriter()
+ }
+
+ def getOutputCommitter(t: NewTaskAttempContext): NewOutputCommitter = {
+ new YetAnotherFakeCommitter()
+ }
+}
+
+object JobID {
+ var jobid = -1
--- End diff --
@steveloughran, this isn't used by committers. It is just used for storing
the jobId. For test purposes here I used -1 to just make sure this variable is
not set.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
