This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6bbfb45 [SPARK-33298][CORE][FOLLOWUP] Add Unstable annotation to
`FileCommitProtocol`
6bbfb45 is described below
commit 6bbfb45ffe75aa6c27a7bf3c3385a596637d1822
Author: Cheng Su <chen...@fb.com>
AuthorDate: Wed Jun 30 16:25:20 2021 +0900
[SPARK-33298][CORE][FOLLOWUP] Add Unstable annotation to
`FileCommitProtocol`
### What changes were proposed in this pull request?
This is the followup from
https://github.com/apache/spark/pull/33012#discussion_r659440833, where we want
to add `Unstable` to `FileCommitProtocol`, to give people a better idea of API.
### Why are the changes needed?
Make it easier for people to follow and understand code. Clean up code.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing unit tests, as no real logic change.
Closes #33148 from c21/bucket-followup.
Authored-by: Cheng Su <chen...@fb.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
.../org/apache/spark/internal/io/FileCommitProtocol.scala | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
index 6465cc7..5cd7397 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
@@ -20,6 +20,7 @@ package org.apache.spark.internal.io
import org.apache.hadoop.fs._
import org.apache.hadoop.mapreduce._
+import org.apache.spark.annotation.Unstable
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
@@ -41,7 +42,11 @@ import org.apache.spark.util.Utils
* (or abortTask if task failed).
* 3. When all necessary tasks completed successfully, the driver calls
commitJob. If the job
* failed to execute (e.g. too many failed tasks), the job should call
abortJob.
+ *
+ * @note This class is exposed as an API considering the usage of many
downstream custom
+ * implementations, but will be subject to be changed and/or moved.
*/
+@Unstable
abstract class FileCommitProtocol extends Logging {
import FileCommitProtocol._
@@ -107,9 +112,7 @@ abstract class FileCommitProtocol extends Logging {
* if a task is going to write out multiple files to the same dir. The file
commit protocol only
* guarantees that files written by different tasks will not conflict.
*
- * This API should be implemented and called, instead of
- * [[newTaskTempFile(taskContest, dir, ext)]]. Provide a default
implementation here to be
- * backward compatible with custom [[FileCommitProtocol]] implementations
before Spark 3.2.0.
+ * @since 3.2.0
*/
def newTaskTempFile(
taskContext: TaskAttemptContext, dir: Option[String], spec:
FileNameSpec): String = {
@@ -144,10 +147,7 @@ abstract class FileCommitProtocol extends Logging {
* if a task is going to write out multiple files to the same dir. The file
commit protocol only
* guarantees that files written by different tasks will not conflict.
*
- * This API should be implemented and called, instead of
- * [[newTaskTempFileAbsPath(taskContest, absoluteDir, ext)]]. Provide a
default implementation
- * here to be backward compatible with custom [[FileCommitProtocol]]
implementations before
- * Spark 3.2.0.
+ * @since 3.2.0
*/
def newTaskTempFileAbsPath(
taskContext: TaskAttemptContext, absoluteDir: String, spec:
FileNameSpec): String = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
