HADOOP-15469. S3A directory committer commit job fails if _temporary directory created under dest. Contributed by Steve Loughran.
(cherry picked from commit 170f1040d46f9b1a084e6637def91e9864446acc) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc3600aa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc3600aa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc3600aa Branch: refs/heads/trunk Commit: cc3600aabdca6e8b14c9fe02fe54073bf4ef7685 Parents: c0ec061 Author: Steve Loughran <ste...@apache.org> Authored: Thu May 17 19:08:05 2018 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Thu May 17 19:08:55 2018 +0100 ---------------------------------------------------------------------- .../fs/s3a/commit/staging/DirectoryStagingCommitter.java | 7 ++----- .../commit/staging/TestStagingDirectoryOutputCommitter.java | 8 +++----- 2 files changed, 5 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc3600aa/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java index 03dfed2..3eda24f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java @@ -93,11 +93,8 @@ public class DirectoryStagingCommitter extends StagingCommitter { Configuration fsConf = fs.getConf(); switch (getConflictResolutionMode(context, fsConf)) { case FAIL: - // this was checked in setupJob, but this avoids some cases where - // output was created while the job was processing - if (fs.exists(outputPath)) { - throw new PathExistsException(outputPath.toString(), E_DEST_EXISTS); - } + // this was checked in setupJob; temporary files may have been + // created, so do not check again. break; case APPEND: // do nothing http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc3600aa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java index 4f0189e..f5b8d1f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java @@ -67,15 +67,14 @@ public class TestStagingDirectoryOutputCommitter pathExists(mockS3, OUTPUT_PATH); final DirectoryStagingCommitter committer = newJobCommitter(); + // this should fail intercept(PathExistsException.class, InternalCommitterConstants.E_DEST_EXISTS, "Should throw an exception because the path exists", () -> committer.setupJob(getJob())); - intercept(PathExistsException.class, - InternalCommitterConstants.E_DEST_EXISTS, - "Should throw an exception because the path exists", - () -> committer.commitJob(getJob())); + // but there are no checks in job commit (HADOOP-15469) + committer.commitJob(getJob()); reset(mockS3); pathDoesNotExist(mockS3, OUTPUT_PATH); @@ -87,7 +86,6 @@ public class TestStagingDirectoryOutputCommitter reset(mockS3); pathDoesNotExist(mockS3, OUTPUT_PATH); committer.commitJob(getJob()); - verifyExistenceChecked(mockS3, OUTPUT_PATH); verifyCompletion(mockS3); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org