Repository: hadoop Updated Branches: refs/heads/HEAD be1ec005f -> 907ef6c28
HDFS-13936. Multipart upload to HDFS to support 0 byte upload. Contributed by Ewan Higgs. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fab6886 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fab6886 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fab6886 Branch: refs/heads/HEAD Commit: 6fab6886f652492573734b832ca0375459a82775 Parents: a383ac4 Author: Ewan Higgs <ehi...@apache.org> Authored: Tue Oct 2 14:03:28 2018 +0200 Committer: Ewan Higgs <ehi...@apache.org> Committed: Tue Oct 2 14:05:35 2018 +0200 ---------------------------------------------------------------------- .../hadoop/fs/FileSystemMultipartUploader.java | 25 +++++++++++++++----- .../AbstractContractMultipartUploaderTest.java | 22 +++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fab6886/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java index 690194d..94c7861 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java @@ -100,6 +100,14 @@ public class FileSystemMultipartUploader extends MultipartUploader { return fs.getPathHandle(status); } + private long totalPartsLen(List<Path> partHandles) throws IOException { + long totalLen = 0; + for (Path p: partHandles) { + totalLen += fs.getFileStatus(p).getLen(); + } + return totalLen; + } + @Override @SuppressWarnings("deprecation") // rename w/ OVERWRITE public PathHandle complete(Path filePath, @@ -127,12 +135,17 @@ public class FileSystemMultipartUploader extends MultipartUploader { .collect(Collectors.toList()); Path collectorPath = createCollectorPath(filePath); - Path filePathInsideCollector = mergePaths(collectorPath, - new Path(Path.SEPARATOR + filePath.getName())); - fs.create(filePathInsideCollector).close(); - fs.concat(filePathInsideCollector, - partHandles.toArray(new Path[handles.size()])); - fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE); + boolean emptyFile = totalPartsLen(partHandles) == 0; + if (emptyFile) { + fs.create(filePath).close(); + } else { + Path filePathInsideCollector = mergePaths(collectorPath, + new Path(Path.SEPARATOR + filePath.getName())); + fs.create(filePathInsideCollector).close(); + fs.concat(filePathInsideCollector, + partHandles.toArray(new Path[handles.size()])); + fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE); + } fs.delete(collectorPath, true); return getPathHandle(filePath); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fab6886/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java index 85a6861..7cee5a6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java @@ -165,6 +165,28 @@ public abstract class AbstractContractMultipartUploaderTest extends } /** + * Assert that a multipart upload is successful when a single empty part is + * uploaded. + * @throws Exception failure + */ + @Test + public void testMultipartUploadEmptyPart() throws Exception { + FileSystem fs = getFileSystem(); + Path file = path("testMultipartUpload"); + MultipartUploader mpu = MultipartUploaderFactory.get(fs, null); + UploadHandle uploadHandle = mpu.initialize(file); + List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>(); + MessageDigest origDigest = DigestUtils.getMd5Digest(); + byte[] payload = new byte[0]; + origDigest.update(payload); + InputStream is = new ByteArrayInputStream(payload); + PartHandle partHandle = mpu.putPart(file, is, 0, uploadHandle, + payload.length); + partHandles.add(Pair.of(0, partHandle)); + completeUpload(file, mpu, uploadHandle, partHandles, origDigest, 0); + } + + /** * Assert that a multipart upload is successful even when the parts are * given in the reverse order. */ --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
