HADOOP-12891. S3AFileSystem should configure Multipart Copy threshold and chunk size. (Andrew Olson via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5df89f9a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5df89f9a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5df89f9a Branch: refs/heads/branch-2.8 Commit: 5df89f9a8a4adc16820f28289878b5f800878447 Parents: adf937f Author: Steve Loughran <[email protected]> Authored: Fri Apr 22 11:24:24 2016 +0100 Committer: Steve Loughran <[email protected]> Committed: Fri Apr 22 11:24:24 2016 +0100 ---------------------------------------------------------------------- .../hadoop-common/src/main/resources/core-default.xml | 5 ++++- .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 ++ .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df89f9a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 83dd37d..51e2df4 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -840,7 +840,10 @@ <property> <name>fs.s3a.multipart.threshold</name> <value>2147483647</value> - <description>Threshold before uploads or copies use parallel multipart operations.</description> + <description>How big (in bytes) to split upload or copy operations up into. + This also controls the partition size in renamed files, as rename() involves + copying the source file(s) + </description> </property> <property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df89f9a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 83d1d53..4cd2783 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -318,6 +318,8 @@ public class S3AFileSystem extends FileSystem { TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMinimumUploadPartSize(partSize); transferConfiguration.setMultipartUploadThreshold(multiPartThreshold); + transferConfiguration.setMultipartCopyPartSize(partSize); + transferConfiguration.setMultipartCopyThreshold(multiPartThreshold); transfers = new TransferManager(s3, threadPoolExecutor); transfers.setConfiguration(transferConfiguration); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df89f9a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 15b9837..9016264 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -315,7 +315,9 @@ this capability. <property> <name>fs.s3a.multipart.size</name> <value>104857600</value> - <description>How big (in bytes) to split upload or copy operations up into.</description> + <description>How big (in bytes) to split upload or copy operations up into. + This also controls the partition size in renamed files, as rename() involves + copying the source file(s)</description> </property> <property> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
