Repository: hadoop Updated Branches: refs/heads/branch-2 970fdc3ad -> 4cd9657b7 refs/heads/trunk 20660b7a6 -> 4e7ad4f0a
HADOOP-11463 Replace method-local TransferManager object with S3AFileSystem#transfers. (Ted Yu via stevel) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cd9657b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cd9657b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cd9657b Branch: refs/heads/branch-2 Commit: 4cd9657b7153b571022495629acfb3fa4ec51b14 Parents: 970fdc3 Author: Steve Loughran <[email protected]> Authored: Thu Feb 5 12:19:49 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Thu Feb 5 12:19:49 2015 +0000 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 30 ++++++++------------ 2 files changed, 15 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cd9657b/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 1eef93e..269c77a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -175,6 +175,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11492. Bump up curator version to 2.7.1. (Arun Suresh and Karthik Kambatla via kasha) + HADOOP-11463 Replace method-local TransferManager object with + S3AFileSystem#transfers. (Ted Yu via stevel) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cd9657b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 0e4d54f..22350bc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -292,7 +292,6 @@ public class S3AFileSystem extends FileSystem { Date purgeBefore = new Date(new Date().getTime() - purgeExistingMultipartAge*1000); transfers.abortMultipartUploads(bucket, purgeBefore); - transfers.shutdownNow(false); } serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM); @@ -995,13 +994,6 @@ public class S3AFileSystem extends FileSystem { LocalFileSystem local = getLocal(getConf()); File srcfile = local.pathToFile(src); - TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); - transferConfiguration.setMinimumUploadPartSize(partSize); - transferConfiguration.setMultipartUploadThreshold(partSizeThreshold); - - TransferManager transfers = new TransferManager(s3); - transfers.setConfiguration(transferConfiguration); - final ObjectMetadata om = new ObjectMetadata(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { om.setServerSideEncryption(serverSideEncryptionAlgorithm); @@ -1029,8 +1021,6 @@ public class S3AFileSystem extends FileSystem { statistics.incrementWriteOps(1); } catch (InterruptedException e) { throw new IOException("Got interrupted, cancelling"); - } finally { - transfers.shutdownNow(false); } // This will delete unnecessary fake parent directories @@ -1041,6 +1031,18 @@ public class S3AFileSystem extends FileSystem { } } + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + if (transfers != null) { + transfers.shutdownNow(true); + transfers = null; + } + } + } + /** * Override getCononicalServiceName because we don't support token in S3A */ @@ -1055,12 +1057,6 @@ public class S3AFileSystem extends FileSystem { LOG.debug("copyFile " + srcKey + " -> " + dstKey); } - TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); - transferConfiguration.setMultipartCopyPartSize(partSize); - - TransferManager transfers = new TransferManager(s3); - transfers.setConfiguration(transferConfiguration); - ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey); final ObjectMetadata dstom = srcom.clone(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { @@ -1089,8 +1085,6 @@ public class S3AFileSystem extends FileSystem { statistics.incrementWriteOps(1); } catch (InterruptedException e) { throw new IOException("Got interrupted, cancelling"); - } finally { - transfers.shutdownNow(false); } }
