Repository: hadoop Updated Branches: refs/heads/branch-3.0.0-alpha1 cc55b18ad -> 1b1e3c10e
HADOOP-13212 Provide an option to set the socket buffers in S3AFileSystem (Rajesh Balamohan) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b1e3c10 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b1e3c10 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b1e3c10 Branch: refs/heads/branch-3.0.0-alpha1 Commit: 1b1e3c10e8fb4267883e9410fabd19cc0d62d685 Parents: cc55b18 Author: Steve Loughran <[email protected]> Authored: Wed Jul 20 13:42:19 2016 +0100 Committer: Steve Loughran <[email protected]> Committed: Wed Jul 20 14:03:44 2016 +0100 ---------------------------------------------------------------------- .../hadoop-common/src/main/resources/core-default.xml | 12 ++++++++++++ .../main/java/org/apache/hadoop/fs/s3a/Constants.java | 8 ++++++++ .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 5 +++++ .../src/site/markdown/tools/hadoop-aws/index.md | 12 ++++++++++++ .../fs/s3a/scale/TestS3AInputStreamPerformance.java | 3 +++ 5 files changed, 40 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 3018884..0d751bb 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -908,6 +908,18 @@ </property> <property> + <name>fs.s3a.socket.send.buffer</name> + <value>8192</value> + <description>Socket send buffer hint to amazon connector. Represented in bytes.</description> +</property> + +<property> + <name>fs.s3a.socket.recv.buffer</name> + <value>8192</value> + <description>Socket receive buffer hint to amazon connector. Represented in bytes.</description> +</property> + +<property> <name>fs.s3a.paging.maximum</name> <value>5000</value> <description>How many keys to request from S3 when doing http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 3f6f347..612b648 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -82,6 +82,14 @@ public final class Constants { public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout"; public static final int DEFAULT_SOCKET_TIMEOUT = 200000; + // socket send buffer to be used in Amazon client + public static final String SOCKET_SEND_BUFFER = "fs.s3a.socket.send.buffer"; + public static final int DEFAULT_SOCKET_SEND_BUFFER = 8 * 1024; + + // socket send buffer to be used in Amazon client + public static final String SOCKET_RECV_BUFFER = "fs.s3a.socket.recv.buffer"; + public static final int DEFAULT_SOCKET_RECV_BUFFER = 8 * 1024; + // number of records to get while paging through a directory listing public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum"; public static final int DEFAULT_MAX_PAGING_KEYS = 5000; http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 513b57c..10595e2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -163,6 +163,11 @@ public class S3AFileSystem extends FileSystem { DEFAULT_ESTABLISH_TIMEOUT, 0)); awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0)); + int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER, + DEFAULT_SOCKET_SEND_BUFFER, 2048); + int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER, + DEFAULT_SOCKET_RECV_BUFFER, 2048); + awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer); String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, ""); if (!signerOverride.isEmpty()) { LOG.debug("Signer override = {}", signerOverride); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 2b64754..01a2bae 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -504,6 +504,18 @@ this capability. </property> <property> + <name>fs.s3a.socket.send.buffer</name> + <value>8192</value> + <description>Socket send buffer hint to amazon connector. Represented in bytes.</description> + </property> + + <property> + <name>fs.s3a.socket.recv.buffer</name> + <value>8192</value> + <description>Socket receive buffer hint to amazon connector. Represented in bytes.</description> + </property> + + <property> <name>fs.s3a.threads.keepalivetime</name> <value>60</value> <description>Number of seconds a thread can be idle before being http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java index 752e374..bddd8e2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java @@ -44,6 +44,7 @@ import java.io.EOFException; import java.io.IOException; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; +import static org.apache.hadoop.fs.s3a.Constants.*; /** * Look at the performance of S3a operations. @@ -71,6 +72,8 @@ public class TestS3AInputStreamPerformance extends S3AScaleTestBase { @Before public void openFS() throws IOException { Configuration conf = getConf(); + conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024); + conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024); String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE); if (testFile.isEmpty()) { assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
