HADOOP-11521. Make connection timeout configurable in s3a. (Thomas Demoor via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00b80958 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00b80958 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00b80958 Branch: refs/heads/trunk Commit: 00b80958d862dbcc08d6f186c09963d3351ba0fd Parents: 6be5670 Author: Steve Loughran <ste...@apache.org> Authored: Tue Feb 17 20:00:00 2015 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Feb 17 20:06:27 2015 +0000 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/main/resources/core-default.xml | 6 ++++++ .../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 4 ++++ .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 ++ .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 6 ++++++ 5 files changed, 21 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 405e821..c1caf5f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -615,6 +615,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel) + HADOOP-11521. Make connection timeout configurable in s3a. + (Thomas Demoor via stevel) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 5eb301a..41121f1 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -758,6 +758,12 @@ for ldap providers in the same way as above does. </property> <property> + <name>fs.s3a.connection.establish.timeout</name> + <value>5000</value> + <description>Socket connection setup timeout in seconds.</description> +</property> + +<property> <name>fs.s3a.connection.timeout</name> <value>50000</value> <description>Socket connection timeout in seconds.</description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index b6863bb..1d4f67b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -42,6 +42,10 @@ public class Constants { // number of times we should retry errors public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum"; public static final int DEFAULT_MAX_ERROR_RETRIES = 10; + + // seconds until we give up trying to establish a connection to s3 + public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout"; + public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000; // seconds until we give up on a connection to s3 public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 22350bc..4de5c13 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -174,6 +174,8 @@ public class S3AFileSystem extends FileSystem { awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES)); + awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, + DEFAULT_ESTABLISH_TIMEOUT)); awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 8f511dd..1e44864 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -211,6 +211,12 @@ If you do any of these: change your credentials immediately! </property> <property> + <name>fs.s3a.connection.establish.timeout</name> + <value>5000</value> + <description>Socket connection setup timeout in seconds.</description> + </property> + + <property> <name>fs.s3a.connection.timeout</name> <value>50000</value> <description>Socket connection timeout in seconds.</description>