Repository: tajo Updated Branches: refs/heads/master 009e8e5ae -> a1d5aeee7
TAJO-1651: Too long fetcher default retries. (jinho) Closes #648 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a1d5aeee Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a1d5aeee Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a1d5aeee Branch: refs/heads/master Commit: a1d5aeee7241281ff64945b7d30399b9c76ad363 Parents: 009e8e5 Author: Jinho Kim <[email protected]> Authored: Fri Jul 24 14:49:03 2015 +0900 Committer: Jinho Kim <[email protected]> Committed: Fri Jul 24 14:49:03 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 ++ .../main/java/org/apache/tajo/conf/TajoConf.java | 5 +++-- .../main/java/org/apache/tajo/worker/Fetcher.java | 17 +++++++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/a1d5aeee/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index ae1e2d8..4403673 100644 --- a/CHANGES +++ b/CHANGES @@ -32,6 +32,8 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1651: Too long fetcher default retries. (jinho) + TAJO-1700: Add better exception handling in TajoMasterClientService. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/a1d5aeee/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 910d6bc..b16e55f 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -210,8 +210,9 @@ public class TajoConf extends Configuration { SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM("tajo.shuffle.fetcher.parallel-execution.max-num", 2, Validators.min("1")), SHUFFLE_FETCHER_CHUNK_MAX_SIZE("tajo.shuffle.fetcher.chunk.max-size", 8192), - SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 120), - SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 20), + SHUFFLE_FETCHER_CONNECT_TIMEOUT("tajo.shuffle.fetcher.connect.timeout-sec", 10, Validators.min("1")), + SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 60, Validators.min("1")), + SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 2, Validators.min("0")), SHUFFLE_HASH_APPENDER_BUFFER_SIZE("tajo.shuffle.hash.appender.buffer.size", 10000), SHUFFLE_HASH_APPENDER_PAGE_VOLUME("tajo.shuffle.hash.appender.page.volumn-mb", 30), HASH_SHUFFLE_PARENT_DIRS("tajo.hash.shuffle.parent.dirs.count", 10), http://git-wip-us.apache.org/repos/asf/tajo/blob/a1d5aeee/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java b/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java index 94488d0..ff85a4b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java @@ -88,14 +88,15 @@ public class Fetcher { if (!useLocalFile) { bootstrap = new Bootstrap() - .group( - RpcChannelFactory.getSharedClientEventloopGroup(RpcChannelFactory.ClientChannelId.FETCHER, - conf.getIntVar(TajoConf.ConfVars.SHUFFLE_RPC_CLIENT_WORKER_THREAD_NUM))) - .channel(NioSocketChannel.class) - .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 5000) // set 5 sec - .option(ChannelOption.SO_RCVBUF, 1048576) // set 1M - .option(ChannelOption.TCP_NODELAY, true); + .group( + RpcChannelFactory.getSharedClientEventloopGroup(RpcChannelFactory.ClientChannelId.FETCHER, + conf.getIntVar(TajoConf.ConfVars.SHUFFLE_RPC_CLIENT_WORKER_THREAD_NUM))) + .channel(NioSocketChannel.class) + .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, + conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_CONNECT_TIMEOUT) * 1000) + .option(ChannelOption.SO_RCVBUF, 1048576) // set 1M + .option(ChannelOption.TCP_NODELAY, true); ChannelInitializer<Channel> initializer = new HttpClientChannelInitializer(fileChunk.getFile()); bootstrap.handler(initializer);
