MAPREDUCE-6473. Job submission can take a long time during Cluster initialization. Contributed by Kuhu Shukla
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f657b542 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f657b542 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f657b542 Branch: refs/heads/HDFS-1312 Commit: f657b54281d517c69df0a23b97b47befa77d1dc2 Parents: 126705f Author: Jason Lowe <[email protected]> Authored: Tue Jan 12 23:56:23 2016 +0000 Committer: Jason Lowe <[email protected]> Committed: Tue Jan 12 23:56:23 2016 +0000 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 + .../org/apache/hadoop/mapreduce/Cluster.java | 67 ++++++++++++-------- 2 files changed, 43 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f657b542/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 20fc08b..fa5c086 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -468,6 +468,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6394. Speed up Task processing loop in HsTasksBlock#render() (Ray Chiang via jlowe) + MAPREDUCE-6473. Job submission can take a long time during Cluster + initialization (Kuhu Shukla via jlowe) + BUG FIXES MAPREDUCE-6314. TestPipeApplication fails on trunk. http://git-wip-us.apache.org/repos/asf/hadoop/blob/f657b542/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java index fc228e6..9563c0b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java @@ -66,7 +66,23 @@ public class Cluster { private static ServiceLoader<ClientProtocolProvider> frameworkLoader = ServiceLoader.load(ClientProtocolProvider.class); - + private volatile List<ClientProtocolProvider> providerList = null; + + private void initProviderList() { + if (providerList == null) { + synchronized (frameworkLoader) { + if (providerList == null) { + List<ClientProtocolProvider> localProviderList = + new ArrayList<ClientProtocolProvider>(); + for (ClientProtocolProvider provider : frameworkLoader) { + localProviderList.add(provider); + } + providerList = localProviderList; + } + } + } + } + static { ConfigUtil.loadResources(); } @@ -85,34 +101,31 @@ public class Cluster { private void initialize(InetSocketAddress jobTrackAddr, Configuration conf) throws IOException { - synchronized (frameworkLoader) { - for (ClientProtocolProvider provider : frameworkLoader) { - LOG.debug("Trying ClientProtocolProvider : " - + provider.getClass().getName()); - ClientProtocol clientProtocol = null; - try { - if (jobTrackAddr == null) { - clientProtocol = provider.create(conf); - } else { - clientProtocol = provider.create(jobTrackAddr, conf); - } + initProviderList(); + for (ClientProtocolProvider provider : providerList) { + LOG.debug("Trying ClientProtocolProvider : " + + provider.getClass().getName()); + ClientProtocol clientProtocol = null; + try { + if (jobTrackAddr == null) { + clientProtocol = provider.create(conf); + } else { + clientProtocol = provider.create(jobTrackAddr, conf); + } - if (clientProtocol != null) { - clientProtocolProvider = provider; - client = clientProtocol; - LOG.debug("Picked " + provider.getClass().getName() - + " as the ClientProtocolProvider"); - break; - } - else { - LOG.debug("Cannot pick " + provider.getClass().getName() - + " as the ClientProtocolProvider - returned null protocol"); - } - } - catch (Exception e) { - LOG.info("Failed to use " + provider.getClass().getName() - + " due to error: ", e); + if (clientProtocol != null) { + clientProtocolProvider = provider; + client = clientProtocol; + LOG.debug("Picked " + provider.getClass().getName() + + " as the ClientProtocolProvider"); + break; + } else { + LOG.debug("Cannot pick " + provider.getClass().getName() + + " as the ClientProtocolProvider - returned null protocol"); } + } catch (Exception e) { + LOG.info("Failed to use " + provider.getClass().getName() + + " due to error: ", e); } }
