Repository: tez Updated Branches: refs/heads/branch-0.5 4bf4c4409 -> 8e94986f0
TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option (rbalamohan) (cherry picked from commit a405ce95762ffc314269a0428e260b966a1627da) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8e94986f Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8e94986f Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8e94986f Branch: refs/heads/branch-0.5 Commit: 8e94986f08c0c46b81728ba9a9ac2309db3f2c04 Parents: 4bf4c44 Author: Rajesh Balamohan <[email protected]> Authored: Mon Aug 17 04:23:21 2015 +0530 Committer: Rajesh Balamohan <[email protected]> Committed: Mon Aug 17 04:32:48 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../tez/runtime/library/common/shuffle/FetchResult.java | 11 +++++++++++ .../tez/runtime/library/common/shuffle/Fetcher.java | 6 +----- .../tez/runtime/library/common/shuffle/InputHost.java | 9 +++++++++ .../library/common/shuffle/impl/ShuffleManager.java | 3 ++- 5 files changed, 24 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7831fb9..f519ee6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-2552. CRC errors can cause job to run for very long time in large jobs. ALL CHANGES: + TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option TEZ-2630. TezChild receives IP address instead of FQDN. (hitesh) TEZ-2635. Limit number of attempts being downloaded in unordered fetch. TEZ-2636. MRInput and MultiMRInput should work for cases when there are 0 physical inputs. http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java index 1c39a24..d9595f0 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java @@ -43,13 +43,20 @@ public class FetchResult { private final int port; private final int partition; private final Iterable<InputAttemptIdentifier> pendingInputs; + private final String additionalInfo; public FetchResult(String host, int port, int partition, Iterable<InputAttemptIdentifier> pendingInputs) { + this(host, port, partition, pendingInputs, null); + } + + public FetchResult(String host, int port, int partition, + Iterable<InputAttemptIdentifier> pendingInputs, String additionalInfo) { this.host = host; this.port = port; this.partition = partition; this.pendingInputs = pendingInputs; + this.additionalInfo = additionalInfo; } public String getHost() { @@ -67,4 +74,8 @@ public class FetchResult { public Iterable<InputAttemptIdentifier> getPendingInputs() { return pendingInputs; } + + public String getAdditionalInfo() { + return additionalInfo; + } } http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java index 6875966..e4cf071 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java @@ -208,8 +208,6 @@ public class Fetcher implements Callable<FetchResult> { if (!multiplex) { throw new IOException("server didn't return all expected map outputs: " + remaining.size() + " left."); - } else { - LOG.info("Shared fetch failed to return " + remaining.size() + " inputs on this try"); } } @@ -355,10 +353,8 @@ public class Fetcher implements Callable<FetchResult> { lock = getLock(); if (lock == null) { // re-queue until we get a lock - LOG.info("Requeuing " + host + ":" + port - + " downloads because we didn't get a lock"); return new HostFetchResult(new FetchResult(host, port, partition, - remaining), null, false); + remaining, "Requeuing as we didn't get a lock"), null, false); } else { if (findInputs() == srcAttempts.size()) { // double checked after lock http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java index 6638633..b3382ea 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java @@ -38,6 +38,7 @@ public class InputHost { private final int port; private final int srcPhysicalIndex; private final String identifier; + private String additionalInfo; private final BlockingQueue<InputAttemptIdentifier> inputs = new LinkedBlockingQueue<InputAttemptIdentifier>(); @@ -64,6 +65,14 @@ public class InputHost { return this.identifier; } + public void setAdditionalInfo(String additionalInfo) { + this.additionalInfo = additionalInfo; + } + + public String getAdditionalInfo() { + return (additionalInfo == null) ? "" : additionalInfo; + } + public int getSrcPhysicalIndex() { return this.srcPhysicalIndex; } http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java index b032bdc..d93de20 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java @@ -310,7 +310,6 @@ public class ShuffleManager implements FetcherCallback { LOG.debug("Processing pending host: " + inputHost.toDetailedString()); } if (inputHost.getNumPendingInputs() > 0 && !isShutdown.get()) { - LOG.info("Scheduling fetch for inputHost: " + inputHost.getIdentifier()); Fetcher fetcher = constructFetcherForHost(inputHost, conf); runningFetchers.add(fetcher); if (isShutdown.get()) { @@ -396,6 +395,7 @@ public class ShuffleManager implements FetcherCallback { fetcherBuilder.assignWork(inputHost.getHost(), inputHost.getPort(), inputHost.getSrcPhysicalIndex(), pendingInputsForHost); LOG.info("Created Fetcher for host: " + inputHost.getHost() + + ", info: " + inputHost.getAdditionalInfo() + ", with inputs: " + pendingInputsForHost); return fetcherBuilder.build(); } @@ -799,6 +799,7 @@ public class ShuffleManager implements FetcherCallback { for (InputAttemptIdentifier input : pendingInputs) { inputHost.addKnownInput(input); } + inputHost.setAdditionalInfo(result.getAdditionalInfo()); pendingHosts.add(inputHost); } doBookKeepingForFetcherComplete();
