Repository: tez Updated Branches: refs/heads/master c82b2eade -> 0207281b3
TEZ-3852. Optimize ContainerContext.isSuperSet to speed container reuse decisions. Contributed by Jonathan Eagles Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0207281b Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0207281b Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0207281b Branch: refs/heads/master Commit: 0207281b3cea4f78b5a57fb81fef0cf14d8c24e1 Parents: c82b2ea Author: Jason Lowe <[email protected]> Authored: Thu Oct 12 09:33:14 2017 -0500 Committer: Jason Lowe <[email protected]> Committed: Thu Oct 12 09:33:14 2017 -0500 ---------------------------------------------------------------------- .../apache/tez/dag/app/ContainerContext.java | 52 ++++++++------------ .../tez/dag/app/rm/TestContainerReuse.java | 4 ++ 2 files changed, 24 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/0207281b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java index f00b27b..d88daff 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java @@ -129,26 +129,24 @@ public class ContainerContext { // classpath modification private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs, Map<String, LocalResource> reqLRs) { - Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs); - for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) { - LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey()); - if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) { + for (Entry<String, LocalResource> reqLREntry : reqLRs.entrySet()) { + LocalResource requestedLocalResource = srcLRs.get(reqLREntry.getKey()); + if (requestedLocalResource == null) { + LocalResource lr = reqLREntry.getValue(); + if (!LocalResourceType.FILE.equals(lr.getType())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot match container: Additional local resource needed is not of type FILE" + + ", resourceName: " + reqLREntry.getKey() + + ", resourceDetails: " + reqLREntry); + } + return false; + } + } else if(!reqLREntry.getValue().equals(requestedLocalResource)) { if (LOG.isDebugEnabled()) { LOG.debug("Cannot match container: Attempting to use same target resource name: " - + srcLREntry.getKey() + + reqLREntry.getKey() + ", but with different source resources. Already localized: " - + srcLREntry.getValue() + ", requested: " + requestedLocalResource); - } - return false; - } - } - for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) { - LocalResource lr = additionalLREntry.getValue(); - if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) { - if (LOG.isDebugEnabled()) { - LOG.debug("Cannot match container: Additional local resource needed is not of type FILE" - + ", resourceName: " + additionalLREntry.getKey() - + ", resourceDetails: " + additionalLREntry); + + requestedLocalResource + ", requested: " + reqLREntry.getValue()); } return false; } @@ -161,24 +159,14 @@ public class ContainerContext { for (Entry<K, V> oEntry : matchMap.entrySet()) { K oKey = oEntry.getKey(); V oVal = oEntry.getValue(); - if (srcMap.containsKey(oKey)) { - if (!oVal.equals(srcMap.get(oKey))) { - if (LOG.isDebugEnabled()) { - LOG.debug("Incompatible container context" + V srcVal = srcMap.get(oKey); + if (!oVal.equals(srcVal)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Incompatible container context" + ", matchInfo=" + matchInfo + ", thisKey=" + oKey - + ", thisVal=" + srcMap.get(oKey) + + ", thisVal=" + srcVal + ", otherVal=" + oVal); - } - return false; - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Incompatible container context" - + ", matchInfo=" + matchInfo - + ", thisKey=" + oKey - + ", thisVal=null" - + ", otherVal=" + oVal); } return false; } http://git-wip-us.apache.org/repos/asf/tez/blob/0207281b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java index 7e9e9ab..4e29dd5 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java @@ -38,6 +38,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.tez.common.TezUtils; import org.apache.tez.serviceplugins.api.TaskScheduler; import org.mockito.ArgumentCaptor; @@ -1127,8 +1128,11 @@ public class TestContainerReuse { String rsrc1 = "rsrc1"; String rsrc2 = "rsrc2"; LocalResource lr1 = mock(LocalResource.class); + doReturn(LocalResourceType.FILE).when(lr1).getType(); LocalResource lr2 = mock(LocalResource.class); + doReturn(LocalResourceType.FILE).when(lr2).getType(); LocalResource lr3 = mock(LocalResource.class); + doReturn(LocalResourceType.FILE).when(lr3).getType(); AMContainerEventAssignTA assignEvent = null;
