Repository: tez Updated Branches: refs/heads/branch-0.8 ec89a6332 -> 9a4e7eabc
TEZ-3609. Improve ATSv15 performance for DAG entities read calls. Contributed by Harish Jaiprakash. (cherry picked from commit 11815a7ccd80ac3cb052f34a420d44de9f083542) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9a4e7eab Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9a4e7eab Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9a4e7eab Branch: refs/heads/branch-0.8 Commit: 9a4e7eabc3a06aaaf39fb615e47383f5be97c565 Parents: ec89a63 Author: Siddharth Seth <[email protected]> Authored: Tue Feb 7 22:54:08 2017 -0800 Committer: Siddharth Seth <[email protected]> Committed: Tue Feb 7 22:55:39 2017 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../logging/ats/TimelineCachePluginImpl.java | 1 - .../ats/TestTimelineCachePluginImpl.java | 41 ++++++++++---------- 3 files changed, 22 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/9a4e7eab/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index aa18dcc..7a401b2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3609. Improve ATSv15 performance for DAG entities read calls. TEZ-3244. Allow overlap of input and output memory when they are not concurrent TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels TEZ-3582. Exception swallowed in PipelinedSorter causing incorrect results. http://git-wip-us.apache.org/repos/asf/tez/blob/9a4e7eab/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java index b4217a1..8269714 100644 --- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java @@ -72,7 +72,6 @@ public class TimelineCachePluginImpl extends TimelineEntityGroupPlugin implement private Set<TimelineEntityGroupId> createTimelineEntityGroupIds(TezDAGID dagId) { ApplicationId appId = dagId.getApplicationId(); HashSet<TimelineEntityGroupId> groupIds = Sets.newHashSet( - TimelineEntityGroupId.newInstance(appId, appId.toString()), TimelineEntityGroupId.newInstance(appId, dagId.toString())); for (int numGroupsPerDag : allNumGroupsPerDag) { groupIds.add(TimelineEntityGroupId.newInstance(appId, dagId.getGroupId(numGroupsPerDag))); http://git-wip-us.apache.org/repos/asf/tez/blob/9a4e7eab/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java index 6f819ba..3d1af63 100644 --- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java @@ -124,12 +124,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(3, groupIds.size()); + Assert.assertEquals(2, groupIds.size()); Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId1, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId())); } } } @@ -143,12 +143,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(2, groupIds.size()); + Assert.assertEquals(1, groupIds.size()); Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId1, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId())); } } } @@ -162,12 +162,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(2, groupIds.size()); + Assert.assertEquals(1, groupIds.size()); Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId1, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId())); } } } @@ -181,12 +181,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(3, groupIds.size()); + Assert.assertEquals(2, groupIds.size()); Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId1, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId())); } } } @@ -200,12 +200,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(4, groupIds.size()); + Assert.assertEquals(3, groupIds.size()); Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId2, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 50).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID2, 100, 50).contains(groupId.getTimelineEntityGroupId())); } } } @@ -219,12 +219,13 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(5, groupIds.size()); + Assert.assertEquals(4, groupIds.size()); Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId2, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue( + getGroupIds(dagID2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId())); } } } @@ -238,12 +239,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(3, groupIds.size()); + Assert.assertEquals(2, groupIds.size()); Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId2, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID2, appId2, 100).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID2, 100).contains(groupId.getTimelineEntityGroupId())); } } } @@ -261,21 +262,21 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(6, groupIds.size()); + Assert.assertEquals(4, groupIds.size()); int found = 0; Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); if (groupId.getApplicationId().equals(appId1)) { String entityGroupId = groupId.getTimelineEntityGroupId(); - if (getGroupIds(dagID1, appId1, 100).contains(entityGroupId)) { + if (getGroupIds(dagID1, 100).contains(entityGroupId)) { ++found; } else { Assert.fail("Unexpected group id: " + entityGroupId); } } else if (groupId.getApplicationId().equals(appId2)) { String entityGroupId = groupId.getTimelineEntityGroupId(); - if (getGroupIds(dagID2, appId2, 100).contains(entityGroupId)) { + if (getGroupIds(dagID2, 100).contains(entityGroupId)) { ++found; } else { Assert.fail("Unexpected group id: " + entityGroupId); @@ -284,7 +285,7 @@ public class TestTimelineCachePluginImpl { Assert.fail("Unexpected appId: " + groupId.getApplicationId()); } } - Assert.assertEquals("All groupIds not returned", 6, found); + Assert.assertEquals("All groupIds not returned", 4, found); } } @@ -374,8 +375,8 @@ public class TestTimelineCachePluginImpl { Assert.assertEquals("All groupIds not returned", 1, found); } - private Set<String> getGroupIds(TezDAGID dagId, ApplicationId appId, int ... allNumDagsPerGroup) { - HashSet<String> groupIds = Sets.newHashSet(dagId.toString(), appId.toString()); + private Set<String> getGroupIds(TezDAGID dagId, int ... allNumDagsPerGroup) { + HashSet<String> groupIds = Sets.newHashSet(dagId.toString()); for (int numDagsPerGroup : allNumDagsPerGroup) { groupIds.add(dagId.getGroupId(numDagsPerGroup)); }
