TEZ-2179. Timeline relatedentries missing cause exaggerated warning. (Chang Li via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/19df2778 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/19df2778 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/19df2778 Branch: refs/heads/TEZ-2003 Commit: 19df277868e2775bb92ac80c6c7fd147724f9268 Parents: e0e1912 Author: Hitesh Shah <[email protected]> Authored: Thu Mar 12 07:46:43 2015 -0700 Committer: Hitesh Shah <[email protected]> Committed: Thu Mar 12 07:47:47 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../ats/HistoryEventTimelineConversion.java | 32 +++++------- .../ats/TestHistoryEventTimelineConversion.java | 53 ++++++++------------ 3 files changed, 35 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/19df2778/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 44a13b7..d29cb5d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -80,6 +80,7 @@ Release 0.6.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2179. Timeline relatedentries missing cause exaggerated warning. TEZ-2168. Fix application dependencies on mutually exclusive artifacts: tez-yarn-timeline-history and tez-yarn-timeline-history-with-acls. TEZ-2190. TestOrderedWordCount fails when generateSplitsInClient set to true. http://git-wip-us.apache.org/repos/asf/tez/blob/19df2778/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java index ca47b92..e1c2a72 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java +++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java @@ -135,14 +135,13 @@ public class HistoryEventTimelineConversion { + event.getApplicationId().toString()); atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION.name()); - atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID, - event.getApplicationId().toString()); - atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser()); - atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser()); atsEntity.addOtherInfo(ATSConstants.CONFIG, DAGUtils.convertConfigurationToATSMap(event.getConf())); + atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID, + event.getApplicationId().toString()); + atsEntity.addOtherInfo(ATSConstants.USER, event.getUser()); atsEntity.setStartTime(event.getLaunchTime()); @@ -160,12 +159,6 @@ public class HistoryEventTimelineConversion { + event.getApplicationAttemptId().toString()); atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); - atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID, - event.getApplicationAttemptId().getApplicationId().toString()); - atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ATTEMPT_ID, - event.getApplicationAttemptId().toString()); - atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser()); - atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser()); atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID, event.getApplicationAttemptId().getApplicationId().toString()); @@ -178,6 +171,11 @@ public class HistoryEventTimelineConversion { atsEntity.addEvent(launchEvt); atsEntity.addOtherInfo(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime()); + atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID, + event.getApplicationAttemptId().getApplicationId().toString()); + atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID, + event.getApplicationAttemptId().toString()); + atsEntity.addOtherInfo(ATSConstants.USER, event.getUser()); return atsEntity; } @@ -208,12 +206,12 @@ public class HistoryEventTimelineConversion { atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), "tez_" + event.getApplicationAttemptId().toString()); - atsEntity.addRelatedEntity(ATSConstants.CONTAINER_ID, - event.getContainerId().toString()); atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID, event.getApplicationAttemptId().getApplicationId().toString()); + atsEntity.addOtherInfo(ATSConstants.CONTAINER_ID, + event.getContainerId().toString()); atsEntity.setStartTime(event.getLaunchTime()); TimelineEvent launchEvt = new TimelineEvent(); @@ -341,11 +339,6 @@ public class HistoryEventTimelineConversion { "tez_" + event.getApplicationAttemptId().getApplicationId().toString()); atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), "tez_" + event.getApplicationAttemptId().toString()); - atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID, - event.getApplicationAttemptId().getApplicationId().toString()); - atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ATTEMPT_ID, - event.getApplicationAttemptId().toString()); - atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser()); TimelineEvent submitEvt = new TimelineEvent(); submitEvt.setEventType(HistoryEventType.DAG_SUBMITTED.name()); @@ -367,6 +360,9 @@ public class HistoryEventTimelineConversion { } atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID, event.getApplicationAttemptId().getApplicationId().toString()); + atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID, + event.getApplicationAttemptId().toString()); + atsEntity.addOtherInfo(ATSConstants.USER, event.getUser()); return atsEntity; } @@ -412,8 +408,6 @@ public class HistoryEventTimelineConversion { atsEntity.setStartTime(event.getStartTime()); - atsEntity.addRelatedEntity(ATSConstants.NODE_ID, event.getNodeId().toString()); - atsEntity.addRelatedEntity(ATSConstants.CONTAINER_ID, event.getContainerId().toString()); atsEntity.addRelatedEntity(EntityTypes.TEZ_TASK_ID.name(), event.getTaskAttemptID().getTaskID().toString()); http://git-wip-us.apache.org/repos/asf/tez/blob/19df2778/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java index 6d713c5..a2b0f89 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java @@ -222,18 +222,17 @@ public class TestHistoryEventTimelineConversion { Assert.assertEquals(EntityTypes.TEZ_APPLICATION.name(), timelineEntity.getEntityType()); Assert.assertEquals("tez_" + applicationId.toString(), timelineEntity.getEntityId()); - Assert.assertEquals(2, timelineEntity.getRelatedEntities().size()); - Assert.assertTrue(timelineEntity.getRelatedEntities().get(ATSConstants.USER).contains(user)); - Assert.assertTrue( - timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ID).contains( - applicationId.toString())); + Assert.assertEquals(0, timelineEntity.getRelatedEntities().size()); Assert.assertEquals(1, timelineEntity.getPrimaryFilters().size()); Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user)); - Assert.assertEquals(2, timelineEntity.getOtherInfo().size()); + Assert.assertEquals(4, timelineEntity.getOtherInfo().size()); Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.CONFIG)); Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.TEZ_VERSION)); + Assert.assertEquals(user, timelineEntity.getOtherInfo().get(ATSConstants.USER)); + Assert.assertEquals(applicationId.toString(), + timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID)); Map<String, String> config = (Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.CONFIG); @@ -263,12 +262,7 @@ public class TestHistoryEventTimelineConversion { Assert.assertEquals(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), timelineEntity.getEntityType()); final Map<String, Set<String>> relatedEntities = timelineEntity.getRelatedEntities(); - Assert.assertEquals(3, relatedEntities.size()); - Assert.assertTrue(relatedEntities.get(ATSConstants.APPLICATION_ID) - .contains(applicationId.toString())); - Assert.assertTrue(relatedEntities.get(ATSConstants.APPLICATION_ATTEMPT_ID) - .contains(applicationAttemptId.toString())); - Assert.assertTrue(relatedEntities.get(ATSConstants.USER).contains(user)); + Assert.assertEquals(0, relatedEntities.size()); final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters(); Assert.assertEquals(2, primaryFilters.size()); @@ -284,8 +278,11 @@ public class TestHistoryEventTimelineConversion { Assert.assertEquals(launchTime, evt.getTimestamp()); final Map<String, Object> otherInfo = timelineEntity.getOtherInfo(); - Assert.assertEquals(1, otherInfo.size()); + Assert.assertEquals(4, otherInfo.size()); Assert.assertEquals(submitTime, otherInfo.get(ATSConstants.APP_SUBMIT_TIME)); + Assert.assertEquals(applicationId.toString(), otherInfo.get(ATSConstants.APPLICATION_ID)); + Assert.assertEquals(applicationAttemptId.toString(), otherInfo.get(ATSConstants.APPLICATION_ATTEMPT_ID)); + Assert.assertEquals(user, otherInfo.get(ATSConstants.USER)); } @Test(timeout = 5000) @@ -298,9 +295,7 @@ public class TestHistoryEventTimelineConversion { Assert.assertEquals(EntityTypes.TEZ_CONTAINER_ID.name(), timelineEntity.getEntityType()); Assert.assertEquals("tez_" + containerId.toString(), timelineEntity.getEntityId()); - Assert.assertEquals(2, timelineEntity.getRelatedEntities().size()); - Assert.assertTrue(timelineEntity.getRelatedEntities().get(ATSConstants.CONTAINER_ID).contains( - containerId.toString())); + Assert.assertEquals(1, timelineEntity.getRelatedEntities().size()); Assert.assertTrue( timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()).contains( "tez_" + applicationAttemptId.toString())); @@ -309,6 +304,8 @@ public class TestHistoryEventTimelineConversion { Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.APPLICATION_ID).contains( applicationAttemptId.getApplicationId().toString())); + Assert.assertEquals(containerId.toString(),timelineEntity.getOtherInfo().get(ATSConstants.CONTAINER_ID)); + Assert.assertEquals(launchTime, timelineEntity.getStartTime().longValue()); Assert.assertEquals(1, timelineEntity.getEvents().size()); @@ -389,21 +386,13 @@ public class TestHistoryEventTimelineConversion { Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType()); Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId()); - Assert.assertEquals(5, timelineEntity.getRelatedEntities().size()); + Assert.assertEquals(2, timelineEntity.getRelatedEntities().size()); Assert.assertTrue( timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION.name()).contains( "tez_" + applicationId.toString())); Assert.assertTrue( timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()).contains( "tez_" + applicationAttemptId.toString())); - Assert.assertTrue( - timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ATTEMPT_ID).contains( - applicationAttemptId.toString())); - Assert.assertTrue( - timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ID).contains( - applicationAttemptId.getApplicationId().toString())); - Assert.assertTrue( - timelineEntity.getRelatedEntities().get(ATSConstants.USER).contains(user)); Assert.assertEquals(1, timelineEntity.getEvents().size()); TimelineEvent timelineEvent = timelineEntity.getEvents().get(0); @@ -426,7 +415,12 @@ public class TestHistoryEventTimelineConversion { Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.DAG_PLAN)); Assert.assertEquals(applicationId.toString(), timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID)); - + Assert.assertEquals(applicationAttemptId.toString(), + timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ATTEMPT_ID)); + Assert.assertEquals(applicationAttemptId.getApplicationId().toString(), + timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID)); + Assert.assertEquals(user, + timelineEntity.getOtherInfo().get(ATSConstants.USER)); } @Test(timeout = 5000) @@ -721,12 +715,7 @@ public class TestHistoryEventTimelineConversion { Assert.assertEquals(startTime, timelineEntity.getStartTime().longValue()); - Assert.assertEquals(3, timelineEntity.getRelatedEntities().size()); - Assert.assertTrue( - timelineEntity.getRelatedEntities().get(ATSConstants.NODE_ID).contains(nodeId.toString())); - Assert.assertTrue( - timelineEntity.getRelatedEntities().get(ATSConstants.CONTAINER_ID).contains( - containerId.toString())); + Assert.assertEquals(1, timelineEntity.getRelatedEntities().size()); Assert.assertTrue( timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_TASK_ID.name()).contains( tezTaskID.toString()));
