YARN-4238. createdTime and modifiedTime is not reported while publishing entities to ATSv2. (Varun Saxena via Naganarasimha G R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd369a54 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd369a54 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd369a54 Branch: refs/heads/YARN-2928 Commit: fd369a54e3cc08b812746f7c339826b0c4c7977f Parents: 2269e78 Author: Naganarasimha <naganarasimha...@apache.com> Authored: Wed Jan 27 11:59:40 2016 +0530 Committer: Li Lu <gtcarre...@apache.org> Committed: Wed May 4 16:24:12 2016 -0700 ---------------------------------------------------------------------- .../jobhistory/JobHistoryEventHandler.java | 47 ++++++---- .../records/timelineservice/TimelineEntity.java | 19 ---- .../TestTimelineServiceRecords.java | 1 - .../monitor/ContainersMonitorImpl.java | 6 +- .../timelineservice/NMTimelinePublisher.java | 6 +- .../TestSystemMetricsPublisherForV2.java | 34 +++++-- .../reader/TimelineReaderManager.java | 7 +- .../reader/TimelineReaderWebServices.java | 51 ++++------- .../storage/FileSystemTimelineReaderImpl.java | 21 +---- .../storage/HBaseTimelineReaderImpl.java | 8 +- .../storage/HBaseTimelineWriterImpl.java | 4 - .../PhoenixOfflineAggregationWriterImpl.java | 9 +- .../timelineservice/storage/TimelineReader.java | 32 +++---- .../storage/application/ApplicationColumn.java | 5 -- .../storage/application/ApplicationTable.java | 11 +-- .../storage/entity/EntityColumn.java | 5 -- .../storage/entity/EntityTable.java | 11 +-- .../storage/reader/ApplicationEntityReader.java | 23 +---- .../reader/FlowActivityEntityReader.java | 7 +- .../storage/reader/FlowRunEntityReader.java | 7 +- .../storage/reader/GenericEntityReader.java | 22 +---- .../storage/reader/TimelineEntityReader.java | 5 -- .../reader/TimelineEntityReaderFactory.java | 27 +++--- .../reader/TestTimelineReaderWebServices.java | 48 +--------- .../TestFileSystemTimelineReaderImpl.java | 92 +++++-------------- .../TestFileSystemTimelineWriterImpl.java | 3 +- .../storage/TestHBaseTimelineStorage.java | 95 +++++++------------- ...TestPhoenixOfflineAggregationWriterImpl.java | 1 - .../storage/flow/TestFlowDataGenerator.java | 1 - .../flow/TestHBaseStorageFlowActivity.java | 6 +- .../storage/flow/TestHBaseStorageFlowRun.java | 9 +- 31 files changed, 197 insertions(+), 426 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index d88588c..a10872a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -1135,10 +1135,10 @@ public class JobHistoryEventHandler extends AbstractService // jobId, timestamp and entityType. private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createJobEntity(HistoryEvent event, long timestamp, JobId jobId, - String entityType) { - - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = - createBaseEntity(event, timestamp, entityType); + String entityType, boolean setCreatedTime) { + + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = + createBaseEntity(event, timestamp, entityType, setCreatedTime); entity.setId(jobId.toString()); return entity; } @@ -1146,8 +1146,9 @@ public class JobHistoryEventHandler extends AbstractService // create BaseEntity from HistoryEvent with adding other info, like: // timestamp and entityType. private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity - createBaseEntity(HistoryEvent event, long timestamp, String entityType) { - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent = + createBaseEntity(HistoryEvent event, long timestamp, String entityType, + boolean setCreatedTime) { + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent = event.toTimelineEvent(); tEvent.setTimestamp(timestamp); @@ -1155,6 +1156,9 @@ public class JobHistoryEventHandler extends AbstractService new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); entity.addEvent(tEvent); entity.setType(entityType); + if (setCreatedTime) { + entity.setCreatedTime(timestamp); + } return entity; } @@ -1162,9 +1166,10 @@ public class JobHistoryEventHandler extends AbstractService // taskId, jobId, timestamp, entityType and relatedJobEntity. private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createTaskEntity(HistoryEvent event, long timestamp, String taskId, - String entityType, String relatedJobEntity, JobId jobId) { - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = - createBaseEntity(event, timestamp, entityType); + String entityType, String relatedJobEntity, JobId jobId, + boolean setCreatedTime) { + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = + createBaseEntity(event, timestamp, entityType, setCreatedTime); entity.setId(taskId); entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString()); return entity; @@ -1175,9 +1180,9 @@ public class JobHistoryEventHandler extends AbstractService private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createTaskAttemptEntity(HistoryEvent event, long timestamp, String taskAttemptId, String entityType, String relatedTaskEntity, - String taskId) { - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = - createBaseEntity(event, timestamp, entityType); + String taskId, boolean setCreatedTime) { + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = + createBaseEntity(event, timestamp, entityType, setCreatedTime); entity.setId(taskAttemptId); entity.addIsRelatedToEntity(relatedTaskEntity, taskId); return entity; @@ -1188,10 +1193,13 @@ public class JobHistoryEventHandler extends AbstractService org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity = null; String taskId = null; String taskAttemptId = null; + boolean setCreatedTime = false; switch (event.getEventType()) { // Handle job events case JOB_SUBMITTED: + setCreatedTime = true; + break; case JOB_STATUS_CHANGED: case JOB_INFO_CHANGED: case JOB_INITED: @@ -1206,6 +1214,7 @@ public class JobHistoryEventHandler extends AbstractService break; // Handle task events case TASK_STARTED: + setCreatedTime = true; taskId = ((TaskStartedEvent)event).getTaskId().toString(); break; case TASK_FAILED: @@ -1218,8 +1227,13 @@ public class JobHistoryEventHandler extends AbstractService taskId = ((TaskFinishedEvent)event).getTaskId().toString(); break; case MAP_ATTEMPT_STARTED: - case CLEANUP_ATTEMPT_STARTED: case REDUCE_ATTEMPT_STARTED: + setCreatedTime = true; + taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString(); + taskAttemptId = ((TaskAttemptStartedEvent)event). + getTaskAttemptId().toString(); + break; + case CLEANUP_ATTEMPT_STARTED: case SETUP_ATTEMPT_STARTED: taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString(); taskAttemptId = ((TaskAttemptStartedEvent)event). @@ -1258,17 +1272,18 @@ public class JobHistoryEventHandler extends AbstractService if (taskId == null) { // JobEntity tEntity = createJobEntity(event, timestamp, jobId, - MAPREDUCE_JOB_ENTITY_TYPE); + MAPREDUCE_JOB_ENTITY_TYPE, setCreatedTime); } else { if (taskAttemptId == null) { // TaskEntity tEntity = createTaskEntity(event, timestamp, taskId, - MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE, jobId); + MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE, + jobId, setCreatedTime); } else { // TaskAttemptEntity tEntity = createTaskAttemptEntity(event, timestamp, taskAttemptId, MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE, MAPREDUCE_TASK_ENTITY_TYPE, - taskId); + taskId, setCreatedTime); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java index 0701001..dcf2473 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java @@ -140,7 +140,6 @@ public class TimelineEntity implements Comparable<TimelineEntity> { private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>(); private HashMap<String, Set<String>> relatesToEntities = new HashMap<>(); private long createdTime; - private long modifiedTime; public TimelineEntity() { identifier = new Identifier(); @@ -505,24 +504,6 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } } - @XmlElement(name = "modifiedtime") - public long getModifiedTime() { - if (real == null) { - return modifiedTime; - } else { - return real.getModifiedTime(); - } - } - - @JsonSetter("modifiedtime") - public void setModifiedTime(long modifiedTime) { - if (real == null) { - this.modifiedTime = modifiedTime; - } else { - real.setModifiedTime(modifiedTime); - } - } - public boolean isValid() { return (getId() != null && getType() != null); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java index 7c9acf2..51ec762 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java @@ -139,7 +139,6 @@ public class TestTimelineServiceRecords { event1, event2); entity.setCreatedTime(0L); - entity.setModifiedTime(1L); entity.addRelatesToEntity("test type 2", "test id 2"); entity.addRelatesToEntity("test type 3", "test id 3"); entity.addIsRelatedToEntity("test type 4", "test id 4"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 739d363..da4aac2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -463,7 +463,6 @@ public class ContainersMonitorImpl extends AbstractService implements pTree.updateProcessTree(); // update process-tree long currentVmemUsage = pTree.getVirtualMemorySize(); long currentPmemUsage = pTree.getRssMemorySize(); - long currentTime = System.currentTimeMillis(); // if machine has 6 cores and 3 are used, // cpuUsagePercentPerCore should be 300% and @@ -568,9 +567,8 @@ public class ContainersMonitorImpl extends AbstractService implements NMTimelinePublisher nmMetricsPublisher = container.getNMTimelinePublisher(); if (nmMetricsPublisher != null) { - nmMetricsPublisher.reportContainerResourceUsage( - container, currentTime, pId, currentPmemUsage, - cpuUsageTotalCoresPercentage); + nmMetricsPublisher.reportContainerResourceUsage(container, pId, + currentPmemUsage, cpuUsageTotalCoresPercentage); } } catch (Exception e) { // Log the exception and proceed to the next container. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index 69de433..affaaae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -113,9 +113,8 @@ public class NMTimelinePublisher extends CompositeService { } @SuppressWarnings("unchecked") - public void reportContainerResourceUsage(Container container, - long createdTime, String pId, Long pmemUsage, - Float cpuUsageTotalCoresPercentage) { + public void reportContainerResourceUsage(Container container, String pId, + Long pmemUsage, Float cpuUsageTotalCoresPercentage) { if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE || cpuUsageTotalCoresPercentage != ResourceCalculatorProcessTree.UNAVAILABLE) { @@ -164,6 +163,7 @@ public class NMTimelinePublisher extends CompositeService { tEvent.setTimestamp(timestamp); entity.addEvent(tEvent); + entity.setCreatedTime(timestamp); putEntity(entity, containerId.getApplicationAttemptId().getApplicationId()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 9100366..1a58cb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -44,10 +45,15 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; +import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; @@ -58,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector; +import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.junit.AfterClass; @@ -201,8 +208,7 @@ public class TestSystemMetricsPublisherForV2 { + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); - Assert.assertEquals("Expected 3 events to be published", 3, - getNumOfNonEmptyLines(appFile)); + verifyEntity(appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE); } @Test(timeout = 10000) @@ -236,8 +242,7 @@ public class TestSystemMetricsPublisherForV2 { + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); - Assert.assertEquals("Expected 2 events to be published", 2, - getNumOfNonEmptyLines(appFile)); + verifyEntity(appFile,2, AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE); } @Test(timeout = 10000) @@ -268,8 +273,8 @@ public class TestSystemMetricsPublisherForV2 { + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); - Assert.assertEquals("Expected 2 events to be published", 2, - getNumOfNonEmptyLines(appFile)); + verifyEntity(appFile, 2, + ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE); } private RMApp createAppAndRegister(ApplicationId appId) { @@ -282,20 +287,31 @@ public class TestSystemMetricsPublisherForV2 { return app; } - private long getNumOfNonEmptyLines(File entityFile) throws IOException { + private static void verifyEntity(File entityFile, long expectedEvents, + String eventForCreatedTime) throws IOException { BufferedReader reader = null; String strLine; long count = 0; try { reader = new BufferedReader(new FileReader(entityFile)); while ((strLine = reader.readLine()) != null) { - if (strLine.trim().length() > 0) + if (strLine.trim().length() > 0) { + TimelineEntity entity = FileSystemTimelineReaderImpl. + getTimelineRecordFromJSON(strLine.trim(), TimelineEntity.class); + for (TimelineEvent event : entity.getEvents()) { + if (event.getId().equals(eventForCreatedTime)) { + assertTrue(entity.getCreatedTime() > 0); + break; + } + } count++; + } } } finally { reader.close(); } - return count; + assertEquals("Expected " + expectedEvents + " events to be published", + count, expectedEvents); } private String getTimelineEntityDir(RMApp app) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java index 37f1671..65d251d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java @@ -68,16 +68,15 @@ public class TimelineReaderManager extends AbstractService { Set<TimelineEntity> getEntities(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, EnumSet<Field> fieldsToRetrieve) throws IOException { String cluster = getClusterID(clusterId, getConfig()); return reader.getEntities(userId, cluster, flowName, flowRunId, appId, - entityType, limit, createdTimeBegin, createdTimeEnd, modifiedTimeBegin, - modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters, - metricFilters, eventFilters, null, null, fieldsToRetrieve); + entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo, + isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters, + null, null, fieldsToRetrieve); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index f0283a5..a054ee5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -331,8 +331,6 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, - @QueryParam("modifiedtimestart") String modifiedTimeStart, - @QueryParam("modifiedtimeend") String modifiedTimeEnd, @QueryParam("relatesto") String relatesTo, @QueryParam("isrelatedto") String isRelatedTo, @QueryParam("infofilters") String infofilters, @@ -341,9 +339,9 @@ public class TimelineReaderWebServices { @QueryParam("eventfilters") String eventfilters, @QueryParam("fields") String fields) { return getEntities(req, res, null, appId, entityType, userId, flowName, - flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart, - modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, - metricfilters, eventfilters, fields); + flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, + isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, + fields); } /** @@ -364,8 +362,6 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, - @QueryParam("modifiedtimestart") String modifiedTimeStart, - @QueryParam("modifiedtimeend") String modifiedTimeEnd, @QueryParam("relatesto") String relatesTo, @QueryParam("isrelatedto") String isRelatedTo, @QueryParam("infofilters") String infofilters, @@ -387,8 +383,7 @@ public class TimelineReaderWebServices { parseStr(userId), parseStr(clusterId), parseStr(flowName), parseLongStr(flowRunId), parseStr(appId), parseStr(entityType), parseLongStr(limit), parseLongStr(createdTimeStart), - parseLongStr(createdTimeEnd), parseLongStr(modifiedTimeStart), - parseLongStr(modifiedTimeEnd), + parseLongStr(createdTimeEnd), parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER), parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER), parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER), @@ -398,7 +393,7 @@ public class TimelineReaderWebServices { parseFieldsStr(fields, COMMA_DELIMITER)); } catch (Exception e) { handleException(e, url, startTime, - "createdTime or modifiedTime start/end or limit or flowrunid"); + "createdTime start/end or limit or flowrunid"); } long endTime = Time.monotonicNow(); if (entities == null) { @@ -585,8 +580,8 @@ public class TimelineReaderWebServices { entities = timelineReaderManager.getEntities( parseStr(userId), parseStr(clusterId), parseStr(flowName), null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), parseLongStr(limit), - parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd), null, - null, null, null, null, null, null, null, + parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd), + null, null, null, null, null, null, parseFieldsStr(fields, COMMA_DELIMITER)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit"); @@ -668,7 +663,7 @@ public class TimelineReaderWebServices { null, parseStr(clusterId), null, null, null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), parseLongStr(limit), range.dateStart, range.dateEnd, null, null, null, null, null, null, - null, null, parseFieldsStr(fields, COMMA_DELIMITER)); + parseFieldsStr(fields, COMMA_DELIMITER)); } catch (Exception e) { handleException(e, url, startTime, "limit"); } @@ -760,8 +755,6 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, - @QueryParam("modifiedtimestart") String modifiedTimeStart, - @QueryParam("modifiedtimeend") String modifiedTimeEnd, @QueryParam("relatesto") String relatesTo, @QueryParam("isrelatedto") String isRelatedTo, @QueryParam("infofilters") String infofilters, @@ -771,9 +764,9 @@ public class TimelineReaderWebServices { @QueryParam("fields") String fields) { return getEntities(req, res, null, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, - flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart, - modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, - metricfilters, eventfilters, fields); + flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, + isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, + fields); } /** @@ -794,8 +787,6 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, - @QueryParam("modifiedtimestart") String modifiedTimeStart, - @QueryParam("modifiedtimeend") String modifiedTimeEnd, @QueryParam("relatesto") String relatesTo, @QueryParam("isrelatedto") String isRelatedTo, @QueryParam("infofilters") String infofilters, @@ -805,9 +796,9 @@ public class TimelineReaderWebServices { @QueryParam("fields") String fields) { return getEntities(req, res, clusterId, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, - flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart, - modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, - metricfilters, eventfilters, fields); + flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, + isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, + fields); } /** @@ -827,8 +818,6 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, - @QueryParam("modifiedtimestart") String modifiedTimeStart, - @QueryParam("modifiedtimeend") String modifiedTimeEnd, @QueryParam("relatesto") String relatesTo, @QueryParam("isrelatedto") String isRelatedTo, @QueryParam("infofilters") String infofilters, @@ -838,9 +827,8 @@ public class TimelineReaderWebServices { @QueryParam("fields") String fields) { return getEntities(req, res, null, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, - null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart, - modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, - metricfilters, eventfilters, fields); + null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, + infofilters, conffilters, metricfilters, eventfilters, fields); } /** @@ -860,8 +848,6 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, - @QueryParam("modifiedtimestart") String modifiedTimeStart, - @QueryParam("modifiedtimeend") String modifiedTimeEnd, @QueryParam("relatesto") String relatesTo, @QueryParam("isrelatedto") String isRelatedTo, @QueryParam("infofilters") String infofilters, @@ -871,8 +857,7 @@ public class TimelineReaderWebServices { @QueryParam("fields") String fields) { return getEntities(req, res, clusterId, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, - null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart, - modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, - metricfilters, eventfilters, fields); + null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, + infofilters, conffilters, metricfilters, eventfilters, fields); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index 36b3f0d..72c9249 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -188,7 +188,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService TimelineEntity entityToBeReturned = new TimelineEntity(); entityToBeReturned.setIdentifier(entity.getIdentifier()); entityToBeReturned.setCreatedTime(entity.getCreatedTime()); - entityToBeReturned.setModifiedTime(entity.getModifiedTime()); if (fieldsToRetrieve != null) { fillFields(entityToBeReturned, entity, fieldsToRetrieve); } @@ -206,9 +205,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService if (entity2.getCreatedTime() > 0) { entity1.setCreatedTime(entity2.getCreatedTime()); } - if (entity2.getModifiedTime() > 0) { - entity1.setModifiedTime(entity2.getModifiedTime()); - } for (Entry<String, String> configEntry : entity2.getConfigs().entrySet()) { entity1.addConfig(configEntry.getKey(), configEntry.getValue()); } @@ -268,8 +264,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService } private Set<TimelineEntity> getEntities(File dir, String entityType, - Long limit, Long createdTimeBegin, - Long createdTimeEnd, Long modifiedTimeBegin, Long modifiedTimeEnd, + Long limit, Long createdTimeBegin, Long createdTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, @@ -284,12 +279,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService if (createdTimeEnd == null || createdTimeEnd <= 0) { createdTimeEnd = Long.MAX_VALUE; } - if (modifiedTimeBegin == null || modifiedTimeBegin <= 0) { - modifiedTimeBegin = 0L; - } - if (modifiedTimeEnd == null || modifiedTimeEnd <= 0) { - modifiedTimeEnd = Long.MAX_VALUE; - } // First sort the selected entities based on created/start time. Map<Long, Set<TimelineEntity>> sortedEntities = @@ -318,10 +307,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService createdTimeEnd)) { continue; } - if (!isTimeInRange(entity.getModifiedTime(), modifiedTimeBegin, - modifiedTimeEnd)) { - continue; - } if (relatesTo != null && !relatesTo.isEmpty() && !TimelineStorageUtils .matchRelations(entity.getRelatesToEntities(), relatesTo)) { @@ -413,7 +398,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService public Set<TimelineEntity> getEntities(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, @@ -424,8 +408,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService File dir = new File(new File(rootPath, ENTITIES_DIR), clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType); - return getEntities(dir, entityType, limit, - createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd, + return getEntities(dir, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java index bc48cbe..0ce9a94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java @@ -81,7 +81,6 @@ public class HBaseTimelineReaderImpl public Set<TimelineEntity> getEntities(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, @@ -90,10 +89,9 @@ public class HBaseTimelineReaderImpl TimelineEntityReader reader = TimelineEntityReaderFactory.createMultipleEntitiesReader(userId, clusterId, flowName, flowRunId, appId, entityType, limit, - createdTimeBegin, createdTimeEnd, modifiedTimeBegin, - modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters, - metricFilters, eventFilters, confsToRetrieve, metricsToRetrieve, - fieldsToRetrieve); + createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, + infoFilters, configFilters, metricFilters, eventFilters, + confsToRetrieve, metricsToRetrieve, fieldsToRetrieve); return reader.readEntities(hbaseConf, conn); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java index a57be55..2a82ccb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java @@ -315,8 +315,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements ApplicationColumn.ID.store(rowKey, applicationTable, null, te.getId()); ApplicationColumn.CREATED_TIME.store(rowKey, applicationTable, null, te.getCreatedTime()); - ApplicationColumn.MODIFIED_TIME.store(rowKey, applicationTable, null, - te.getModifiedTime()); ApplicationColumn.FLOW_VERSION.store(rowKey, applicationTable, null, flowVersion); Map<String, Object> info = te.getInfo(); @@ -331,8 +329,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType()); EntityColumn.CREATED_TIME.store(rowKey, entityTable, null, te.getCreatedTime()); - EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null, - te.getModifiedTime()); EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion); Map<String, Object> info = te.getInfo(); if (info != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java index 4c1352c..b5834c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java @@ -143,10 +143,10 @@ public class PhoenixOfflineAggregationWriterImpl TimelineWriteResponse response = new TimelineWriteResponse(); String sql = "UPSERT INTO " + info.getTableName() + " (" + StringUtils.join(info.getPrimaryKeyList(), ",") - + ", created_time, modified_time, metric_names) " + + ", created_time, metric_names) " + "VALUES (" + StringUtils.repeat("?,", info.getPrimaryKeyList().length) - + "?, ?, ?)"; + + "?, ?)"; if (LOG.isDebugEnabled()) { LOG.debug("TimelineEntity write SQL: " + sql); } @@ -162,7 +162,6 @@ public class PhoenixOfflineAggregationWriterImpl } int idx = info.setStringsForPrimaryKey(ps, context, null, 1); ps.setLong(idx++, entity.getCreatedTime()); - ps.setLong(idx++, entity.getModifiedTime()); ps.setString(idx++, StringUtils.join(formattedMetrics.keySet().toArray(), AGGREGATION_STORAGE_SEPARATOR)); ps.execute(); @@ -197,7 +196,7 @@ public class PhoenixOfflineAggregationWriterImpl + OfflineAggregationInfo.FLOW_AGGREGATION_TABLE_NAME + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, " + "flow_name VARCHAR NOT NULL, " - + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, " + + "created_time UNSIGNED_LONG, " + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, " + "metric_names VARCHAR, info_keys VARCHAR " + "CONSTRAINT pk PRIMARY KEY(" @@ -206,7 +205,7 @@ public class PhoenixOfflineAggregationWriterImpl sql = "CREATE TABLE IF NOT EXISTS " + OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, " - + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, " + + "created_time UNSIGNED_LONG, " + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, " + "metric_names VARCHAR, info_keys VARCHAR " + "CONSTRAINT pk PRIMARY KEY(user, cluster))"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java index 56bd3a0..a5a271e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java @@ -87,8 +87,8 @@ public interface TimelineReader extends Service { * @param fieldsToRetrieve * Specifies which fields of the entity object to retrieve(optional), see * {@link Field}. If null, retrieves 4 fields namely entity id, - * entity type, entity created time and entity modified time. All - * entities will be returned if {@link Field#ALL} is specified. + * entity type and entity created time. All entities will be returned if + * {@link Field#ALL} is specified. * @return a {@link TimelineEntity} instance or null. The entity will * contain the metadata plus the given fields to retrieve. * @throws IOException @@ -101,13 +101,13 @@ public interface TimelineReader extends Service { /** * <p>The API to search for a set of entities of the given the entity type in * the scope of the given context which matches the given predicates. The - * predicates include the created/modified time window, limit to number of - * entities to be returned, and the entities can be filtered by checking - * whether they contain the given info/configs entries in the form of - * key/value pairs, given metrics in the form of metricsIds and its relation - * with metric values given events in the form of the Ids, and whether they - * relate to/are related to other entities. For those parameters which have - * multiple entries, the qualified entity needs to meet all or them.</p> + * predicates include the created time window, limit to number of entities to + * be returned, and the entities can be filtered by checking whether they + * contain the given info/configs entries in the form of key/value pairs, + * given metrics in the form of metricsIds and its relation with metric + * values, given events in the form of the Ids, and whether they relate to/are + * related to other entities. For those parameters which have multiple + * entries, the qualified entity needs to meet all or them.</p> * * @param userId * Context user Id(optional). @@ -130,12 +130,6 @@ public interface TimelineReader extends Service { * @param createdTimeEnd * Matched entities should not be created after this timestamp (optional). * If null or <=0, defaults to {@link Long#MAX_VALUE}. - * @param modifiedTimeBegin - * Matched entities should not be modified before this timestamp - * (optional). If null or <=0, defaults to 0. - * @param modifiedTimeEnd - * Matched entities should not be modified after this timestamp (optional). - * If null or <=0, defaults to {@link Long#MAX_VALUE}. * @param relatesTo * Matched entities should relate to given entities (optional). * @param isRelatedTo @@ -173,19 +167,17 @@ public interface TimelineReader extends Service { * @param fieldsToRetrieve * Specifies which fields of the entity object to retrieve(optional), see * {@link Field}. If null, retrieves 4 fields namely entity id, - * entity type, entity created time and entity modified time. All - * entities will be returned if {@link Field#ALL} is specified. + * entity type and entity created time. All entities will be returned if + * {@link Field#ALL} is specified. * @return A set of {@link TimelineEntity} instances of the given entity type * in the given context scope which matches the given predicates * ordered by created time, descending. Each entity will only contain the - * metadata(id, type, created and modified times) plus the given fields to - * retrieve. + * metadata(id, type and created time) plus the given fields to retrieve. * @throws IOException */ Set<TimelineEntity> getEntities(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java index 802626d..c03c9b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java @@ -44,11 +44,6 @@ public enum ApplicationColumn implements Column<ApplicationTable> { CREATED_TIME(ApplicationColumnFamily.INFO, "created_time"), /** - * When it was modified. - */ - MODIFIED_TIME(ApplicationColumnFamily.INFO, "modified_time"), - - /** * The version of the flow that this app belongs to. */ FLOW_VERSION(ApplicationColumnFamily.INFO, "flow_version"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java index ee5ff83..681c200 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java @@ -50,15 +50,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas * | flowName! | created_time: | @timestamp1 | | * | flowRunId! | 1392993084018 | | configKey2: | * | AppId | | metriciD1: | configValue2 | - * | | modified_time: | metricValue2 | | - * | | 1392995081012 | @timestamp2 | | + * | | i!infoKey: | metricValue2 | | + * | | infoValue | @timestamp2 | | * | | | | | - * | | i!infoKey: | metricId2: | | - * | | infoValue | metricValue1 | | + * | | r!relatesToKey: | metricId2: | | + * | | id3=id4=id5 | metricValue1 | | * | | | @timestamp2 | | - * | | r!relatesToKey: | | | - * | | id3=id4=id5 | | | - * | | | | | * | | s!isRelatedToKey: | | | * | | id7=id9=id6 | | | * | | | | | http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java index e12b6e0..deb8bd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java @@ -49,11 +49,6 @@ public enum EntityColumn implements Column<EntityTable> { CREATED_TIME(EntityColumnFamily.INFO, "created_time"), /** - * When it was modified. - */ - MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"), - - /** * The version of the flow that this entity belongs to. */ FLOW_VERSION(EntityColumnFamily.INFO, "flow_version"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java index 15d4465..d7d770b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java @@ -48,16 +48,13 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas * | clusterId! | | metricValue1 | configValue1 | * | flowName! | type:entityType | @timestamp1 | | * | flowRunId! | | | configKey2: | - * | AppId! | created_time: | metriciD1: | configValue2 | + * | AppId! | created_time: | metricId1: | configValue2 | * | entityType!| 1392993084018 | metricValue2 | | * | entityId | | @timestamp2 | | - * | | modified_time: | | | - * | | 1392995081012 | metricId2: | | + * | | i!infoKey: | | | + * | | infoValue | metricId1: | | * | | | metricValue1 | | - * | | i!infoKey: | @timestamp2 | | - * | | infoValue | | | - * | | | | | - * | | r!relatesToKey: | | | + * | | r!relatesToKey: | @timestamp2 | | * | | id3=id4=id5 | | | * | | | | | * | | s!isRelatedToKey | | | http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java index 181ec81..4d61076 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java @@ -63,17 +63,15 @@ class ApplicationEntityReader extends GenericEntityReader { public ApplicationEntityReader(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) { super(userId, clusterId, flowName, flowRunId, appId, entityType, limit, - createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd, - relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters, - eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve, - true); + createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters, + configFilters, metricFilters, eventFilters, confsToRetrieve, + metricsToRetrieve, fieldsToRetrieve, true); } public ApplicationEntityReader(String userId, String clusterId, @@ -230,12 +228,6 @@ class ApplicationEntityReader extends GenericEntityReader { if (createdTimeEnd == null) { createdTimeEnd = DEFAULT_END_TIME; } - if (modifiedTimeBegin == null) { - modifiedTimeBegin = DEFAULT_BEGIN_TIME; - } - if (modifiedTimeEnd == null) { - modifiedTimeEnd = DEFAULT_END_TIME; - } } } @@ -278,15 +270,6 @@ class ApplicationEntityReader extends GenericEntityReader { return null; } - // fetch modified time - Number modifiedTime = - (Number)ApplicationColumn.MODIFIED_TIME.readResult(result); - entity.setModifiedTime(modifiedTime.longValue()); - if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin || - entity.getModifiedTime() > modifiedTimeEnd)) { - return null; - } - // fetch is related to entities boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0; if (fieldsToRetrieve.contains(Field.ALL) || http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java index 52ceef8..048f608 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java @@ -52,15 +52,14 @@ class FlowActivityEntityReader extends TimelineEntityReader { public FlowActivityEntityReader(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, EnumSet<Field> fieldsToRetrieve) { super(userId, clusterId, flowName, flowRunId, appId, entityType, limit, - createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd, - relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters, - eventFilters, null, null, fieldsToRetrieve, true); + createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters, + configFilters, metricFilters, eventFilters, null, null, + fieldsToRetrieve, true); } public FlowActivityEntityReader(String userId, String clusterId, http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java index 6286ee1..4f50b02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java @@ -61,16 +61,15 @@ class FlowRunEntityReader extends TimelineEntityReader { public FlowRunEntityReader(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) { super(userId, clusterId, flowName, flowRunId, appId, entityType, limit, - createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd, - relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters, - eventFilters, null, metricsToRetrieve, fieldsToRetrieve, true); + createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters, + configFilters, metricFilters, eventFilters, null, metricsToRetrieve, + fieldsToRetrieve, true); } public FlowRunEntityReader(String userId, String clusterId, http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java index f3f380c..990856f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java @@ -79,17 +79,15 @@ class GenericEntityReader extends TimelineEntityReader { public GenericEntityReader(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) { super(userId, clusterId, flowName, flowRunId, appId, entityType, limit, - createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd, - relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters, - eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve, - sortedKeys); + createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters, + configFilters, metricFilters, eventFilters, confsToRetrieve, + metricsToRetrieve, fieldsToRetrieve, sortedKeys); } public GenericEntityReader(String userId, String clusterId, @@ -257,12 +255,6 @@ class GenericEntityReader extends TimelineEntityReader { if (createdTimeEnd == null) { createdTimeEnd = DEFAULT_END_TIME; } - if (modifiedTimeBegin == null) { - modifiedTimeBegin = DEFAULT_BEGIN_TIME; - } - if (modifiedTimeEnd == null) { - modifiedTimeEnd = DEFAULT_END_TIME; - } } } @@ -314,14 +306,6 @@ class GenericEntityReader extends TimelineEntityReader { return null; } - // fetch modified time - Number modifiedTime = (Number)EntityColumn.MODIFIED_TIME.readResult(result); - entity.setModifiedTime(modifiedTime.longValue()); - if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin || - entity.getModifiedTime() > modifiedTimeEnd)) { - return null; - } - // fetch is related to entities boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0; if (fieldsToRetrieve.contains(Field.ALL) || http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java index e801466..bc86b6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java @@ -64,8 +64,6 @@ public abstract class TimelineEntityReader { protected Long limit; protected Long createdTimeBegin; protected Long createdTimeEnd; - protected Long modifiedTimeBegin; - protected Long modifiedTimeEnd; protected Map<String, Set<String>> relatesTo; protected Map<String, Set<String>> isRelatedTo; protected Map<String, Object> infoFilters; @@ -94,7 +92,6 @@ public abstract class TimelineEntityReader { protected TimelineEntityReader(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, @@ -112,8 +109,6 @@ public abstract class TimelineEntityReader { this.limit = limit; this.createdTimeBegin = createdTimeBegin; this.createdTimeEnd = createdTimeEnd; - this.modifiedTimeBegin = modifiedTimeBegin; - this.modifiedTimeEnd = modifiedTimeEnd; this.relatesTo = relatesTo; this.isRelatedTo = isRelatedTo; this.infoFilters = infoFilters; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java index c77897a..2e2c652 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java @@ -62,7 +62,6 @@ public class TimelineEntityReaderFactory { public static TimelineEntityReader createMultipleEntitiesReader(String userId, String clusterId, String flowName, Long flowRunId, String appId, String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd, - Long modifiedTimeBegin, Long modifiedTimeEnd, Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, Map<String, Object> infoFilters, Map<String, String> configFilters, Set<String> metricFilters, Set<String> eventFilters, @@ -72,29 +71,25 @@ public class TimelineEntityReaderFactory { // table are application, flow run, and flow activity entities if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) { return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId, - appId, entityType, limit, createdTimeBegin, createdTimeEnd, - modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo, - infoFilters, configFilters, metricFilters, eventFilters, confs, - metrics, fieldsToRetrieve); + appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo, + isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters, + confs, metrics, fieldsToRetrieve); } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) { return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId, - appId, entityType, limit, createdTimeBegin, createdTimeEnd, - modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo, - infoFilters, configFilters, metricFilters, eventFilters, + appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo, + isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters, fieldsToRetrieve); } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) { return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId, - appId, entityType, limit, createdTimeBegin, createdTimeEnd, - modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo, - infoFilters, configFilters, metricFilters, eventFilters, confs, - metrics, fieldsToRetrieve); + appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo, + isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters, + confs, metrics, fieldsToRetrieve); } else { // assume we're dealing with a generic entity read return new GenericEntityReader(userId, clusterId, flowName, flowRunId, - appId, entityType, limit, createdTimeBegin, createdTimeEnd, - modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo, - infoFilters, configFilters, metricFilters, eventFilters, confs, - metrics, fieldsToRetrieve, false); + appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo, + isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters, + confs, metrics, fieldsToRetrieve, false); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java index 9c74e2d..1aefdca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java @@ -174,9 +174,8 @@ public class TestTimelineReaderWebServices { assertEquals("id_1", entity.getId()); assertEquals("app", entity.getType()); assertEquals(1425016502000L, entity.getCreatedTime()); - assertEquals(1425016503000L, entity.getModifiedTime()); // Default view i.e. when no fields are specified, entity contains only - // entity id, entity type, created and modified time. + // entity id, entity type and created time. assertEquals(0, entity.getConfigs().size()); assertEquals(0, entity.getMetrics().size()); } finally { @@ -198,7 +197,6 @@ public class TestTimelineReaderWebServices { assertEquals("id_1", entity.getId()); assertEquals("app", entity.getType()); assertEquals(1425016502000L, entity.getCreatedTime()); - assertEquals(1425016503000L, entity.getModifiedTime()); } finally { client.destroy(); } @@ -382,50 +380,6 @@ public class TestTimelineReaderWebServices { } @Test - public void testGetEntitiesBasedOnModifiedTime() throws Exception { - Client client = createClient(); - try { - URI uri = URI.create("http://localhost:"; + serverPort + "/ws/v2/" + - "timeline/entities/cluster1/app1/app?modifiedtimestart=1425016502090" - + "&modifiedtimeend=1425016503020"); - ClientResponse resp = getResponse(client, uri); - Set<TimelineEntity> entities = - resp.getEntity(new GenericType<Set<TimelineEntity>>(){}); - assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); - assertNotNull(entities); - assertEquals(2, entities.size()); - assertTrue("Entities with id_1 and id_4 should have been" + - " present in response.", - entities.contains(newEntity("app", "id_1")) && - entities.contains(newEntity("app", "id_4"))); - - uri = URI.create("http://localhost:"; + serverPort + "/ws/v2/timeline/" + - "entities/cluster1/app1/app?modifiedtimeend=1425016502090"); - resp = getResponse(client, uri); - entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){}); - assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); - assertNotNull(entities); - assertEquals(2, entities.size()); - assertTrue("Entities with id_2 and id_3 should have been " + - "present in response.", - entities.contains(newEntity("app", "id_2")) && - entities.contains(newEntity("app", "id_3"))); - - uri = URI.create("http://localhost:"; + serverPort + "/ws/v2/timeline/" + - "entities/cluster1/app1/app?modifiedtimestart=1425016503005"); - resp = getResponse(client, uri); - entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){}); - assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); - assertNotNull(entities); - assertEquals(1, entities.size()); - assertTrue("Entity with id_4 should have been present in response.", - entities.contains(newEntity("app", "id_4"))); - } finally { - client.destroy(); - } - } - - @Test public void testGetEntitiesByRelations() throws Exception { Client client = createClient(); try { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
