Repository: eagle Updated Branches: refs/heads/master 4a5c4a43c -> 45a649399
[EAGLE-917] add Job error category and error message mapping Author: wujinhu <[email protected]> Closes #827 from wujinhu/EAGLE-917. Project: http://git-wip-us.apache.org/repos/asf/eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/45a64939 Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/45a64939 Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/45a64939 Branch: refs/heads/master Commit: 45a6493999853328b89c16cd1dbc462a4315953a Parents: 4a5c4a4 Author: wujinhu <[email protected]> Authored: Tue Feb 21 15:57:20 2017 +0800 Committer: wujinhu <[email protected]> Committed: Tue Feb 21 15:57:20 2017 +0800 ---------------------------------------------------------------------- .../mr/MRMetricsAggregateContainer.java | 3 +- .../UnExpectedLongDurationJobProcessor.java | 2 +- .../mr/historyentity/JPAEntityRepository.java | 1 + .../JobConfigurationAPIEntity.java | 2 +- .../jpm/mr/historyentity/JobCountEntity.java | 2 +- .../JobErrorCategoryMappingAPIEntity.java | 62 ++++++++++++++++++++ .../jpm/mr/historyentity/JobEventAPIEntity.java | 2 +- .../mr/historyentity/JobExecutionAPIEntity.java | 4 +- .../JobProcessTimeStampEntity.java | 2 +- .../historyentity/JobSuggestionAPIEntity.java | 4 +- .../TaskAttemptCounterAPIEntity.java | 2 +- .../TaskAttemptErrorCategoryEntity.java | 2 +- .../TaskAttemptExecutionAPIEntity.java | 5 +- .../historyentity/TaskExecutionAPIEntity.java | 2 +- .../TaskFailureCountAPIEntity.java | 2 +- .../mr/runningentity/JobExecutionAPIEntity.java | 2 +- .../TaskAttemptExecutionAPIEntity.java | 2 +- .../runningentity/TaskExecutionAPIEntity.java | 2 +- .../MRHistoryJobApplicationHealthCheck.java | 2 +- .../mr/history/MRHistoryJobDailyReporter.java | 8 +-- .../mr/history/parser/JHFEventReaderBase.java | 30 ++++++++++ .../service/jpm/MRJobExecutionResource.java | 16 ++--- .../service/jpm/MRTaskExecutionResource.java | 12 ++-- .../eagle/service/jpm/RunningQueueResource.java | 8 +-- .../org/apache/eagle/jpm/util/Constants.java | 29 ++++----- 25 files changed, 150 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java index dd1980b..be139fb 100644 --- a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java +++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java @@ -21,7 +21,6 @@ package org.apache.eagle.jpm.aggregation.mr; import org.apache.eagle.jpm.aggregation.AggregationConfig; import org.apache.eagle.jpm.aggregation.common.MetricAggregator; import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer; -import org.apache.eagle.jpm.mr.historyentity.JobProcessTimeStampEntity; import org.apache.eagle.jpm.util.Constants; import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; import org.apache.eagle.service.client.IEagleServiceClient; @@ -62,7 +61,7 @@ public class MRMetricsAggregateContainer implements MetricsAggregateContainer, S eagleServiceConfig.password); String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}", - Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME, + Constants.MR_JOB_PROCESS_TIME_STAMP_NAME, appConfig.getStormConfig().site); GenericServiceAPIResponseEntity response = client http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java index 8f655ba..f7748f8 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java @@ -90,7 +90,7 @@ public class UnExpectedLongDurationJobProcessor implements Processor, Serializab } String query = String.format("%s[@site=\"%s\" and @jobDefId=\"%s\"]<@site>{avg(durationTime)}", - org.apache.eagle.jpm.util.Constants.JPA_JOB_EXECUTION_SERVICE_NAME, + org.apache.eagle.jpm.util.Constants.MR_JOB_EXECUTION_SERVICE_NAME, mrJobAnalysisEntity.getSiteId(), URLEncoder.encode(mrJobAnalysisEntity.getJobDefId())); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java index 8c65adf..54383bd 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java @@ -30,6 +30,7 @@ public class JPAEntityRepository extends EntityRepository { entitySet.add(JobConfigurationAPIEntity.class); entitySet.add(JobEventAPIEntity.class); entitySet.add(JobExecutionAPIEntity.class); + entitySet.add(JobErrorCategoryMappingAPIEntity.class); entitySet.add(TaskAttemptExecutionAPIEntity.class); entitySet.add(TaskExecutionAPIEntity.class); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java index 6298fe8..36875e3 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa") @ColumnFamily("f") @Prefix("jconf") -@Service(Constants.JPA_JOB_CONFIG_SERVICE_NAME) +@Service(Constants.MR_JOB_CONFIG_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java index 5b4163a..0fde454 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejcount") @ColumnFamily("f") @Prefix("jcount") -@Service(Constants.JPA_JOB_COUNT_SERVICE_NAME) +@Service(Constants.MR_JOB_COUNT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class JobCountEntity extends TaggedLogAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java new file mode 100644 index 0000000..91c2c68 --- /dev/null +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.eagle.jpm.mr.historyentity; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.eagle.jpm.util.Constants; +import org.apache.eagle.log.entity.meta.*; + +import java.util.List; + +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@Table("eaglejpa") +@ColumnFamily("f") +@Prefix("jecm") +@Service(Constants.MR_JOB_ERROR_MAPPING_SERVICE_NAME) +@TimeSeries(true) +@Partition({"site"}) +@Indexes({ + @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false), + @Index(name = "Index_1_jobDefId", columns = { "jobDefId" }, unique = false), + @Index(name = "Index_1_jobIdAndErrorCategory", columns = { "jobId", "errorCategory" }, unique = true) + }) +public class JobErrorCategoryMappingAPIEntity extends JobBaseAPIEntity { + @Column("a") + private String error; + @Column("b") + private List<String> taskAttempts; + + public String getError() { + return error; + } + + public void setError(String error) { + this.error = error; + valueChanged("error"); + } + + public List<String> getTaskAttempts() { + return taskAttempts; + } + + public void setTaskAttempts(List<String> taskAttempts) { + this.taskAttempts = taskAttempts; + valueChanged("taskAttempts"); + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java index aef3750..56f541b 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa") @ColumnFamily("f") @Prefix("jevent") -@Service(Constants.JPA_JOB_EVENT_SERVICE_NAME) +@Service(Constants.MR_JOB_EVENT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class JobEventAPIEntity extends JobBaseAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java index df7b271..746843d 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java @@ -23,13 +23,11 @@ import org.apache.eagle.jpm.util.jobcounter.JobCounters; import org.apache.eagle.log.entity.meta.*; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.util.Map; - @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @Table("eaglejpa") @ColumnFamily("f") @Prefix("jexec") -@Service(Constants.JPA_JOB_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_JOB_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java index ee632f4..1490390 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_process") @ColumnFamily("f") @Prefix("process") -@Service(Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME) +@Service(Constants.MR_JOB_PROCESS_TIME_STAMP_NAME) @TimeSeries(true) @Partition({"site"}) public class JobProcessTimeStampEntity extends TaggedLogAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java index 3863a5d..e433672 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java @@ -23,13 +23,13 @@ import org.apache.eagle.log.entity.meta.*; import java.util.List; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import static org.apache.eagle.jpm.util.Constants.JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME; @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @Table("eaglejpa") @ColumnFamily("f") @Prefix("jsuggestion") -@Service(JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME) +@Service(MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java index 04fe7d7..33a5102 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_anomaly") @ColumnFamily("f") @Prefix("tacount") -@Service(Constants.JPA_TASK_ATTEMPT_COUNTER_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_COUNTER_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class TaskAttemptCounterAPIEntity extends JobBaseAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java index 37f6451..1228f4e 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java @@ -28,7 +28,7 @@ import org.apache.eagle.log.entity.meta.*; @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("taexece") -@Service(Constants.JPA_TASK_ATTEMPT_ERROR_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_ERROR_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java index 46fcf5e..b4bb916 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java @@ -27,11 +27,12 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("taexec") -@Service(Constants.JPA_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ - @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false) + @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false), + @Index(name = "Index_1_taskAttemptId", columns = { "taskAttemptId" }, unique = true) }) public class TaskAttemptExecutionAPIEntity extends JobBaseAPIEntity { @Column("a") http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java index 9600fe2..9f85555 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("texec") -@Service(Constants.JPA_TASK_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_TASK_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java index 3e5e997..f82bcaf 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_anomaly") @ColumnFamily("f") @Prefix("taskfailurecount") -@Service(Constants.JPA_TASK_FAILURE_COUNT_SERVICE_NAME) +@Service(Constants.MR_TASK_FAILURE_COUNT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class TaskFailureCountAPIEntity extends JobBaseAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java index 082612b..3327a18 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eagleMRRunningJobs") @ColumnFamily("f") @Prefix("jobs") -@Service(Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java index 0d1703a..4fa7775 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eagleMRRunningTasks") @ColumnFamily("f") @Prefix("tasks_exec_attempt") -@Service(Constants.JPA_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java index 61c92c1..e9f0fcf 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java @@ -28,7 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eagleMRRunningTasks") @ColumnFamily("f") @Prefix("tasks_exec") -@Service(Constants.JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_TASK_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java index 4016e6d..90544e7 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java @@ -61,7 +61,7 @@ public class MRHistoryJobApplicationHealthCheck extends ApplicationHealthCheckBa } String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}", - Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME, + Constants.MR_JOB_PROCESS_TIME_STAMP_NAME, mrHistoryJobConfig.getJobHistoryEndpointConfig().site); GenericServiceAPIResponseEntity response = client http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java index 9aef640..541d352 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java @@ -208,16 +208,16 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService { private Map<String, Object> buildJobSummery(String site, long startTime, long endTime) { Map<String, Object> data = new HashMap<>(); - String query = String.format(STATUS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String query = String.format(STATUS_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); Map<String, Long> jobSummery = queryGroupByMetrics(query, startTime, endTime, Integer.MAX_VALUE); if (jobSummery == null || jobSummery.isEmpty()) { LOG.warn("Result set is empty for query={}", query); return data; } Long totalJobs = jobSummery.values().stream().reduce((a, b) -> a + b).get(); - String finishedJobQuery = String.format(FINISHED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); - String failedJobQuery = String.format(FAILED_JOBS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); - String succeededJobQuery = String.format(SUCCEEDED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, jobOvertimeLimit * DateTimeUtil.ONEHOUR, endTime); + String finishedJobQuery = String.format(FINISHED_JOB_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String failedJobQuery = String.format(FAILED_JOBS_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String succeededJobQuery = String.format(SUCCEEDED_JOB_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, jobOvertimeLimit * DateTimeUtil.ONEHOUR, endTime); data.put(SUMMARY_INFO_KEY, processResult(jobSummery, totalJobs)); data.put(FAILED_JOB_USERS_KEY, buildJobSummery(failedJobQuery, startTime, endTime, jobSummery.get(Constants.JobState.FAILED.toString()))); data.put(SUCCEEDED_JOB_USERS_KEY, buildJobSummery(succeededJobQuery, startTime, endTime, jobSummery.get(Constants.JobState.SUCCEEDED.toString()))); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index d58eadc..afcd7be 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -57,6 +57,9 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl // taskAttemptID to task attempt startTime protected Map<String, Long> taskAttemptStartTime; + //errorCategory, <taskId, taskAttemptId(last attempt)> + protected Map<String, Map<String, String>> errorCategoryTaskMapping; + // taskID to host mapping, for task it's the host where the last attempt runs on protected Map<String, String> taskRunningHosts; // hostname to rack mapping @@ -117,6 +120,7 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl taskStartTime = new HashMap<>(); taskAttemptStartTime = new HashMap<>(); + errorCategoryTaskMapping = new HashMap<>(); this.configuration = configuration; @@ -305,6 +309,25 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl formatDiagnostics(values.get(Keys.DIAGNOSTICS)); entityCreated(jobExecutionEntity); + for (String errorCategory : errorCategoryTaskMapping.keySet()) { + JobErrorCategoryMappingAPIEntity jobErrorCategoryMappingAPIEntity = new JobErrorCategoryMappingAPIEntity(); + jobErrorCategoryMappingAPIEntity.setTags(new HashMap<>(jobExecutionEntity.getTags())); + jobErrorCategoryMappingAPIEntity.getTags().put(MRJobTagName.ERROR_CATEGORY.toString(), errorCategory); + jobErrorCategoryMappingAPIEntity.setTimestamp(jobExecutionEntity.getTimestamp()); + + for (String taskId : errorCategoryTaskMapping.get(errorCategory).keySet()) { + jobErrorCategoryMappingAPIEntity.getTaskAttempts().add(errorCategoryTaskMapping.get(errorCategory).get(taskId)); + } + + if (jobErrorCategoryMappingAPIEntity.getTaskAttempts().size() > 0) { + jobErrorCategoryMappingAPIEntity.setError( + attempt2ErrorMsg.get( + jobErrorCategoryMappingAPIEntity.getTaskAttempts().get(0) + ).getRight() + ); + } + entityCreated(jobErrorCategoryMappingAPIEntity); + } if (configuration != null && totalCounters != null) { JobCounters parsedTotalCounters = parseCounters(totalCounters); JobCounters parsedMapCounters = parseCounters(mapCounters); @@ -491,6 +514,13 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl taskAttemptErrorCategoryEntity.setEndTime(entity.getEndTime()); taskAttemptErrorCategoryEntity.setTimestamp(entity.getTimestamp()); entityCreated(taskAttemptErrorCategoryEntity); + + String errorCategory = entity.getTags().get(MRJobTagName.ERROR_CATEGORY.toString()); + if (!errorCategoryTaskMapping.containsKey(errorCategory)) { + errorCategoryTaskMapping.put(errorCategory, new HashMap<>()); + } + + errorCategoryTaskMapping.get(errorCategory).put(taskID, taskAttemptID); } taskAttemptStartTime.remove(taskAttemptID); } else { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java index 28e6bb3..5804e0c 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java @@ -69,7 +69,7 @@ public class MRJobExecutionResource { StopWatch stopWatch = new StopWatch(); stopWatch.start(); - String jobQuery = String.format(query, Constants.JPA_JOB_EXECUTION_SERVICE_NAME); + String jobQuery = String.format(query, Constants.MR_JOB_EXECUTION_SERVICE_NAME); GenericServiceAPIResponseEntity<JobExecutionAPIEntity> res = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose); if (res.isSuccess() && res.getObj() != null) { @@ -80,7 +80,7 @@ public class MRJobExecutionResource { jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString())); } } - jobQuery = String.format(query, Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME); + jobQuery = String.format(query, Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME); GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity> runningRes = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose); if (runningRes.isSuccess() && runningRes.getObj() != null) { @@ -150,7 +150,7 @@ public class MRJobExecutionResource { StopWatch stopWatch = new StopWatch(); stopWatch.start(); String queryFormat = "%s[%s]{*}"; - String queryString = String.format(queryFormat, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, condition); + String queryString = String.format(queryFormat, Constants.MR_JOB_EXECUTION_SERVICE_NAME, condition); GenericServiceAPIResponseEntity<TaggedLogAPIEntity> res = ResourceUtils.getQueryResult(queryString, null, null); if (res.isSuccess() && res.getObj() != null) { for (TaggedLogAPIEntity o : res.getObj()) { @@ -158,7 +158,7 @@ public class MRJobExecutionResource { jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString())); } } - queryString = String.format(queryFormat, Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, condition); + queryString = String.format(queryFormat, Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, condition); res = ResourceUtils.getQueryResult(queryString, null, null); if (res.isSuccess() && res.getObj() != null) { for (TaggedLogAPIEntity o : res.getObj()) { @@ -214,13 +214,13 @@ public class MRJobExecutionResource { response.errMessage = e.getMessage(); return response; } - String query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site); + String query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.MR_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity<JobExecutionAPIEntity> historyRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { response.errMessage = String.format("Catch an exception during fetch history jobs: %s with query=%s", historyRes.getException(), query); return response; } - query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, site); + query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity> runningRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!runningRes.isSuccess() || runningRes.getObj() == null) { response.errMessage = String.format("Catch an exception during fetch running jobs: %s with query=%s", runningRes.getException(), query); @@ -289,7 +289,7 @@ public class MRJobExecutionResource { return response; } String query = String.format("%s[@site=\"%s\" AND @startTime<=\"%s\" AND @endTime>=\"%s\"]{@startTime,@endTime}", - Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, timePointsInMills, timePointsInMills); + Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, timePointsInMills, timePointsInMills); GenericServiceAPIResponseEntity<JobExecutionAPIEntity> historyRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { return historyRes; @@ -333,7 +333,7 @@ public class MRJobExecutionResource { response.errMessage = "IllegalArgument: site, startTime, endTime, or timeDistInSecs is null"; return response; } - String query = String.format("%s[@site=\"%s\"]{@durationTime,@jobType}", Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site); + String query = String.format("%s[@site=\"%s\"]{@durationTime,@jobType}", Constants.MR_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity<JobExecutionAPIEntity> historyRes = ResourceUtils.getQueryResult(query, startTime, endTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { response.errMessage = String.format("Catch an exception: %s with query=%s", historyRes.getException(), query); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java index 1125387..d81e7d9 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java @@ -60,7 +60,7 @@ public class MRTaskExecutionResource { List<MRJobTaskCountResponse.UnitTaskCount> finishedTaskCount = new ArrayList<>(); List<Long> times = ResourceUtils.parseDistributionList(timeDistInSecs); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, jobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity> historyRes = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (historyRes.isSuccess() && historyRes.getObj() != null && historyRes.getObj().size() > 0) { @@ -72,7 +72,7 @@ public class MRTaskExecutionResource { counter.entities.add(o); } } else { - query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME, site, jobId); + query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_RUNNING_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.runningentity.TaskExecutionAPIEntity> runningRes = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (runningRes.isSuccess() && runningRes.getObj() != null) { @@ -105,7 +105,7 @@ public class MRTaskExecutionResource { @QueryParam("shortJob_id") String shortDurationJobId, @QueryParam("longJob_id") String longDurationJobId) { MRTaskExecutionResponse.TaskGroupResponse result = new MRTaskExecutionResponse.TaskGroupResponse(); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, shortDurationJobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, shortDurationJobId); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> smallResponse = ResourceUtils.getQueryResult(query, null, null); if (!smallResponse.isSuccess() || smallResponse.getObj() == null) { result.errMessage = smallResponse.getException(); @@ -117,7 +117,7 @@ public class MRTaskExecutionResource { longestDuration = entity.getDuration(); } } - query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, longDurationJobId); + query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, longDurationJobId); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> largeResponse = ResourceUtils.getQueryResult(query, null, null); if (!largeResponse.isSuccess() || largeResponse.getObj() == null) { result.errMessage = largeResponse.getException(); @@ -193,7 +193,7 @@ public class MRTaskExecutionResource { return result; } - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{@startTime,@endTime,@taskType}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, jobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{@startTime,@endTime,@taskType}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> response = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (!response.isSuccess() || response.getObj() == null) { result.errMessage = response.getException(); @@ -221,7 +221,7 @@ public class MRTaskExecutionResource { @PathParam("counterName") String counterName, @QueryParam("distRange") String distRange) { MRTaskExecutionResponse.TaskDistributionResponse result = new MRTaskExecutionResponse.TaskDistributionResponse(); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\" AND @taskType=\"%s\"]{@jobCounters}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\" AND @taskType=\"%s\"]{@jobCounters}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId, Constants.TaskType.MAP.toString()); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> response = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (!response.isSuccess() || response.getObj() == null) { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java index 2632423..2b02052 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java @@ -31,8 +31,8 @@ import java.text.ParseException; import java.util.*; import static org.apache.eagle.hadoop.queue.common.HadoopClusterConstants.QUEUE_MAPPING_SERVICE_NAME; -import static org.apache.eagle.jpm.util.Constants.JPA_JOB_EXECUTION_SERVICE_NAME; -import static org.apache.eagle.jpm.util.Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_JOB_EXECUTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME; import static org.apache.eagle.jpm.util.MRJobTagName.JOB_ID; import static org.apache.eagle.jpm.util.MRJobTagName.JOB_QUEUE; import static org.apache.eagle.jpm.util.MRJobTagName.USER; @@ -87,7 +87,7 @@ public class RunningQueueResource { private List<JobExecutionAPIEntity> getRunningJobs(String site, long currentTime, String startTime, String endTime) throws Exception { GenericEntityServiceResource resource = new GenericEntityServiceResource(); String query = String.format("%s[@site=\"%s\" and @startTime<=%s and (@internalState=\"RUNNING\" or @endTime>%s)]{@jobId, @user, @queue, @allocatedMB}", - JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); + MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); GenericServiceAPIResponseEntity<JobExecutionAPIEntity> runningJobResponse = resource.search(query, startTime, endTime, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); if (!runningJobResponse.isSuccess() || runningJobResponse.getObj() == null) { @@ -102,7 +102,7 @@ public class RunningQueueResource { String startTime, String endTime) throws Exception { GenericEntityServiceResource resource = new GenericEntityServiceResource(); - String query = String.format("%s[@site=\"%s\" and @startTime<=%s and @endTime>%s]{@jobId}", JPA_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); + String query = String.format("%s[@site=\"%s\" and @startTime<=%s and @endTime>%s]{@jobId}", MR_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity> response = resource.search(query, startTime, endTime, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java index 4ee58a1..b87c41d 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java @@ -104,20 +104,21 @@ public class Constants { public static final String TASK_FINISHED = "FINISHED"; //MR - public static final String JPA_JOB_CONFIG_SERVICE_NAME = "JobConfigService"; - public static final String JPA_JOB_EVENT_SERVICE_NAME = "JobEventService"; - public static final String JPA_JOB_EXECUTION_SERVICE_NAME = "JobExecutionService"; - public static final String JPA_JOB_COUNT_SERVICE_NAME = "JobCountService"; - public static final String JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME = "RunningJobExecutionService"; - public static final String JPA_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "TaskAttemptExecutionService"; - public static final String JPA_TASK_ATTEMPT_ERROR_SERVICE_NAME = "TaskAttemptErrorCategoryService"; - public static final String JPA_TASK_FAILURE_COUNT_SERVICE_NAME = "TaskFailureCountService"; - public static final String JPA_TASK_ATTEMPT_COUNTER_SERVICE_NAME = "TaskAttemptCounterService"; - public static final String JPA_TASK_EXECUTION_SERVICE_NAME = "TaskExecutionService"; - public static final String JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME = "RunningTaskExecutionService"; - public static final String JPA_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "RunningTaskAttemptExecutionService"; - public static final String JPA_JOB_PROCESS_TIME_STAMP_NAME = "JobProcessTimeStampService"; - public static final String JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME = "JobOptimizerSuggestionService"; + public static final String MR_JOB_CONFIG_SERVICE_NAME = "JobConfigService"; + public static final String MR_JOB_EVENT_SERVICE_NAME = "JobEventService"; + public static final String MR_JOB_EXECUTION_SERVICE_NAME = "JobExecutionService"; + public static final String MR_JOB_ERROR_MAPPING_SERVICE_NAME = "JobErrorMappingService"; + public static final String MR_JOB_COUNT_SERVICE_NAME = "JobCountService"; + public static final String MR_RUNNING_JOB_EXECUTION_SERVICE_NAME = "RunningJobExecutionService"; + public static final String MR_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "TaskAttemptExecutionService"; + public static final String MR_TASK_ATTEMPT_ERROR_SERVICE_NAME = "TaskAttemptErrorCategoryService"; + public static final String MR_TASK_FAILURE_COUNT_SERVICE_NAME = "TaskFailureCountService"; + public static final String MR_TASK_ATTEMPT_COUNTER_SERVICE_NAME = "TaskAttemptCounterService"; + public static final String MR_TASK_EXECUTION_SERVICE_NAME = "TaskExecutionService"; + public static final String MR_RUNNING_TASK_EXECUTION_SERVICE_NAME = "RunningTaskExecutionService"; + public static final String MR_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "RunningTaskAttemptExecutionService"; + public static final String MR_JOB_PROCESS_TIME_STAMP_NAME = "JobProcessTimeStampService"; + public static final String MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME = "JobOptimizerSuggestionService"; public static final String JOB_TASK_TYPE_TAG = "taskType";
