[1/2] incubator-eagle git commit: EAGLE-477 eagle-data-process project clean up to contain only common processing eagle-data-process project clean up to contain only common processing

2016-08-19 Thread yonzhang2012 
Repository: incubator-eagle
Updated Branches:
  refs/heads/develop d6ec142d3 -> b31bac50b


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
--
diff --git 
a/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
 
b/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
new file mode 100644
index 000..7d69bbd
--- /dev/null
+++ 
b/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.dataproc.impl.storm;
+
+import backtype.storm.topology.base.BaseRichSpout;
+
+import com.typesafe.config.Config;
+
+/**
+ * Normally storm spout is a special part of storm topology and it is 
implemented in underlying spout implementation
+ * which can be retrieved from getSpout method.
+ */
+public interface StormSpoutProvider {
+   BaseRichSpout getSpout(Config context);
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/ValuesArray.java
--
diff --git 
a/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/ValuesArray.java
 
b/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/ValuesArray.java
new file mode 100644
index 000..de06105
--- /dev/null
+++ 
b/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/ValuesArray.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.dataproc.impl.storm;
+
+import java.util.ArrayList;
+
+/**
+ * multiple datapoints are stored within one ValuesArray object
+ * sent out
+ */
+public class ValuesArray extends ArrayList{
+   private static final long serialVersionUID = -8218427810421668178L;
+
+   public ValuesArray() {
+
+}
+
+public ValuesArray(Object... vals) {
+super(vals.length);
+for(Object o: vals) {
+add(o);
+}
+}
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
--
diff --git 
a/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
 
b/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
new file mode 100644
index 000..7454cc2
--- /dev/null
+++ 
b/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a

[1/4] incubator-eagle git commit: [EAGLE-467] Job list apis for querying jobs regardless of the status [Forced Update!]

2016-08-19 Thread qingwzhao 
Repository: incubator-eagle
Updated Branches:
  refs/heads/develop a20656b5e -> d6ec142d3 (forced update)


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d6ec142d/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
index 9f993a6..2accad8 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
@@ -20,12 +20,13 @@ package org.apache.eagle.jpm.mr.running.parser;
 
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.eagle.jpm.mr.running.config.MRRunningConfigManager;
-import org.apache.eagle.jpm.mr.running.entities.JobConfig;
-import org.apache.eagle.jpm.mr.running.entities.JobExecutionAPIEntity;
-import org.apache.eagle.jpm.mr.running.entities.TaskAttemptExecutionAPIEntity;
-import org.apache.eagle.jpm.mr.running.entities.TaskExecutionAPIEntity;
 import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager;
+import org.apache.eagle.jpm.mr.runningentity.JobConfig;
+import org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity;
+import org.apache.eagle.jpm.mr.runningentity.TaskAttemptExecutionAPIEntity;
+import org.apache.eagle.jpm.mr.runningentity.TaskExecutionAPIEntity;
 import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.jpm.util.JobNameNormalization;
 import org.apache.eagle.jpm.util.MRJobTagName;
 import org.apache.eagle.jpm.util.Utils;
 import org.apache.eagle.jpm.util.resourceFetch.ResourceFetcher;
@@ -118,7 +119,7 @@ public class MRJobParser implements Runnable {
 
 private void finishMRJob(String mrJobId) {
 JobExecutionAPIEntity jobExecutionAPIEntity = 
mrJobEntityMap.get(mrJobId);
-
jobExecutionAPIEntity.setStatus(Constants.AppState.FINISHED.toString());
+
jobExecutionAPIEntity.setCurrentState(Constants.AppState.FINISHED.toString());
 mrJobConfigs.remove(mrJobId);
 if (mrJobConfigs.size() == 0) {
 this.parserStatus = ParserStatus.APP_FINISHED;
@@ -186,19 +187,20 @@ public class MRJobParser implements Runnable {
 mrJobEntityMap.put(id, new JobExecutionAPIEntity());
 }
 
+String jobDefId = 
JobNameNormalization.getInstance().normalize(mrJob.getName());
 JobExecutionAPIEntity jobExecutionAPIEntity = 
mrJobEntityMap.get(id);
 jobExecutionAPIEntity.setTags(new HashMap<>(commonTags));
 
jobExecutionAPIEntity.getTags().put(MRJobTagName.JOB_ID.toString(), id);
 
jobExecutionAPIEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), 
mrJob.getName());
-
jobExecutionAPIEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), 
mrJob.getName());
+
jobExecutionAPIEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), 
jobDefId);
 jobExecutionAPIEntity.setTimestamp(app.getStartedTime());
 jobExecutionAPIEntity.setSubmissionTime(app.getStartedTime());
 jobExecutionAPIEntity.setStartTime(mrJob.getStartTime());
-jobExecutionAPIEntity.setElapsedTime(mrJob.getElapsedTime());
-jobExecutionAPIEntity.setStatus(mrJob.getState());
-jobExecutionAPIEntity.setMapsTotal(mrJob.getMapsTotal());
+jobExecutionAPIEntity.setDurationTime(mrJob.getElapsedTime());
+jobExecutionAPIEntity.setCurrentState(mrJob.getState());
+jobExecutionAPIEntity.setNumTotalMaps(mrJob.getMapsTotal());
 jobExecutionAPIEntity.setMapsCompleted(mrJob.getMapsCompleted());
-jobExecutionAPIEntity.setReducesTotal(mrJob.getReducesTotal());
+jobExecutionAPIEntity.setNumTotalReduces(mrJob.getReducesTotal());
 
jobExecutionAPIEntity.setReducesCompleted(mrJob.getReducesCompleted());
 jobExecutionAPIEntity.setMapProgress(mrJob.getMapProgress());
 jobExecutionAPIEntity.setReduceProgress(mrJob.getReduceProgress());
@@ -220,7 +222,6 @@ public class MRJobParser implements Runnable {
 jobExecutionAPIEntity.setAllocatedMB(app.getAllocatedMB());
 jobExecutionAPIEntity.setAllocatedVCores(app.getAllocatedVCores());
 
jobExecutionAPIEntity.setRunningContainers(app.getRunningContainers());
-runningJobManager.update(app.getId(), id, jobExecutionAPIEntity);
 }
 
 return true;
@@ -434,10 +435,10 @@ public class MRJobParser implements Runnable {
 
 taskExecutionAPIEntity.setTimestamp(app.getStartedTime());
 taskExecutionAPIEntity.setStartTime(task.getStartTime());
-

[4/4] incubator-eagle git commit: [EAGLE-467] Job list apis for querying jobs regardless of the status

2016-08-19 Thread qingwzhao 
[EAGLE-467] Job list apis for querying jobs regardless of the status

https://issues.apache.org/jira/browse/EAGLE-467

Author: Qingwen Zhao 

Closes #348 from qingwen220/jobAPI.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/d6ec142d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/d6ec142d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/d6ec142d

Branch: refs/heads/develop
Commit: d6ec142d3b1a6de28787e5fcda94f8f28cf02a3e
Parents: acee5cb
Author: Qingwen Zhao 
Authored: Fri Aug 19 17:46:42 2016 +0800
Committer: Qingwen Zhao 
Committed: Fri Aug 19 17:49:50 2016 +0800

--
 .../eagle/jpm/entity/JPMEntityRepository.java   |  30 --
 .../org/apache/eagle/jpm/entity/JobConfig.java  |  38 --
 .../org/apache/eagle/jpm/entity/SparkApp.java   | 428 --
 .../apache/eagle/jpm/entity/SparkExecutor.java  | 233 --
 .../org/apache/eagle/jpm/entity/SparkJob.java   | 178 
 .../org/apache/eagle/jpm/entity/SparkStage.java | 299 -
 .../org/apache/eagle/jpm/entity/SparkTask.java  | 290 
 .../mr/historyentity/JPAEntityRepository.java   |  40 ++
 .../jpm/mr/historyentity/JobBaseAPIEntity.java  |  24 +
 .../eagle/jpm/mr/historyentity/JobConfig.java   |  38 ++
 .../jpm/mr/historyentity/JobConfigSerDeser.java |  62 +++
 .../JobConfigurationAPIEntity.java  |  66 +++
 .../jpm/mr/historyentity/JobEventAPIEntity.java |  44 ++
 .../mr/historyentity/JobExecutionAPIEntity.java | 230 ++
 .../JobProcessTimeStampEntity.java  |  44 ++
 .../TaskAttemptCounterAPIEntity.java|  61 +++
 .../TaskAttemptExecutionAPIEntity.java  | 101 +
 .../historyentity/TaskExecutionAPIEntity.java   |  92 
 .../TaskFailureCountAPIEntity.java  |  67 +++
 .../mr/runningentity/JPMEntityRepository.java   |  33 ++
 .../eagle/jpm/mr/runningentity/JobConfig.java   |  26 ++
 .../jpm/mr/runningentity/JobConfigSerDeser.java |  46 ++
 .../mr/runningentity/JobExecutionAPIEntity.java | 437 +++
 .../TaskAttemptExecutionAPIEntity.java  | 137 ++
 .../runningentity/TaskExecutionAPIEntity.java   | 127 ++
 .../jpm/spark/crawl/JHFSparkEventReader.java|   6 +-
 .../jpm/spark/entity/JPMEntityRepository.java   |  32 ++
 .../eagle/jpm/spark/entity/JobConfig.java   |  39 ++
 .../jpm/spark/entity/JobConfigSerDeser.java |  46 ++
 .../apache/eagle/jpm/spark/entity/SparkApp.java | 429 ++
 .../eagle/jpm/spark/entity/SparkExecutor.java   | 234 ++
 .../apache/eagle/jpm/spark/entity/SparkJob.java | 179 
 .../eagle/jpm/spark/entity/SparkStage.java  | 300 +
 .../eagle/jpm/spark/entity/SparkTask.java   | 291 
 eagle-jpm/eagle-jpm-mr-history/pom.xml  |   5 +
 .../eagle/jpm/mr/history/MRHistoryJobMain.java  |   5 +-
 .../crawler/JobHistoryContentFilter.java|   6 +-
 .../crawler/JobHistoryContentFilterBuilder.java |   8 +
 .../crawler/JobHistoryContentFilterImpl.java|  11 +
 .../history/entities/JPAEntityRepository.java   |  40 --
 .../mr/history/entities/JobBaseAPIEntity.java   |  24 -
 .../jpm/mr/history/entities/JobConfig.java  |  38 --
 .../mr/history/entities/JobConfigSerDeser.java  |  62 ---
 .../entities/JobConfigurationAPIEntity.java |  66 ---
 .../mr/history/entities/JobEventAPIEntity.java  |  44 --
 .../history/entities/JobExecutionAPIEntity.java | 220 --
 .../entities/JobProcessTimeStampEntity.java |  44 --
 .../entities/TaskAttemptCounterAPIEntity.java   |  61 ---
 .../entities/TaskAttemptExecutionAPIEntity.java | 101 -
 .../entities/TaskExecutionAPIEntity.java|  89 
 .../entities/TaskFailureCountAPIEntity.java |  67 ---
 .../HistoryJobEntityCreationListener.java   |   2 +-
 .../HistoryJobEntityLifecycleListener.java  |   2 +-
 .../mr/history/parser/JHFEventReaderBase.java   |  14 +-
 .../mr/history/parser/JHFMRVer1EventReader.java |   2 +-
 ...JobConfigurationCreationServiceListener.java |   4 +-
 .../JobEntityCreationEagleServiceListener.java  |   2 +-
 .../parser/JobEntityCreationPublisher.java  |   2 +-
 .../parser/JobEntityLifecycleAggregator.java|   6 +-
 .../parser/TaskAttemptCounterListener.java  |   6 +-
 .../mr/history/parser/TaskFailureListener.java  |   6 +-
 .../jpm/mr/history/storm/JobHistorySpout.java   |   2 +-
 .../src/main/resources/application.conf |  35 +-
 eagle-jpm/eagle-jpm-mr-running/pom.xml  |   5 +
 .../eagle/jpm/mr/running/MRRunningJobMain.java  |   1 +
 .../running/entities/JPMEntityRepository.java   |  32 --
 .../jpm/mr/running/entities/JobConfig.java  |  25 --
 .../running/entities/JobExecutionAPIEntity.java | 437 ---

[2/4] incubator-eagle git commit: [EAGLE-467] Job list apis for querying jobs regardless of the status

2016-08-19 Thread qingwzhao 
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d6ec142d/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JPAEntityRepository.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JPAEntityRepository.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JPAEntityRepository.java
deleted file mode 100755
index 964d68a..000
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JPAEntityRepository.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.eagle.jpm.mr.history.entities;
-
-import org.apache.eagle.jpm.util.jobcounter.JobCounters;
-import org.apache.eagle.jpm.util.jobcounter.JobCountersSerDeser;
-import org.apache.eagle.log.entity.repo.EntityRepository;
-
-public class JPAEntityRepository extends EntityRepository {
-
-public JPAEntityRepository() {
-serDeserMap.put(JobCounters.class, new JobCountersSerDeser());
-serDeserMap.put(JobConfig.class, new JobConfigSerDeser());
-entitySet.add(JobConfigurationAPIEntity.class);
-entitySet.add(JobEventAPIEntity.class);
-entitySet.add(JobExecutionAPIEntity.class);
-
-entitySet.add(TaskAttemptExecutionAPIEntity.class);
-entitySet.add(TaskExecutionAPIEntity.class);
-entitySet.add(TaskFailureCountAPIEntity.class);
-entitySet.add(TaskAttemptCounterAPIEntity.class);
-entitySet.add(JobProcessTimeStampEntity.class);
-}
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d6ec142d/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobBaseAPIEntity.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobBaseAPIEntity.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobBaseAPIEntity.java
deleted file mode 100644
index 32c6f7c..000
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobBaseAPIEntity.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.eagle.jpm.mr.history.entities;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-
-public class JobBaseAPIEntity extends TaggedLogAPIEntity {
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d6ec142d/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobConfig.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobConfig.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobConfig.java
deleted file mode 100644
index f1dc375..000
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobConfig.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright

[2/4] incubator-eagle git commit: [EAGLE-467] Job list apis for querying jobs regardless of the status

2016-08-19 Thread qingwzhao 
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a20656b5/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JPAEntityRepository.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JPAEntityRepository.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JPAEntityRepository.java
deleted file mode 100755
index 964d68a..000
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JPAEntityRepository.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.eagle.jpm.mr.history.entities;
-
-import org.apache.eagle.jpm.util.jobcounter.JobCounters;
-import org.apache.eagle.jpm.util.jobcounter.JobCountersSerDeser;
-import org.apache.eagle.log.entity.repo.EntityRepository;
-
-public class JPAEntityRepository extends EntityRepository {
-
-public JPAEntityRepository() {
-serDeserMap.put(JobCounters.class, new JobCountersSerDeser());
-serDeserMap.put(JobConfig.class, new JobConfigSerDeser());
-entitySet.add(JobConfigurationAPIEntity.class);
-entitySet.add(JobEventAPIEntity.class);
-entitySet.add(JobExecutionAPIEntity.class);
-
-entitySet.add(TaskAttemptExecutionAPIEntity.class);
-entitySet.add(TaskExecutionAPIEntity.class);
-entitySet.add(TaskFailureCountAPIEntity.class);
-entitySet.add(TaskAttemptCounterAPIEntity.class);
-entitySet.add(JobProcessTimeStampEntity.class);
-}
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a20656b5/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobBaseAPIEntity.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobBaseAPIEntity.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobBaseAPIEntity.java
deleted file mode 100644
index 32c6f7c..000
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobBaseAPIEntity.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.eagle.jpm.mr.history.entities;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-
-public class JobBaseAPIEntity extends TaggedLogAPIEntity {
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a20656b5/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobConfig.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobConfig.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobConfig.java
deleted file mode 100644
index f1dc375..000
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/entities/JobConfig.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright

[1/4] incubator-eagle git commit: [EAGLE-467] Job list apis for querying jobs regardless of the status

2016-08-19 Thread qingwzhao 
Repository: incubator-eagle
Updated Branches:
  refs/heads/develop acee5cb33 -> a20656b5e


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a20656b5/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
index 9f993a6..2accad8 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
@@ -20,12 +20,13 @@ package org.apache.eagle.jpm.mr.running.parser;
 
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.eagle.jpm.mr.running.config.MRRunningConfigManager;
-import org.apache.eagle.jpm.mr.running.entities.JobConfig;
-import org.apache.eagle.jpm.mr.running.entities.JobExecutionAPIEntity;
-import org.apache.eagle.jpm.mr.running.entities.TaskAttemptExecutionAPIEntity;
-import org.apache.eagle.jpm.mr.running.entities.TaskExecutionAPIEntity;
 import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager;
+import org.apache.eagle.jpm.mr.runningentity.JobConfig;
+import org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity;
+import org.apache.eagle.jpm.mr.runningentity.TaskAttemptExecutionAPIEntity;
+import org.apache.eagle.jpm.mr.runningentity.TaskExecutionAPIEntity;
 import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.jpm.util.JobNameNormalization;
 import org.apache.eagle.jpm.util.MRJobTagName;
 import org.apache.eagle.jpm.util.Utils;
 import org.apache.eagle.jpm.util.resourceFetch.ResourceFetcher;
@@ -118,7 +119,7 @@ public class MRJobParser implements Runnable {
 
 private void finishMRJob(String mrJobId) {
 JobExecutionAPIEntity jobExecutionAPIEntity = 
mrJobEntityMap.get(mrJobId);
-
jobExecutionAPIEntity.setStatus(Constants.AppState.FINISHED.toString());
+
jobExecutionAPIEntity.setCurrentState(Constants.AppState.FINISHED.toString());
 mrJobConfigs.remove(mrJobId);
 if (mrJobConfigs.size() == 0) {
 this.parserStatus = ParserStatus.APP_FINISHED;
@@ -186,19 +187,20 @@ public class MRJobParser implements Runnable {
 mrJobEntityMap.put(id, new JobExecutionAPIEntity());
 }
 
+String jobDefId = 
JobNameNormalization.getInstance().normalize(mrJob.getName());
 JobExecutionAPIEntity jobExecutionAPIEntity = 
mrJobEntityMap.get(id);
 jobExecutionAPIEntity.setTags(new HashMap<>(commonTags));
 
jobExecutionAPIEntity.getTags().put(MRJobTagName.JOB_ID.toString(), id);
 
jobExecutionAPIEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), 
mrJob.getName());
-
jobExecutionAPIEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), 
mrJob.getName());
+
jobExecutionAPIEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), 
jobDefId);
 jobExecutionAPIEntity.setTimestamp(app.getStartedTime());
 jobExecutionAPIEntity.setSubmissionTime(app.getStartedTime());
 jobExecutionAPIEntity.setStartTime(mrJob.getStartTime());
-jobExecutionAPIEntity.setElapsedTime(mrJob.getElapsedTime());
-jobExecutionAPIEntity.setStatus(mrJob.getState());
-jobExecutionAPIEntity.setMapsTotal(mrJob.getMapsTotal());
+jobExecutionAPIEntity.setDurationTime(mrJob.getElapsedTime());
+jobExecutionAPIEntity.setCurrentState(mrJob.getState());
+jobExecutionAPIEntity.setNumTotalMaps(mrJob.getMapsTotal());
 jobExecutionAPIEntity.setMapsCompleted(mrJob.getMapsCompleted());
-jobExecutionAPIEntity.setReducesTotal(mrJob.getReducesTotal());
+jobExecutionAPIEntity.setNumTotalReduces(mrJob.getReducesTotal());
 
jobExecutionAPIEntity.setReducesCompleted(mrJob.getReducesCompleted());
 jobExecutionAPIEntity.setMapProgress(mrJob.getMapProgress());
 jobExecutionAPIEntity.setReduceProgress(mrJob.getReduceProgress());
@@ -220,7 +222,6 @@ public class MRJobParser implements Runnable {
 jobExecutionAPIEntity.setAllocatedMB(app.getAllocatedMB());
 jobExecutionAPIEntity.setAllocatedVCores(app.getAllocatedVCores());
 
jobExecutionAPIEntity.setRunningContainers(app.getRunningContainers());
-runningJobManager.update(app.getId(), id, jobExecutionAPIEntity);
 }
 
 return true;
@@ -434,10 +435,10 @@ public class MRJobParser implements Runnable {
 
 taskExecutionAPIEntity.setTimestamp(app.getStartedTime());
 taskExecutionAPIEntity.setStartTime(task.getStartTime());
-

[4/4] incubator-eagle git commit: [EAGLE-467] Job list apis for querying jobs regardless of the status

2016-08-19 Thread qingwzhao 
[EAGLE-467] Job list apis for querying jobs regardless of the status

https://issues.apache.org/jira/browse/EAGLE-467

Author: Qingwen Zhao 
Author: Qingwen Zhao 

Closes #348 from qingwen220/jobAPI.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a20656b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a20656b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a20656b5

Branch: refs/heads/develop
Commit: a20656b5e26e3644a7d9a7cd591c4cdf91f8695c
Parents: acee5cb
Author: Qingwen Zhao 
Authored: Fri Aug 19 17:46:42 2016 +0800
Committer: Qingwen Zhao 
Committed: Fri Aug 19 17:46:42 2016 +0800

--
 .../eagle/jpm/entity/JPMEntityRepository.java   |  30 --
 .../org/apache/eagle/jpm/entity/JobConfig.java  |  38 --
 .../org/apache/eagle/jpm/entity/SparkApp.java   | 428 --
 .../apache/eagle/jpm/entity/SparkExecutor.java  | 233 --
 .../org/apache/eagle/jpm/entity/SparkJob.java   | 178 
 .../org/apache/eagle/jpm/entity/SparkStage.java | 299 -
 .../org/apache/eagle/jpm/entity/SparkTask.java  | 290 
 .../mr/historyentity/JPAEntityRepository.java   |  40 ++
 .../jpm/mr/historyentity/JobBaseAPIEntity.java  |  24 +
 .../eagle/jpm/mr/historyentity/JobConfig.java   |  38 ++
 .../jpm/mr/historyentity/JobConfigSerDeser.java |  62 +++
 .../JobConfigurationAPIEntity.java  |  66 +++
 .../jpm/mr/historyentity/JobEventAPIEntity.java |  44 ++
 .../mr/historyentity/JobExecutionAPIEntity.java | 230 ++
 .../JobProcessTimeStampEntity.java  |  44 ++
 .../TaskAttemptCounterAPIEntity.java|  61 +++
 .../TaskAttemptExecutionAPIEntity.java  | 101 +
 .../historyentity/TaskExecutionAPIEntity.java   |  92 
 .../TaskFailureCountAPIEntity.java  |  67 +++
 .../mr/runningentity/JPMEntityRepository.java   |  33 ++
 .../eagle/jpm/mr/runningentity/JobConfig.java   |  26 ++
 .../jpm/mr/runningentity/JobConfigSerDeser.java |  46 ++
 .../mr/runningentity/JobExecutionAPIEntity.java | 437 +++
 .../TaskAttemptExecutionAPIEntity.java  | 137 ++
 .../runningentity/TaskExecutionAPIEntity.java   | 127 ++
 .../jpm/spark/crawl/JHFSparkEventReader.java|   6 +-
 .../jpm/spark/entity/JPMEntityRepository.java   |  32 ++
 .../eagle/jpm/spark/entity/JobConfig.java   |  39 ++
 .../jpm/spark/entity/JobConfigSerDeser.java |  46 ++
 .../apache/eagle/jpm/spark/entity/SparkApp.java | 429 ++
 .../eagle/jpm/spark/entity/SparkExecutor.java   | 234 ++
 .../apache/eagle/jpm/spark/entity/SparkJob.java | 179 
 .../eagle/jpm/spark/entity/SparkStage.java  | 300 +
 .../eagle/jpm/spark/entity/SparkTask.java   | 291 
 eagle-jpm/eagle-jpm-mr-history/pom.xml  |   5 +
 .../eagle/jpm/mr/history/MRHistoryJobMain.java  |   5 +-
 .../crawler/JobHistoryContentFilter.java|   6 +-
 .../crawler/JobHistoryContentFilterBuilder.java |   8 +
 .../crawler/JobHistoryContentFilterImpl.java|  11 +
 .../history/entities/JPAEntityRepository.java   |  40 --
 .../mr/history/entities/JobBaseAPIEntity.java   |  24 -
 .../jpm/mr/history/entities/JobConfig.java  |  38 --
 .../mr/history/entities/JobConfigSerDeser.java  |  62 ---
 .../entities/JobConfigurationAPIEntity.java |  66 ---
 .../mr/history/entities/JobEventAPIEntity.java  |  44 --
 .../history/entities/JobExecutionAPIEntity.java | 220 --
 .../entities/JobProcessTimeStampEntity.java |  44 --
 .../entities/TaskAttemptCounterAPIEntity.java   |  61 ---
 .../entities/TaskAttemptExecutionAPIEntity.java | 101 -
 .../entities/TaskExecutionAPIEntity.java|  89 
 .../entities/TaskFailureCountAPIEntity.java |  67 ---
 .../HistoryJobEntityCreationListener.java   |   2 +-
 .../HistoryJobEntityLifecycleListener.java  |   2 +-
 .../mr/history/parser/JHFEventReaderBase.java   |  14 +-
 .../mr/history/parser/JHFMRVer1EventReader.java |   2 +-
 ...JobConfigurationCreationServiceListener.java |   4 +-
 .../JobEntityCreationEagleServiceListener.java  |   2 +-
 .../parser/JobEntityCreationPublisher.java  |   2 +-
 .../parser/JobEntityLifecycleAggregator.java|   6 +-
 .../parser/TaskAttemptCounterListener.java  |   6 +-
 .../mr/history/parser/TaskFailureListener.java  |   6 +-
 .../jpm/mr/history/storm/JobHistorySpout.java   |   2 +-
 .../src/main/resources/application.conf |  35 +-
 eagle-jpm/eagle-jpm-mr-running/pom.xml  |   5 +
 .../eagle/jpm/mr/running/MRRunningJobMain.java  |   1 +
 .../running/entities/JPMEntityRepository.java   |  32 --
 .../jpm/mr/running/entities/JobConfig.java  |  25 --
 .../running/entities/JobExecutionAPIEntity.java | 437 ---