http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounterException.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounterException.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounterException.java new file mode 100644 index 0000000..e60e1d4 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounterException.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.eagle.jpm.util.jobcounter; + +public class JobCounterException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -4525162176188266862L; + + /** + * Default constructor of JobCounterException + */ + public JobCounterException() { + super(); + } + + /** + * Constructor of JobCounterException + * + * @param message error message + */ + public JobCounterException(String message) { + super(message); + } + + /** + * Constructor of JobCounterException + * + * @param message error message + * @param cause the cause of the exception + * + */ + public JobCounterException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor of JobCounterException + * + * @param cause the cause of the exception + */ + public JobCounterException(Throwable cause) { + super(cause); + } +}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java new file mode 100644 index 0000000..5ecda8c --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.eagle.jpm.util.jobcounter; + +import java.io.Serializable; +import java.util.Map; +import java.util.TreeMap; + + +public final class JobCounters implements Serializable { + + private Map<String, Map<String, Long>> counters = new TreeMap<>(); + + public Map<String, Map<String, Long>> getCounters() { + return counters; + } + + public void setCounters(Map<String, Map<String, Long>> counters) { + this.counters = counters; + } + + public String toString(){ + return counters.toString(); + } + + public void clear() { + for (Map.Entry<String, Map<String, Long>> entry : counters.entrySet()) { + entry.getValue().clear(); + } + counters.clear(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCountersSerDeser.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCountersSerDeser.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCountersSerDeser.java new file mode 100755 index 0000000..89b9edd --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCountersSerDeser.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.eagle.jpm.util.jobcounter; + +import org.apache.eagle.log.entity.meta.EntitySerDeser; +import org.apache.hadoop.hbase.util.Bytes; + +import java.util.Arrays; +import java.util.Map; +import java.util.TreeMap; + +public class JobCountersSerDeser implements EntitySerDeser<JobCounters> { + + private CounterGroupDictionary dictionary = null; + + @Override + public JobCounters deserialize(byte[] bytes) { + JobCounters counters = new JobCounters(); + final int length = bytes.length; + if (length < 4) { + return counters; + } + + final Map<String, Map<String, Long> > groupMap = counters.getCounters(); + int pos = 0; + final int totalGroups = Bytes.toInt(bytes, pos); + pos += 4; + + for (int i = 0; i < totalGroups; ++i) { + final int groupIndex = Bytes.toInt(bytes, pos); + pos += 4; + final int totalCounters = Bytes.toInt(bytes, pos); + pos += 4; + final int nextGroupPos = pos + (totalCounters * 12); + try { + final CounterGroupKey groupKey = getCounterGroup(groupIndex); + if (groupKey == null) { + throw new JobCounterException("Group index " + groupIndex + " is not defined"); + } + final Map<String, Long> counterMap = new TreeMap<String, Long>(); + groupMap.put(groupKey.getName(), counterMap); + for (int j = 0; j < totalCounters; ++j) { + final int counterIndex = Bytes.toInt(bytes, pos); + pos += 4; + final long value = Bytes.toLong(bytes, pos); + pos += 8; + final CounterKey counterKey = groupKey.getCounterKeyByID(counterIndex); + if (counterKey == null) { + continue; + } + counterMap.put(counterKey.getNames().get(0), value); + } + } catch (JobCounterException ex) { + // skip the group + pos = nextGroupPos; + } + } + return counters; + } + + @Override + public byte[] serialize(JobCounters counters) { + + final Map<String, Map<String, Long>> groupMap = counters.getCounters(); + int totalSize = 4; + for (Map<String, Long> counterMap : groupMap.values()) { + final int counterCount = counterMap.size(); + totalSize += counterCount * 12 + 8; + } + byte[] buffer = new byte[totalSize]; + + int totalGroups = 0; + int pos = 0; + int totalGroupNumberPos = pos; + pos += 4; + int nextGroupPos = pos; + + for (Map.Entry<String, Map<String, Long>> entry : groupMap.entrySet()) { + final String groupName = entry.getKey(); + final Map<String, Long> counterMap = entry.getValue(); + try { + nextGroupPos = pos = serializeGroup(buffer, pos, groupName, counterMap); + ++totalGroups; + } catch (JobCounterException ex) { + pos = nextGroupPos; + } + } + + Bytes.putInt(buffer, totalGroupNumberPos, totalGroups); + if (pos < totalSize) { + buffer = Arrays.copyOf(buffer, pos); + } + return buffer; + } + + @Override + public Class<JobCounters> type() { + return JobCounters.class; + } + + private int serializeGroup(byte[] buffer, int currentPos, String groupName, Map<String, Long> counterMap) throws JobCounterException { + int pos = currentPos; + final CounterGroupKey groupKey = getCounterGroup(groupName); + if (groupKey == null) { + throw new JobCounterException("Group name " + groupName + " is not defined"); + } + Bytes.putInt(buffer, pos, groupKey.getIndex()); + pos += 4; + int totalCounterNumberPos = pos; + pos += 4; + int totalCounters = 0; + + for (Map.Entry<String, Long> entry : counterMap.entrySet()) { + final String counterName = entry.getKey(); + final CounterKey counterKey = groupKey.getCounterKeyByName(counterName); + if (counterKey == null) { + continue; + } + final Long counterValue = entry.getValue(); + Bytes.putInt(buffer, pos, counterKey.getIndex()); + pos += 4; + Bytes.putLong(buffer, pos, counterValue); + pos += 8; + ++totalCounters; + } + Bytes.putInt(buffer, totalCounterNumberPos, totalCounters); + return pos; + } + + private CounterGroupKey getCounterGroup(String groupName) throws JobCounterException { + if (dictionary == null) { + dictionary = CounterGroupDictionary.getInstance(); + } + final CounterGroupKey groupKey = dictionary.getCounterGroupByName(groupName); + if (groupKey == null) { + throw new JobCounterException("Invalid counter group name: " + groupName); + } + return groupKey; + } + + private CounterGroupKey getCounterGroup(int groupIndex) throws JobCounterException { + if (dictionary == null) { + dictionary = CounterGroupDictionary.getInstance(); + } + return dictionary.getCounterGroupByIndex(groupIndex); + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java new file mode 100644 index 0000000..55ffc17 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.eagle.jpm.util.jobrecover; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.eagle.jpm.util.resourceFetch.model.AppInfo; +import org.apache.zookeeper.CreateMode; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serializable; +import java.util.*; + +public class RunningJobManager implements Serializable { + public static final Logger LOG = LoggerFactory.getLogger(RunningJobManager.class); + private String zkRoot; + private CuratorFramework curator; + private final static String ENTITY_TAGS_KEY = "entityTags"; + private final static String APP_INFO_KEY = "appInfo"; + + private CuratorFramework newCurator(String zkQuorum, int zkSessionTimeoutMs, int zkRetryTimes, int zkRetryInterval) throws Exception { + return CuratorFrameworkFactory.newClient( + zkQuorum, + zkSessionTimeoutMs, + 15000, + new RetryNTimes(zkRetryTimes, zkRetryInterval) + ); + } + + public RunningJobManager(String zkQuorum, int zkSessionTimeoutMs, int zkRetryTimes, int zkRetryInterval, String zkRoot) { + this.zkRoot = zkRoot; + + try { + curator = newCurator(zkQuorum, zkSessionTimeoutMs, zkRetryTimes, zkRetryInterval); + curator.start(); + if (curator.checkExists().forPath(this.zkRoot) == null) { + curator.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(this.zkRoot); + } + } catch (Exception e) { + } + } + + public Map<String, Pair<Map<String, String>, AppInfo>> recoverYarnApp(String yarnAppId) throws Exception { + Map<String, Pair<Map<String, String>, AppInfo>> result = new HashMap<>(); + String path = this.zkRoot + "/" + yarnAppId; + List<String> jobIds = curator.getChildren().forPath(path); + /*if (jobIds.size() == 0) { + LOG.info("delete empty path {}", path); + delete(yarnAppId); + }*/ + + for (String jobId : jobIds) { + String jobPath = path + "/" + jobId; + LOG.info("recover path {}", jobPath); + String fields = new String(curator.getData().forPath(jobPath), "UTF-8"); + if (fields.length() == 0) { + //LOG.info("delete empty path {}", jobPath); + //delete(yarnAppId, jobId); + continue; + } + JSONObject object = new JSONObject(fields); + Map<String, Map<String, String>> parseResult = parse(object); + Map<String, String> tags = parseResult.get(ENTITY_TAGS_KEY); + + Map<String, String> appInfoMap = parseResult.get(APP_INFO_KEY); + AppInfo appInfo = new AppInfo(); + appInfo.setId(appInfoMap.get("id")); + appInfo.setUser(appInfoMap.get("user")); + appInfo.setName(appInfoMap.get("name")); + appInfo.setQueue(appInfoMap.get("queue")); + appInfo.setState(appInfoMap.get("state")); + appInfo.setFinalStatus(appInfoMap.get("finalStatus")); + appInfo.setProgress(Double.parseDouble(appInfoMap.get("progress"))); + appInfo.setTrackingUI(appInfoMap.get("trackingUI")); + appInfo.setDiagnostics(appInfoMap.get("diagnostics")); + appInfo.setTrackingUrl(appInfoMap.get("trackingUrl")); + appInfo.setClusterId(appInfoMap.get("clusterId")); + appInfo.setApplicationType(appInfoMap.get("applicationType")); + appInfo.setStartedTime(Long.parseLong(appInfoMap.get("startedTime"))); + appInfo.setFinishedTime(Long.parseLong(appInfoMap.get("finishedTime"))); + appInfo.setElapsedTime(Long.parseLong(appInfoMap.get("elapsedTime"))); + appInfo.setAmContainerLogs(appInfoMap.get("amContainerLogs")); + appInfo.setAmHostHttpAddress(appInfoMap.get("amHostHttpAddress")); + appInfo.setAllocatedMB(Long.parseLong(appInfoMap.get("allocatedMB"))); + appInfo.setAllocatedVCores(Integer.parseInt(appInfoMap.get("allocatedVCores"))); + appInfo.setRunningContainers(Integer.parseInt(appInfoMap.get("runningContainers"))); + + result.put(jobId, Pair.of(tags, appInfo)); + } + return result; + } + + public Map<String, Map<String, Pair<Map<String, String>, AppInfo>>> recover() { + //we need read from zookeeper, path looks like /apps/x/running/yarnAppId/jobId/ + //content of path /apps/x/running/yarnAppId/jobId is Pair<Map<String, String>, AppInfo> + //Pair is entity tags and AppInfo + //as we know, a yarn application may contains many mr jobs or spark applications + //so, the returned results is a Map-Map + //<yarnAppId, <jobId, Pair<<Map<String, String>, AppInfo>>> + Map<String, Map<String, Pair<Map<String, String>, AppInfo>>> result = new HashMap<>(); + try { + List<String> yarnAppIds = curator.getChildren().forPath(this.zkRoot); + for (String yarnAppId : yarnAppIds) { + if (!result.containsKey(yarnAppId)) { + result.put(yarnAppId, new HashMap<>()); + } + + result.put(yarnAppId, recoverYarnApp(yarnAppId)); + } + } catch (Exception e) { + LOG.error("fail to recover", e); + throw new RuntimeException(e); + } + return result; + } + + public boolean update(String yarnAppId, String jobId, Map<String, String> tags, AppInfo app) { + String path = this.zkRoot + "/" + yarnAppId + "/" + jobId; + //InterProcessMutex lock = new InterProcessMutex(curator, path); + Map<String, String> fields = new HashMap<>(); + Map<String, String> appInfo = new HashMap<>(); + appInfo.put("id", app.getId()); + appInfo.put("user", app.getUser()); + appInfo.put("name", app.getName()); + appInfo.put("queue", app.getQueue()); + appInfo.put("state", app.getState()); + appInfo.put("finalStatus", app.getFinalStatus()); + appInfo.put("progress", app.getProgress() + ""); + appInfo.put("trackingUI", app.getTrackingUI()); + appInfo.put("diagnostics", app.getDiagnostics()); + appInfo.put("trackingUrl", app.getTrackingUrl()); + appInfo.put("clusterId", app.getClusterId()); + appInfo.put("applicationType", app.getApplicationType()); + appInfo.put("startedTime", app.getStartedTime() + ""); + appInfo.put("finishedTime", app.getFinishedTime() + ""); + appInfo.put("elapsedTime", app.getElapsedTime() + ""); + appInfo.put("amContainerLogs", app.getAmContainerLogs()); + appInfo.put("amHostHttpAddress", app.getAmHostHttpAddress()); + appInfo.put("allocatedMB", app.getAllocatedMB() + ""); + appInfo.put("allocatedVCores", app.getAllocatedVCores() + ""); + appInfo.put("runningContainers", app.getRunningContainers() + ""); + + fields.put(ENTITY_TAGS_KEY, (new JSONObject(tags)).toString()); + fields.put(APP_INFO_KEY, (new JSONObject(appInfo)).toString()); + try { + //lock.acquire(); + JSONObject object = new JSONObject(fields); + if (curator.checkExists().forPath(path) == null) { + curator.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(path); + } + curator.setData().forPath(path, object.toString().getBytes("UTF-8")); + + } catch (Exception e) { + LOG.error("failed to update job {} for yarn app {} ", jobId, yarnAppId); + } finally { + try { + //lock.release(); + } catch (Exception e) { + LOG.error("fail releasing lock", e); + } + } + return true; + } + + public void delete(String yarnAppId, String jobId) { + String path = this.zkRoot + "/" + yarnAppId + "/" + jobId; + //InterProcessMutex lock = new InterProcessMutex(curator, path); + try { + //lock.acquire(); + if (curator.checkExists().forPath(path) != null) { + curator.delete().deletingChildrenIfNeeded().forPath(path); + LOG.info("delete job {} for yarn app {}, path {} ", jobId, yarnAppId, path); + if (curator.getChildren().forPath(path).size() == 0) { + delete(yarnAppId); + } + } + } catch (Exception e) { + LOG.error("failed to delete job {} for yarn app {}, path {}, {}", jobId, yarnAppId, path, e); + } finally { + try { + //lock.release(); + } catch (Exception e) { + LOG.error("fail releasing lock", e); + + } + } + } + + public void delete(String yarnAppId) { + String path = this.zkRoot + "/" + yarnAppId; + //InterProcessMutex lock = new InterProcessMutex(curator, path); + try { + //lock.acquire(); + if (curator.checkExists().forPath(path) != null) { + curator.delete().forPath(path); + LOG.info("delete yarn app {}, path {} ", yarnAppId, path); + } + } catch (Exception e) { + LOG.error("failed to delete yarn app {}, path {} ", yarnAppId, path); + } finally { + try { + //lock.release(); + } catch (Exception e) { + LOG.error("fail releasing lock", e); + } + } + } + + public Map<String, Map<String, String>> parse(JSONObject object) throws JSONException { + Map<String, Map<String, String>> result = new HashMap<>(); + + Iterator<String> keysItr = object.keys(); + while (keysItr.hasNext()) { + String key = keysItr.next(); + result.put(key, new HashMap<>()); + String value = (String)object.get(key); + + JSONObject jsonObject = new JSONObject(value); + Map<String, String> items = result.get(key); + Iterator<String> keyItemItr = jsonObject.keys(); + while (keyItemItr.hasNext()) { + String itemKey = keyItemItr.next(); + items.put(itemKey, (String)jsonObject.get(itemKey)); + } + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/RMResourceFetcher.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/RMResourceFetcher.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/RMResourceFetcher.java index d5147b6..1b29f0c 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/RMResourceFetcher.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/RMResourceFetcher.java @@ -25,9 +25,12 @@ import org.apache.eagle.jpm.util.resourceFetch.ha.HAURLSelector; import org.apache.eagle.jpm.util.resourceFetch.ha.HAURLSelectorImpl; import org.apache.eagle.jpm.util.resourceFetch.model.AppInfo; import org.apache.eagle.jpm.util.resourceFetch.model.AppsWrapper; +import org.apache.eagle.jpm.util.resourceFetch.model.ClusterInfo; +import org.apache.eagle.jpm.util.resourceFetch.model.ClusterInfoWrapper; import org.apache.eagle.jpm.util.resourceFetch.url.JobListServiceURLBuilderImpl; import org.apache.eagle.jpm.util.resourceFetch.url.ServiceURLBuilder; import org.apache.eagle.jpm.util.resourceFetch.url.SparkCompleteJobServiceURLBuilderImpl; +import org.apache.hadoop.mapreduce.Cluster; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; @@ -35,10 +38,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; import java.util.List; -public class RMResourceFetcher implements ResourceFetcher{ +public class RMResourceFetcher implements ResourceFetcher<AppInfo> { private static final Logger LOG = LoggerFactory.getLogger(RMResourceFetcher.class); private final HAURLSelector selector; @@ -64,8 +66,8 @@ public class RMResourceFetcher implements ResourceFetcher{ } } - private List<Object> doFetchSparkFinishApplicationsList(String lastFinishTime) throws Exception { - List<AppInfo> result = null; + private List<AppInfo> doFetchSparkFinishApplicationsList(String lastFinishTime) throws Exception { + List<AppInfo> result; InputStream is = null; try { checkUrl(); @@ -76,23 +78,100 @@ public class RMResourceFetcher implements ResourceFetcher{ if (appWrapper != null && appWrapper.getApps() != null && appWrapper.getApps().getApp() != null) { result = appWrapper.getApps().getApp(); - return Arrays.asList((Object)result); + return result; } return null; - }finally { - if (is != null) { try {is.close();} catch (Exception e){} } + } finally { + if (is != null) { try { is.close();} catch (Exception e) { } } } } - - - public List<Object> getResource(Constants.ResourceType resoureType, Object... parameter) throws Exception{ + private String getSparkRunningJobURL() { + StringBuilder sb = new StringBuilder(); + sb.append(selector.getSelectedUrl()).append("/").append(Constants.V2_APPS_URL); + sb.append("?applicationTypes=SPARK&state=RUNNING&"); + sb.append(Constants.ANONYMOUS_PARAMETER); + return sb.toString(); + } + + private List<AppInfo> doFetchSparkRunningApplicationsList() throws Exception { + List<AppInfo> result; + InputStream is = null; + try { + checkUrl(); + final String urlString = getSparkRunningJobURL(); + LOG.info("Going to call yarn api to fetch running spark job list: " + urlString); + is = InputStreamUtils.getInputStream(urlString, null, Constants.CompressionType.NONE); + final AppsWrapper appWrapper = OBJ_MAPPER.readValue(is, AppsWrapper.class); + if (appWrapper != null && appWrapper.getApps() != null && appWrapper.getApps().getApp() != null) { + result = appWrapper.getApps().getApp(); + return result; + } + return null; + } finally { + if (is != null) { try { is.close();} catch (Exception e) { } } + } + } + + private String getMRRunningJobURL() { + return String.format("%s/%s?applicationTypes=MAPREDUCE&state=RUNNING&%s", + selector.getSelectedUrl(), + Constants.V2_APPS_URL, + Constants.ANONYMOUS_PARAMETER); + } + + private List<AppInfo> doFetchMRRunningApplicationsList() throws Exception { + List<AppInfo> result; + InputStream is = null; + try { + checkUrl(); + final String urlString = getMRRunningJobURL(); + LOG.info("Going to call yarn api to fetch running mr job list: " + urlString); + is = InputStreamUtils.getInputStream(urlString, null, Constants.CompressionType.GZIP); + final AppsWrapper appWrapper = OBJ_MAPPER.readValue(is, AppsWrapper.class); + if (appWrapper != null && appWrapper.getApps() != null && appWrapper.getApps().getApp() != null) { + result = appWrapper.getApps().getApp(); + return result; + } + return null; + } finally { + if (is != null) { try { is.close();} catch (Exception e) { } } + } + } + + public List<AppInfo> getResource(Constants.ResourceType resoureType, Object... parameter) throws Exception{ switch(resoureType) { case COMPLETE_SPARK_JOB: return doFetchSparkFinishApplicationsList((String)parameter[0]); - + case RUNNING_SPARK_JOB: + return doFetchSparkRunningApplicationsList(); + case RUNNING_MR_JOB: + return doFetchMRRunningApplicationsList(); default: - throw new Exception("Not support ressourceType :" + resoureType); + throw new Exception("Not support resourceType :" + resoureType); + } + } + + private String getClusterInfoURL() { + StringBuilder sb = new StringBuilder(); + sb.append(selector.getSelectedUrl()).append("/").append(Constants.YARN_API_CLUSTER_INFO).append("?" + Constants.ANONYMOUS_PARAMETER); + return sb.toString(); + } + + public ClusterInfo getClusterInfo() throws Exception { + InputStream is = null; + try { + checkUrl(); + final String urlString = getClusterInfoURL(); + LOG.info("Calling yarn api to fetch cluster info: " + urlString); + is = InputStreamUtils.getInputStream(urlString, null, Constants.CompressionType.GZIP); + final ClusterInfoWrapper clusterInfoWrapper = OBJ_MAPPER.readValue(is, ClusterInfoWrapper.class); + if (clusterInfoWrapper != null && clusterInfoWrapper.getClusterInfo() != null) { + return clusterInfoWrapper.getClusterInfo(); + } + return null; + } finally { + if (is != null) { try { is.close();} catch (Exception e) { } } } } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ResourceFetcher.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ResourceFetcher.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ResourceFetcher.java index b21d030..cd475e7 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ResourceFetcher.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ResourceFetcher.java @@ -20,8 +20,8 @@ import org.apache.eagle.jpm.util.Constants; import java.util.List; -public interface ResourceFetcher { - - List<Object> getResource(Constants.ResourceType resoureType, Object... parameter) throws Exception; +public interface ResourceFetcher<T> { + //continue to refactor later + List<T> getResource(Constants.ResourceType resoureType, Object... parameter) throws Exception; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/SparkHistoryServerResourceFetcher.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/SparkHistoryServerResourceFetcher.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/SparkHistoryServerResourceFetcher.java index c13bee0..689d2f3 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/SparkHistoryServerResourceFetcher.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/SparkHistoryServerResourceFetcher.java @@ -35,7 +35,7 @@ import java.io.InputStream; import java.util.Arrays; import java.util.List; -public class SparkHistoryServerResourceFetcher implements ResourceFetcher{ +public class SparkHistoryServerResourceFetcher implements ResourceFetcher<SparkApplication> { private static final Logger LOG = LoggerFactory.getLogger(SparkHistoryServerResourceFetcher.class); @@ -55,14 +55,14 @@ public class SparkHistoryServerResourceFetcher implements ResourceFetcher{ this.auth = "Basic " + new String(new Base64().encode(String.format("%s:%s", userName, pwd).getBytes()));; } - private List<Object> doFetchSparkApplicationDetail(String appId) throws Exception { + private List<SparkApplication> doFetchSparkApplicationDetail(String appId) throws Exception { InputStream is = null; try { final String urlString = sparkDetailJobServiceURLBuilder.build(this.historyServerURL, appId); LOG.info("Going to call spark history server api to fetch spark job: " + urlString); is = InputStreamUtils.getInputStream(urlString, auth, Constants.CompressionType.NONE); SparkApplication app = OBJ_MAPPER.readValue(is, SparkApplication.class); - return Arrays.asList((Object)app); + return Arrays.asList(app); } catch (FileNotFoundException e) { return null; } finally { @@ -70,7 +70,7 @@ public class SparkHistoryServerResourceFetcher implements ResourceFetcher{ } } - public List<Object> getResource(Constants.ResourceType resoureType, Object... parameter) throws Exception{ + public List<SparkApplication> getResource(Constants.ResourceType resoureType, Object... parameter) throws Exception{ switch(resoureType) { case SPARK_JOB_DETAIL: return doFetchSparkApplicationDetail((String)parameter[0]); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppInfo.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppInfo.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppInfo.java index 463ce1e..5f13616 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppInfo.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppInfo.java @@ -19,26 +19,33 @@ package org.apache.eagle.jpm.util.resourceFetch.model; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; +import java.io.Serializable; + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -public class AppInfo { - String id; - String user; - String name; - String queue; - String state; - String finalStatus; - double progress; - String trackingUI; - String trackingUrl; - String diagnostics; - String clusterId; - String applicationType; - long startedTime; - long finishedTime; - long elapsedTime; - String amContainerLogs; - String amHostHttpAddress; +public class AppInfo implements Serializable { + private static final long serialVersionUID = 1L; + + private String id; + private String user; + private String name; + private String queue; + private String state; + private String finalStatus; + private double progress; + private String trackingUI; + private String trackingUrl; + private String diagnostics; + private String clusterId; + private String applicationType; + private long startedTime; + private long finishedTime; + private long elapsedTime; + private String amContainerLogs; + private String amHostHttpAddress; + private long allocatedMB; + private int allocatedVCores; + private int runningContainers; public String getId() { return id; @@ -143,4 +150,28 @@ public class AppInfo { this.amHostHttpAddress = amHostHttpAddress; } + public long getAllocatedMB() { + return allocatedMB; + } + + public void setAllocatedMB(long allocatedMB) { + this.allocatedMB = allocatedMB; + } + + public int getAllocatedVCores() { + return allocatedVCores; + } + + public void setAllocatedVCores(int allocatedVCores) { + this.allocatedVCores = allocatedVCores; + } + + public int getRunningContainers() { + return runningContainers; + } + + public void setRunningContainers(int runningContainers) { + this.runningContainers = runningContainers; + } + } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppsWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppsWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppsWrapper.java index d791685..42ff4f8 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppsWrapper.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/AppsWrapper.java @@ -22,7 +22,8 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) public class AppsWrapper { - + private static final long serialVersionUID = 1L; + private Applications apps; public Applications getApps() { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/ClusterInfo.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/ClusterInfo.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/ClusterInfo.java new file mode 100644 index 0000000..75a1e8a --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/ClusterInfo.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.io.Serializable; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class ClusterInfo implements Serializable { + private static final long serialVersionUID = 1L; + + private long id; + private long startedOn; + private String state; + private String haState; + private String resourceManagerVersion; + private String resourceManagerBuildVersion; + private String resourceManagerVersionBuiltOn; + private String hadoopVersion; + private String hadoopBuildVersion; + private String hadoopVersionBuiltOn; + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public long getStartedOn() { + return startedOn; + } + + public void setStartedOn(long startedOn) { + this.startedOn = startedOn; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getHaState() { + return haState; + } + + public void setHaState(String haState) { + this.haState = haState; + } + + public String getResourceManagerVersion() { + return resourceManagerVersion; + } + + public void setResourceManagerVersion(String resourceManagerVersion) { + this.resourceManagerVersion = resourceManagerVersion; + } + + public String getResourceManagerBuildVersion() { + return resourceManagerBuildVersion; + } + + public void setResourceManagerBuildVersion(String resourceManagerBuildVersion) { + this.resourceManagerBuildVersion = resourceManagerBuildVersion; + } + + public String getResourceManagerVersionBuiltOn() { + return resourceManagerVersionBuiltOn; + } + + public void setResourceManagerVersionBuiltOn(String resourceManagerVersionBuiltOn) { + this.resourceManagerVersionBuiltOn = resourceManagerVersionBuiltOn; + } + + public String getHadoopVersion() { + return hadoopVersion; + } + + public void setHadoopVersion(String hadoopVersion) { + this.hadoopVersion = hadoopVersion; + } + + public String getHadoopBuildVersion() { + return hadoopBuildVersion; + } + + public void setHadoopBuildVersion(String hadoopBuildVersion) { + this.hadoopBuildVersion = hadoopBuildVersion; + } + + public String getHadoopVersionBuiltOn() { + return hadoopVersionBuiltOn; + } + + public void setHadoopVersionBuiltOn(String hadoopVersionBuiltOn) { + this.hadoopVersionBuiltOn = hadoopVersionBuiltOn; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/ClusterInfoWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/ClusterInfoWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/ClusterInfoWrapper.java new file mode 100644 index 0000000..301487e --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/ClusterInfoWrapper.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class ClusterInfoWrapper { + private ClusterInfo clusterInfo; + + public ClusterInfo getClusterInfo() { + return clusterInfo; + } + + public void setClusterInfo(ClusterInfo clusterInfo) { + this.clusterInfo = clusterInfo; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounterGroup.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounterGroup.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounterGroup.java new file mode 100644 index 0000000..102843f --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounterGroup.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.List; +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class JobCounterGroup { + private String counterGroupName; + private List<JobCounterItem> counter; + + public String getCounterGroupName() { + return counterGroupName; + } + public void setCounterGroupName(String counterGroupName) { + this.counterGroupName = counterGroupName; + } + public List<JobCounterItem> getCounter() { + return counter; + } + public void setCounter(List<JobCounterItem> counter) { + this.counter = counter; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounterItem.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounterItem.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounterItem.java new file mode 100644 index 0000000..76969a4 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounterItem.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class JobCounterItem { + private String name; + private long totalCounterValue; + private long mapCounterValue; + private long reduceCounterValue; + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public long getTotalCounterValue() { + return totalCounterValue; + } + public void setTotalCounterValue(long totalCounterValue) { + this.totalCounterValue = totalCounterValue; + } + public long getMapCounterValue() { + return mapCounterValue; + } + public void setMapCounterValue(long mapCounterValue) { + this.mapCounterValue = mapCounterValue; + } + public long getReduceCounterValue() { + return reduceCounterValue; + } + public void setReduceCounterValue(long reduceCounterValue) { + this.reduceCounterValue = reduceCounterValue; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounters.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounters.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounters.java new file mode 100644 index 0000000..e212ea0 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCounters.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.List; +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class JobCounters { + private String id; + private List<JobCounterGroup> counterGroup; + + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public List<JobCounterGroup> getCounterGroup() { + return counterGroup; + } + public void setCounterGroup(List<JobCounterGroup> counterGroup) { + this.counterGroup = counterGroup; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCountersWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCountersWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCountersWrapper.java new file mode 100644 index 0000000..1178f52 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/JobCountersWrapper.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class JobCountersWrapper { + public JobCounters getJobCounters() { + return jobCounters; + } + + public void setJobCounters(JobCounters jobCounters) { + this.jobCounters = jobCounters; + } + + private JobCounters jobCounters; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRJob.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRJob.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRJob.java new file mode 100644 index 0000000..b175152 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRJob.java @@ -0,0 +1,289 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MRJob { + private long startTime; + private long finishTime; + private long elapsedTime; + private String id; + private String name; + private String user; + private String state; + private int mapsTotal; + private int mapsCompleted; + private int reducesTotal; + private int reducesCompleted; + private double mapProgress; + private double reduceProgress; + private int mapsPending; + private int mapsRunning; + private int reducesPending; + private int reducesRunning; + private boolean uberized; + private String diagnostics; + private int newReduceAttempts; + private int runningReduceAttempts; + private int failedReduceAttempts; + private int killedReduceAttempts; + private int successfulReduceAttempts; + private int newMapAttempts; + private int runningMapAttempts; + private int failedMapAttempts; + private int killedMapAttempts; + private int successfulMapAttempts; + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public long getFinishTime() { + return finishTime; + } + + public void setFinishTime(long finishTime) { + this.finishTime = finishTime; + } + + public long getElapsedTime() { + return elapsedTime; + } + + public void setElapsedTime(long elapsedTime) { + this.elapsedTime = elapsedTime; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public int getMapsTotal() { + return mapsTotal; + } + + public void setMapsTotal(int mapsTotal) { + this.mapsTotal = mapsTotal; + } + + public int getMapsCompleted() { + return mapsCompleted; + } + + public void setMapsCompleted(int mapsCompleted) { + this.mapsCompleted = mapsCompleted; + } + + public int getReducesTotal() { + return reducesTotal; + } + + public void setReducesTotal(int reducesTotal) { + this.reducesTotal = reducesTotal; + } + + public int getReducesCompleted() { + return reducesCompleted; + } + + public void setReducesCompleted(int reducesCompleted) { + this.reducesCompleted = reducesCompleted; + } + + public double getMapProgress() { + return mapProgress; + } + + public void setMapProgress(double mapProgress) { + this.mapProgress = mapProgress; + } + + public double getReduceProgress() { + return reduceProgress; + } + + public void setReduceProgress(double reduceProgress) { + this.reduceProgress = reduceProgress; + } + + public int getMapsPending() { + return mapsPending; + } + + public void setMapsPending(int mapsPending) { + this.mapsPending = mapsPending; + } + + public int getMapsRunning() { + return mapsRunning; + } + + public void setMapsRunning(int mapsRunning) { + this.mapsRunning = mapsRunning; + } + + public int getReducesPending() { + return reducesPending; + } + + public void setReducesPending(int reducesPending) { + this.reducesPending = reducesPending; + } + + public int getReducesRunning() { + return reducesRunning; + } + + public void setReducesRunning(int reducesRunning) { + this.reducesRunning = reducesRunning; + } + + public boolean isUberized() { + return uberized; + } + + public void setUberized(boolean uberized) { + this.uberized = uberized; + } + + public String getDiagnostics() { + return diagnostics; + } + + public void setDiagnostics(String diagnostics) { + this.diagnostics = diagnostics; + } + + public int getNewReduceAttempts() { + return newReduceAttempts; + } + + public void setNewReduceAttempts(int newReduceAttempts) { + this.newReduceAttempts = newReduceAttempts; + } + + public int getRunningReduceAttempts() { + return runningReduceAttempts; + } + + public void setRunningReduceAttempts(int runningReduceAttempts) { + this.runningReduceAttempts = runningReduceAttempts; + } + + public int getFailedReduceAttempts() { + return failedReduceAttempts; + } + + public void setFailedReduceAttempts(int failedReduceAttempts) { + this.failedReduceAttempts = failedReduceAttempts; + } + + public int getKilledReduceAttempts() { + return killedReduceAttempts; + } + + public void setKilledReduceAttempts(int killedReduceAttempts) { + this.killedReduceAttempts = killedReduceAttempts; + } + + public int getSuccessfulReduceAttempts() { + return successfulReduceAttempts; + } + + public void setSuccessfulReduceAttempts(int successfulReduceAttempts) { + this.successfulReduceAttempts = successfulReduceAttempts; + } + + public int getNewMapAttempts() { + return newMapAttempts; + } + + public void setNewMapAttempts(int newMapAttempts) { + this.newMapAttempts = newMapAttempts; + } + + public int getRunningMapAttempts() { + return runningMapAttempts; + } + + public void setRunningMapAttempts(int runningMapAttempts) { + this.runningMapAttempts = runningMapAttempts; + } + + public int getFailedMapAttempts() { + return failedMapAttempts; + } + + public void setFailedMapAttempts(int failedMapAttempts) { + this.failedMapAttempts = failedMapAttempts; + } + + public int getKilledMapAttempts() { + return killedMapAttempts; + } + + public void setKilledMapAttempts(int killedMapAttempts) { + this.killedMapAttempts = killedMapAttempts; + } + + public int getSuccessfulMapAttempts() { + return successfulMapAttempts; + } + + public void setSuccessfulMapAttempts(int successfulMapAttempts) { + this.successfulMapAttempts = successfulMapAttempts; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRJobsWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRJobsWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRJobsWrapper.java new file mode 100644 index 0000000..ec16506 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRJobsWrapper.java @@ -0,0 +1,37 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MRJobsWrapper { + public MrJobs getJobs() { + return jobs; + } + + public void setJobs(MrJobs jobs) { + this.jobs = jobs; + } + + private MrJobs jobs; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTask.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTask.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTask.java new file mode 100644 index 0000000..d6e34ca --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTask.java @@ -0,0 +1,109 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MRTask { + private long startTime; + private long finishTime; + private long elapsedTime; + private double progress; + private String id; + private String state; + private String type; + private String successfulAttempt; + private String status; + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public long getFinishTime() { + return finishTime; + } + + public void setFinishTime(long finishTime) { + this.finishTime = finishTime; + } + + public long getElapsedTime() { + return elapsedTime; + } + + public void setElapsedTime(long elapsedTime) { + this.elapsedTime = elapsedTime; + } + + public double getProgress() { + return progress; + } + + public void setProgress(double progress) { + this.progress = progress; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getSuccessfulAttempt() { + return successfulAttempt; + } + + public void setSuccessfulAttempt(String successfulAttempt) { + this.successfulAttempt = successfulAttempt; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempt.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempt.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempt.java new file mode 100644 index 0000000..b29e312 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempt.java @@ -0,0 +1,136 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MRTaskAttempt { + private long startTime; + private long finishTime; + private long elapsedTime; + private double progress; + private String id; + private String rack; + private String state; + private String status; + private String nodeHttpAddress; + private String diagnostics; + private String type; + private String assignedContainerId; + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public long getFinishTime() { + return finishTime; + } + + public void setFinishTime(long finishTime) { + this.finishTime = finishTime; + } + + public long getElapsedTime() { + return elapsedTime; + } + + public void setElapsedTime(long elapsedTime) { + this.elapsedTime = elapsedTime; + } + + public double getProgress() { + return progress; + } + + public void setProgress(double progress) { + this.progress = progress; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getRack() { + return rack; + } + + public void setRack(String rack) { + this.rack = rack; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getNodeHttpAddress() { + return nodeHttpAddress; + } + + public void setNodeHttpAddress(String nodeHttpAddress) { + this.nodeHttpAddress = nodeHttpAddress; + } + + public String getDiagnostics() { + return diagnostics; + } + + public void setDiagnostics(String diagnostics) { + this.diagnostics = diagnostics; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getAssignedContainerId() { + return assignedContainerId; + } + + public void setAssignedContainerId(String assignedContainerId) { + this.assignedContainerId = assignedContainerId; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttemptWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttemptWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttemptWrapper.java new file mode 100644 index 0000000..fbf3f21 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttemptWrapper.java @@ -0,0 +1,37 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MRTaskAttemptWrapper { + public MRTaskAttempts getTaskAttempts() { + return taskAttempts; + } + + public void setTaskAttempts(MRTaskAttempts taskAttempts) { + this.taskAttempts = taskAttempts; + } + + private MRTaskAttempts taskAttempts; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempts.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempts.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempts.java new file mode 100644 index 0000000..606ee05 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempts.java @@ -0,0 +1,39 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.List; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MRTaskAttempts { + public List<MRTaskAttempt> getTaskAttempt() { + return taskAttempt; + } + + public void setTaskAttempt(List<MRTaskAttempt> taskAttempt) { + this.taskAttempt = taskAttempt; + } + + private List<MRTaskAttempt> taskAttempt; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasks.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasks.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasks.java new file mode 100644 index 0000000..faf46fc --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasks.java @@ -0,0 +1,40 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.List; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MRTasks { + public List<MRTask> getTask() { + return task; + } + + public void setTask(List<MRTask> task) { + this.task = task; + } + + private List<MRTask> task; + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasksWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasksWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasksWrapper.java new file mode 100644 index 0000000..c541e89 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasksWrapper.java @@ -0,0 +1,37 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MRTasksWrapper { + public MRTasks getTasks() { + return tasks; + } + + public void setTasks(MRTasks tasks) { + this.tasks = tasks; + } + + private MRTasks tasks; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MrJobs.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MrJobs.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MrJobs.java new file mode 100644 index 0000000..9165550 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MrJobs.java @@ -0,0 +1,39 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.List; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class MrJobs { + public List<MRJob> getJob() { + return job; + } + + public void setJobs(List<MRJob> job) { + this.job = job; + } + + private List<MRJob> job; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkExecutor.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkExecutor.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkExecutor.java new file mode 100644 index 0000000..db84fa1 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkExecutor.java @@ -0,0 +1,155 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SparkExecutor { + private String id; + private String hostPort; + private int rddBlocks; + private long memoryUsed; + private long diskUsed; + private int activeTasks; + private int failedTasks; + private int completedTasks; + private int totalTasks; + private long totalDuration; + private long totalInputBytes; + private long totalShuffleRead; + private long totalShuffleWrite; + private long maxMemory; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getHostPort() { + return hostPort; + } + + public void setHostPort(String hostPort) { + this.hostPort = hostPort; + } + + public int getRddBlocks() { + return rddBlocks; + } + + public void setRddBlocks(int rddBlocks) { + this.rddBlocks = rddBlocks; + } + + public long getMemoryUsed() { + return memoryUsed; + } + + public void setMemoryUsed(long memoryUsed) { + this.memoryUsed = memoryUsed; + } + + public long getDiskUsed() { + return diskUsed; + } + + public void setDiskUsed(long diskUsed) { + this.diskUsed = diskUsed; + } + + public int getActiveTasks() { + return activeTasks; + } + + public void setActiveTasks(int activeTasks) { + this.activeTasks = activeTasks; + } + + public int getFailedTasks() { + return failedTasks; + } + + public void setFailedTasks(int failedTasks) { + this.failedTasks = failedTasks; + } + + public int getCompletedTasks() { + return completedTasks; + } + + public void setCompletedTasks(int completedTasks) { + this.completedTasks = completedTasks; + } + + public int getTotalTasks() { + return totalTasks; + } + + public void setTotalTasks(int totalTasks) { + this.totalTasks = totalTasks; + } + + public long getTotalDuration() { + return totalDuration; + } + + public void setTotalDuration(long totalDuration) { + this.totalDuration = totalDuration; + } + + public long getTotalInputBytes() { + return totalInputBytes; + } + + public void setTotalInputBytes(long totalInputBytes) { + this.totalInputBytes = totalInputBytes; + } + + public long getTotalShuffleRead() { + return totalShuffleRead; + } + + public void setTotalShuffleRead(long totalShuffleRead) { + this.totalShuffleRead = totalShuffleRead; + } + + public long getTotalShuffleWrite() { + return totalShuffleWrite; + } + + public void setTotalShuffleWrite(long totalShuffleWrite) { + this.totalShuffleWrite = totalShuffleWrite; + } + + public long getMaxMemory() { + return maxMemory; + } + + public void setMaxMemory(long maxMemory) { + this.maxMemory = maxMemory; + } + +}
