Repository: incubator-eagle Updated Branches: refs/heads/develop 5c0db6aff -> 5bf2c62d2
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkJob.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkJob.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkJob.java new file mode 100644 index 0000000..018185a --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkJob.java @@ -0,0 +1,165 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.List; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SparkJob { + private int jobId; + private String name; + private String submissionTime; + private String completionTime; + private List<Integer> stageIds; + private String status; + private int numTasks; + private int numActiveTasks; + private int numCompletedTasks; + private int numSkippedTasks; + private int numFailedTasks; + private int numActiveStages; + private int numCompletedStages; + private int numSkippedStages; + private int numFailedStages; + + public int getJobId() { + return jobId; + } + + public void setJobId(int jobId) { + this.jobId = jobId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSubmissionTime() { + return submissionTime; + } + + public void setSubmissionTime(String submissionTime) { + this.submissionTime = submissionTime; + } + + public String getCompletionTime() { + return completionTime; + } + + public void setCompletionTime(String completionTime) { + this.completionTime = completionTime; + } + + public List<Integer> getStageIds() { + return stageIds; + } + + public void setStageIds(List<Integer> stageIds) { + this.stageIds = stageIds; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public int getNumTasks() { + return numTasks; + } + + public void setNumTasks(int numTasks) { + this.numTasks = numTasks; + } + + public int getNumActiveTasks() { + return numActiveTasks; + } + + public void setNumActiveTasks(int numActiveTasks) { + this.numActiveTasks = numActiveTasks; + } + + public int getNumCompletedTasks() { + return numCompletedTasks; + } + + public void setNumCompletedTasks(int numCompletedTasks) { + this.numCompletedTasks = numCompletedTasks; + } + + public int getNumSkippedTasks() { + return numSkippedTasks; + } + + public void setNumSkippedTasks(int numSkippedTasks) { + this.numSkippedTasks = numSkippedTasks; + } + + public int getNumFailedTasks() { + return numFailedTasks; + } + + public void setNumFailedTasks(int numFailedTasks) { + this.numFailedTasks = numFailedTasks; + } + + public int getNumActiveStages() { + return numActiveStages; + } + + public void setNumActiveStages(int numActiveStages) { + this.numActiveStages = numActiveStages; + } + + public int getNumCompletedStages() { + return numCompletedStages; + } + + public void setNumCompletedStages(int numCompletedStages) { + this.numCompletedStages = numCompletedStages; + } + + public int getNumSkippedStages() { + return numSkippedStages; + } + + public void setNumSkippedStages(int numSkippedStages) { + this.numSkippedStages = numSkippedStages; + } + + public int getNumFailedStages() { + return numFailedStages; + } + + public void setNumFailedStages(int numFailedStages) { + this.numFailedStages = numFailedStages; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkStage.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkStage.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkStage.java new file mode 100644 index 0000000..4b49e9a --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkStage.java @@ -0,0 +1,211 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.Map; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SparkStage { + private String status; + private int stageId; + private int attemptId; + private int numActiveTasks; + private int numCompleteTasks; + private int numFailedTasks; + private long executorRunTime; + private long inputBytes; + private long inputRecords; + private long outputBytes; + private long outputRecords; + private long shuffleReadBytes; + private long shuffleReadRecords; + private long shuffleWriteBytes; + private long shuffleWriteRecords; + private long memoryBytesSpilled; + private long diskBytesSpilled; + private String name; + private String schedulingPool; + //taskID + private Map<String, SparkTask> tasks; + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public int getStageId() { + return stageId; + } + + public void setStageId(int stageId) { + this.stageId = stageId; + } + + public int getAttemptId() { + return attemptId; + } + + public void setAttemptId(int attemptId) { + this.attemptId = attemptId; + } + + public int getNumActiveTasks() { + return numActiveTasks; + } + + public void setNumActiveTasks(int numActiveTasks) { + this.numActiveTasks = numActiveTasks; + } + + public int getNumCompleteTasks() { + return numCompleteTasks; + } + + public void setNumCompleteTasks(int numCompleteTasks) { + this.numCompleteTasks = numCompleteTasks; + } + + public int getNumFailedTasks() { + return numFailedTasks; + } + + public void setNumFailedTasks(int numFailedTasks) { + this.numFailedTasks = numFailedTasks; + } + + public long getExecutorRunTime() { + return executorRunTime; + } + + public void setExecutorRunTime(long executorRunTime) { + this.executorRunTime = executorRunTime; + } + + public long getInputBytes() { + return inputBytes; + } + + public void setInputBytes(long inputBytes) { + this.inputBytes = inputBytes; + } + + public long getInputRecords() { + return inputRecords; + } + + public void setInputRecords(long inputRecords) { + this.inputRecords = inputRecords; + } + + public long getOutputBytes() { + return outputBytes; + } + + public void setOutputBytes(long outputBytes) { + this.outputBytes = outputBytes; + } + + public long getOutputRecords() { + return outputRecords; + } + + public void setOutputRecords(long outputRecords) { + this.outputRecords = outputRecords; + } + + public long getShuffleReadBytes() { + return shuffleReadBytes; + } + + public void setShuffleReadBytes(long shuffleReadBytes) { + this.shuffleReadBytes = shuffleReadBytes; + } + + public long getShuffleReadRecords() { + return shuffleReadRecords; + } + + public void setShuffleReadRecords(long shuffleReadRecords) { + this.shuffleReadRecords = shuffleReadRecords; + } + + public long getShuffleWriteBytes() { + return shuffleWriteBytes; + } + + public void setShuffleWriteBytes(long shuffleWriteBytes) { + this.shuffleWriteBytes = shuffleWriteBytes; + } + + public long getShuffleWriteRecords() { + return shuffleWriteRecords; + } + + public void setShuffleWriteRecords(long shuffleWriteRecords) { + this.shuffleWriteRecords = shuffleWriteRecords; + } + + public long getMemoryBytesSpilled() { + return memoryBytesSpilled; + } + + public void setMemoryBytesSpilled(long memoryBytesSpilled) { + this.memoryBytesSpilled = memoryBytesSpilled; + } + + public long getDiskBytesSpilled() { + return diskBytesSpilled; + } + + public void setDiskBytesSpilled(long diskBytesSpilled) { + this.diskBytesSpilled = diskBytesSpilled; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSchedulingPool() { + return schedulingPool; + } + + public void setSchedulingPool(String schedulingPool) { + this.schedulingPool = schedulingPool; + } + + public Map<String, SparkTask> getTasks() { + return tasks; + } + + public void setTasks(Map<String, SparkTask> tasks) { + this.tasks = tasks; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTask.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTask.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTask.java new file mode 100644 index 0000000..6ee665e --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTask.java @@ -0,0 +1,111 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SparkTask { + private int taskId; + private int index; + private int attempt; + private String launchTime; + private String executorId; + private String host; + private String taskLocality; + private boolean speculative; + private SparkTaskMetrics taskMetrics; + + public int getTaskId() { + return taskId; + } + + public void setTaskId(int taskId) { + this.taskId = taskId; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + public int getAttempt() { + return attempt; + } + + public void setAttempt(int attempt) { + this.attempt = attempt; + } + + public String getLaunchTime() { + return launchTime; + } + + public void setLaunchTime(String launchTime) { + this.launchTime = launchTime; + } + + public String getExecutorId() { + return executorId; + } + + public void setExecutorId(String executorId) { + this.executorId = executorId; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getTaskLocality() { + return taskLocality; + } + + public void setTaskLocality(String taskLocality) { + this.taskLocality = taskLocality; + } + + public boolean isSpeculative() { + return speculative; + } + + public void setSpeculative(boolean speculative) { + this.speculative = speculative; + } + + public SparkTaskMetrics getTaskMetrics() { + return taskMetrics; + } + + public void setTaskMetrics(SparkTaskMetrics taskMetrics) { + this.taskMetrics = taskMetrics; + } +} + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskInputMetrics.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskInputMetrics.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskInputMetrics.java new file mode 100644 index 0000000..6fcb132 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskInputMetrics.java @@ -0,0 +1,46 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SparkTaskInputMetrics { + private long bytesRead; + private long recordsRead; + + public long getBytesRead() { + return bytesRead; + } + + public void setBytesRead(long bytesRead) { + this.bytesRead = bytesRead; + } + + public long getRecordsRead() { + return recordsRead; + } + + public void setRecordsRead(long recordsRead) { + this.recordsRead = recordsRead; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskMetrics.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskMetrics.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskMetrics.java new file mode 100644 index 0000000..4cd7934 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskMetrics.java @@ -0,0 +1,118 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SparkTaskMetrics { + private long executorDeserializeTime; + private long executorRunTime; + private long resultSize; + private long jvmGcTime; + private long resultSerializationTime; + private long memoryBytesSpilled; + private long diskBytesSpilled; + private SparkTaskInputMetrics inputMetrics; + private SparkTaskShuffleWriteMetrics shuffleWriteMetrics; + private SparkTaskShuffleReadMetrics shuffleReadMetrics; + + public long getExecutorDeserializeTime() { + return executorDeserializeTime; + } + + public void setExecutorDeserializeTime(long executorDeserializeTime) { + this.executorDeserializeTime = executorDeserializeTime; + } + + public long getExecutorRunTime() { + return executorRunTime; + } + + public void setExecutorRunTime(long executorRunTime) { + this.executorRunTime = executorRunTime; + } + + public long getResultSize() { + return resultSize; + } + + public void setResultSize(long resultSize) { + this.resultSize = resultSize; + } + + public long getJvmGcTime() { + return jvmGcTime; + } + + public void setJvmGcTime(long jvmGcTime) { + this.jvmGcTime = jvmGcTime; + } + + public long getResultSerializationTime() { + return resultSerializationTime; + } + + public void setResultSerializationTime(long resultSerializationTime) { + this.resultSerializationTime = resultSerializationTime; + } + + public long getMemoryBytesSpilled() { + return memoryBytesSpilled; + } + + public void setMemoryBytesSpilled(long memoryBytesSpilled) { + this.memoryBytesSpilled = memoryBytesSpilled; + } + + public long getDiskBytesSpilled() { + return diskBytesSpilled; + } + + public void setDiskBytesSpilled(long diskBytesSpilled) { + this.diskBytesSpilled = diskBytesSpilled; + } + + public SparkTaskInputMetrics getInputMetrics() { + return inputMetrics; + } + + public void setInputMetrics(SparkTaskInputMetrics inputMetrics) { + this.inputMetrics = inputMetrics; + } + + public SparkTaskShuffleWriteMetrics getShuffleWriteMetrics() { + return shuffleWriteMetrics; + } + + public void setShuffleWriteMetrics(SparkTaskShuffleWriteMetrics shuffleWriteMetrics) { + this.shuffleWriteMetrics = shuffleWriteMetrics; + } + + public SparkTaskShuffleReadMetrics getShuffleReadMetrics() { + return shuffleReadMetrics; + } + + public void setShuffleReadMetrics(SparkTaskShuffleReadMetrics shuffleReadMetrics) { + this.shuffleReadMetrics = shuffleReadMetrics; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleReadMetrics.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleReadMetrics.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleReadMetrics.java new file mode 100644 index 0000000..457898b --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleReadMetrics.java @@ -0,0 +1,82 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SparkTaskShuffleReadMetrics { + private long remoteBlocksFetched; + private long localBlocksFetched; + private long fetchWaitTime; + private long remoteBytesRead; + private long totalBlocksFetched; + private long recordsRead; + + public long getRemoteBlocksFetched() { + return remoteBlocksFetched; + } + + public void setRemoteBlocksFetched(long remoteBlocksFetched) { + this.remoteBlocksFetched = remoteBlocksFetched; + } + + public long getLocalBlocksFetched() { + return localBlocksFetched; + } + + public void setLocalBlocksFetched(long localBlocksFetched) { + this.localBlocksFetched = localBlocksFetched; + } + + public long getFetchWaitTime() { + return fetchWaitTime; + } + + public void setFetchWaitTime(long fetchWaitTime) { + this.fetchWaitTime = fetchWaitTime; + } + + public long getRemoteBytesRead() { + return remoteBytesRead; + } + + public void setRemoteBytesRead(long remoteBytesRead) { + this.remoteBytesRead = remoteBytesRead; + } + + public long getTotalBlocksFetched() { + return totalBlocksFetched; + } + + public void setTotalBlocksFetched(long totalBlocksFetched) { + this.totalBlocksFetched = totalBlocksFetched; + } + + public long getRecordsRead() { + return recordsRead; + } + + public void setRecordsRead(long recordsRead) { + this.recordsRead = recordsRead; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleWriteMetrics.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleWriteMetrics.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleWriteMetrics.java new file mode 100644 index 0000000..00de09e --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleWriteMetrics.java @@ -0,0 +1,55 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SparkTaskShuffleWriteMetrics { + private long bytesWritten; + private long writeTime; + private long recordsWritten; + + public long getBytesWritten() { + return bytesWritten; + } + + public void setBytesWritten(long bytesWritten) { + this.bytesWritten = bytesWritten; + } + + public long getWriteTime() { + return writeTime; + } + + public void setWriteTime(long writeTime) { + this.writeTime = writeTime; + } + + public long getRecordsWritten() { + return recordsWritten; + } + + public void setRecordsWritten(long recordsWritten) { + this.recordsWritten = recordsWritten; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterGroup.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterGroup.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterGroup.java new file mode 100644 index 0000000..9561d56 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterGroup.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.List; +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class TaskCounterGroup { + public String getCounterGroupName() { + return counterGroupName; + } + + public void setCounterGroupName(String counterGroupName) { + this.counterGroupName = counterGroupName; + } + + public List<TaskCounterItem> getCounter() { + return counter; + } + + public void setCounter(List<TaskCounterItem> counter) { + this.counter = counter; + } + + private String counterGroupName; + private List<TaskCounterItem> counter; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterItem.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterItem.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterItem.java new file mode 100644 index 0000000..dee038a --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterItem.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class TaskCounterItem { + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getValue() { + return value; + } + + public void setValue(long value) { + this.value = value; + } + + private String name; + private long value; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounters.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounters.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounters.java new file mode 100644 index 0000000..eade729 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounters.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.List; +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class TaskCounters { + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public List<TaskCounterGroup> getTaskCounterGroup() { + return taskCounterGroup; + } + + public void setTaskCounterGroup(List<TaskCounterGroup> taskCounterGroup) { + this.taskCounterGroup = taskCounterGroup; + } + + private String id; + private List<TaskCounterGroup> taskCounterGroup; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCountersWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCountersWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCountersWrapper.java new file mode 100644 index 0000000..3624b73 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCountersWrapper.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.model; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class TaskCountersWrapper { + public TaskCounters getJobTaskCounters() { + return jobTaskCounters; + } + + public void setJobTaskCounters(TaskCounters jobTaskCounters) { + this.jobTaskCounters = jobTaskCounters; + } + + private TaskCounters jobTaskCounters; +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java index 3e21e8d..e3beb9f 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java @@ -21,17 +21,32 @@ import org.apache.eagle.jpm.util.Constants; public class JobListServiceURLBuilderImpl implements ServiceURLBuilder { public String build(String ... parameters) { - // {rmUrl}/ws/v1/cluster/apps?state=RUNNING + /** + * {rmUrl}/ws/v1/cluster/apps?state=RUNNING + * We need to remove tailing slashes to avoid "url//ws/v1" + * because it would not be found and would be redirected to + * history server ui. + */ + String rmUrl = URLUtil.removeTrailingSlash(parameters[0]); + + String restApi = null; String jobState = parameters[1]; + if (jobState.equals(Constants.JobState.RUNNING.name())) { - return parameters[0] + "/" + Constants.V2_APPS_RUNNING_URL + "&" + Constants.ANONYMOUS_PARAMETER; + restApi = Constants.V2_APPS_RUNNING_URL; + } else if (jobState.equals(Constants.JobState.FINISHED.name())) { + restApi = Constants.V2_APPS_COMPLETED_URL; + } else if (jobState.equals(Constants.JobState.ALL.name())) { + restApi = Constants.V2_APPS_URL; } - else if (jobState.equals(Constants.JobState.COMPLETED.name())) { - return parameters[0] + "/" + Constants.V2_APPS_COMPLETED_URL + "&" + Constants.ANONYMOUS_PARAMETER; + if (restApi == null) { + return null; } - else if (jobState.equals(Constants.JobState.ALL.name())) { - return parameters[0] + "/" + Constants.V2_APPS_URL + "&" + Constants.ANONYMOUS_PARAMETER; - } - return null; + // "/ws/v1/cluster/apps?state=RUNNING" + StringBuilder sb = new StringBuilder(); + sb.append(rmUrl).append("/").append(restApi); + sb.append("&").append(Constants.ANONYMOUS_PARAMETER); + + return sb.toString(); } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java index d4d235f..6339d14 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java @@ -24,6 +24,13 @@ import org.apache.eagle.jpm.util.Constants; public class SparkCompleteJobServiceURLBuilderImpl implements ServiceURLBuilder { public String build(String... parameters) { - return String.format("%s/%s?applicationTypes=SPARK&state=FINISHED&finishedTimeBegin=%s&%s", parameters[0], Constants.V2_APPS_URL, parameters[1], Constants.ANONYMOUS_PARAMETER); + String url = URLUtil.removeTrailingSlash(parameters[0]); + + StringBuilder sb = new StringBuilder(); + sb.append(url).append("/").append(Constants.V2_APPS_URL); + sb.append("?applicationTypes=SPARK&state=FINISHED&finishedTimeBegin="); + sb.append(parameters[1]).append("&").append(Constants.ANONYMOUS_PARAMETER); + + return sb.toString(); } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java index c60fb3d..009731f 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java @@ -24,6 +24,8 @@ import org.apache.eagle.jpm.util.Constants; public class SparkJobServiceURLBuilderImpl implements ServiceURLBuilder { public String build(String... parameters) { - return String.format("%s/%s/%s", parameters[0], Constants.SPARK_APPS_URL, parameters[1]); + String serverAddress = URLUtil.removeTrailingSlash(parameters[0]); + + return serverAddress + Constants.SPARK_APPS_URL + parameters[1]; } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/URLUtil.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/URLUtil.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/URLUtil.java new file mode 100644 index 0000000..5ae2e8d --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/URLUtil.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util.resourceFetch.url; + +/** + * URL utils. + */ +public class URLUtil { + public static String removeTrailingSlash(String url) { + int i = url.length() - 1; + // Skip all slashes from the end. + while (i >= 0 && url.charAt(i) == '/') { + i--; + } + + return url.substring(0, i + 1); + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5bf2c62d/eagle-jpm/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/pom.xml b/eagle-jpm/pom.xml index eada67f..c48cdeb 100644 --- a/eagle-jpm/pom.xml +++ b/eagle-jpm/pom.xml @@ -38,6 +38,7 @@ <module>eagle-jpm-entity</module> <module>eagle-hadoop-queue</module> <module>eagle-jpm-util</module> + <module>eagle-jpm-mr-running</module> <module>eagle-jpm-app</module> </modules>
