http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempts.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempts.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempts.java deleted file mode 100644 index 03af0d7..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTaskAttempts.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class MRTaskAttempts { - public List<MRTaskAttempt> getTaskAttempt() { - return taskAttempt; - } - - public void setTaskAttempt(List<MRTaskAttempt> taskAttempt) { - this.taskAttempt = taskAttempt; - } - - private List<MRTaskAttempt> taskAttempt; -}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasks.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasks.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasks.java deleted file mode 100644 index b23b23d..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasks.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class MRTasks { - public List<MRTask> getTask() { - return task; - } - - public void setTask(List<MRTask> task) { - this.task = task; - } - - private List<MRTask> task; - -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasksWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasksWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasksWrapper.java deleted file mode 100644 index f92234a..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MRTasksWrapper.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class MRTasksWrapper { - public MRTasks getTasks() { - return tasks; - } - - public void setTasks(MRTasks tasks) { - this.tasks = tasks; - } - - private MRTasks tasks; -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MrJobs.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MrJobs.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MrJobs.java deleted file mode 100644 index 1d4871c..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/MrJobs.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class MrJobs { - public List<MRJob> getJob() { - return job; - } - - public void setJobs(List<MRJob> job) { - this.job = job; - } - - private List<MRJob> job; -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplication.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplication.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplication.java deleted file mode 100644 index 299934e..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplication.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkApplication { - String id; - String name; - List<SparkApplicationAttempt> attempts; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public List<SparkApplicationAttempt> getAttempts() { - return attempts; - } - - public void setAttempts(List<SparkApplicationAttempt> attempts) { - this.attempts = attempts; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplicationAttempt.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplicationAttempt.java deleted file mode 100644 index b437cc2..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplicationAttempt.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkApplicationAttempt { - String attemptId; - String sparkUser; - String startTime; - String endTime; - boolean completed; - - public String getAttemptId() { - return attemptId; - } - - public void setAttemptId(String attemptId) { - this.attemptId = attemptId; - } - - public String getSparkUser() { - return sparkUser; - } - - public void setSparkUser(String sparkUser) { - this.sparkUser = sparkUser; - } - - public String getStartTime() { - return startTime; - } - - public void setStartTime(String startTime) { - this.startTime = startTime; - } - - public String getEndTime() { - return endTime; - } - - public void setEndTime(String endTime) { - this.endTime = endTime; - } - - public boolean isCompleted() { - return completed; - } - - public void setCompleted(boolean completed) { - this.completed = completed; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplicationWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplicationWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplicationWrapper.java deleted file mode 100644 index 0b04f31..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkApplicationWrapper.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkApplicationWrapper { - - SparkApplication app; - - public SparkApplication getApp() { - return app; - } - - public void setApp(SparkApplication app) { - this.app = app; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkExecutor.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkExecutor.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkExecutor.java deleted file mode 100644 index c2a2115..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkExecutor.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkExecutor { - private String id; - private String hostPort; - private int rddBlocks; - private long memoryUsed; - private long diskUsed; - private int activeTasks; - private int failedTasks; - private int completedTasks; - private int totalTasks; - private long totalDuration; - private long totalInputBytes; - private long totalShuffleRead; - private long totalShuffleWrite; - private long maxMemory; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getHostPort() { - return hostPort; - } - - public void setHostPort(String hostPort) { - this.hostPort = hostPort; - } - - public int getRddBlocks() { - return rddBlocks; - } - - public void setRddBlocks(int rddBlocks) { - this.rddBlocks = rddBlocks; - } - - public long getMemoryUsed() { - return memoryUsed; - } - - public void setMemoryUsed(long memoryUsed) { - this.memoryUsed = memoryUsed; - } - - public long getDiskUsed() { - return diskUsed; - } - - public void setDiskUsed(long diskUsed) { - this.diskUsed = diskUsed; - } - - public int getActiveTasks() { - return activeTasks; - } - - public void setActiveTasks(int activeTasks) { - this.activeTasks = activeTasks; - } - - public int getFailedTasks() { - return failedTasks; - } - - public void setFailedTasks(int failedTasks) { - this.failedTasks = failedTasks; - } - - public int getCompletedTasks() { - return completedTasks; - } - - public void setCompletedTasks(int completedTasks) { - this.completedTasks = completedTasks; - } - - public int getTotalTasks() { - return totalTasks; - } - - public void setTotalTasks(int totalTasks) { - this.totalTasks = totalTasks; - } - - public long getTotalDuration() { - return totalDuration; - } - - public void setTotalDuration(long totalDuration) { - this.totalDuration = totalDuration; - } - - public long getTotalInputBytes() { - return totalInputBytes; - } - - public void setTotalInputBytes(long totalInputBytes) { - this.totalInputBytes = totalInputBytes; - } - - public long getTotalShuffleRead() { - return totalShuffleRead; - } - - public void setTotalShuffleRead(long totalShuffleRead) { - this.totalShuffleRead = totalShuffleRead; - } - - public long getTotalShuffleWrite() { - return totalShuffleWrite; - } - - public void setTotalShuffleWrite(long totalShuffleWrite) { - this.totalShuffleWrite = totalShuffleWrite; - } - - public long getMaxMemory() { - return maxMemory; - } - - public void setMaxMemory(long maxMemory) { - this.maxMemory = maxMemory; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkJob.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkJob.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkJob.java deleted file mode 100644 index 280cf67..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkJob.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkJob { - private int jobId; - private String name; - private String submissionTime; - private String completionTime; - private List<Integer> stageIds; - private String status; - private int numTasks; - private int numActiveTasks; - private int numCompletedTasks; - private int numSkippedTasks; - private int numFailedTasks; - private int numActiveStages; - private int numCompletedStages; - private int numSkippedStages; - private int numFailedStages; - - public int getJobId() { - return jobId; - } - - public void setJobId(int jobId) { - this.jobId = jobId; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getSubmissionTime() { - return submissionTime; - } - - public void setSubmissionTime(String submissionTime) { - this.submissionTime = submissionTime; - } - - public String getCompletionTime() { - return completionTime; - } - - public void setCompletionTime(String completionTime) { - this.completionTime = completionTime; - } - - public List<Integer> getStageIds() { - return stageIds; - } - - public void setStageIds(List<Integer> stageIds) { - this.stageIds = stageIds; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public int getNumTasks() { - return numTasks; - } - - public void setNumTasks(int numTasks) { - this.numTasks = numTasks; - } - - public int getNumActiveTasks() { - return numActiveTasks; - } - - public void setNumActiveTasks(int numActiveTasks) { - this.numActiveTasks = numActiveTasks; - } - - public int getNumCompletedTasks() { - return numCompletedTasks; - } - - public void setNumCompletedTasks(int numCompletedTasks) { - this.numCompletedTasks = numCompletedTasks; - } - - public int getNumSkippedTasks() { - return numSkippedTasks; - } - - public void setNumSkippedTasks(int numSkippedTasks) { - this.numSkippedTasks = numSkippedTasks; - } - - public int getNumFailedTasks() { - return numFailedTasks; - } - - public void setNumFailedTasks(int numFailedTasks) { - this.numFailedTasks = numFailedTasks; - } - - public int getNumActiveStages() { - return numActiveStages; - } - - public void setNumActiveStages(int numActiveStages) { - this.numActiveStages = numActiveStages; - } - - public int getNumCompletedStages() { - return numCompletedStages; - } - - public void setNumCompletedStages(int numCompletedStages) { - this.numCompletedStages = numCompletedStages; - } - - public int getNumSkippedStages() { - return numSkippedStages; - } - - public void setNumSkippedStages(int numSkippedStages) { - this.numSkippedStages = numSkippedStages; - } - - public int getNumFailedStages() { - return numFailedStages; - } - - public void setNumFailedStages(int numFailedStages) { - this.numFailedStages = numFailedStages; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkStage.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkStage.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkStage.java deleted file mode 100644 index 14edeee..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkStage.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.Map; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkStage { - private String status; - private int stageId; - private int attemptId; - private int numActiveTasks; - private int numCompleteTasks; - private int numFailedTasks; - private long executorRunTime; - private long inputBytes; - private long inputRecords; - private long outputBytes; - private long outputRecords; - private long shuffleReadBytes; - private long shuffleReadRecords; - private long shuffleWriteBytes; - private long shuffleWriteRecords; - private long memoryBytesSpilled; - private long diskBytesSpilled; - private String name; - private String schedulingPool; - //taskID - private Map<String, SparkTask> tasks; - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public int getStageId() { - return stageId; - } - - public void setStageId(int stageId) { - this.stageId = stageId; - } - - public int getAttemptId() { - return attemptId; - } - - public void setAttemptId(int attemptId) { - this.attemptId = attemptId; - } - - public int getNumActiveTasks() { - return numActiveTasks; - } - - public void setNumActiveTasks(int numActiveTasks) { - this.numActiveTasks = numActiveTasks; - } - - public int getNumCompleteTasks() { - return numCompleteTasks; - } - - public void setNumCompleteTasks(int numCompleteTasks) { - this.numCompleteTasks = numCompleteTasks; - } - - public int getNumFailedTasks() { - return numFailedTasks; - } - - public void setNumFailedTasks(int numFailedTasks) { - this.numFailedTasks = numFailedTasks; - } - - public long getExecutorRunTime() { - return executorRunTime; - } - - public void setExecutorRunTime(long executorRunTime) { - this.executorRunTime = executorRunTime; - } - - public long getInputBytes() { - return inputBytes; - } - - public void setInputBytes(long inputBytes) { - this.inputBytes = inputBytes; - } - - public long getInputRecords() { - return inputRecords; - } - - public void setInputRecords(long inputRecords) { - this.inputRecords = inputRecords; - } - - public long getOutputBytes() { - return outputBytes; - } - - public void setOutputBytes(long outputBytes) { - this.outputBytes = outputBytes; - } - - public long getOutputRecords() { - return outputRecords; - } - - public void setOutputRecords(long outputRecords) { - this.outputRecords = outputRecords; - } - - public long getShuffleReadBytes() { - return shuffleReadBytes; - } - - public void setShuffleReadBytes(long shuffleReadBytes) { - this.shuffleReadBytes = shuffleReadBytes; - } - - public long getShuffleReadRecords() { - return shuffleReadRecords; - } - - public void setShuffleReadRecords(long shuffleReadRecords) { - this.shuffleReadRecords = shuffleReadRecords; - } - - public long getShuffleWriteBytes() { - return shuffleWriteBytes; - } - - public void setShuffleWriteBytes(long shuffleWriteBytes) { - this.shuffleWriteBytes = shuffleWriteBytes; - } - - public long getShuffleWriteRecords() { - return shuffleWriteRecords; - } - - public void setShuffleWriteRecords(long shuffleWriteRecords) { - this.shuffleWriteRecords = shuffleWriteRecords; - } - - public long getMemoryBytesSpilled() { - return memoryBytesSpilled; - } - - public void setMemoryBytesSpilled(long memoryBytesSpilled) { - this.memoryBytesSpilled = memoryBytesSpilled; - } - - public long getDiskBytesSpilled() { - return diskBytesSpilled; - } - - public void setDiskBytesSpilled(long diskBytesSpilled) { - this.diskBytesSpilled = diskBytesSpilled; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getSchedulingPool() { - return schedulingPool; - } - - public void setSchedulingPool(String schedulingPool) { - this.schedulingPool = schedulingPool; - } - - public Map<String, SparkTask> getTasks() { - return tasks; - } - - public void setTasks(Map<String, SparkTask> tasks) { - this.tasks = tasks; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTask.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTask.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTask.java deleted file mode 100644 index a5dd08f..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTask.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkTask { - private int taskId; - private int index; - private int attempt; - private String launchTime; - private String executorId; - private String host; - private String taskLocality; - private boolean speculative; - private SparkTaskMetrics taskMetrics; - - public int getTaskId() { - return taskId; - } - - public void setTaskId(int taskId) { - this.taskId = taskId; - } - - public int getIndex() { - return index; - } - - public void setIndex(int index) { - this.index = index; - } - - public int getAttempt() { - return attempt; - } - - public void setAttempt(int attempt) { - this.attempt = attempt; - } - - public String getLaunchTime() { - return launchTime; - } - - public void setLaunchTime(String launchTime) { - this.launchTime = launchTime; - } - - public String getExecutorId() { - return executorId; - } - - public void setExecutorId(String executorId) { - this.executorId = executorId; - } - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - public String getTaskLocality() { - return taskLocality; - } - - public void setTaskLocality(String taskLocality) { - this.taskLocality = taskLocality; - } - - public boolean isSpeculative() { - return speculative; - } - - public void setSpeculative(boolean speculative) { - this.speculative = speculative; - } - - public SparkTaskMetrics getTaskMetrics() { - return taskMetrics; - } - - public void setTaskMetrics(SparkTaskMetrics taskMetrics) { - this.taskMetrics = taskMetrics; - } -} - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskInputMetrics.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskInputMetrics.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskInputMetrics.java deleted file mode 100644 index d079694..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskInputMetrics.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkTaskInputMetrics { - private long bytesRead; - private long recordsRead; - - public long getBytesRead() { - return bytesRead; - } - - public void setBytesRead(long bytesRead) { - this.bytesRead = bytesRead; - } - - public long getRecordsRead() { - return recordsRead; - } - - public void setRecordsRead(long recordsRead) { - this.recordsRead = recordsRead; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskMetrics.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskMetrics.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskMetrics.java deleted file mode 100644 index cdb913d..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskMetrics.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkTaskMetrics { - private long executorDeserializeTime; - private long executorRunTime; - private long resultSize; - private long jvmGcTime; - private long resultSerializationTime; - private long memoryBytesSpilled; - private long diskBytesSpilled; - private SparkTaskInputMetrics inputMetrics; - private SparkTaskShuffleWriteMetrics shuffleWriteMetrics; - private SparkTaskShuffleReadMetrics shuffleReadMetrics; - - public long getExecutorDeserializeTime() { - return executorDeserializeTime; - } - - public void setExecutorDeserializeTime(long executorDeserializeTime) { - this.executorDeserializeTime = executorDeserializeTime; - } - - public long getExecutorRunTime() { - return executorRunTime; - } - - public void setExecutorRunTime(long executorRunTime) { - this.executorRunTime = executorRunTime; - } - - public long getResultSize() { - return resultSize; - } - - public void setResultSize(long resultSize) { - this.resultSize = resultSize; - } - - public long getJvmGcTime() { - return jvmGcTime; - } - - public void setJvmGcTime(long jvmGcTime) { - this.jvmGcTime = jvmGcTime; - } - - public long getResultSerializationTime() { - return resultSerializationTime; - } - - public void setResultSerializationTime(long resultSerializationTime) { - this.resultSerializationTime = resultSerializationTime; - } - - public long getMemoryBytesSpilled() { - return memoryBytesSpilled; - } - - public void setMemoryBytesSpilled(long memoryBytesSpilled) { - this.memoryBytesSpilled = memoryBytesSpilled; - } - - public long getDiskBytesSpilled() { - return diskBytesSpilled; - } - - public void setDiskBytesSpilled(long diskBytesSpilled) { - this.diskBytesSpilled = diskBytesSpilled; - } - - public SparkTaskInputMetrics getInputMetrics() { - return inputMetrics; - } - - public void setInputMetrics(SparkTaskInputMetrics inputMetrics) { - this.inputMetrics = inputMetrics; - } - - public SparkTaskShuffleWriteMetrics getShuffleWriteMetrics() { - return shuffleWriteMetrics; - } - - public void setShuffleWriteMetrics(SparkTaskShuffleWriteMetrics shuffleWriteMetrics) { - this.shuffleWriteMetrics = shuffleWriteMetrics; - } - - public SparkTaskShuffleReadMetrics getShuffleReadMetrics() { - return shuffleReadMetrics; - } - - public void setShuffleReadMetrics(SparkTaskShuffleReadMetrics shuffleReadMetrics) { - this.shuffleReadMetrics = shuffleReadMetrics; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleReadMetrics.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleReadMetrics.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleReadMetrics.java deleted file mode 100644 index 3aa94bb..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleReadMetrics.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkTaskShuffleReadMetrics { - private long remoteBlocksFetched; - private long localBlocksFetched; - private long fetchWaitTime; - private long remoteBytesRead; - private long totalBlocksFetched; - private long recordsRead; - - public long getRemoteBlocksFetched() { - return remoteBlocksFetched; - } - - public void setRemoteBlocksFetched(long remoteBlocksFetched) { - this.remoteBlocksFetched = remoteBlocksFetched; - } - - public long getLocalBlocksFetched() { - return localBlocksFetched; - } - - public void setLocalBlocksFetched(long localBlocksFetched) { - this.localBlocksFetched = localBlocksFetched; - } - - public long getFetchWaitTime() { - return fetchWaitTime; - } - - public void setFetchWaitTime(long fetchWaitTime) { - this.fetchWaitTime = fetchWaitTime; - } - - public long getRemoteBytesRead() { - return remoteBytesRead; - } - - public void setRemoteBytesRead(long remoteBytesRead) { - this.remoteBytesRead = remoteBytesRead; - } - - public long getTotalBlocksFetched() { - return totalBlocksFetched; - } - - public void setTotalBlocksFetched(long totalBlocksFetched) { - this.totalBlocksFetched = totalBlocksFetched; - } - - public long getRecordsRead() { - return recordsRead; - } - - public void setRecordsRead(long recordsRead) { - this.recordsRead = recordsRead; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleWriteMetrics.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleWriteMetrics.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleWriteMetrics.java deleted file mode 100644 index 3d8f025..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/SparkTaskShuffleWriteMetrics.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class SparkTaskShuffleWriteMetrics { - private long bytesWritten; - private long writeTime; - private long recordsWritten; - - public long getBytesWritten() { - return bytesWritten; - } - - public void setBytesWritten(long bytesWritten) { - this.bytesWritten = bytesWritten; - } - - public long getWriteTime() { - return writeTime; - } - - public void setWriteTime(long writeTime) { - this.writeTime = writeTime; - } - - public long getRecordsWritten() { - return recordsWritten; - } - - public void setRecordsWritten(long recordsWritten) { - this.recordsWritten = recordsWritten; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterGroup.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterGroup.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterGroup.java deleted file mode 100644 index 9803d11..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterGroup.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class TaskCounterGroup { - public String getCounterGroupName() { - return counterGroupName; - } - - public void setCounterGroupName(String counterGroupName) { - this.counterGroupName = counterGroupName; - } - - public List<TaskCounterItem> getCounter() { - return counter; - } - - public void setCounter(List<TaskCounterItem> counter) { - this.counter = counter; - } - - private String counterGroupName; - private List<TaskCounterItem> counter; -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterItem.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterItem.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterItem.java deleted file mode 100644 index f8ea22c..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounterItem.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class TaskCounterItem { - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public long getValue() { - return value; - } - - public void setValue(long value) { - this.value = value; - } - - private String name; - private long value; -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounters.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounters.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounters.java deleted file mode 100644 index ef99547..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCounters.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class TaskCounters { - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public List<TaskCounterGroup> getTaskCounterGroup() { - return taskCounterGroup; - } - - public void setTaskCounterGroup(List<TaskCounterGroup> taskCounterGroup) { - this.taskCounterGroup = taskCounterGroup; - } - - private String id; - private List<TaskCounterGroup> taskCounterGroup; -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCountersWrapper.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCountersWrapper.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCountersWrapper.java deleted file mode 100644 index 80808a3..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/model/TaskCountersWrapper.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.jpm.util.resourcefetch.model; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class TaskCountersWrapper { - public TaskCounters getJobTaskCounters() { - return jobTaskCounters; - } - - public void setJobTaskCounters(TaskCounters jobTaskCounters) { - this.jobTaskCounters = jobTaskCounters; - } - - private TaskCounters jobTaskCounters; -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java deleted file mode 100644 index 5513771..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/JobListServiceURLBuilderImpl.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.jpm.util.resourcefetch.url; - -import org.apache.eagle.jpm.util.Constants; - -public class JobListServiceURLBuilderImpl implements ServiceURLBuilder { - - public String build(String... parameters) { - /** - * {rmUrl}/ws/v1/cluster/apps?state=RUNNING. - * We need to remove tailing slashes to avoid "url//ws/v1" - * because it would not be found and would be redirected to - * history server ui. - */ - String rmUrl = URLUtil.removeTrailingSlash(parameters[0]); - - String restApi = null; - String jobState = parameters[1]; - - if (jobState.equals(Constants.JobState.RUNNING.name())) { - restApi = Constants.V2_APPS_RUNNING_URL; - } else if (jobState.equals(Constants.JobState.FINISHED.name())) { - restApi = Constants.V2_APPS_COMPLETED_URL; - } else if (jobState.equals(Constants.JobState.ALL.name())) { - restApi = Constants.V2_APPS_URL; - } - if (restApi == null) { - return null; - } - // "/ws/v1/cluster/apps?state=RUNNING" - StringBuilder sb = new StringBuilder(); - sb.append(rmUrl).append("/").append(restApi); - sb.append("&").append(Constants.ANONYMOUS_PARAMETER); - - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/ServiceURLBuilder.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/ServiceURLBuilder.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/ServiceURLBuilder.java deleted file mode 100644 index 09fea2f..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/ServiceURLBuilder.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.jpm.util.resourcefetch.url; - -public interface ServiceURLBuilder { - String build(String... parameters); -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java deleted file mode 100644 index ca6e938..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkCompleteJobServiceURLBuilderImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.url; - -import org.apache.eagle.jpm.util.Constants; - -public class SparkCompleteJobServiceURLBuilderImpl implements ServiceURLBuilder { - - public String build(String... parameters) { - String url = URLUtil.removeTrailingSlash(parameters[0]); - - return url + "/" + Constants.V2_APPS_URL - + "?applicationTypes=SPARK&state=FINISHED&finishedTimeBegin=" - + parameters[1] + "&" + Constants.ANONYMOUS_PARAMETER; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java deleted file mode 100644 index c5ec67a..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/SparkJobServiceURLBuilderImpl.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.eagle.jpm.util.resourcefetch.url; - -import org.apache.eagle.jpm.util.Constants; - -public class SparkJobServiceURLBuilderImpl implements ServiceURLBuilder { - - public String build(String... parameters) { - String serverAddress = URLUtil.removeTrailingSlash(parameters[0]); - - return serverAddress + Constants.SPARK_APPS_URL + parameters[1]; - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/URLUtil.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/URLUtil.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/URLUtil.java deleted file mode 100644 index 11cde80..0000000 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/url/URLUtil.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.jpm.util.resourcefetch.url; - -/** - * URL utils. - */ -public class URLUtil { - public static String removeTrailingSlash(String url) { - int i = url.length() - 1; - // Skip all slashes from the end. - while (i >= 0 && url.charAt(i) == '/') { - i--; - } - - return url.substring(0, i + 1); - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/RMResourceFetcher.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/RMResourceFetcher.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/RMResourceFetcher.java new file mode 100644 index 0000000..b1881ef --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/RMResourceFetcher.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * + */ +package org.apache.eagle.jpm.util.resourcefetch; + +import org.apache.eagle.jpm.util.Constants; +import org.apache.eagle.jpm.util.resourcefetch.connection.InputStreamUtils; +import org.apache.eagle.jpm.util.resourcefetch.ha.HAURLSelector; +import org.apache.eagle.jpm.util.resourcefetch.ha.HAURLSelectorImpl; +import org.apache.eagle.jpm.util.resourcefetch.model.AppInfo; +import org.apache.eagle.jpm.util.resourcefetch.model.AppsWrapper; +import org.apache.eagle.jpm.util.resourcefetch.model.ClusterInfo; +import org.apache.eagle.jpm.util.resourcefetch.model.ClusterInfoWrapper; +import org.apache.eagle.jpm.util.resourcefetch.url.JobListServiceURLBuilderImpl; +import org.apache.eagle.jpm.util.resourcefetch.url.ServiceURLBuilder; +import org.apache.eagle.jpm.util.resourcefetch.url.SparkCompleteJobServiceURLBuilderImpl; +import org.apache.eagle.jpm.util.resourcefetch.url.URLUtil; +import org.codehaus.jackson.JsonParser; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +public class RMResourceFetcher implements ResourceFetcher<AppInfo> { + + private static final Logger LOG = LoggerFactory.getLogger(RMResourceFetcher.class); + private final HAURLSelector selector; + private final ServiceURLBuilder jobListServiceURLBuilder; + private final ServiceURLBuilder sparkCompleteJobServiceURLBuilder; + private static final ObjectMapper OBJ_MAPPER = new ObjectMapper(); + + static { + OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); + } + + public RMResourceFetcher(String[] rmBasePaths) { + this.jobListServiceURLBuilder = new JobListServiceURLBuilderImpl(); + this.sparkCompleteJobServiceURLBuilder = new SparkCompleteJobServiceURLBuilderImpl(); + + this.selector = new HAURLSelectorImpl(rmBasePaths, jobListServiceURLBuilder, Constants.CompressionType.GZIP); + } + + private void checkUrl() throws IOException { + if (!selector.checkUrl(jobListServiceURLBuilder.build(selector.getSelectedUrl(), Constants.JobState.RUNNING.name()))) { + selector.reSelectUrl(); + } + } + + private List<AppInfo> doFetchFinishApplicationsList(String urlString, Constants.CompressionType compressionType) throws Exception { + List<AppInfo> result; + InputStream is = null; + try { + checkUrl(); + LOG.info("Going to call yarn api to fetch finished application list: " + urlString); + is = InputStreamUtils.getInputStream(urlString, null, compressionType); + final AppsWrapper appWrapper = OBJ_MAPPER.readValue(is, AppsWrapper.class); + if (appWrapper != null && appWrapper.getApps() != null + && appWrapper.getApps().getApp() != null) { + result = appWrapper.getApps().getApp(); + return result; + } + return null; + } finally { + if (is != null) { + try { + is.close(); + } catch (Exception e) { + LOG.warn("{}", e); + } + } + } + } + + private String getSparkRunningJobURL() { + return selector.getSelectedUrl() + + "/" + + Constants.V2_APPS_URL + + "?applicationTypes=SPARK&state=RUNNING&" + + Constants.ANONYMOUS_PARAMETER; + } + + private String getMRRunningJobURL() { + return String.format("%s/%s?applicationTypes=MAPREDUCE&state=RUNNING&%s", + selector.getSelectedUrl(), + Constants.V2_APPS_URL, + Constants.ANONYMOUS_PARAMETER); + } + + private String getMRFinishedJobURL(String lastFinishedTime) { + String url = URLUtil.removeTrailingSlash(selector.getSelectedUrl()); + return url + "/" + "Constants.V2_APPS_URL" + + "?applicationTypes=MAPREDUCE&state=FINISHED&finishedTimeBegin=" + + lastFinishedTime + "&" + Constants.ANONYMOUS_PARAMETER; + } + + private List<AppInfo> doFetchRunningApplicationsList(String urlString, Constants.CompressionType compressionType) throws Exception { + List<AppInfo> result; + InputStream is = null; + try { + checkUrl(); + LOG.info("Going to call yarn api to fetch running application list: " + urlString); + is = InputStreamUtils.getInputStream(urlString, null, compressionType); + final AppsWrapper appWrapper = OBJ_MAPPER.readValue(is, AppsWrapper.class); + if (appWrapper != null && appWrapper.getApps() != null && appWrapper.getApps().getApp() != null) { + result = appWrapper.getApps().getApp(); + return result; + } + return null; + } finally { + if (is != null) { + try { + is.close(); + } catch (Exception e) { + LOG.warn("{}", e); + } + } + } + } + + private List<AppInfo> getResource(Constants.ResourceType resourceType, Constants.CompressionType compressionType, Object... parameter) throws Exception { + switch (resourceType) { + case COMPLETE_SPARK_JOB: + final String urlString = sparkCompleteJobServiceURLBuilder.build(selector.getSelectedUrl(), (String) parameter[0]); + return doFetchFinishApplicationsList(urlString, compressionType); + case RUNNING_SPARK_JOB: + return doFetchRunningApplicationsList(getSparkRunningJobURL(), compressionType); + case RUNNING_MR_JOB: + return doFetchRunningApplicationsList(getMRRunningJobURL(), compressionType); + case COMPLETE_MR_JOB: + return doFetchFinishApplicationsList(getMRFinishedJobURL((String) parameter[0]), compressionType); + default: + throw new Exception("Not support resourceType :" + resourceType); + } + } + + public List<AppInfo> getResource(Constants.ResourceType resourceType, Object... parameter) throws Exception { + try { + return getResource(resourceType, Constants.CompressionType.GZIP, parameter); + } catch (java.util.zip.ZipException ex) { + return getResource(resourceType, Constants.CompressionType.NONE, parameter); + } + } + + private String getClusterInfoURL() { + return selector.getSelectedUrl() + "/" + Constants.YARN_API_CLUSTER_INFO + "?" + Constants.ANONYMOUS_PARAMETER; + } + + public ClusterInfo getClusterInfo() throws Exception { + InputStream is = null; + try { + checkUrl(); + final String urlString = getClusterInfoURL(); + LOG.info("Calling yarn api to fetch cluster info: " + urlString); + is = InputStreamUtils.getInputStream(urlString, null, Constants.CompressionType.GZIP); + final ClusterInfoWrapper clusterInfoWrapper = OBJ_MAPPER.readValue(is, ClusterInfoWrapper.class); + if (clusterInfoWrapper != null && clusterInfoWrapper.getClusterInfo() != null) { + return clusterInfoWrapper.getClusterInfo(); + } + return null; + } finally { + if (is != null) { + try { + is.close(); + } catch (Exception e) { + LOG.warn("{}", e); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/ResourceFetcher.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/ResourceFetcher.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/ResourceFetcher.java new file mode 100644 index 0000000..f920ddb --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/ResourceFetcher.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.jpm.util.resourcefetch; + +import org.apache.eagle.jpm.util.Constants; + +import java.util.List; + +public interface ResourceFetcher<T> { + //continue to refactor later + List<T> getResource(Constants.ResourceType resoureType, Object... parameter) throws Exception; + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/SparkHistoryServerResourceFetcher.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/SparkHistoryServerResourceFetcher.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/SparkHistoryServerResourceFetcher.java new file mode 100644 index 0000000..ce2d9b8 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/SparkHistoryServerResourceFetcher.java @@ -0,0 +1,88 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.eagle.jpm.util.resourcefetch; + +import org.apache.eagle.jpm.util.Constants; +import org.apache.eagle.jpm.util.resourcefetch.connection.InputStreamUtils; +import org.apache.eagle.jpm.util.resourcefetch.model.SparkApplication; +import org.apache.eagle.jpm.util.resourcefetch.url.ServiceURLBuilder; +import org.apache.eagle.jpm.util.resourcefetch.url.SparkJobServiceURLBuilderImpl; +import org.apache.commons.codec.binary.Base64; +import org.codehaus.jackson.JsonParser; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; + +public class SparkHistoryServerResourceFetcher implements ResourceFetcher<SparkApplication> { + + private static final Logger LOG = LoggerFactory.getLogger(SparkHistoryServerResourceFetcher.class); + + private static final ObjectMapper OBJ_MAPPER = new ObjectMapper(); + + private String historyServerURL; + private final ServiceURLBuilder sparkDetailJobServiceURLBuilder; + private String auth; + + static { + OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); + } + + public SparkHistoryServerResourceFetcher(String historyServerURL, String userName, String pwd) { + this.historyServerURL = historyServerURL; + this.sparkDetailJobServiceURLBuilder = new SparkJobServiceURLBuilderImpl(); + this.auth = "Basic " + new String(new Base64().encode(String.format("%s:%s", userName, pwd).getBytes())); + ; + } + + private List<SparkApplication> doFetchSparkApplicationDetail(String appId) throws Exception { + InputStream is = null; + try { + final String urlString = sparkDetailJobServiceURLBuilder.build(this.historyServerURL, appId); + LOG.info("Going to call spark history server api to fetch spark job: " + urlString); + is = InputStreamUtils.getInputStream(urlString, auth, Constants.CompressionType.NONE); + SparkApplication app = OBJ_MAPPER.readValue(is, SparkApplication.class); + return Arrays.asList(app); + } catch (FileNotFoundException e) { + return null; + } finally { + if (is != null) { + try { + is.close(); + } catch (Exception e) { + LOG.warn("{}", e); + } + } + } + } + + public List<SparkApplication> getResource(Constants.ResourceType resoureType, Object... parameter) throws Exception { + switch (resoureType) { + case SPARK_JOB_DETAIL: + return doFetchSparkApplicationDetail((String) parameter[0]); + default: + throw new Exception("Not support resourceType :" + resoureType); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/connection/InputStreamUtils.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/connection/InputStreamUtils.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/connection/InputStreamUtils.java new file mode 100644 index 0000000..e99e2aa --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/connection/InputStreamUtils.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.jpm.util.resourcefetch.connection; + +import org.apache.eagle.jpm.util.Constants; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.zip.GZIPInputStream; + +public class InputStreamUtils { + + private static final int CONNECTION_TIMEOUT = 10 * 1000; + private static final int READ_TIMEOUT = 5 * 60 * 1000; + private static final String GZIP_HTTP_HEADER = "Accept-Encoding"; + private static final String GZIP_COMPRESSION = "gzip"; + + private static InputStream openGZIPInputStream(URL url, String auth, int timeout) throws IOException { + final URLConnection connection = url.openConnection(); + connection.setConnectTimeout(CONNECTION_TIMEOUT); + connection.setReadTimeout(timeout); + connection.addRequestProperty(GZIP_HTTP_HEADER, GZIP_COMPRESSION); + if (null != auth) { + connection.setRequestProperty("Authorization", auth); + } + return new GZIPInputStream(connection.getInputStream()); + } + + private static InputStream openInputStream(URL url, String auth, int timeout) throws IOException { + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(timeout); + if (null != auth) { + connection.setRequestProperty("Authorization", auth); + } + + return connection.getInputStream(); + } + + public static InputStream getInputStream(String urlString, String auth, Constants.CompressionType compressionType, int timeout) throws Exception { + final URL url = URLConnectionUtils.getUrl(urlString); + if (compressionType.equals(Constants.CompressionType.GZIP)) { + return openGZIPInputStream(url, auth, timeout); + } else { // CompressionType.NONE + return openInputStream(url, auth, timeout); + } + } + + public static InputStream getInputStream(String urlString, String auth, Constants.CompressionType compressionType) throws Exception { + return getInputStream(urlString, auth, compressionType, READ_TIMEOUT); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b5fd782c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/connection/JobUtils.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/connection/JobUtils.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/connection/JobUtils.java new file mode 100644 index 0000000..44d27e8 --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/connection/JobUtils.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.jpm.util.resourcefetch.connection; + +import org.apache.eagle.jpm.util.Constants; + +public class JobUtils { + + public static String checkAndAddLastSlash(String urlBase) { + if (!urlBase.endsWith("/")) { + return urlBase + "/"; + } + return urlBase; + } + + public static String getJobIDByAppID(String appID) { + if (appID.startsWith(Constants.APPLICATION_PREFIX)) { + return appID.replace(Constants.APPLICATION_PREFIX, Constants.JOB_PREFIX); + } + return null; + } + + public static String getAppIDByJobID(String jobID) { + if (jobID.startsWith(Constants.JOB_PREFIX)) { + return jobID.replace(Constants.JOB_PREFIX, Constants.APPLICATION_PREFIX); + } + return null; + } +}