http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala deleted file mode 100644 index fd24aea..0000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -import javax.ws.rs._ -import javax.ws.rs.core.MediaType - -import org.apache.spark.SparkException -import org.apache.spark.scheduler.StageInfo -import org.apache.spark.status.api.v1.StageStatus._ -import org.apache.spark.status.api.v1.TaskSorting._ -import org.apache.spark.ui.SparkUI -import org.apache.spark.ui.jobs.JobProgressListener -import org.apache.spark.ui.jobs.UIData.StageUIData - -@Produces(Array(MediaType.APPLICATION_JSON)) -private[v1] class OneStageResource(ui: SparkUI) { - - @GET - @Path("") - def stageData(@PathParam("stageId") stageId: Int): Seq[StageData] = { - withStage(stageId){ stageAttempts => - stageAttempts.map { stage => - AllStagesResource.stageUiToStageData(stage.status, stage.info, stage.ui, - includeDetails = true) - } - } - } - - @GET - @Path("/{stageAttemptId: \\d+}") - def oneAttemptData( - @PathParam("stageId") stageId: Int, - @PathParam("stageAttemptId") stageAttemptId: Int): StageData = { - withStageAttempt(stageId, stageAttemptId) { stage => - AllStagesResource.stageUiToStageData(stage.status, stage.info, stage.ui, - includeDetails = true) - } - } - - @GET - @Path("/{stageAttemptId: \\d+}/taskSummary") - def taskSummary( - @PathParam("stageId") stageId: Int, - @PathParam("stageAttemptId") stageAttemptId: Int, - @DefaultValue("0.05,0.25,0.5,0.75,0.95") @QueryParam("quantiles") quantileString: String) - : TaskMetricDistributions = { - withStageAttempt(stageId, stageAttemptId) { stage => - val quantiles = quantileString.split(",").map { s => - try { - s.toDouble - } catch { - case nfe: NumberFormatException => - throw new BadParameterException("quantiles", "double", s) - } - } - AllStagesResource.taskMetricDistributions(stage.ui.taskData.values, quantiles) - } - } - - @GET - @Path("/{stageAttemptId: \\d+}/taskList") - def taskList( - @PathParam("stageId") stageId: Int, - @PathParam("stageAttemptId") stageAttemptId: Int, - @DefaultValue("0") @QueryParam("offset") offset: Int, - @DefaultValue("20") @QueryParam("length") length: Int, - @DefaultValue("ID") @QueryParam("sortBy") sortBy: TaskSorting): Seq[TaskData] = { - withStageAttempt(stageId, stageAttemptId) { stage => - val tasks = stage.ui.taskData.values.map{AllStagesResource.convertTaskData}.toIndexedSeq - .sorted(OneStageResource.ordering(sortBy)) - tasks.slice(offset, offset + length) - } - } - - private case class StageStatusInfoUi(status: StageStatus, info: StageInfo, ui: StageUIData) - - private def withStage[T](stageId: Int)(f: Seq[StageStatusInfoUi] => T): T = { - val stageAttempts = findStageStatusUIData(ui.jobProgressListener, stageId) - if (stageAttempts.isEmpty) { - throw new NotFoundException("unknown stage: " + stageId) - } else { - f(stageAttempts) - } - } - - private def findStageStatusUIData( - listener: JobProgressListener, - stageId: Int): Seq[StageStatusInfoUi] = { - listener.synchronized { - def getStatusInfoUi(status: StageStatus, infos: Seq[StageInfo]): Seq[StageStatusInfoUi] = { - infos.filter { _.stageId == stageId }.map { info => - val ui = listener.stageIdToData.getOrElse((info.stageId, info.attemptId), - // this is an internal error -- we should always have uiData - throw new SparkException( - s"no stage ui data found for stage: ${info.stageId}:${info.attemptId}") - ) - StageStatusInfoUi(status, info, ui) - } - } - getStatusInfoUi(ACTIVE, listener.activeStages.values.toSeq) ++ - getStatusInfoUi(COMPLETE, listener.completedStages) ++ - getStatusInfoUi(FAILED, listener.failedStages) ++ - getStatusInfoUi(PENDING, listener.pendingStages.values.toSeq) - } - } - - private def withStageAttempt[T]( - stageId: Int, - stageAttemptId: Int) - (f: StageStatusInfoUi => T): T = { - withStage(stageId) { attempts => - val oneAttempt = attempts.find { stage => stage.info.attemptId == stageAttemptId } - oneAttempt match { - case Some(stage) => - f(stage) - case None => - val stageAttempts = attempts.map { _.info.attemptId } - throw new NotFoundException(s"unknown attempt for stage $stageId. " + - s"Found attempts: ${stageAttempts.mkString("[", ",", "]")}") - } - } - } -} - -object OneStageResource { - def ordering(taskSorting: TaskSorting): Ordering[TaskData] = { - val extractor: (TaskData => Long) = td => - taskSorting match { - case ID => td.taskId - case INCREASING_RUNTIME => td.taskMetrics.map{_.executorRunTime}.getOrElse(-1L) - case DECREASING_RUNTIME => -td.taskMetrics.map{_.executorRunTime}.getOrElse(-1L) - } - Ordering.by(extractor) - } -}
http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/status/api/v1/SecurityFilter.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/SecurityFilter.scala b/core/src/main/scala/org/apache/spark/status/api/v1/SecurityFilter.scala deleted file mode 100644 index 95fbd96..0000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/SecurityFilter.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -import javax.ws.rs.WebApplicationException -import javax.ws.rs.core.Response - -import com.sun.jersey.spi.container.{ContainerRequest, ContainerRequestFilter} - -private[v1] class SecurityFilter extends ContainerRequestFilter with UIRootFromServletContext { - def filter(req: ContainerRequest): ContainerRequest = { - val user = Option(req.getUserPrincipal).map { _.getName }.orNull - if (uiRoot.securityManager.checkUIViewPermissions(user)) { - req - } else { - throw new WebApplicationException( - Response - .status(Response.Status.FORBIDDEN) - .entity(raw"""user "$user"is not authorized""") - .build() - ) - } - } -} http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/status/api/v1/SimpleDateParam.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/SimpleDateParam.scala b/core/src/main/scala/org/apache/spark/status/api/v1/SimpleDateParam.scala deleted file mode 100644 index cee2978..0000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/SimpleDateParam.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -import java.text.SimpleDateFormat -import java.util.TimeZone -import javax.ws.rs.WebApplicationException -import javax.ws.rs.core.Response -import javax.ws.rs.core.Response.Status - -import scala.util.Try - -private[v1] class SimpleDateParam(val originalValue: String) { - val timestamp: Long = { - SimpleDateParam.formats.collectFirst { - case fmt if Try(fmt.parse(originalValue)).isSuccess => - fmt.parse(originalValue).getTime() - }.getOrElse( - throw new WebApplicationException( - Response - .status(Status.BAD_REQUEST) - .entity("Couldn't parse date: " + originalValue) - .build() - ) - ) - } -} - -private[v1] object SimpleDateParam { - - val formats: Seq[SimpleDateFormat] = { - - val gmtDay = new SimpleDateFormat("yyyy-MM-dd") - gmtDay.setTimeZone(TimeZone.getTimeZone("GMT")) - - Seq( - new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz"), - gmtDay - ) - } -} http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/status/api/v1/api.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala deleted file mode 100644 index ef3c857..0000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -import java.util.Date - -import scala.collection.Map - -import org.apache.spark.JobExecutionStatus - -class ApplicationInfo private[spark]( - val id: String, - val name: String, - val attempts: Seq[ApplicationAttemptInfo]) - -class ApplicationAttemptInfo private[spark]( - val attemptId: Option[String], - val startTime: Date, - val endTime: Date, - val sparkUser: String, - val completed: Boolean = false) - -class ExecutorStageSummary private[spark]( - val taskTime : Long, - val failedTasks : Int, - val succeededTasks : Int, - val inputBytes : Long, - val outputBytes : Long, - val shuffleRead : Long, - val shuffleWrite : Long, - val memoryBytesSpilled : Long, - val diskBytesSpilled : Long) - -class ExecutorSummary private[spark]( - val id: String, - val hostPort: String, - val rddBlocks: Int, - val memoryUsed: Long, - val diskUsed: Long, - val activeTasks: Int, - val failedTasks: Int, - val completedTasks: Int, - val totalTasks: Int, - val totalDuration: Long, - val totalInputBytes: Long, - val totalShuffleRead: Long, - val totalShuffleWrite: Long, - val maxMemory: Long, - val executorLogs: Map[String, String]) - -class JobData private[spark]( - val jobId: Int, - val name: String, - val description: Option[String], - val submissionTime: Option[Date], - val completionTime: Option[Date], - val stageIds: Seq[Int], - val jobGroup: Option[String], - val status: JobExecutionStatus, - val numTasks: Int, - val numActiveTasks: Int, - val numCompletedTasks: Int, - val numSkippedTasks: Int, - val numFailedTasks: Int, - val numActiveStages: Int, - val numCompletedStages: Int, - val numSkippedStages: Int, - val numFailedStages: Int) - -// Q: should Tachyon size go in here as well? currently the UI only shows it on the overall storage -// page ... does anybody pay attention to it? -class RDDStorageInfo private[spark]( - val id: Int, - val name: String, - val numPartitions: Int, - val numCachedPartitions: Int, - val storageLevel: String, - val memoryUsed: Long, - val diskUsed: Long, - val dataDistribution: Option[Seq[RDDDataDistribution]], - val partitions: Option[Seq[RDDPartitionInfo]]) - -class RDDDataDistribution private[spark]( - val address: String, - val memoryUsed: Long, - val memoryRemaining: Long, - val diskUsed: Long) - -class RDDPartitionInfo private[spark]( - val blockName: String, - val storageLevel: String, - val memoryUsed: Long, - val diskUsed: Long, - val executors: Seq[String]) - -class StageData private[spark]( - val status: StageStatus, - val stageId: Int, - val attemptId: Int, - val numActiveTasks: Int , - val numCompleteTasks: Int, - val numFailedTasks: Int, - - val executorRunTime: Long, - - val inputBytes: Long, - val inputRecords: Long, - val outputBytes: Long, - val outputRecords: Long, - val shuffleReadBytes: Long, - val shuffleReadRecords: Long, - val shuffleWriteBytes: Long, - val shuffleWriteRecords: Long, - val memoryBytesSpilled: Long, - val diskBytesSpilled: Long, - - val name: String, - val details: String, - val schedulingPool: String, - - val accumulatorUpdates: Seq[AccumulableInfo], - val tasks: Option[Map[Long, TaskData]], - val executorSummary:Option[Map[String,ExecutorStageSummary]]) - -class TaskData private[spark]( - val taskId: Long, - val index: Int, - val attempt: Int, - val launchTime: Date, - val executorId: String, - val host: String, - val taskLocality: String, - val speculative: Boolean, - val accumulatorUpdates: Seq[AccumulableInfo], - val errorMessage: Option[String] = None, - val taskMetrics: Option[TaskMetrics] = None) - -class TaskMetrics private[spark]( - val executorDeserializeTime: Long, - val executorRunTime: Long, - val resultSize: Long, - val jvmGcTime: Long, - val resultSerializationTime: Long, - val memoryBytesSpilled: Long, - val diskBytesSpilled: Long, - val inputMetrics: Option[InputMetrics], - val outputMetrics: Option[OutputMetrics], - val shuffleReadMetrics: Option[ShuffleReadMetrics], - val shuffleWriteMetrics: Option[ShuffleWriteMetrics]) - -class InputMetrics private[spark]( - val bytesRead: Long, - val recordsRead: Long) - -class OutputMetrics private[spark]( - val bytesWritten: Long, - val recordsWritten: Long) - -class ShuffleReadMetrics private[spark]( - val remoteBlocksFetched: Int, - val localBlocksFetched: Int, - val fetchWaitTime: Long, - val remoteBytesRead: Long, - val totalBlocksFetched: Int, - val recordsRead: Long) - -class ShuffleWriteMetrics private[spark]( - val bytesWritten: Long, - val writeTime: Long, - val recordsWritten: Long) - -class TaskMetricDistributions private[spark]( - val quantiles: IndexedSeq[Double], - - val executorDeserializeTime: IndexedSeq[Double], - val executorRunTime: IndexedSeq[Double], - val resultSize: IndexedSeq[Double], - val jvmGcTime: IndexedSeq[Double], - val resultSerializationTime: IndexedSeq[Double], - val memoryBytesSpilled: IndexedSeq[Double], - val diskBytesSpilled: IndexedSeq[Double], - - val inputMetrics: Option[InputMetricDistributions], - val outputMetrics: Option[OutputMetricDistributions], - val shuffleReadMetrics: Option[ShuffleReadMetricDistributions], - val shuffleWriteMetrics: Option[ShuffleWriteMetricDistributions]) - -class InputMetricDistributions private[spark]( - val bytesRead: IndexedSeq[Double], - val recordsRead: IndexedSeq[Double]) - -class OutputMetricDistributions private[spark]( - val bytesWritten: IndexedSeq[Double], - val recordsWritten: IndexedSeq[Double]) - -class ShuffleReadMetricDistributions private[spark]( - val readBytes: IndexedSeq[Double], - val readRecords: IndexedSeq[Double], - val remoteBlocksFetched: IndexedSeq[Double], - val localBlocksFetched: IndexedSeq[Double], - val fetchWaitTime: IndexedSeq[Double], - val remoteBytesRead: IndexedSeq[Double], - val totalBlocksFetched: IndexedSeq[Double]) - -class ShuffleWriteMetricDistributions private[spark]( - val writeBytes: IndexedSeq[Double], - val writeRecords: IndexedSeq[Double], - val writeTime: IndexedSeq[Double]) - -class AccumulableInfo private[spark]( - val id: Long, - val name: String, - val update: Option[String], - val value: String) http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala index ec71148..7d75929 100644 --- a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala +++ b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala @@ -25,17 +25,13 @@ import org.apache.spark.scheduler._ /** * :: DeveloperApi :: * A SparkListener that maintains executor storage status. - * - * This class is thread-safe (unlike JobProgressListener) */ @DeveloperApi class StorageStatusListener extends SparkListener { // This maintains only blocks that are cached (i.e. storage level is not StorageLevel.NONE) private[storage] val executorIdToStorageStatus = mutable.Map[String, StorageStatus]() - def storageStatusList: Seq[StorageStatus] = synchronized { - executorIdToStorageStatus.values.toSeq - } + def storageStatusList: Seq[StorageStatus] = executorIdToStorageStatus.values.toSeq /** Update storage status list to reflect updated block statuses */ private def updateStorageStatus(execId: String, updatedBlocks: Seq[(BlockId, BlockStatus)]) { http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/SparkUI.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index bfe4a18..a5271f0 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -17,9 +17,6 @@ package org.apache.spark.ui -import java.util.Date - -import org.apache.spark.status.api.v1.{ApplicationAttemptInfo, ApplicationInfo, JsonRootResource, UIRoot} import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext} import org.apache.spark.scheduler._ import org.apache.spark.storage.StorageStatusListener @@ -36,7 +33,7 @@ import org.apache.spark.ui.scope.RDDOperationGraphListener private[spark] class SparkUI private ( val sc: Option[SparkContext], val conf: SparkConf, - securityManager: SecurityManager, + val securityManager: SecurityManager, val environmentListener: EnvironmentListener, val storageStatusListener: StorageStatusListener, val executorsListener: ExecutorsListener, @@ -44,27 +41,22 @@ private[spark] class SparkUI private ( val storageListener: StorageListener, val operationGraphListener: RDDOperationGraphListener, var appName: String, - val basePath: String, - val startTime: Long) + val basePath: String) extends WebUI(securityManager, SparkUI.getUIPort(conf), conf, basePath, "SparkUI") - with Logging - with UIRoot { + with Logging { val killEnabled = sc.map(_.conf.getBoolean("spark.ui.killEnabled", true)).getOrElse(false) - - val stagesTab = new StagesTab(this) - /** Initialize all components of the server. */ def initialize() { attachTab(new JobsTab(this)) + val stagesTab = new StagesTab(this) attachTab(stagesTab) attachTab(new StorageTab(this)) attachTab(new EnvironmentTab(this)) attachTab(new ExecutorsTab(this)) attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static")) attachHandler(createRedirectHandler("/", "/jobs", basePath = basePath)) - attachHandler(JsonRootResource.getJsonServlet(this)) // This should be POST only, but, the YARN AM proxy won't proxy POSTs attachHandler(createRedirectHandler( "/stages/stage/kill", "/stages", stagesTab.handleKillRequest, @@ -91,24 +83,6 @@ private[spark] class SparkUI private ( private[spark] def appUIHostPort = publicHostName + ":" + boundPort private[spark] def appUIAddress = s"http://$appUIHostPort"; - - def getSparkUI(appId: String): Option[SparkUI] = { - if (appId == appName) Some(this) else None - } - - def getApplicationInfoList: Iterator[ApplicationInfo] = { - Iterator(new ApplicationInfo( - id = appName, - name = appName, - attempts = Seq(new ApplicationAttemptInfo( - attemptId = None, - startTime = new Date(startTime), - endTime = new Date(-1), - sparkUser = "", - completed = false - )) - )) - } } private[spark] abstract class SparkUITab(parent: SparkUI, prefix: String) @@ -135,10 +109,9 @@ private[spark] object SparkUI { listenerBus: SparkListenerBus, jobProgressListener: JobProgressListener, securityManager: SecurityManager, - appName: String, - startTime: Long): SparkUI = { + appName: String): SparkUI = { create(Some(sc), conf, listenerBus, securityManager, appName, - jobProgressListener = Some(jobProgressListener), startTime = startTime) + jobProgressListener = Some(jobProgressListener)) } def createHistoryUI( @@ -146,9 +119,8 @@ private[spark] object SparkUI { listenerBus: SparkListenerBus, securityManager: SecurityManager, appName: String, - basePath: String, - startTime: Long): SparkUI = { - create(None, conf, listenerBus, securityManager, appName, basePath, startTime = startTime) + basePath: String): SparkUI = { + create(None, conf, listenerBus, securityManager, appName, basePath) } /** @@ -165,8 +137,7 @@ private[spark] object SparkUI { securityManager: SecurityManager, appName: String, basePath: String = "", - jobProgressListener: Option[JobProgressListener] = None, - startTime: Long): SparkUI = { + jobProgressListener: Option[JobProgressListener] = None): SparkUI = { val _jobProgressListener: JobProgressListener = jobProgressListener.getOrElse { val listener = new JobProgressListener(conf) @@ -188,6 +159,6 @@ private[spark] object SparkUI { new SparkUI(sc, conf, securityManager, environmentListener, storageStatusListener, executorsListener, _jobProgressListener, storageListener, operationGraphListener, - appName, basePath, startTime) + appName, basePath) } } http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/WebUI.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala index 384f2ad..f9860d1 100644 --- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala @@ -37,7 +37,7 @@ import org.apache.spark.{Logging, SecurityManager, SparkConf} * pages. The use of tabs is optional, however; a WebUI may choose to include pages directly. */ private[spark] abstract class WebUI( - val securityManager: SecurityManager, + securityManager: SecurityManager, port: Int, conf: SparkConf, basePath: String = "", @@ -77,9 +77,15 @@ private[spark] abstract class WebUI( val pagePath = "/" + page.prefix val renderHandler = createServletHandler(pagePath, (request: HttpServletRequest) => page.render(request), securityManager, basePath) + val renderJsonHandler = createServletHandler(pagePath.stripSuffix("/") + "/json", + (request: HttpServletRequest) => page.renderJson(request), securityManager, basePath) attachHandler(renderHandler) + attachHandler(renderJsonHandler) pageToHandlers.getOrElseUpdate(page, ArrayBuffer[ServletContextHandler]()) .append(renderHandler) + pageToHandlers.getOrElseUpdate(page, ArrayBuffer[ServletContextHandler]()) + .append(renderJsonHandler) + } /** Attach a handler to this UI. */ http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index b247e4c..956608d 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -22,11 +22,11 @@ import javax.servlet.http.HttpServletRequest import scala.xml.Node -import org.apache.spark.status.api.v1.ExecutorSummary import org.apache.spark.ui.{ToolTips, UIUtils, WebUIPage} import org.apache.spark.util.Utils -// This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive +/** Summary information about an executor to display in the UI. */ +// Needs to be private[ui] because of a false positive MiMa failure. private[ui] case class ExecutorSummaryInfo( id: String, hostPort: String, @@ -44,7 +44,6 @@ private[ui] case class ExecutorSummaryInfo( maxMemory: Long, executorLogs: Map[String, String]) - private[ui] class ExecutorsPage( parent: ExecutorsTab, threadDumpEnabled: Boolean) @@ -56,8 +55,7 @@ private[ui] class ExecutorsPage( val maxMem = storageStatusList.map(_.maxMem).sum val memUsed = storageStatusList.map(_.memUsed).sum val diskUsed = storageStatusList.map(_.diskUsed).sum - val execInfo = for (statusId <- 0 until storageStatusList.size) yield - ExecutorsPage.getExecInfo(listener, statusId) + val execInfo = for (statusId <- 0 until storageStatusList.size) yield getExecInfo(statusId) val execInfoSorted = execInfo.sortBy(_.id) val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty @@ -113,7 +111,7 @@ private[ui] class ExecutorsPage( } /** Render an HTML row representing an executor */ - private def execRow(info: ExecutorSummary, logsExist: Boolean): Seq[Node] = { + private def execRow(info: ExecutorSummaryInfo, logsExist: Boolean): Seq[Node] = { val maximumMemory = info.maxMemory val memoryUsed = info.memoryUsed val diskUsed = info.diskUsed @@ -172,11 +170,8 @@ private[ui] class ExecutorsPage( </tr> } -} - -private[spark] object ExecutorsPage { /** Represent an executor's info as a map given a storage status index */ - def getExecInfo(listener: ExecutorsListener, statusId: Int): ExecutorSummary = { + private def getExecInfo(statusId: Int): ExecutorSummaryInfo = { val status = listener.storageStatusList(statusId) val execId = status.blockManagerId.executorId val hostPort = status.blockManagerId.hostPort @@ -194,7 +189,7 @@ private[spark] object ExecutorsPage { val totalShuffleWrite = listener.executorToShuffleWrite.getOrElse(execId, 0L) val executorLogs = listener.executorToLogUrls.getOrElse(execId, Map.empty) - new ExecutorSummary( + new ExecutorSummaryInfo( execId, hostPort, rddBlocks, http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 09323d1..f6abf27 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -271,12 +271,6 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { val shouldShowCompletedJobs = completedJobs.nonEmpty val shouldShowFailedJobs = failedJobs.nonEmpty - val completedJobNumStr = if (completedJobs.size == listener.numCompletedJobs) { - s"${completedJobs.size}" - } else { - s"${listener.numCompletedJobs}, only showing ${completedJobs.size}" - } - val summary: NodeSeq = <div> <ul class="unstyled"> @@ -301,9 +295,9 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { } { if (shouldShowCompletedJobs) { - <li id="completed-summary"> + <li> <a href="#completed"><strong>Completed Jobs:</strong></a> - {completedJobNumStr} + {completedJobs.size} </li> } } @@ -311,7 +305,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { if (shouldShowFailedJobs) { <li> <a href="#failed"><strong>Failed Jobs:</strong></a> - {listener.numFailedJobs} + {failedJobs.size} </li> } } @@ -328,7 +322,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { activeJobsTable } if (shouldShowCompletedJobs) { - content ++= <h4 id="completed">Completed Jobs ({completedJobNumStr})</h4> ++ + content ++= <h4 id="completed">Completed Jobs ({completedJobs.size})</h4> ++ completedJobsTable } if (shouldShowFailedJobs) { http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala index a37f739..236bc8e 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala @@ -64,12 +64,6 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { val shouldShowCompletedStages = completedStages.nonEmpty val shouldShowFailedStages = failedStages.nonEmpty - val completedStageNumStr = if (numCompletedStages == completedStages.size) { - s"$numCompletedStages" - } else { - s"$numCompletedStages, only showing ${completedStages.size}" - } - val summary: NodeSeq = <div> <ul class="unstyled"> @@ -104,9 +98,9 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { } { if (shouldShowCompletedStages) { - <li id="completed-summary"> + <li> <a href="#completed"><strong>Completed Stages:</strong></a> - {completedStageNumStr} + {numCompletedStages} </li> } } @@ -138,7 +132,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { pendingStagesTable.toNodeSeq } if (shouldShowCompletedStages) { - content ++= <h4 id="completed">Completed Stages ({completedStageNumStr})</h4> ++ + content ++= <h4 id="completed">Completed Stages ({numCompletedStages})</h4> ++ completedStagesTable.toNodeSeq } if (shouldShowFailedStages) { http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala index 7163217..96cc3d7 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala @@ -187,7 +187,7 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") { val jobDataOption = listener.jobIdToData.get(jobId) if (jobDataOption.isEmpty) { val content = - <div id="no-info"> + <div> <p>No information to display for job {jobId}</p> </div> return UIUtils.headerSparkPage( http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index 246e191..8f9aa9f 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -74,8 +74,6 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { // JobProgressListener's retention limits. var numCompletedStages = 0 var numFailedStages = 0 - var numCompletedJobs = 0 - var numFailedJobs = 0 // Misc: val executorIdToBlockManagerId = HashMap[ExecutorId, BlockManagerId]() @@ -219,12 +217,10 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { completedJobs += jobData trimJobsIfNecessary(completedJobs) jobData.status = JobExecutionStatus.SUCCEEDED - numCompletedJobs += 1 case JobFailed(exception) => failedJobs += jobData trimJobsIfNecessary(failedJobs) jobData.status = JobExecutionStatus.FAILED - numFailedJobs += 1 } for (stageId <- jobData.stageIds) { stageIdToActiveJobIds.get(stageId).foreach { jobsUsingStage => http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala index f3e0b38..d725b9d 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala @@ -21,7 +21,7 @@ import javax.servlet.http.HttpServletRequest import scala.xml.Node -import org.apache.spark.scheduler.StageInfo +import org.apache.spark.scheduler.{Schedulable, StageInfo} import org.apache.spark.ui.{WebUIPage, UIUtils} /** Page showing specific pool details */ http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 89d175b..5793100 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -25,11 +25,11 @@ import scala.xml.{Elem, Node, Unparsed} import org.apache.commons.lang3.StringEscapeUtils import org.apache.spark.executor.TaskMetrics -import org.apache.spark.scheduler.{AccumulableInfo, TaskInfo} import org.apache.spark.ui.{ToolTips, WebUIPage, UIUtils} import org.apache.spark.ui.jobs.UIData._ import org.apache.spark.ui.scope.RDDOperationGraph import org.apache.spark.util.{Utils, Distribution} +import org.apache.spark.scheduler.{AccumulableInfo, TaskInfo} /** Page showing statistics and task list for a given stage */ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { @@ -48,22 +48,14 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { val stageAttemptId = parameterAttempt.toInt val stageDataOption = progressListener.stageIdToData.get((stageId, stageAttemptId)) - val stageHeader = s"Details for Stage $stageId (Attempt $stageAttemptId)" - if (stageDataOption.isEmpty) { - val content = - <div id="no-info"> - <p>No information to display for Stage {stageId} (Attempt {stageAttemptId})</p> - </div> - return UIUtils.headerSparkPage(stageHeader, content, parent) - - } - if (stageDataOption.get.taskData.isEmpty) { + if (stageDataOption.isEmpty || stageDataOption.get.taskData.isEmpty) { val content = <div> <h4>Summary Metrics</h4> No tasks have started yet <h4>Tasks</h4> No tasks have started yet </div> - return UIUtils.headerSparkPage(stageHeader, content, parent) + return UIUtils.headerSparkPage( + s"Details for Stage $stageId (Attempt $stageAttemptId)", content, parent) } val stageData = stageDataOption.get @@ -454,7 +446,8 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { maybeAccumulableTable ++ <h4>Tasks</h4> ++ taskTable - UIUtils.headerSparkPage(stageHeader, content, parent, showVisualization = true) + UIUtils.headerSparkPage( + "Details for Stage %d".format(stageId), content, parent, showVisualization = true) } } http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala index 05f94a7..199f731 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala @@ -21,8 +21,8 @@ import javax.servlet.http.HttpServletRequest import scala.xml.Node -import org.apache.spark.status.api.v1.{AllRDDResource, RDDDataDistribution, RDDPartitionInfo} -import org.apache.spark.ui.{UIUtils, WebUIPage} +import org.apache.spark.storage.{BlockId, BlockStatus, StorageStatus, StorageUtils} +import org.apache.spark.ui.{WebUIPage, UIUtils} import org.apache.spark.util.Utils /** Page showing storage details for a given RDD */ @@ -32,19 +32,28 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") { def render(request: HttpServletRequest): Seq[Node] = { val parameterId = request.getParameter("id") require(parameterId != null && parameterId.nonEmpty, "Missing id parameter") + val rddId = parameterId.toInt - val rddStorageInfo = AllRDDResource.getRDDStorageInfo(rddId, listener,includeDetails = true) - .getOrElse { - // Rather than crashing, render an "RDD Not Found" page - return UIUtils.headerSparkPage("RDD Not Found", Seq[Node](), parent) - } + val storageStatusList = listener.storageStatusList + val rddInfo = listener.rddInfoList.find(_.id == rddId).getOrElse { + // Rather than crashing, render an "RDD Not Found" page + return UIUtils.headerSparkPage("RDD Not Found", Seq[Node](), parent) + } // Worker table - val workerTable = UIUtils.listingTable(workerHeader, workerRow, - rddStorageInfo.dataDistribution.get, id = Some("rdd-storage-by-worker-table")) + val workers = storageStatusList.map((rddId, _)) + val workerTable = UIUtils.listingTable(workerHeader, workerRow, workers, + id = Some("rdd-storage-by-worker-table")) // Block table - val blockTable = UIUtils.listingTable(blockHeader, blockRow, rddStorageInfo.partitions.get, + val blockLocations = StorageUtils.getRddBlockLocations(rddId, storageStatusList) + val blocks = storageStatusList + .flatMap(_.rddBlocksById(rddId)) + .sortWith(_._1.name < _._1.name) + .map { case (blockId, status) => + (blockId, status, blockLocations.get(blockId).getOrElse(Seq[String]("Unknown"))) + } + val blockTable = UIUtils.listingTable(blockHeader, blockRow, blocks, id = Some("rdd-storage-by-block-table")) val content = @@ -53,23 +62,23 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") { <ul class="unstyled"> <li> <strong>Storage Level:</strong> - {rddStorageInfo.storageLevel} + {rddInfo.storageLevel.description} </li> <li> <strong>Cached Partitions:</strong> - {rddStorageInfo.numCachedPartitions} + {rddInfo.numCachedPartitions} </li> <li> <strong>Total Partitions:</strong> - {rddStorageInfo.numPartitions} + {rddInfo.numPartitions} </li> <li> <strong>Memory Size:</strong> - {Utils.bytesToString(rddStorageInfo.memoryUsed)} + {Utils.bytesToString(rddInfo.memSize)} </li> <li> <strong>Disk Size:</strong> - {Utils.bytesToString(rddStorageInfo.diskUsed)} + {Utils.bytesToString(rddInfo.diskSize)} </li> </ul> </div> @@ -77,19 +86,19 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") { <div class="row-fluid"> <div class="span12"> - <h4> Data Distribution on {rddStorageInfo.dataDistribution.size} Executors </h4> + <h4> Data Distribution on {workers.size} Executors </h4> {workerTable} </div> </div> <div class="row-fluid"> <div class="span12"> - <h4> {rddStorageInfo.partitions.size} Partitions </h4> + <h4> {blocks.size} Partitions </h4> {blockTable} </div> </div>; - UIUtils.headerSparkPage("RDD Storage Info for " + rddStorageInfo.name, content, parent) + UIUtils.headerSparkPage("RDD Storage Info for " + rddInfo.name, content, parent) } /** Header fields for the worker table */ @@ -107,32 +116,34 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") { "Executors") /** Render an HTML row representing a worker */ - private def workerRow(worker: RDDDataDistribution): Seq[Node] = { + private def workerRow(worker: (Int, StorageStatus)): Seq[Node] = { + val (rddId, status) = worker <tr> - <td>{worker.address}</td> + <td>{status.blockManagerId.host + ":" + status.blockManagerId.port}</td> <td> - {Utils.bytesToString(worker.memoryUsed)} - ({Utils.bytesToString(worker.memoryRemaining)} Remaining) + {Utils.bytesToString(status.memUsedByRdd(rddId))} + ({Utils.bytesToString(status.memRemaining)} Remaining) </td> - <td>{Utils.bytesToString(worker.diskUsed)}</td> + <td>{Utils.bytesToString(status.diskUsedByRdd(rddId))}</td> </tr> } /** Render an HTML row representing a block */ - private def blockRow(row: RDDPartitionInfo): Seq[Node] = { + private def blockRow(row: (BlockId, BlockStatus, Seq[String])): Seq[Node] = { + val (id, block, locations) = row <tr> - <td>{row.blockName}</td> + <td>{id}</td> <td> - {row.storageLevel} + {block.storageLevel.description} </td> - <td sorttable_customkey={row.memoryUsed.toString}> - {Utils.bytesToString(row.memoryUsed)} + <td sorttable_customkey={block.memSize.toString}> + {Utils.bytesToString(block.memSize)} </td> - <td sorttable_customkey={row.diskUsed.toString}> - {Utils.bytesToString(row.diskUsed)} + <td sorttable_customkey={block.diskSize.toString}> + {Utils.bytesToString(block.diskSize)} </td> <td> - {row.executors.map(l => <span>{l}<br/></span>)} + {locations.map(l => <span>{l}<br/></span>)} </td> </tr> } http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala index 07db783..59dc6b5 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala @@ -22,7 +22,7 @@ import javax.servlet.http.HttpServletRequest import scala.xml.Node import org.apache.spark.storage.RDDInfo -import org.apache.spark.ui.{UIUtils, WebUIPage} +import org.apache.spark.ui.{WebUIPage, UIUtils} import org.apache.spark.util.Utils /** Page showing list of RDD's currently stored in the cluster */ http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala index 0351749..045bd78 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala @@ -35,8 +35,6 @@ private[ui] class StorageTab(parent: SparkUI) extends SparkUITab(parent, "storag /** * :: DeveloperApi :: * A SparkListener that prepares information to be displayed on the BlockManagerUI. - * - * This class is thread-safe (unlike JobProgressListener) */ @DeveloperApi class StorageListener(storageStatusListener: StorageStatusListener) extends SparkListener { @@ -45,9 +43,7 @@ class StorageListener(storageStatusListener: StorageStatusListener) extends Spar def storageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList /** Filter RDD info to include only those with cached partitions */ - def rddInfoList: Seq[RDDInfo] = synchronized { - _rddInfoMap.values.filter(_.numCachedPartitions > 0).toSeq - } + def rddInfoList: Seq[RDDInfo] = _rddInfoMap.values.filter(_.numCachedPartitions > 0).toSeq /** Update the storage info of the RDDs whose blocks are among the given updated blocks */ private def updateRDDInfo(updatedBlocks: Seq[(BlockId, BlockStatus)]): Unit = { http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/json_expectation deleted file mode 100644 index 6101177..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/json_expectation +++ /dev/null @@ -1,53 +0,0 @@ -[ { - "id" : "local-1427397477963", - "name" : "Spark shell", - "attempts" : [ { - "startTime" : "2015-03-26T19:17:57.184GMT", - "endTime" : "2015-03-26T19:20:02.949GMT", - "sparkUser" : "irashid", - "completed" : true - } ] -}, { - "id" : "local-1426533911241", - "name" : "Spark shell", - "attempts" : [ { - "attemptId" : "2", - "startTime" : "2015-03-17T23:11:50.242GMT", - "endTime" : "2015-03-17T23:12:25.177GMT", - "sparkUser" : "irashid", - "completed" : true - }, { - "attemptId" : "1", - "startTime" : "2015-03-16T19:25:10.242GMT", - "endTime" : "2015-03-16T19:25:45.177GMT", - "sparkUser" : "irashid", - "completed" : true - } ] -}, { - "id" : "local-1425081759269", - "name" : "Spark shell", - "attempts" : [ { - "startTime" : "2015-02-28T00:02:38.277GMT", - "endTime" : "2015-02-28T00:02:46.912GMT", - "sparkUser" : "irashid", - "completed" : true - } ] -}, { - "id" : "local-1422981780767", - "name" : "Spark shell", - "attempts" : [ { - "startTime" : "2015-02-03T16:42:59.720GMT", - "endTime" : "2015-02-03T16:43:08.731GMT", - "sparkUser" : "irashid", - "completed" : true - } ] -}, { - "id" : "local-1422981759269", - "name" : "Spark shell", - "attempts" : [ { - "startTime" : "2015-02-03T16:42:38.277GMT", - "endTime" : "2015-02-03T16:42:46.912GMT", - "sparkUser" : "irashid", - "completed" : true - } ] -} ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/executors/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/executors/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/executors/json_expectation deleted file mode 100644 index cb622e1..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/executors/json_expectation +++ /dev/null @@ -1,17 +0,0 @@ -[ { - "id" : "<driver>", - "hostPort" : "localhost:57971", - "rddBlocks" : 8, - "memoryUsed" : 28000128, - "diskUsed" : 0, - "activeTasks" : 0, - "failedTasks" : 1, - "completedTasks" : 31, - "totalTasks" : 32, - "totalDuration" : 8820, - "totalInputBytes" : 28000288, - "totalShuffleRead" : 0, - "totalShuffleWrite" : 13180, - "maxMemory" : 278302556, - "executorLogs" : { } -} ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs/0/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs/0/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs/0/json_expectation deleted file mode 100644 index 4a29072..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs/0/json_expectation +++ /dev/null @@ -1,15 +0,0 @@ -{ - "jobId" : 0, - "name" : "count at <console>:15", - "stageIds" : [ 0 ], - "status" : "SUCCEEDED", - "numTasks" : 8, - "numActiveTasks" : 0, - "numCompletedTasks" : 8, - "numSkippedTasks" : 8, - "numFailedTasks" : 0, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 0 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs/json_expectation deleted file mode 100644 index cab4750..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs/json_expectation +++ /dev/null @@ -1,43 +0,0 @@ -[ { - "jobId" : 2, - "name" : "count at <console>:17", - "stageIds" : [ 3 ], - "status" : "SUCCEEDED", - "numTasks" : 8, - "numActiveTasks" : 0, - "numCompletedTasks" : 8, - "numSkippedTasks" : 8, - "numFailedTasks" : 0, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 0 -}, { - "jobId" : 1, - "name" : "count at <console>:20", - "stageIds" : [ 1, 2 ], - "status" : "FAILED", - "numTasks" : 16, - "numActiveTasks" : 0, - "numCompletedTasks" : 15, - "numSkippedTasks" : 15, - "numFailedTasks" : 1, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 1 -}, { - "jobId" : 0, - "name" : "count at <console>:15", - "stageIds" : [ 0 ], - "status" : "SUCCEEDED", - "numTasks" : 8, - "numActiveTasks" : 0, - "numCompletedTasks" : 8, - "numSkippedTasks" : 8, - "numFailedTasks" : 0, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 0 -} ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs?status=succeeded&status=failed/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs?status=succeeded&status=failed/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs?status=succeeded&status=failed/json_expectation deleted file mode 100644 index cab4750..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs?status=succeeded&status=failed/json_expectation +++ /dev/null @@ -1,43 +0,0 @@ -[ { - "jobId" : 2, - "name" : "count at <console>:17", - "stageIds" : [ 3 ], - "status" : "SUCCEEDED", - "numTasks" : 8, - "numActiveTasks" : 0, - "numCompletedTasks" : 8, - "numSkippedTasks" : 8, - "numFailedTasks" : 0, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 0 -}, { - "jobId" : 1, - "name" : "count at <console>:20", - "stageIds" : [ 1, 2 ], - "status" : "FAILED", - "numTasks" : 16, - "numActiveTasks" : 0, - "numCompletedTasks" : 15, - "numSkippedTasks" : 15, - "numFailedTasks" : 1, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 1 -}, { - "jobId" : 0, - "name" : "count at <console>:15", - "stageIds" : [ 0 ], - "status" : "SUCCEEDED", - "numTasks" : 8, - "numActiveTasks" : 0, - "numCompletedTasks" : 8, - "numSkippedTasks" : 8, - "numFailedTasks" : 0, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 0 -} ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs?status=succeeded/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs?status=succeeded/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs?status=succeeded/json_expectation deleted file mode 100644 index 6fd25be..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/jobs?status=succeeded/json_expectation +++ /dev/null @@ -1,29 +0,0 @@ -[ { - "jobId" : 2, - "name" : "count at <console>:17", - "stageIds" : [ 3 ], - "status" : "SUCCEEDED", - "numTasks" : 8, - "numActiveTasks" : 0, - "numCompletedTasks" : 8, - "numSkippedTasks" : 8, - "numFailedTasks" : 0, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 0 -}, { - "jobId" : 0, - "name" : "count at <console>:15", - "stageIds" : [ 0 ], - "status" : "SUCCEEDED", - "numTasks" : 8, - "numActiveTasks" : 0, - "numCompletedTasks" : 8, - "numSkippedTasks" : 8, - "numFailedTasks" : 0, - "numActiveStages" : 0, - "numCompletedStages" : 1, - "numSkippedStages" : 0, - "numFailedStages" : 0 -} ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/json_expectation deleted file mode 100644 index 07489ad..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/json_expectation +++ /dev/null @@ -1,10 +0,0 @@ -{ - "id" : "local-1422981780767", - "name" : "Spark shell", - "attempts" : [ { - "startTime" : "2015-02-03T16:42:59.720GMT", - "endTime" : "2015-02-03T16:43:08.731GMT", - "sparkUser" : "irashid", - "completed" : true - } ] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/1/0/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/1/0/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/1/0/json_expectation deleted file mode 100644 index 111cb81..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/1/0/json_expectation +++ /dev/null @@ -1,270 +0,0 @@ -{ - "status" : "COMPLETE", - "stageId" : 1, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 8, - "numFailedTasks" : 0, - "executorRunTime" : 3476, - "inputBytes" : 28000128, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 13180, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "map at <console>:14", - "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIM ain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ], - "tasks" : { - "8" : { - "taskId" : 8, - "index" : 0, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.829GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 1, - "executorRunTime" : 435, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 2, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 94000, - "recordsWritten" : 0 - } - } - }, - "11" : { - "taskId" : 11, - "index" : 3, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1647, - "writeTime" : 83000, - "recordsWritten" : 0 - } - } - }, - "14" : { - "taskId" : 14, - "index" : 6, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.832GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 88000, - "recordsWritten" : 0 - } - } - }, - "13" : { - "taskId" : 13, - "index" : 5, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.831GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 2, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 73000, - "recordsWritten" : 0 - } - } - }, - "10" : { - "taskId" : 10, - "index" : 2, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 76000, - "recordsWritten" : 0 - } - } - }, - "9" : { - "taskId" : 9, - "index" : 1, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 1, - "executorRunTime" : 436, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 98000, - "recordsWritten" : 0 - } - } - }, - "12" : { - "taskId" : 12, - "index" : 4, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.831GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1645, - "writeTime" : 101000, - "recordsWritten" : 0 - } - } - }, - "15" : { - "taskId" : 15, - "index" : 7, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.833GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 1, - "executorRunTime" : 435, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 79000, - "recordsWritten" : 0 - } - } - } - }, - "executorSummary" : { - "<driver>" : { - "taskTime" : 3624, - "failedTasks" : 0, - "succeededTasks" : 8, - "inputBytes" : 28000128, - "outputBytes" : 0, - "shuffleRead" : 0, - "shuffleWrite" : 13180, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0 - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/1/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/1/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/1/json_expectation deleted file mode 100644 index ef339f8..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/1/json_expectation +++ /dev/null @@ -1,270 +0,0 @@ -[ { - "status" : "COMPLETE", - "stageId" : 1, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 8, - "numFailedTasks" : 0, - "executorRunTime" : 3476, - "inputBytes" : 28000128, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 13180, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "map at <console>:14", - "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIM ain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ], - "tasks" : { - "8" : { - "taskId" : 8, - "index" : 0, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.829GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 1, - "executorRunTime" : 435, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 2, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 94000, - "recordsWritten" : 0 - } - } - }, - "11" : { - "taskId" : 11, - "index" : 3, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1647, - "writeTime" : 83000, - "recordsWritten" : 0 - } - } - }, - "14" : { - "taskId" : 14, - "index" : 6, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.832GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 88000, - "recordsWritten" : 0 - } - } - }, - "13" : { - "taskId" : 13, - "index" : 5, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.831GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 2, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 73000, - "recordsWritten" : 0 - } - } - }, - "10" : { - "taskId" : 10, - "index" : 2, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 76000, - "recordsWritten" : 0 - } - } - }, - "9" : { - "taskId" : 9, - "index" : 1, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 1, - "executorRunTime" : 436, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 98000, - "recordsWritten" : 0 - } - } - }, - "12" : { - "taskId" : 12, - "index" : 4, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.831GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 2, - "executorRunTime" : 434, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1645, - "writeTime" : 101000, - "recordsWritten" : 0 - } - } - }, - "15" : { - "taskId" : 15, - "index" : 7, - "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.833GMT", - "executorId" : "<driver>", - "host" : "localhost", - "taskLocality" : "PROCESS_LOCAL", - "speculative" : false, - "accumulatorUpdates" : [ ], - "taskMetrics" : { - "executorDeserializeTime" : 1, - "executorRunTime" : 435, - "resultSize" : 1902, - "jvmGcTime" : 19, - "resultSerializationTime" : 1, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "inputMetrics" : { - "bytesRead" : 3500016, - "recordsRead" : 0 - }, - "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 79000, - "recordsWritten" : 0 - } - } - } - }, - "executorSummary" : { - "<driver>" : { - "taskTime" : 3624, - "failedTasks" : 0, - "succeededTasks" : 8, - "inputBytes" : 28000128, - "outputBytes" : 0, - "shuffleRead" : 0, - "shuffleWrite" : 13180, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0 - } - } -} ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/json_expectation deleted file mode 100644 index 056fac7..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages/json_expectation +++ /dev/null @@ -1,89 +0,0 @@ -[ { - "status" : "COMPLETE", - "stageId" : 3, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 8, - "numFailedTasks" : 0, - "executorRunTime" : 162, - "inputBytes" : 160, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 0, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "count at <console>:17", - "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line19.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:17)\n$line19.$read$$iwC$$iwC$$iwC.<init>(<console>:22)\n$line19.$read$$iwC$$iwC.<init>(<console>:24)\n$line19.$read$$iwC.<init>(<console>:26)\n$line19.$read.<init>(<console>:28)\n$line19.$read$.<init>(<console>:32)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.<init>(<console>:7)\n$line19.$eval$.<clinit>(<console>)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.Spark IMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ] -}, { - "status" : "COMPLETE", - "stageId" : 1, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 8, - "numFailedTasks" : 0, - "executorRunTime" : 3476, - "inputBytes" : 28000128, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 13180, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "map at <console>:14", - "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIM ain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ] -}, { - "status" : "COMPLETE", - "stageId" : 0, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 8, - "numFailedTasks" : 0, - "executorRunTime" : 4338, - "inputBytes" : 0, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 0, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "count at <console>:15", - "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:15)\n$line9.$read$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line9.$read$$iwC$$iwC.<init>(<console>:22)\n$line9.$read$$iwC.<init>(<console>:24)\n$line9.$read.<init>(<console>:26)\n$line9.$read$.<init>(<console>:30)\n$line9.$read$.<clinit>(<console>)\n$line9.$eval$.<init>(<console>:7)\n$line9.$eval$.<clinit>(<console>)\n$line9.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.inte rpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ] -}, { - "status" : "FAILED", - "stageId" : 2, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 7, - "numFailedTasks" : 1, - "executorRunTime" : 278, - "inputBytes" : 0, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 0, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "count at <console>:20", - "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line11.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line11.$read$$iwC$$iwC$$iwC.<init>(<console>:25)\n$line11.$read$$iwC$$iwC.<init>(<console>:27)\n$line11.$read$$iwC.<init>(<console>:29)\n$line11.$read.<init>(<console>:31)\n$line11.$read$.<init>(<console>:35)\n$line11.$read$.<clinit>(<console>)\n$line11.$eval$.<init>(<console>:7)\n$line11.$eval$.<clinit>(<console>)\n$line11.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.Spark IMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ] -} ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/765f6e11/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages?status=complete/json_expectation ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages?status=complete/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages?status=complete/json_expectation deleted file mode 100644 index 31ac9be..0000000 --- a/core/src/test/resources/HistoryServerExpectations/applications/local-1422981780767/stages?status=complete/json_expectation +++ /dev/null @@ -1,67 +0,0 @@ -[ { - "status" : "COMPLETE", - "stageId" : 3, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 8, - "numFailedTasks" : 0, - "executorRunTime" : 162, - "inputBytes" : 160, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 0, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "count at <console>:17", - "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line19.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:17)\n$line19.$read$$iwC$$iwC$$iwC.<init>(<console>:22)\n$line19.$read$$iwC$$iwC.<init>(<console>:24)\n$line19.$read$$iwC.<init>(<console>:26)\n$line19.$read.<init>(<console>:28)\n$line19.$read$.<init>(<console>:32)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.<init>(<console>:7)\n$line19.$eval$.<clinit>(<console>)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.Spark IMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ] -}, { - "status" : "COMPLETE", - "stageId" : 1, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 8, - "numFailedTasks" : 0, - "executorRunTime" : 3476, - "inputBytes" : 28000128, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 13180, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "map at <console>:14", - "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIM ain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ] -}, { - "status" : "COMPLETE", - "stageId" : 0, - "attemptId" : 0, - "numActiveTasks" : 0, - "numCompleteTasks" : 8, - "numFailedTasks" : 0, - "executorRunTime" : 4338, - "inputBytes" : 0, - "inputRecords" : 0, - "outputBytes" : 0, - "outputRecords" : 0, - "shuffleReadBytes" : 0, - "shuffleReadRecords" : 0, - "shuffleWriteBytes" : 0, - "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, - "name" : "count at <console>:15", - "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:15)\n$line9.$read$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line9.$read$$iwC$$iwC.<init>(<console>:22)\n$line9.$read$$iwC.<init>(<console>:24)\n$line9.$read.<init>(<console>:26)\n$line9.$read$.<init>(<console>:30)\n$line9.$read$.<clinit>(<console>)\n$line9.$eval$.<init>(<console>:7)\n$line9.$eval$.<clinit>(<console>)\n$line9.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.inte rpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", - "schedulingPool" : "default", - "accumulatorUpdates" : [ ] -} ] \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
