GitHub user scwf opened a pull request:
https://github.com/apache/spark/pull/212
java.nio.charset.MalformedInputException
Source.fromInputStream throws " java.nio.charset.MalformedInputException"
when spark running on windows. there are two class (Utils and PipedRDD) use the
method. fixed it
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/scwf/spark master
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/spark/pull/212.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #212
----
commit 0be1e3ddf0730d0d607e73a408c9bf859d9cf9d1
Author: scwf <[email protected]>
Date: 2014-03-21T03:18:49Z
Update StageTable.scala
fix the problem that the numbers on progress bar move along with tasks
progress until invisible
commit eee724686e2696a3c1988804640882ec77400b47
Author: scwf <[email protected]>
Date: 2014-03-21T05:16:56Z
Update StageTable.scala
commit ab92b12c98aa3225ec38c543ccd2ca7d9f9285db
Author: scwf <root@kf-thinkpad-sl410.(none)>
Date: 2014-03-23T14:25:55Z
Merge branch 'master' of https://github.com/apache/spark
Conflicts:
core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
commit e4a379875db7bb3c6f9068ceda1071f8f749f5db
Author: scwf <root@kf-thinkpad-sl410.(none)>
Date: 2014-03-23T14:29:38Z
delete stage table
commit 85a30c8e5fff1d8aaa60c6c5ff479b04a0620595
Author: scwf <root@kf-thinkpad-sl410.(none)>
Date: 2014-03-23T14:57:04Z
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.ui.jobs
import java.util.Date
import scala.collection.mutable.HashMap
import scala.xml.Node
import org.apache.spark.scheduler.{StageInfo, TaskInfo}
import org.apache.spark.ui.{WebUI, UIUtils}
import org.apache.spark.util.Utils
/** Page showing list of all ongoing and recently finished stages */
private[ui] class StageTable(stages: Seq[StageInfo], parent: JobProgressUI)
{
private val basePath = parent.basePath
private lazy val listener = parent.listener
private lazy val isFairScheduler = parent.isFairScheduler
def toNodeSeq: Seq[Node] = {
listener.synchronized {
stageTable(stageRow, stages)
}
}
/** Special table that merges two header cells. */
private def stageTable[T](makeRow: T => Seq[Node], rows: Seq[T]):
Seq[Node] = {
<table class="table table-bordered table-striped table-condensed
sortable">
<thead>
<th>Stage Id</th>
{if (isFairScheduler) {<th>Pool Name</th>} else {}}
<th>Description</th>
<th>Submitted</th>
<th>Duration</th>
<th>Tasks: Succeeded/Total</th>
<th>Shuffle Read</th>
<th>Shuffle Write</th>
</thead>
<tbody>
{rows.map(r => makeRow(r))}
</tbody>
</table>
}
private def makeProgressBar(started: Int, completed: Int, failed: String,
total: Int): Seq[Node] =
{
val completeWidth = "width: %s%%".format((completed.toDouble/total)*100)
val startWidth = "width: %s%%".format((started.toDouble/total)*100)
<div class="progress">
<span style="text-align:center; position:absolute; width:100%;
left:0;">
{completed}/{total} {failed}
</span>
<div class="bar bar-completed" style={completeWidth}></div>
<div class="bar bar-running" style={startWidth}></div>
</div>
}
/** Render an HTML row that represents a stage */
private def stageRow(s: StageInfo): Seq[Node] = {
val poolName = listener.stageIdToPool.get(s.stageId)
val nameLink =
<a
href={"%s/stages/stage?id=%s".format(UIUtils.prependBaseUri(basePath),
s.stageId)}>
{s.name}
</a>
val description = listener.stageIdToDescription.get(s.stageId)
.map(d =>
<div><em>{d}</em></div><div>{nameLink}</div>).getOrElse(nameLink)
val submissionTime = s.submissionTime match {
case Some(t) => WebUI.formatDate(new Date(t))
case None => "Unknown"
}
val finishTime = s.completionTime.getOrElse(System.currentTimeMillis)
val duration = s.submissionTime.map { t =>
if (finishTime > t) finishTime - t else System.currentTimeMillis - t
}
val formattedDuration = duration.map(d =>
parent.formatDuration(d)).getOrElse("Unknown")
val startedTasks =
listener.stageIdToTasksActive.getOrElse(s.stageId, HashMap[Long,
TaskInfo]()).size
val completedTasks =
listener.stageIdToTasksComplete.getOrElse(s.stageId, 0)
val failedTasks = listener.stageIdToTasksFailed.getOrElse(s.stageId, 0)
match {
case f if f > 0 => "(%s failed)".format(f)
case _ => ""
}
val totalTasks = s.numTasks
val shuffleReadSortable =
listener.stageIdToShuffleRead.getOrElse(s.stageId, 0L)
val shuffleRead = shuffleReadSortable match {
case 0 => ""
case b => Utils.bytesToString(b)
}
val shuffleWriteSortable =
listener.stageIdToShuffleWrite.getOrElse(s.stageId, 0L)
val shuffleWrite = shuffleWriteSortable match {
case 0 => ""
case b => Utils.bytesToString(b)
}
<tr>
<td>{s.stageId}</td>
{if (isFairScheduler) {
<td>
<a href={"%s/stages/pool?poolname=%s"
.format(UIUtils.prependBaseUri(basePath), poolName.get)}>
{poolName.get}
</a>
</td>
}}
<td>{description}</td>
<td valign="middle">{submissionTime}</td>
<td
sorttable_customkey={duration.getOrElse(-1).toString}>{formattedDuration}</td>
<td class="progress-cell">
{makeProgressBar(startedTasks, completedTasks, failedTasks,
totalTasks)}
</td>
<td
sorttable_customekey={shuffleReadSortable.toString}>{shuffleRead}</td>
<td
sorttable_customekey={shuffleWriteSortable.toString}>{shuffleWrite}</td>
</tr>
}
}
commit 307b6c568a1eeb7af488b3a6fa280ea734a6d36c
Author: scwf <[email protected]>
Date: 2014-03-24T02:37:50Z
fix MalformedInputException
Source.fromInputStream throws MalformedInputException when spark running on
windows
commit 09a43ed0f76cf756b34eaacaacf5331292f95f00
Author: scwf <[email protected]>
Date: 2014-03-24T02:40:05Z
fix MalformedInputException
commit d80a40e3cda295904e92ae0a660b57e1c6c8d590
Author: scwf <[email protected]>
Date: 2014-03-24T02:52:17Z
Update StageTable.scala
commit 27a809a47c94e9734e10673c79aacf86a6dfb7d1
Author: scwf <[email protected]>
Date: 2014-03-24T02:52:52Z
Update StageTable.scala
commit 98255a54875f95e37788cf4a09435115dd343670
Author: scwf <[email protected]>
Date: 2014-03-24T02:54:01Z
fix java.nio.charset.MalformedInputException
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
