Repository: spark
Updated Branches:
refs/heads/master 693102203 -> 083cf2235
[SPARK-21033][CORE][FOLLOW-UP] Update Spillable
## What changes were proposed in this pull request?
Update
```scala
SparkEnv.get.conf.getLong("spark.shuffle.spill.numElementsForceSpillThreshold",
Long.MaxValue)
```
to
```scala
SparkEnv.get.conf.get(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD)
```
because of `SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD`'s default value
is `Integer.MAX_VALUE`:
https://github.com/apache/spark/blob/c99fc9ad9b600095baba003053dbf84304ca392b/core/src/main/scala/org/apache/spark/internal/config/package.scala#L503-L511
## How was this patch tested?
N/A
Author: Yuming Wang <yumw...@ebay.com>
Closes #21077 from wangyum/SPARK-21033.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/083cf223
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/083cf223
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/083cf223
Branch: refs/heads/master
Commit: 083cf223569b7896e35ff1d53a73498a4971b28d
Parents: 6931022
Author: Yuming Wang <yumw...@ebay.com>
Authored: Mon Apr 16 23:50:50 2018 +0800
Committer: Wenchen Fan <wenc...@databricks.com>
Committed: Mon Apr 16 23:50:50 2018 +0800
----------------------------------------------------------------------
.../scala/org/apache/spark/util/collection/Spillable.scala | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/083cf223/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
----------------------------------------------------------------------
diff --git
a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
index 8183f82..81457b5 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
@@ -19,6 +19,7 @@ package org.apache.spark.util.collection
import org.apache.spark.SparkEnv
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
import org.apache.spark.memory.{MemoryConsumer, MemoryMode, TaskMemoryManager}
/**
@@ -41,7 +42,7 @@ private[spark] abstract class Spillable[C](taskMemoryManager:
TaskMemoryManager)
protected def forceSpill(): Boolean
// Number of elements read from input since last spill
- protected def elementsRead: Long = _elementsRead
+ protected def elementsRead: Int = _elementsRead
// Called by subclasses every time a record is read
// It's used for checking spilling frequency
@@ -54,15 +55,15 @@ private[spark] abstract class
Spillable[C](taskMemoryManager: TaskMemoryManager)
// Force this collection to spill when there are this many elements in memory
// For testing only
- private[this] val numElementsForceSpillThreshold: Long =
-
SparkEnv.get.conf.getLong("spark.shuffle.spill.numElementsForceSpillThreshold",
Long.MaxValue)
+ private[this] val numElementsForceSpillThreshold: Int =
+ SparkEnv.get.conf.get(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD)
// Threshold for this collection's size in bytes before we start tracking
its memory usage
// To avoid a large number of small spills, initialize this to a value
orders of magnitude > 0
@volatile private[this] var myMemoryThreshold = initialMemoryThreshold
// Number of elements read from input since last spill
- private[this] var _elementsRead = 0L
+ private[this] var _elementsRead = 0
// Number of bytes spilled in total
@volatile private[this] var _memoryBytesSpilled = 0L
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
