spark git commit: [CORE] Protect additional test vars from early GC
Repository: spark
Updated Branches:
refs/heads/master b1b9d5802 - 270d4b518
[CORE] Protect additional test vars from early GC
Fix more places in which some test variables could be collected early by
aggressive JVM optimization.
Added a couple of comments to note where existing references are sufficient in
the same test pattern.
Author: Tim Ellison t.p.elli...@gmail.com
Closes #6187 from tellison/DefeatEarlyGC and squashes the following commits:
27329d9 [Tim Ellison] [CORE] Protect additional test vars from early GC
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/270d4b51
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/270d4b51
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/270d4b51
Branch: refs/heads/master
Commit: 270d4b5181b95e3f1f131b1d65dde00a7e5b9d6e
Parents: b1b9d58
Author: Tim Ellison t.p.elli...@gmail.com
Authored: Fri May 15 11:27:24 2015 -0700
Committer: Andrew Or and...@databricks.com
Committed: Fri May 15 11:27:24 2015 -0700
--
.../test/scala/org/apache/spark/ContextCleanerSuite.scala | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/270d4b51/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
--
diff --git a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
index cb30e1f..0922a2c 100644
--- a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
@@ -165,6 +165,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
}
// Test that GC causes RDD cleanup after dereferencing the RDD
+// Note rdd is used after previous GC to avoid early collection by the JVM
val postGCTester = new CleanerTester(sc, rddIds = Seq(rdd.id))
rdd = null // Make RDD out of scope
runGC()
@@ -181,9 +182,9 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
intercept[Exception] {
preGCTester.assertCleanup()(timeout(1000 millis))
}
+rdd.count() // Defeat early collection by the JVM
// Test that GC causes shuffle cleanup after dereferencing the RDD
-rdd.count() // Defeat any early collection of rdd variable by the JVM
val postGCTester = new CleanerTester(sc, shuffleIds = Seq(0))
rdd = null // Make RDD out of scope, so that corresponding shuffle goes
out of scope
runGC()
@@ -201,6 +202,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
}
// Test that GC causes broadcast cleanup after dereferencing the broadcast
variable
+// Note broadcast is used after previous GC to avoid early collection by
the JVM
val postGCTester = new CleanerTester(sc, broadcastIds = Seq(broadcast.id))
broadcast = null // Make broadcast variable out of scope
runGC()
@@ -226,7 +228,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
// the checkpoint is not cleaned by default (without the configuration set)
var postGCTester = new CleanerTester(sc, Seq(rddId), Nil, Nil, Nil)
-rdd = null // Make RDD out of scope
+rdd = null // Make RDD out of scope, ok if collected earlier
runGC()
postGCTester.assertCleanup()
assert(fs.exists(RDDCheckpointData.rddCheckpointDataPath(sc, rddId).get))
@@ -245,6 +247,9 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
// Confirm the checkpoint directory exists
assert(fs.exists(RDDCheckpointData.rddCheckpointDataPath(sc, rddId).get))
+// Reference rdd to defeat any early collection by the JVM
+rdd.count()
+
// Test that GC causes checkpoint data cleanup after dereferencing the RDD
postGCTester = new CleanerTester(sc, Seq(rddId), Nil, Nil, Seq(rddId))
rdd = null // Make RDD out of scope
@@ -352,6 +357,7 @@ class SortShuffleContextCleanerSuite extends
ContextCleanerSuiteBase(classOf[Sor
intercept[Exception] {
preGCTester.assertCleanup()(timeout(1000 millis))
}
+rdd.count() // Defeat early collection by the JVM
// Test that GC causes shuffle cleanup after dereferencing the RDD
val postGCTester = new CleanerTester(sc, shuffleIds = Seq(0))
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [CORE] Protect additional test vars from early GC
Repository: spark
Updated Branches:
refs/heads/branch-1.4 c58b9c616 - 866e4b520
[CORE] Protect additional test vars from early GC
Fix more places in which some test variables could be collected early by
aggressive JVM optimization.
Added a couple of comments to note where existing references are sufficient in
the same test pattern.
Author: Tim Ellison t.p.elli...@gmail.com
Closes #6187 from tellison/DefeatEarlyGC and squashes the following commits:
27329d9 [Tim Ellison] [CORE] Protect additional test vars from early GC
(cherry picked from commit 270d4b5181b95e3f1f131b1d65dde00a7e5b9d6e)
Signed-off-by: Andrew Or and...@databricks.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/866e4b52
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/866e4b52
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/866e4b52
Branch: refs/heads/branch-1.4
Commit: 866e4b52047f5d9da9de2fd81864a0706af9b99f
Parents: c58b9c6
Author: Tim Ellison t.p.elli...@gmail.com
Authored: Fri May 15 11:27:24 2015 -0700
Committer: Andrew Or and...@databricks.com
Committed: Fri May 15 11:27:35 2015 -0700
--
.../test/scala/org/apache/spark/ContextCleanerSuite.scala | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/866e4b52/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
--
diff --git a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
index cb30e1f..0922a2c 100644
--- a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
@@ -165,6 +165,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
}
// Test that GC causes RDD cleanup after dereferencing the RDD
+// Note rdd is used after previous GC to avoid early collection by the JVM
val postGCTester = new CleanerTester(sc, rddIds = Seq(rdd.id))
rdd = null // Make RDD out of scope
runGC()
@@ -181,9 +182,9 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
intercept[Exception] {
preGCTester.assertCleanup()(timeout(1000 millis))
}
+rdd.count() // Defeat early collection by the JVM
// Test that GC causes shuffle cleanup after dereferencing the RDD
-rdd.count() // Defeat any early collection of rdd variable by the JVM
val postGCTester = new CleanerTester(sc, shuffleIds = Seq(0))
rdd = null // Make RDD out of scope, so that corresponding shuffle goes
out of scope
runGC()
@@ -201,6 +202,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
}
// Test that GC causes broadcast cleanup after dereferencing the broadcast
variable
+// Note broadcast is used after previous GC to avoid early collection by
the JVM
val postGCTester = new CleanerTester(sc, broadcastIds = Seq(broadcast.id))
broadcast = null // Make broadcast variable out of scope
runGC()
@@ -226,7 +228,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
// the checkpoint is not cleaned by default (without the configuration set)
var postGCTester = new CleanerTester(sc, Seq(rddId), Nil, Nil, Nil)
-rdd = null // Make RDD out of scope
+rdd = null // Make RDD out of scope, ok if collected earlier
runGC()
postGCTester.assertCleanup()
assert(fs.exists(RDDCheckpointData.rddCheckpointDataPath(sc, rddId).get))
@@ -245,6 +247,9 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
// Confirm the checkpoint directory exists
assert(fs.exists(RDDCheckpointData.rddCheckpointDataPath(sc, rddId).get))
+// Reference rdd to defeat any early collection by the JVM
+rdd.count()
+
// Test that GC causes checkpoint data cleanup after dereferencing the RDD
postGCTester = new CleanerTester(sc, Seq(rddId), Nil, Nil, Seq(rddId))
rdd = null // Make RDD out of scope
@@ -352,6 +357,7 @@ class SortShuffleContextCleanerSuite extends
ContextCleanerSuiteBase(classOf[Sor
intercept[Exception] {
preGCTester.assertCleanup()(timeout(1000 millis))
}
+rdd.count() // Defeat early collection by the JVM
// Test that GC causes shuffle cleanup after dereferencing the RDD
val postGCTester = new CleanerTester(sc, shuffleIds = Seq(0))
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
