Github user juliuszsompolski commented on a diff in the pull request:
https://github.com/apache/spark/pull/19181#discussion_r141843046
--- Diff:
core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
---
@@ -503,6 +511,39 @@ public void testGetIterator() throws Exception {
verifyIntIterator(sorter.getIterator(279), 279, 300);
}
+ @Test
+ public void testOOMDuringSpill() throws Exception {
+ final UnsafeExternalSorter sorter = newSorter();
+ for (int i = 0; sorter.hasSpaceForAnotherRecord(); ++i) {
+ insertNumber(sorter, i);
+ }
+ // todo: this might actually not be zero if pageSize is somehow
configured differently,
+ // so we actually have to compute the expected spill size according to
the configured page size
+ assertEquals(0, sorter.getSpillSize());
+ // we expect the next insert to attempt growing the pointerssArray
+ // first allocation is expected to fail, then a spill is triggered
which attempts another allocation
+ // which also fails and we expect to see this OOM here.
+ // the original code messed with a released array within the spill code
+ // and ended up with a failed assertion.
+ // we also expect the location of the OOM to be
org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset
+ memoryManager.markConseqOOM(2);
+ OutOfMemoryError expectedOOM = null;
+ try {
+ insertNumber(sorter, 1024);
+ }
+ // we expect an OutOfMemoryError here, anything else (i.e the original
NPE is a failure)
+ catch( OutOfMemoryError oom ){
+ expectedOOM = oom;
+ }
+
+ assertNotNull("expected OutOfMmoryError but it seems operation
surprisingly succeeded"
+ ,expectedOOM);
+ String oomStackTrace = Utils.exceptionString(expectedOOM);
+ assertThat("expected OutOfMemoryError in
org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset"
+ , oomStackTrace
+ ,
Matchers.containsString("org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset"));
--- End diff --
nit: move commas to end of line (3x)
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
