Github user hvanhovell commented on a diff in the pull request:
https://github.com/apache/spark/pull/19181#discussion_r143770192
--- Diff:
core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
---
@@ -503,6 +504,41 @@ public void testGetIterator() throws Exception {
verifyIntIterator(sorter.getIterator(279), 279, 300);
}
+ @Test
+ public void testOOMDuringSpill() throws Exception {
+ final UnsafeExternalSorter sorter = newSorter();
+ // we assume that given default configuration, the size of
+ // the data we insert to the sorter (ints) and assuming we shouldn't
+ // spill before pointers array is exhausted (memory manager is not
configured to throw at this point)
+ // - so this loop run a reasonable number of loops (<2000)
+ // test test indeed completed within <30ms (on a quad i7 laptop).
+ for (int i = 0; sorter.hasSpaceForAnotherRecord(); ++i) {
+ insertNumber(sorter, i);
+ }
+ // we expect the next insert to attempt growing the pointerssArray
+ // first allocation is expected to fail, then a spill is triggered
which attempts another allocation
+ // which also fails and we expect to see this OOM here.
+ // the original code messed with a released array within the spill code
+ // and ended up with a failed assertion.
+ // we also expect the location of the OOM to be
org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset
+ memoryManager.markconsequentOOM(2);
+ OutOfMemoryError expectedOOM = null;
+ try {
+ insertNumber(sorter, 1024);
+ }
+ // we expect an OutOfMemoryError here, anything else (i.e the original
NPE is a failure)
+ catch (OutOfMemoryError oom ){
--- End diff --
Nit weird spacing
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
