Github user hvanhovell commented on a diff in the pull request:
https://github.com/apache/spark/pull/19181#discussion_r143771458
--- Diff:
core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
---
@@ -139,4 +139,49 @@ public int compare(
}
assertEquals(dataToSort.length, iterLength);
}
+
+ @Test
+ public void freeAfterOOM() {
+ final SparkConf sparkConf =
+ new SparkConf()
+ .set("spark.memory.offHeap.enabled",
+ "false");
+ final TestMemoryManager testMemoryManager =
+ new TestMemoryManager(sparkConf);
+ final TaskMemoryManager memoryManager = new TaskMemoryManager(
+ testMemoryManager, 0);
+ final TestMemoryConsumer consumer = new
TestMemoryConsumer(memoryManager);
+ final MemoryBlock dataPage = memoryManager.allocatePage(2048,
consumer);
+ final Object baseObject = dataPage.getBaseObject();
+ // Write the records into the data page:
+ long position = dataPage.getBaseOffset();
+
+ final HashPartitioner hashPartitioner = new HashPartitioner(4);
+ // Use integer comparison for comparing prefixes (which are partition
ids, in this case)
+ final PrefixComparator prefixComparator = PrefixComparators.LONG;
+ final RecordComparator recordComparator = new RecordComparator() {
+ @Override
+ public int compare(
+ Object leftBaseObject,
+ long leftBaseOffset,
+ Object rightBaseObject,
+ long rightBaseOffset) {
+ return 0;
+ }
+ };
+ UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer,
memoryManager,
+ recordComparator, prefixComparator, 100, shouldUseRadixSort());
+
+ testMemoryManager.markExecutionAsOutOfMemoryOnce();
+ try {
+ sorter.reset();
+ } catch (OutOfMemoryError oom) {
+ // as expected
--- End diff --
Make sure you fail on an unexpected success
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
