This is an automated email from the ASF dual-hosted git repository. prasanthj pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 80e0d0d HIVE-23477: LLAP: mmap allocation interruptions fails to notify other threads (Prasanth Jayachandran reviewed by Gopal V) 80e0d0d is described below commit 80e0d0d954d4d5680d659f17a06276b90b8571ad Author: Prasanth Jayachandran <prasan...@apache.org> AuthorDate: Thu May 21 12:22:50 2020 -0700 HIVE-23477: LLAP: mmap allocation interruptions fails to notify other threads (Prasanth Jayachandran reviewed by Gopal V) --- .../hadoop/hive/llap/cache/BuddyAllocator.java | 39 ++++++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java index 6934dba..c7a6402 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.llap.cache; +import java.nio.channels.ClosedByInterruptException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -217,7 +218,11 @@ public final class BuddyAllocator } int initCount = doPreallocate ? maxArenas : 1; for (int i = 0; i < initCount; ++i) { - arenas[i].init(i); + try { + arenas[i].init(i); + } catch (ClosedByInterruptException e) { + throw new RuntimeException("Failed pre-allocating buddy allocator arena. ", e); + } metrics.incrAllocatedArena(); } allocatedArenas.set(initCount); @@ -851,7 +856,7 @@ public final class BuddyAllocator return isDirect; } - private ByteBuffer preallocateArenaBuffer(int arenaSize) { + private ByteBuffer preallocateArenaBuffer(int arenaSize) throws ClosedByInterruptException { if (isMapped) { RandomAccessFile rwf = null; File rf = null; @@ -863,6 +868,15 @@ public final class BuddyAllocator // Use RW, not PRIVATE because the copy-on-write is irrelevant for a deleted file // see discussion in YARN-5551 for the memory accounting discussion return rwf.getChannel().map(MapMode.READ_WRITE, 0, arenaSize); + } catch (ClosedByInterruptException cbi) { + LlapIoImpl.LOG.warn("Interrupted while trying to allocate memory mapped arena", cbi); + // finally may not execute on thread interrupts so cleanup the arena file as it may be unmapped + IOUtils.closeQuietly(rwf); + if (rf != null) { + rf.delete(); + rf = null; + } + throw cbi; } catch (IOException ioe) { LlapIoImpl.LOG.warn("Failed trying to allocate memory mapped arena", ioe); // fail similarly when memory allocations fail @@ -892,7 +906,7 @@ public final class BuddyAllocator private byte[] headers; // Free list indices of each unallocated block, for quick lookup. private FreeList[] freeLists; - void init(int arenaIx) { + void init(int arenaIx) throws ClosedByInterruptException { this.arenaIx = arenaIx; try { data = preallocateArenaBuffer(arenaSize); @@ -1453,13 +1467,24 @@ public final class BuddyAllocator continue; // CAS race, look again. } assert data == null; - init(arenaIx); - boolean isCommited = allocatedArenas.compareAndSet(-arenaCount - 1, arenaCount + 1); - assert isCommited; + try { + init(arenaIx); + // if init did not throw interrupt exception then allocation succeeded and so increment and commit the arena + boolean isCommited = allocatedArenas.compareAndSet(-arenaCount - 1, arenaCount + 1); + assert isCommited; + metrics.incrAllocatedArena(); + } catch (ClosedByInterruptException e) { + LlapIoImpl.LOG.info("Received interrupt during arena {} allocation.. Ignoring..", arenaIx); + // not doing the notify in finally() block as thread interruptions may not execute finally + synchronized (this) { + this.notifyAll(); + } + continue; + } + synchronized (this) { this.notifyAll(); } - metrics.incrAllocatedArena(); return allocateWithSplit(freeListIx, dest, null, ix, dest.length, size, -1); } }