[
https://issues.apache.org/jira/browse/HIVE-22277?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
László Bodor updated HIVE-22277:
--------------------------------
Description:
It's more than flaky. Recently, in most of the cases it's the only failing test
in otherwise green runs.
full hive logs: [^hive.log.tar.gz]
{code}
Error Message
java.util.concurrent.ExecutionException:
org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
or an internal issue)
Stacktrace
java.lang.RuntimeException: java.util.concurrent.ExecutionException:
org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
or an internal issue)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.testMTT(TestBuddyAllocator.java:149)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at org.junit.runners.Suite.runChild(Suite.java:127)
at org.junit.runners.Suite.runChild(Suite.java:26)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:379)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:340)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:125)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:413)
Caused by: java.util.concurrent.ExecutionException:
org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
or an internal issue)
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.testMTT(TestBuddyAllocator.java:145)
... 33 more
Caused by:
org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
or an internal issue)
at
org.apache.hadoop.hive.llap.cache.BuddyAllocator.allocateMultiple(BuddyAllocator.java:454)
at
org.apache.hadoop.hive.llap.cache.BuddyAllocator.allocateMultiple(BuddyAllocator.java:299)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.allocateAndUseBuffer(TestBuddyAllocator.java:254)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.allocateUp(TestBuddyAllocator.java:231)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.access$000(TestBuddyAllocator.java:43)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator$1.call(TestBuddyAllocator.java:119)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator$1.call(TestBuddyAllocator.java:116)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Standard Error
2019-09-30T03:04:43,170 INFO [main] LlapIoImpl: Buddy allocator with direct
buffers; memory mapped off
/home/hiveptest/104.197.225.41-hiveptest-1/apache-github-source-source/llap-server/target/tmp/llap-5698481178354861395;
allocation sizes 8 - 256, arena size 256, total size 6144
2019-09-30T03:04:43,174 WARN [pool-6-thread-3] LlapIoImpl: Failed to allocate
[3]X[256] bytes after [15] attempt, evicted [0] bytes and partially allocated
[256] bytes
2019-09-30T03:04:43,174 WARN [pool-6-thread-2] LlapIoImpl: Failed to allocate
[3]X[256] bytes after [15] attempt, evicted [0] bytes and partially allocated
[0] bytes
2019-09-30T03:04:43,175 ERROR [pool-6-thread-2] LlapIoImpl: Failed to allocate
255; at 1 out of 3 (entire cache is fragmented and locked, or an internal issue)
2019-09-30T03:04:43,175 ERROR [pool-6-thread-3] LlapIoImpl: Failed to allocate
255; at 1 out of 3 (entire cache is fragmented and locked, or an internal issue)
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 256, allocated
Arena:
free list for size 8: 5,
free list for size 16: 0,
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 16, free
block 2 at 16: size 8, allocated
block 3 at 24: size 8, allocated
block 4 at 32: size 8, allocated
block 5 at 40: size 8, free
block 6 at 48: size 16, allocated
block 8 at 64: size 64, allocated
block 16 at 128: size 128, allocated
Arena:
free list for size 8:
free list for size 16: 6,
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 16, free
block 2 at 16: size 16, allocated
block 4 at 32: size 16, allocated
block 6 at 48: size 16, free
block 8 at 64: size 64, allocated
block 16 at 128: size 128, allocated
Arena:
free list for size 8:
free list for size 16:
free list for size 32: 0,
free list for size 64:
free list for size 128: 16,
free list for size 256:
block 0 at 0: size 32, free
block 4 at 32: size 32, allocated
block 8 at 64: size 32, allocated
block 12 at 96: size 32, allocated
block 16 at 128: size 128, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 64, allocated
block 8 at 64: size 64, allocated
block 16 at 128: size 128, allocated
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 64, allocated
block 8 at 64: size 64, allocated
block 16 at 128: size 64, allocated
block 24 at 192: size 64, allocated
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64: 8,
free list for size 128: 16,
free list for size 256:
block 0 at 0: size 64, free
block 8 at 64: size 64, free
block 16 at 128: size 128, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
2019-09-30T03:04:43,176 ERROR [pool-6-thread-2] cache.TestBuddyAllocator:
Failed to allocate 3 of 255; {[256.@0]},
{[16.@0][8*@2][8*@3][8*@4][8.@5][16*@6][64*@8][128*@16]},
{[16.@0][16*@2][16*@4][16.@6][64*@8][128*@16]},
{[32.@0][32*@4][32*@8][32*@12][128.@16]}, {[64.@0][64*@8][128*@16]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]},
2019-09-30T03:04:43,177 ERROR [pool-6-thread-3] cache.TestBuddyAllocator:
Failed to allocate 3 of 255; {[256.@0]},
{[16.@0][8*@2][8*@3][8*@4][8.@5][16*@6][64*@8][128*@16]},
{[16.@0][16*@2][16*@4][16.@6][64*@8][128*@16]},
{[32.@0][32*@4][32*@8][32*@12][128.@16]}, {[64.@0][64*@8][128*@16]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]},
{code}
was:
It's more than flaky. Recently, in most of the cases it's the only failing test
in otherwise green runs.
{code}
Error Message
java.util.concurrent.ExecutionException:
org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
or an internal issue)
Stacktrace
java.lang.RuntimeException: java.util.concurrent.ExecutionException:
org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
or an internal issue)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.testMTT(TestBuddyAllocator.java:149)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at org.junit.runners.Suite.runChild(Suite.java:127)
at org.junit.runners.Suite.runChild(Suite.java:26)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:379)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:340)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:125)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:413)
Caused by: java.util.concurrent.ExecutionException:
org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
or an internal issue)
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.testMTT(TestBuddyAllocator.java:145)
... 33 more
Caused by:
org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
or an internal issue)
at
org.apache.hadoop.hive.llap.cache.BuddyAllocator.allocateMultiple(BuddyAllocator.java:454)
at
org.apache.hadoop.hive.llap.cache.BuddyAllocator.allocateMultiple(BuddyAllocator.java:299)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.allocateAndUseBuffer(TestBuddyAllocator.java:254)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.allocateUp(TestBuddyAllocator.java:231)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.access$000(TestBuddyAllocator.java:43)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator$1.call(TestBuddyAllocator.java:119)
at
org.apache.hadoop.hive.llap.cache.TestBuddyAllocator$1.call(TestBuddyAllocator.java:116)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Standard Error
2019-09-30T03:04:43,170 INFO [main] LlapIoImpl: Buddy allocator with direct
buffers; memory mapped off
/home/hiveptest/104.197.225.41-hiveptest-1/apache-github-source-source/llap-server/target/tmp/llap-5698481178354861395;
allocation sizes 8 - 256, arena size 256, total size 6144
2019-09-30T03:04:43,174 WARN [pool-6-thread-3] LlapIoImpl: Failed to allocate
[3]X[256] bytes after [15] attempt, evicted [0] bytes and partially allocated
[256] bytes
2019-09-30T03:04:43,174 WARN [pool-6-thread-2] LlapIoImpl: Failed to allocate
[3]X[256] bytes after [15] attempt, evicted [0] bytes and partially allocated
[0] bytes
2019-09-30T03:04:43,175 ERROR [pool-6-thread-2] LlapIoImpl: Failed to allocate
255; at 1 out of 3 (entire cache is fragmented and locked, or an internal issue)
2019-09-30T03:04:43,175 ERROR [pool-6-thread-3] LlapIoImpl: Failed to allocate
255; at 1 out of 3 (entire cache is fragmented and locked, or an internal issue)
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 256, allocated
Arena:
free list for size 8: 5,
free list for size 16: 0,
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 16, free
block 2 at 16: size 8, allocated
block 3 at 24: size 8, allocated
block 4 at 32: size 8, allocated
block 5 at 40: size 8, free
block 6 at 48: size 16, allocated
block 8 at 64: size 64, allocated
block 16 at 128: size 128, allocated
Arena:
free list for size 8:
free list for size 16: 6,
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 16, free
block 2 at 16: size 16, allocated
block 4 at 32: size 16, allocated
block 6 at 48: size 16, free
block 8 at 64: size 64, allocated
block 16 at 128: size 128, allocated
Arena:
free list for size 8:
free list for size 16:
free list for size 32: 0,
free list for size 64:
free list for size 128: 16,
free list for size 256:
block 0 at 0: size 32, free
block 4 at 32: size 32, allocated
block 8 at 64: size 32, allocated
block 12 at 96: size 32, allocated
block 16 at 128: size 128, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 64, allocated
block 8 at 64: size 64, allocated
block 16 at 128: size 128, allocated
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256:
block 0 at 0: size 64, allocated
block 8 at 64: size 64, allocated
block 16 at 128: size 64, allocated
block 24 at 192: size 64, allocated
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64: 8,
free list for size 128: 16,
free list for size 256:
block 0 at 0: size 64, free
block 8 at 64: size 64, free
block 16 at 128: size 128, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
Arena:
free list for size 8:
free list for size 16:
free list for size 32:
free list for size 64:
free list for size 128:
free list for size 256: 0,
block 0 at 0: size 256, free
2019-09-30T03:04:43,176 ERROR [pool-6-thread-2] cache.TestBuddyAllocator:
Failed to allocate 3 of 255; {[256.@0]},
{[16.@0][8*@2][8*@3][8*@4][8.@5][16*@6][64*@8][128*@16]},
{[16.@0][16*@2][16*@4][16.@6][64*@8][128*@16]},
{[32.@0][32*@4][32*@8][32*@12][128.@16]}, {[64.@0][64*@8][128*@16]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]},
2019-09-30T03:04:43,177 ERROR [pool-6-thread-3] cache.TestBuddyAllocator:
Failed to allocate 3 of 255; {[256.@0]},
{[16.@0][8*@2][8*@3][8*@4][8.@5][16*@6][64*@8][128*@16]},
{[16.@0][16*@2][16*@4][16.@6][64*@8][128*@16]},
{[32.@0][32*@4][32*@8][32*@12][128.@16]}, {[64.@0][64*@8][128*@16]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
{[256.@0]},
{code}
> TestBuddyAllocator.testMTT[2] is flaky
> --------------------------------------
>
> Key: HIVE-22277
> URL: https://issues.apache.org/jira/browse/HIVE-22277
> Project: Hive
> Issue Type: Bug
> Reporter: László Bodor
> Priority: Major
> Attachments: hive.log.tar.gz
>
>
> It's more than flaky. Recently, in most of the cases it's the only failing
> test in otherwise green runs.
> full hive logs: [^hive.log.tar.gz]
> {code}
> Error Message
> java.util.concurrent.ExecutionException:
> org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
> Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
> or an internal issue)
> Stacktrace
> java.lang.RuntimeException: java.util.concurrent.ExecutionException:
> org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
> Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
> or an internal issue)
> at
> org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.testMTT(TestBuddyAllocator.java:149)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
> at org.junit.runners.Suite.runChild(Suite.java:127)
> at org.junit.runners.Suite.runChild(Suite.java:26)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
> at
> org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:379)
> at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:340)
> at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:125)
> at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:413)
> Caused by: java.util.concurrent.ExecutionException:
> org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
> Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
> or an internal issue)
> at java.util.concurrent.FutureTask.report(FutureTask.java:122)
> at java.util.concurrent.FutureTask.get(FutureTask.java:192)
> at
> org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.testMTT(TestBuddyAllocator.java:145)
> ... 33 more
> Caused by:
> org.apache.hadoop.hive.common.io.Allocator$AllocatorOutOfMemoryException:
> Failed to allocate 255; at 1 out of 3 (entire cache is fragmented and locked,
> or an internal issue)
> at
> org.apache.hadoop.hive.llap.cache.BuddyAllocator.allocateMultiple(BuddyAllocator.java:454)
> at
> org.apache.hadoop.hive.llap.cache.BuddyAllocator.allocateMultiple(BuddyAllocator.java:299)
> at
> org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.allocateAndUseBuffer(TestBuddyAllocator.java:254)
> at
> org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.allocateUp(TestBuddyAllocator.java:231)
> at
> org.apache.hadoop.hive.llap.cache.TestBuddyAllocator.access$000(TestBuddyAllocator.java:43)
> at
> org.apache.hadoop.hive.llap.cache.TestBuddyAllocator$1.call(TestBuddyAllocator.java:119)
> at
> org.apache.hadoop.hive.llap.cache.TestBuddyAllocator$1.call(TestBuddyAllocator.java:116)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Standard Error
> 2019-09-30T03:04:43,170 INFO [main] LlapIoImpl: Buddy allocator with direct
> buffers; memory mapped off
> /home/hiveptest/104.197.225.41-hiveptest-1/apache-github-source-source/llap-server/target/tmp/llap-5698481178354861395;
> allocation sizes 8 - 256, arena size 256, total size 6144
> 2019-09-30T03:04:43,174 WARN [pool-6-thread-3] LlapIoImpl: Failed to
> allocate [3]X[256] bytes after [15] attempt, evicted [0] bytes and partially
> allocated [256] bytes
> 2019-09-30T03:04:43,174 WARN [pool-6-thread-2] LlapIoImpl: Failed to
> allocate [3]X[256] bytes after [15] attempt, evicted [0] bytes and partially
> allocated [0] bytes
> 2019-09-30T03:04:43,175 ERROR [pool-6-thread-2] LlapIoImpl: Failed to
> allocate 255; at 1 out of 3 (entire cache is fragmented and locked, or an
> internal issue)
> 2019-09-30T03:04:43,175 ERROR [pool-6-thread-3] LlapIoImpl: Failed to
> allocate 255; at 1 out of 3 (entire cache is fragmented and locked, or an
> internal issue)
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256:
> block 0 at 0: size 256, allocated
> Arena:
> free list for size 8: 5,
> free list for size 16: 0,
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256:
> block 0 at 0: size 16, free
> block 2 at 16: size 8, allocated
> block 3 at 24: size 8, allocated
> block 4 at 32: size 8, allocated
> block 5 at 40: size 8, free
> block 6 at 48: size 16, allocated
> block 8 at 64: size 64, allocated
> block 16 at 128: size 128, allocated
> Arena:
> free list for size 8:
> free list for size 16: 6,
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256:
> block 0 at 0: size 16, free
> block 2 at 16: size 16, allocated
> block 4 at 32: size 16, allocated
> block 6 at 48: size 16, free
> block 8 at 64: size 64, allocated
> block 16 at 128: size 128, allocated
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32: 0,
> free list for size 64:
> free list for size 128: 16,
> free list for size 256:
> block 0 at 0: size 32, free
> block 4 at 32: size 32, allocated
> block 8 at 64: size 32, allocated
> block 12 at 96: size 32, allocated
> block 16 at 128: size 128, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256:
> block 0 at 0: size 64, allocated
> block 8 at 64: size 64, allocated
> block 16 at 128: size 128, allocated
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256:
> block 0 at 0: size 64, allocated
> block 8 at 64: size 64, allocated
> block 16 at 128: size 64, allocated
> block 24 at 192: size 64, allocated
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64: 8,
> free list for size 128: 16,
> free list for size 256:
> block 0 at 0: size 64, free
> block 8 at 64: size 64, free
> block 16 at 128: size 128, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> Arena:
> free list for size 8:
> free list for size 16:
> free list for size 32:
> free list for size 64:
> free list for size 128:
> free list for size 256: 0,
> block 0 at 0: size 256, free
> 2019-09-30T03:04:43,176 ERROR [pool-6-thread-2] cache.TestBuddyAllocator:
> Failed to allocate 3 of 255; {[256.@0]},
> {[16.@0][8*@2][8*@3][8*@4][8.@5][16*@6][64*@8][128*@16]},
> {[16.@0][16*@2][16*@4][16.@6][64*@8][128*@16]},
> {[32.@0][32*@4][32*@8][32*@12][128.@16]}, {[64.@0][64*@8][128*@16]},
> {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
> {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
> {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
> {[256.@0]},
> 2019-09-30T03:04:43,177 ERROR [pool-6-thread-3] cache.TestBuddyAllocator:
> Failed to allocate 3 of 255; {[256.@0]},
> {[16.@0][8*@2][8*@3][8*@4][8.@5][16*@6][64*@8][128*@16]},
> {[16.@0][16*@2][16*@4][16.@6][64*@8][128*@16]},
> {[32.@0][32*@4][32*@8][32*@12][128.@16]}, {[64.@0][64*@8][128*@16]},
> {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
> {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
> {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]}, {[256.@0]},
> {[256.@0]},
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
