[
https://issues.apache.org/jira/browse/HADOOP-10687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14028950#comment-14028950
]
Liang Xie commented on HADOOP-10687:
------------------------------------
another dump:
{code}
"IPC Reader 6 on port 11600" daemon prio=10 tid=0x00007f7ef4b7df20 nid=0x3730
waiting for monitor entry [0x00007f7e07db1000]
java.lang.Thread.State: BLOCKED (on object monitor)
at
org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1495)
- locked <0x0000000278669760> (a java.util.WeakHashMap)
at
org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
at
org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
at
org.apache.hadoop.hbase.io.HbaseObjectWritable.readObject(HbaseObjectWritable.java:688)
at org.apache.hadoop.hbase.ipc.Invocation.readFields(Invocation.java:126)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:618)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
- locked <0x0000000278424e38> (a
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
"IPC Reader 5 on port 11600" daemon prio=10 tid=0x00007f7ef4b63ac0 nid=0x372f
runnable [0x00007f7e07df2000]
java.lang.Thread.State: RUNNABLE
at
org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1495)
- locked <0x0000000278669760> (a java.util.WeakHashMap)
at
org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:94)
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
at
org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
at
org.apache.hadoop.hbase.io.HbaseObjectWritable.readObject(HbaseObjectWritable.java:688)
at org.apache.hadoop.hbase.ipc.Invocation.readFields(Invocation.java:126)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:618)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
- locked <0x0000000278434a78> (a
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
"IPC Reader 4 on port 11600" daemon prio=10 tid=0x00007f7ef4b49310 nid=0x372e
waiting for monitor entry [0x00007f7e07e33000]
java.lang.Thread.State: BLOCKED (on object monitor)
at
org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1495)
- locked <0x0000000278669760> (a java.util.WeakHashMap)
at
org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
at
org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
at
org.apache.hadoop.hbase.io.HbaseObjectWritable.readObject(HbaseObjectWritable.java:688)
at org.apache.hadoop.hbase.ipc.Invocation.readFields(Invocation.java:126)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:618)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
at
org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
- locked <0x0000000278445350> (a
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
{code}
> improve Configuration.CACHE_CLASSES concurrentcy
> ------------------------------------------------
>
> Key: HADOOP-10687
> URL: https://issues.apache.org/jira/browse/HADOOP-10687
> Project: Hadoop Common
> Issue Type: Improvement
> Components: conf
> Affects Versions: 3.0.0, 2.4.0
> Reporter: Liang Xie
> Assignee: Liang Xie
>
> In one of my high concurrent HBase in-memory read only testing scenario, the
> following strace traces show CACHE_CLASSES becoming hotspot, since there's a
> "synchronized" or "synchronizedMap", but for most of hadoop applications,
> when goes into getClassByNameOrNull(), most of code paths should expect
> there's a cached class already, that means it's a read mostly scenario, the
> write should be rare enough. So one possible optimization here is using
> concurrent weakHashMap, the MapMaker could do this.
> ps: i added more debug logging above "cache hit" and "Class.forName" and
> reran my case, shows an expected result (almost all of them go to "cache hit"
> path), and TestConfiguration also passed in my box.
> {code}
> "IPC Reader 3 on port 11600" daemon prio=10 tid=0x00007ff2fcb41190 nid=0x5db4
> runnable [0x00007ff238943000]
> java.lang.Thread.State: RUNNABLE
> at java.util.Collections$SynchronizedMap.get(Collections.java:1979)
> - locked <0x00000002788ab4a0> (a java.util.Collections$SynchronizedMap)
> at
> org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1505)
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
> at
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:617)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
> - locked <0x00000002784211e8> (a
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:662)
> "IPC Reader 1 on port 11600" daemon prio=10 tid=0x00007ff2fc8cb2b0 nid=0x5db2
> waiting for monitor entry [0x00007ff29006c000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at java.util.Collections$SynchronizedMap.get(Collections.java:1979)
> - waiting to lock <0x00000002788ab4a0> (a
> java.util.Collections$SynchronizedMap)
> at
> org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1505)
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
> at
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:617)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
> - locked <0x0000000278434980> (a
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:662)
> "IPC Reader 0 on port 11600" daemon prio=10 tid=0x00007ff2fc9b7630 nid=0x5db1
> waiting for monitor entry [0x00007ff2900ad000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at java.util.Collections$SynchronizedMap.get(Collections.java:1979)
> - waiting to lock <0x00000002788ab4a0> (a
> java.util.Collections$SynchronizedMap)
> at
> org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1505)
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
> at
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:617)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
> - locked <0x00000002784349e0> (a
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:662)
> "IPC Reader 8 on port 11600" daemon prio=10 tid=0x00007ff2fcbc5290 nid=0x5db9
> waiting for monitor entry [0x00007ff2387fe000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at java.util.Collections$SynchronizedMap.get(Collections.java:1979)
> - waiting to lock <0x00000002788ab4a0> (a
> java.util.Collections$SynchronizedMap)
> at
> org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1505)
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
> at
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:617)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
> - locked <0x0000000278424268> (a
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:662)
> "IPC Reader 6 on port 11600" daemon prio=10 tid=0x00007ff2fcb905c0 nid=0x5db7
> waiting for monitor entry [0x00007ff238880000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at java.util.Collections$SynchronizedMap.get(Collections.java:1979)
> - locked <0x00000002788ab4a0> (a java.util.Collections$SynchronizedMap)
> at
> org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1505)
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:94)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
> at
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
> at
> org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
> at
> org.apache.hadoop.hbase.io.HbaseObjectWritable.readObject(HbaseObjectWritable.java:688)
> at org.apache.hadoop.hbase.ipc.Invocation.readFields(Invocation.java:126)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:618)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
> - locked <0x00000002784456d0> (a
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> "IPC Reader 5 on port 11600" daemon prio=10 tid=0x00007ff2fcb75d70 nid=0x5db6
> waiting for monitor entry [0x00007ff2388c1000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at java.util.Collections$SynchronizedMap.get(Collections.java:1979)
> - waiting to lock <0x00000002788ab4a0> (a
> java.util.Collections$SynchronizedMap)
> at
> org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1505)
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
> at
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processData(SecureServer.java:617)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.processOneRpc(SecureServer.java:596)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.saslReadAndProcess(SecureServer.java:362)
> at
> org.apache.hadoop.hbase.ipc.SecureServer$SecureConnection.readAndProcess(SecureServer.java:492)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:775)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:566)
> - locked <0x00000002784242c8> (a
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:541)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:662)
> {code}
--
This message was sent by Atlassian JIRA
(v6.2#6252)
