[ https://issues.apache.org/jira/browse/KAFKA-6343?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16285862#comment-16285862 ]
Alex Dunayevsky edited comment on KAFKA-6343 at 12/11/17 1:02 PM: ------------------------------------------------------------------ Manikumar, sure {code:java} $ ulimit -a core file size (blocks, -c) 0 data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 3093521 max locked memory (kbytes, -l) 64 max memory size (kbytes, -m) unlimited open files (-n) 256000 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 8192 cpu time (seconds, -t) unlimited max user processes (-u) 256000 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited $ ps -ef | grep Kafka pprbusr 30147 1 99 15:44 pts/4 00:02:31 java -Xmx512G -Xms512G -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=/home/pprbusr/kafka/bin/../logs -Dlog4j.configuration=file:/home/pprbusr/kafka/config/log4j.properties -Dcom.sun.management.config.file=/home/pprbusr/kafka/config/jmx/kafka_jmx.properties kafka.Kafka config/server.properties {code} was (Author: alex.dunayevsky): Manikumar, sure {code:bash} $ ulimit -a core file size (blocks, -c) 0 data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 3093521 max locked memory (kbytes, -l) 64 max memory size (kbytes, -m) unlimited open files (-n) 256000 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 8192 cpu time (seconds, -t) unlimited max user processes (-u) 256000 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited $ ps -ef | grep Kafka pprbusr 30147 1 99 15:44 pts/4 00:02:31 java -Xmx512G -Xms512G -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=/home/pprbusr/kafka/bin/../logs -Dlog4j.configuration=file:/home/pprbusr/kafka/config/log4j.properties -Dcom.sun.management.config.file=/home/pprbusr/kafka/config/jmx/kafka_jmx.properties kafka.Kafka config/server.properties {code} > OOM as the result of creation of 5k topics > ------------------------------------------ > > Key: KAFKA-6343 > URL: https://issues.apache.org/jira/browse/KAFKA-6343 > Project: Kafka > Issue Type: Bug > Components: core > Affects Versions: 0.10.1.1 > Environment: RHEL 7, RAM 755GB per host > Reporter: Alex Dunayevsky > > *Reproducing*: Create 5k topics *from the code* quickly, without any delays. > Wait until brokers will finish loading them. This will actually never happen, > since all brokers will go down one by one after approx 10-15 minutes or more, > depending on the hardware. > *Heap*: -Xmx/Xms: 5G, 10G, 50G, 256G > > *Topology*: 3 brokers, 3 zk. > *Code for 5k topic creation:* > {code:java} > package kafka > import kafka.admin.AdminUtils > import kafka.utils.{Logging, ZkUtils} > object TestCreateTopics extends App with Logging { > val zkConnect = "grid978:2185" > var zkUtils = ZkUtils(zkConnect, 6000, 6000, isZkSecurityEnabled = false) > for (topic <- 1 to 5000) { > AdminUtils.createTopic( > topic = s"${topic.toString}", > partitions = 10, > replicationFactor = 2, > zkUtils = zkUtils > ) > logger.info(s"Created topic ${topic.toString}") > } > } > {code} > *Cause of death:* > {code:java} > java.io.IOException: Map failed > at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:920) > at kafka.log.AbstractIndex.<init>(AbstractIndex.scala:61) > at kafka.log.OffsetIndex.<init>(OffsetIndex.scala:52) > at kafka.log.LogSegment.<init>(LogSegment.scala:67) > at kafka.log.Log.loadSegments(Log.scala:255) > at kafka.log.Log.<init>(Log.scala:108) > at kafka.log.LogManager.createLog(LogManager.scala:362) > at kafka.cluster.Partition.getOrCreateReplica(Partition.scala:94) > at > kafka.cluster.Partition$$anonfun$4$$anonfun$apply$2.apply(Partition.scala:174) > at > kafka.cluster.Partition$$anonfun$4$$anonfun$apply$2.apply(Partition.scala:174) > at scala.collection.mutable.HashSet.foreach(HashSet.scala:78) > at kafka.cluster.Partition$$anonfun$4.apply(Partition.scala:174) > at kafka.cluster.Partition$$anonfun$4.apply(Partition.scala:168) > at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) > at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:242) > at kafka.cluster.Partition.makeLeader(Partition.scala:168) > at > kafka.server.ReplicaManager$$anonfun$makeLeaders$4.apply(ReplicaManager.scala:758) > at > kafka.server.ReplicaManager$$anonfun$makeLeaders$4.apply(ReplicaManager.scala:757) > at > scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99) > at > scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99) > at > scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:230) > at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) > at scala.collection.mutable.HashMap.foreach(HashMap.scala:99) > at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:757) > at > kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:703) > at > kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:148) > at kafka.server.KafkaApis.handle(KafkaApis.scala:82) > at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.lang.OutOfMemoryError: Map failed > at sun.nio.ch.FileChannelImpl.map0(Native Method) > at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:917) > ... 28 more > {code} > Broker restart results the same OOM issues. All brokers will not be able to > start again. -- This message was sent by Atlassian JIRA (v6.4.14#64029)