Hi Fedor! Thanks for the reply! i wanted to tell you that i already tried profiling and started node with the --prof parameter but what happened is the following a log file was created for each process and it kept increasing in size like crazy in like 4 hours the file size reached 1.9GB so is there any way to profile the node app at the spike time only in order to be able to process it via the node-tick-processor ? because when i tried to read the 1.9GB file it ran out of memory while reading it and i tried to split but the context was lost so any suggestions?
On Wednesday, December 17, 2014 3:55:03 PM UTC+2, Fedor Indutny wrote: > > Hello Karim, > > I'd suggest profiling the application during such spikes. It seems that > the app is spinning in JS and doing lots of allocations (because of mmap() > calls in strace output). > > Cheers, > Fedor. > > > > On Wed, Dec 17, 2014 at 5:03 PM, Karim Tarek <[email protected] > <javascript:>> wrote: > > We are facing a problem with our Node js (v0.10.33) application with > CLUSTER module on ubuntu amazon EC2 (c3.large) instance that nearly > everyday we get a 100% cpu spike (on the workers) suddenly and it never > goes down until we restart our application the problem is that these spikes > happens randomly and at weird times (at 3AM) when the traffic is minimal, > when i looked at the logs i found that the logs and any APMs stopped > recording any requests probably because the node js app stopped working, > what i did based on what i read so far is that i did a strace on the > process which was at 100% and here is a part of the output of my stack > trace (full strace attached): > > gettimeofday({1418314214, 491214}, NULL) = 0 > > gettimeofday({1418314214, 491327}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > gettimeofday({1418314214, 619975}, NULL) = 0 > gettimeofday({1418314214, 620055}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3b000000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b000000 > mprotect(0xfcc3b005000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b006000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b006000 > mprotect(0xfcc3b0ff000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b100000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b100000 > mprotect(0xfcc3b105000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b106000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b106000 > mprotect(0xfcc3b1ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314214, 731602}, NULL) = 0 > gettimeofday({1418314214, 731682}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3b200000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b200000 > mprotect(0xfcc3b205000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b206000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b206000 > mprotect(0xfcc3b2ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314214, 835071}, NULL) = 0 > gettimeofday({1418314214, 835154}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3b300000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b300000 > mprotect(0xfcc3b305000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b306000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b306000 > mprotect(0xfcc3b3ff000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b400000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b400000 > mprotect(0xfcc3b405000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b406000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b406000 > mprotect(0xfcc3b4ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314214, 938991}, NULL) = 0 > gettimeofday({1418314214, 939072}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3b500000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b500000 > mprotect(0xfcc3b505000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b506000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b506000 > mprotect(0xfcc3b5ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314215, 41662}, NULL) = 0 > gettimeofday({1418314215, 41744}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3b600000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b600000 > mprotect(0xfcc3b605000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b606000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b606000 > mprotect(0xfcc3b6ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314215, 161306}, NULL) = 0 > gettimeofday({1418314215, 161408}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3b700000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b700000 > mprotect(0xfcc3b705000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b706000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b706000 > mprotect(0xfcc3b7ff000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b800000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b800000 > mprotect(0xfcc3b805000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b806000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b806000 > mprotect(0xfcc3b8ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314215, 287996}, NULL) = 0 > gettimeofday({1418314215, 288120}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3b900000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b900000 > mprotect(0xfcc3b905000, 4096, PROT_NONE) = 0 > mmap(0xfcc3b906000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b906000 > mprotect(0xfcc3b9ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314215, 472947}, NULL) = 0 > gettimeofday({1418314215, 473040}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3ba00000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3ba00000 > mprotect(0xfcc3ba05000, 4096, PROT_NONE) = 0 > mmap(0xfcc3ba06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3ba06000 > mprotect(0xfcc3baff000, 4096, PROT_NONE) = 0 > mmap(0xfcc3bb00000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bb00000 > mprotect(0xfcc3bb05000, 4096, PROT_NONE) = 0 > mmap(0xfcc3bb06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bb06000 > mprotect(0xfcc3bbff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314215, 593480}, NULL) = 0 > gettimeofday({1418314215, 593565}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3bc00000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bc00000 > mprotect(0xfcc3bc05000, 4096, PROT_NONE) = 0 > mmap(0xfcc3bc06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bc06000 > mprotect(0xfcc3bcff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314215, 696233}, NULL) = 0 > gettimeofday({1418314215, 696315}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3bd00000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bd00000 > mprotect(0xfcc3bd05000, 4096, PROT_NONE) = 0 > mmap(0xfcc3bd06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bd06000 > mprotect(0xfcc3bdff000, 4096, PROT_NONE) = 0 > mmap(0xfcc3be00000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3be00000 > mprotect(0xfcc3be05000, 4096, PROT_NONE) = 0 > mmap(0xfcc3be06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3be06000 > mprotect(0xfcc3beff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314215, 875801}, NULL) = 0 > gettimeofday({1418314215, 875898}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3bf00000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bf00000 > mprotect(0xfcc3bf05000, 4096, PROT_NONE) = 0 > mmap(0xfcc3bf06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bf06000 > mprotect(0xfcc3bfff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314216, 38282}, NULL) = 0 > gettimeofday({1418314216, 38365}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3c000000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c000000 > mprotect(0xfcc3c005000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c006000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c006000 > mprotect(0xfcc3c0ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314216, 255472}, NULL) = 0 > gettimeofday({1418314216, 255552}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3c100000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c100000 > mprotect(0xfcc3c105000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c106000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c106000 > mprotect(0xfcc3c1ff000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c200000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c200000 > mprotect(0xfcc3c205000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c206000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c206000 > mprotect(0xfcc3c2ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314216, 390157}, NULL) = 0 > gettimeofday({1418314216, 390240}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3c300000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c300000 > mprotect(0xfcc3c305000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c306000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c306000 > mprotect(0xfcc3c3ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314216, 532144}, NULL) = 0 > gettimeofday({1418314216, 532225}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3c400000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c400000 > mprotect(0xfcc3c405000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c406000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c406000 > mprotect(0xfcc3c4ff000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c500000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c500000 > mprotect(0xfcc3c505000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c506000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c506000 > mprotect(0xfcc3c5ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314216, 676703}, NULL) = 0 > gettimeofday({1418314216, 676786}, NULL) = 0 > futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 > mmap(0xfcc3c600000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c600000 > mprotect(0xfcc3c605000, 4096, PROT_NONE) = 0 > mmap(0xfcc3c606000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c606000 > mprotect(0xfcc3c6ff000, 4096, PROT_NONE) = 0 > gettimeofday({1418314216, 832680}, NULL) = 0 > gettimeofday({1418314216, 845863}, NULL) = 0 > gettimeofday({1418314216, 855742}, NULL) = 0 > mmap(0xfcc3c600000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3c600000 > mmap(0xfcc3c500000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3c500000 > mmap(0xfcc3c400000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3c400000 > mmap(0xfcc3c300000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3c300000 > mmap(0xfcc3c200000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3c200000 > mmap(0xfcc3c100000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3c100000 > mmap(0xfcc3c000000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3c000000 > mmap(0xfcc3bf00000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3bf00000 > mmap(0xfcc3be00000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3be00000 > mmap(0xfcc3bd00000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3bd00000 > mmap(0xfcc3bc00000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3bc00000 > mmap(0xfcc3bb00000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3bb00000 > mmap(0xfcc3ba00000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3ba00000 > mmap(0xfcc3b900000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3b900000 > mmap(0xfcc3b800000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3b800000 > mmap(0xfcc3b700000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3b700000 > mmap(0xfcc3b600000, 1048576, PROT_NONE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0xfcc3b600000 > mmap(0xfcc3b500000, > > ... -- Job board: http://jobs.nodejs.org/ New group rules: https://gist.github.com/othiym23/9886289#file-moderation-policy-md Old group rules: https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines --- You received this message because you are subscribed to the Google Groups "nodejs" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To post to this group, send email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/nodejs/630d59ab-5272-4365-a9cd-6bf1895de5fd%40googlegroups.com. For more options, visit https://groups.google.com/d/optout.
