Please consider using some module like: https://www.npmjs.com/package/v8-profiler , to enable and disable profiling in runtime, without logging everything.
So just wait for the 100% CPU peak and turn on profiler for a couple of seconds. On Wed, Dec 17, 2014 at 9:09 PM, Karim Tarek < [email protected]> wrote: > > Hi Fedor! > > Thanks for the reply! i wanted to tell you that i already tried profiling > and started node with the --prof parameter but what happened is the > following a log file was created for each process and it kept increasing in > size like crazy in like 4 hours the file size reached 1.9GB so is there any > way to profile the node app at the spike time only in order to be able to > process it via the node-tick-processor ? because when i tried to read the > 1.9GB file it ran out of memory while reading it and i tried to split but > the context was lost so any suggestions? > > On Wednesday, December 17, 2014 3:55:03 PM UTC+2, Fedor Indutny wrote: >> >> Hello Karim, >> >> I'd suggest profiling the application during such spikes. It seems that >> the app is spinning in JS and doing lots of allocations (because of mmap() >> calls in strace output). >> >> Cheers, >> Fedor. >> >> >> >> On Wed, Dec 17, 2014 at 5:03 PM, Karim Tarek <[email protected]> >> wrote: >> >> We are facing a problem with our Node js (v0.10.33) application with >> CLUSTER module on ubuntu amazon EC2 (c3.large) instance that nearly >> everyday we get a 100% cpu spike (on the workers) suddenly and it never >> goes down until we restart our application the problem is that these spikes >> happens randomly and at weird times (at 3AM) when the traffic is minimal, >> when i looked at the logs i found that the logs and any APMs stopped >> recording any requests probably because the node js app stopped working, >> what i did based on what i read so far is that i did a strace on the >> process which was at 100% and here is a part of the output of my stack >> trace (full strace attached): >> >> gettimeofday({1418314214, 491214}, NULL) = 0 >> >> gettimeofday({1418314214, 491327}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> gettimeofday({1418314214, 619975}, NULL) = 0 >> gettimeofday({1418314214, 620055}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3b000000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b000000 >> mprotect(0xfcc3b005000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b006000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b006000 >> mprotect(0xfcc3b0ff000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b100000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b100000 >> mprotect(0xfcc3b105000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b106000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b106000 >> mprotect(0xfcc3b1ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314214, 731602}, NULL) = 0 >> gettimeofday({1418314214, 731682}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3b200000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b200000 >> mprotect(0xfcc3b205000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b206000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b206000 >> mprotect(0xfcc3b2ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314214, 835071}, NULL) = 0 >> gettimeofday({1418314214, 835154}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3b300000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b300000 >> mprotect(0xfcc3b305000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b306000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b306000 >> mprotect(0xfcc3b3ff000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b400000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b400000 >> mprotect(0xfcc3b405000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b406000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b406000 >> mprotect(0xfcc3b4ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314214, 938991}, NULL) = 0 >> gettimeofday({1418314214, 939072}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3b500000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b500000 >> mprotect(0xfcc3b505000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b506000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b506000 >> mprotect(0xfcc3b5ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314215, 41662}, NULL) = 0 >> gettimeofday({1418314215, 41744}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3b600000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b600000 >> mprotect(0xfcc3b605000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b606000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b606000 >> mprotect(0xfcc3b6ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314215, 161306}, NULL) = 0 >> gettimeofday({1418314215, 161408}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3b700000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b700000 >> mprotect(0xfcc3b705000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b706000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b706000 >> mprotect(0xfcc3b7ff000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b800000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b800000 >> mprotect(0xfcc3b805000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b806000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b806000 >> mprotect(0xfcc3b8ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314215, 287996}, NULL) = 0 >> gettimeofday({1418314215, 288120}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3b900000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b900000 >> mprotect(0xfcc3b905000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3b906000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3b906000 >> mprotect(0xfcc3b9ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314215, 472947}, NULL) = 0 >> gettimeofday({1418314215, 473040}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3ba00000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3ba00000 >> mprotect(0xfcc3ba05000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3ba06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3ba06000 >> mprotect(0xfcc3baff000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3bb00000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bb00000 >> mprotect(0xfcc3bb05000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3bb06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bb06000 >> mprotect(0xfcc3bbff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314215, 593480}, NULL) = 0 >> gettimeofday({1418314215, 593565}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3bc00000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bc00000 >> mprotect(0xfcc3bc05000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3bc06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bc06000 >> mprotect(0xfcc3bcff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314215, 696233}, NULL) = 0 >> gettimeofday({1418314215, 696315}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3bd00000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bd00000 >> mprotect(0xfcc3bd05000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3bd06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bd06000 >> mprotect(0xfcc3bdff000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3be00000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3be00000 >> mprotect(0xfcc3be05000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3be06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3be06000 >> mprotect(0xfcc3beff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314215, 875801}, NULL) = 0 >> gettimeofday({1418314215, 875898}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3bf00000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bf00000 >> mprotect(0xfcc3bf05000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3bf06000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3bf06000 >> mprotect(0xfcc3bfff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314216, 38282}, NULL) = 0 >> gettimeofday({1418314216, 38365}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3c000000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c000000 >> mprotect(0xfcc3c005000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c006000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c006000 >> mprotect(0xfcc3c0ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314216, 255472}, NULL) = 0 >> gettimeofday({1418314216, 255552}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3c100000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c100000 >> mprotect(0xfcc3c105000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c106000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c106000 >> mprotect(0xfcc3c1ff000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c200000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c200000 >> mprotect(0xfcc3c205000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c206000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c206000 >> mprotect(0xfcc3c2ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314216, 390157}, NULL) = 0 >> gettimeofday({1418314216, 390240}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3c300000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c300000 >> mprotect(0xfcc3c305000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c306000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c306000 >> mprotect(0xfcc3c3ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314216, 532144}, NULL) = 0 >> gettimeofday({1418314216, 532225}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3c400000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c400000 >> mprotect(0xfcc3c405000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c406000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c406000 >> mprotect(0xfcc3c4ff000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c500000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c500000 >> mprotect(0xfcc3c505000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c506000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c506000 >> mprotect(0xfcc3c5ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314216, 676703}, NULL) = 0 >> gettimeofday({1418314216, 676786}, NULL) = 0 >> futex(0x7faad80008c8, FUTEX_WAKE_PRIVATE, 1) = 1 >> mmap(0xfcc3c600000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c600000 >> mprotect(0xfcc3c605000, 4096, PROT_NONE) = 0 >> mmap(0xfcc3c606000, 1019904, PROT_READ|PROT_WRITE|PROT_EXEC, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xfcc3c606000 >> mprotect(0xfcc3c6ff000, 4096, PROT_NONE) = 0 >> gettimeofday({1418314216, 832680}, NULL) = 0 >> gettimeofday({1418314216, 845863}, NULL) = 0 >> gettimeofday({1418314216, 855742}, NULL) = 0 >> mmap(0xfcc3c600000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3c600000 >> mmap(0xfcc3c500000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3c500000 >> mmap(0xfcc3c400000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3c400000 >> mmap(0xfcc3c300000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3c300000 >> mmap(0xfcc3c200000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3c200000 >> mmap(0xfcc3c100000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3c100000 >> mmap(0xfcc3c000000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3c000000 >> mmap(0xfcc3bf00000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3bf00000 >> mmap(0xfcc3be00000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3be00000 >> mmap(0xfcc3bd00000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3bd00000 >> mmap(0xfcc3bc00000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3bc00000 >> mmap(0xfcc3bb00000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3bb00000 >> mmap(0xfcc3ba00000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3ba00000 >> mmap(0xfcc3b900000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3b900000 >> mmap(0xfcc3b800000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3b800000 >> mmap(0xfcc3b700000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3b700000 >> mmap(0xfcc3b600000, 1048576, PROT_NONE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, >> -1, 0) = 0xfcc3b600000 >> mmap(0xfcc3b500000, >> >> ... > > -- Job board: http://jobs.nodejs.org/ New group rules: https://gist.github.com/othiym23/9886289#file-moderation-policy-md Old group rules: https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines --- You received this message because you are subscribed to the Google Groups "nodejs" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To post to this group, send email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/nodejs/CAEv2VfKU6Kji4eTiZp6u88WAAe%2BwN7mqsqtwwMApdfFRiNJcgA%40mail.gmail.com. For more options, visit https://groups.google.com/d/optout.
