1. Your async code is incorect as you close file before all write operation complete. 2. WriteStream buffers written data until other end is ready to consume data. So you actually first fill memory with data and then write it to file.
2014-05-20 20:29 GMT+03:00 Jeff Jolma <[email protected]>: > Hello. > > I am working on a node app and found that its performance bottleneck is > logging. And digging deeper, it looks like the root bottleneck is on the > WriteStream. > > I ran some quick benchmarks of the WriteStream performance, and it seems > really slow. Can someone help me understand why it is taking so long and > if I can improve the performance? Or, maybe I am doing something wrong > with my benchmarks. > > I get the following output running on my MacBook: > > > $ node --harmony benchmarks.js > Running benchmarks with 100000 writes > fs.writeSync took 0.302 seconds > fs.write took 0.235 seconds > writeStream.write took 4.565 seconds > > $ cat benchmarks.js > 'use strict'; > const fs = require('fs'); > > const NUM_WRITES = 100 * 1000; > const DUMMY_MESSAGE = 'Dummy log message that is about 100 > characters.....................................................\n'; > const DUMMY_MESSAGE_BUFFER = new Buffer(DUMMY_MESSAGE); > > function benchmarkFileSync() { > var fd = fs.openSync('./dummy.log', 'w'); > > const startTime = new Date(); > for (let i = 0; i < NUM_WRITES; i++) { > fs.writeSync(fd, DUMMY_MESSAGE); > } > const diff = (new Date() - startTime) / 1000; > console.log('fs.writeSync took', diff, 'seconds'); > fs.closeSync(fd); > } > > function benchmarkFileAsync() { > var fd = fs.openSync('./dummyAsync.log', 'w'); > > const startTime = new Date(); > for (let i = 0; i < NUM_WRITES; i++) { > // yes, this is BAD since it isn't waiting for the callback. It > // drops some of these writes on the floor > fs.write(fd, DUMMY_MESSAGE_BUFFER, 0, DUMMY_MESSAGE_BUFFER.length, > null, function cb() { > // ignore the callback > }); > } > const diff = (new Date() - startTime) / 1000; > console.log('fs.write took', diff, 'seconds'); > fs.closeSync(fd); > } > > function benchmarkWriteStream() { > const stream = fs.createWriteStream( > './dummyWriteStream.log', { > encoding: "utf8", > mode: parseInt('0644', 8) > } > ); > > const startTime = new Date(); > for(let i = 0; i < NUM_WRITES; i++) { > let ret = stream.write(DUMMY_MESSAGE); > } > > const interval = setInterval(function checkIfDone() { > if (stream.bytesWritten === DUMMY_MESSAGE.length * NUM_WRITES) { > const diff = (new Date() - startTime) / 1000; > console.log('writeStream.write took', diff, 'seconds'); > clearInterval(interval); > } > }, 100); > } > > console.log('Running benchmarks with', NUM_WRITES, 'writes'); > benchmarkFileSync(); > benchmarkFileAsync(); > benchmarkWriteStream(); > > The performance becomes extremely pathological when I increase the number > of writes beyond this. For example, with 200,000: > > $ node --harmony benchmarks.js > Running benchmarks with 200000 writes > fs.writeSync took 0.646 seconds > fs.write took 0.479 seconds > writeStream.write took 290.4 seconds > > > > Thanks, > Jeff > > -- > Job board: http://jobs.nodejs.org/ > New group rules: > https://gist.github.com/othiym23/9886289#file-moderation-policy-md > Old group rules: > https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines > --- > You received this message because you are subscribed to the Google Groups > "nodejs" group. > To unsubscribe from this group and stop receiving emails from it, send an > email to [email protected]. > To post to this group, send email to [email protected]. > To view this discussion on the web visit > https://groups.google.com/d/msgid/nodejs/2a3f6115-e1f5-40cd-a821-9a326590720e%40googlegroups.com<https://groups.google.com/d/msgid/nodejs/2a3f6115-e1f5-40cd-a821-9a326590720e%40googlegroups.com?utm_medium=email&utm_source=footer> > . > For more options, visit https://groups.google.com/d/optout. > -- Job board: http://jobs.nodejs.org/ New group rules: https://gist.github.com/othiym23/9886289#file-moderation-policy-md Old group rules: https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines --- You received this message because you are subscribed to the Google Groups "nodejs" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To post to this group, send email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/nodejs/CAP8%3DUyRCcyj1LzYS9O4AzndTa%3D2-J67wA6cGqYvnC%2BCyYQBgGw%40mail.gmail.com. For more options, visit https://groups.google.com/d/optout.
