Hello.
I am working on a node app and found that its performance bottleneck is
logging. And digging deeper, it looks like the root bottleneck is on the
WriteStream.
I ran some quick benchmarks of the WriteStream performance, and it seems
really slow. Can someone help me understand why it is taking so long and
if I can improve the performance? Or, maybe I am doing something wrong
with my benchmarks.
I get the following output running on my MacBook:
$ node --harmony benchmarks.js
Running benchmarks with 100000 writes
fs.writeSync took 0.302 seconds
fs.write took 0.235 seconds
writeStream.write took 4.565 seconds
$ cat benchmarks.js
'use strict';
const fs = require('fs');
const NUM_WRITES = 100 * 1000;
const DUMMY_MESSAGE = 'Dummy log message that is about 100
characters.....................................................\n';
const DUMMY_MESSAGE_BUFFER = new Buffer(DUMMY_MESSAGE);
function benchmarkFileSync() {
var fd = fs.openSync('./dummy.log', 'w');
const startTime = new Date();
for (let i = 0; i < NUM_WRITES; i++) {
fs.writeSync(fd, DUMMY_MESSAGE);
}
const diff = (new Date() - startTime) / 1000;
console.log('fs.writeSync took', diff, 'seconds');
fs.closeSync(fd);
}
function benchmarkFileAsync() {
var fd = fs.openSync('./dummyAsync.log', 'w');
const startTime = new Date();
for (let i = 0; i < NUM_WRITES; i++) {
// yes, this is BAD since it isn't waiting for the callback. It
// drops some of these writes on the floor
fs.write(fd, DUMMY_MESSAGE_BUFFER, 0, DUMMY_MESSAGE_BUFFER.length,
null, function cb() {
// ignore the callback
});
}
const diff = (new Date() - startTime) / 1000;
console.log('fs.write took', diff, 'seconds');
fs.closeSync(fd);
}
function benchmarkWriteStream() {
const stream = fs.createWriteStream(
'./dummyWriteStream.log', {
encoding: "utf8",
mode: parseInt('0644', 8)
}
);
const startTime = new Date();
for(let i = 0; i < NUM_WRITES; i++) {
let ret = stream.write(DUMMY_MESSAGE);
}
const interval = setInterval(function checkIfDone() {
if (stream.bytesWritten === DUMMY_MESSAGE.length * NUM_WRITES) {
const diff = (new Date() - startTime) / 1000;
console.log('writeStream.write took', diff, 'seconds');
clearInterval(interval);
}
}, 100);
}
console.log('Running benchmarks with', NUM_WRITES, 'writes');
benchmarkFileSync();
benchmarkFileAsync();
benchmarkWriteStream();
The performance becomes extremely pathological when I increase the number
of writes beyond this. For example, with 200,000:
$ node --harmony benchmarks.js
Running benchmarks with 200000 writes
fs.writeSync took 0.646 seconds
fs.write took 0.479 seconds
writeStream.write took 290.4 seconds
Thanks,
Jeff
--
Job board: http://jobs.nodejs.org/
New group rules:
https://gist.github.com/othiym23/9886289#file-moderation-policy-md
Old group rules:
https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines
---
You received this message because you are subscribed to the Google Groups
"nodejs" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/nodejs/2a3f6115-e1f5-40cd-a821-9a326590720e%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
