Ok, so I spent the last hour running tests, you can see the gradual
deterioration:
the biggest impact seems to be json parsing.
# setup
ms = require('mongoskin')
m = { db : ms.db('localhost:27017/standard') }
# setup one K out
oneK = '';
ppp = 0;
for ppp in [0..1024]
oneK = oneK + 'x';
########################
# app.get('/user/loginA', userRoute.loginA);
# app.get('/user/loginB', userRoute.loginB);
# app.get('/user/loginC', userRoute.loginC);
# app.get('/user/loginD', userRoute.loginD);
# app.get('/user/loginE', userRoute.loginE);
# app.get('/user/loginF', userRoute.loginF);
# etc
# ab -n 1000 -c 20 "http://localhost:3000/user/loginA?name=a&password=a";
# 2205.04 [#/sec] (mean)
exports.loginA = (req, res) ->
res.send(oneK);
# ab -n 1000 -c 20 "http://localhost:3000/user/loginB?name=a&password=a";
# 439.16 [#/sec] (mean)
# when I run db in verbose mode:
# Tue Apr 10 16:56:55 [conn1] query standard.users ntoreturn:1 nreturned:1
reslen:18048 0ms
exports.loginB = (req, res) ->
m.db.collection('users').findOne(
{name:req.query.name},
(err, data) ->
res.send(oneK)
)
exports.getWithName = (name, callback) ->
m.db.collection('users').findOne(
{name:name},
callback
)
# ab -n 1000 -c 20 "http://localhost:3000/user/loginC?name=a&password=a";
# Requests per second: 403.78 [#/sec] (mean)
exports.loginC = (req, res) ->
exports.getWithName(req.query.name, (err, data) ->
res.send(oneK)
)
# ab -n 1000 -c 20 "http://localhost:3000/user/loginD?name=a&password=a";
# Requests per second: 331.83 [#/sec] (mean)
exports.loginD = (req, res) ->
exports.getWithName(req.query.name, (err, data) ->
res.send(data)
)
exports.getWithNamePassword = (name, password, callback) ->
exports.getWithName(
name,
(err, user) ->
if (user.password != password)
callback("validation failed", null)
else
callback(null, user)
)
# ab -n 1000 -c 20 "http://localhost:3000/user/loginE?name=a&password=a";
# Requests per second: 331.35 [#/sec] (mean)
exports.loginE = (req, res) ->
exports.getWithNamePassword(req.query.name, req.query.password, (err, data)
->
res.send(data)
)
# using external service
userService = require('../../service/userService')
# ab -n 1000 -c 20 "http://localhost:3000/user/loginF?name=a&password=a";
# Requests per second: 324.92 [#/sec] (mean)
exports.loginF = (req, res) ->
exports.getWithNamePassword(req.query.name, req.query.password, (err, data)
->
res.send(data)
)
exports.getShop = (callback) ->
m.db.collection('items').find({}).toArray(
(err, data) ->
pruned = (item.shop for item in data)
log.debug("getShop has " + pruned.length + " items")
callback(err, pruned)
)
# ab -n 1000 -c 20 "http://localhost:3000/user/loginG?name=a&password=a";
# Requests per second: 512.91 [#/sec] (mean)
exports.loginG = (req, res) ->
exports.getShop( (err, data) ->
res.send(data)
)
async = require('async')
# ab -n 1000 -c 20 "http://localhost:3000/user/loginH?name=a&password=a";
# Requests per second: 279.72 [#/sec] (mean)
exports.loginH = (req, res) ->
async.auto({
user: (callback) ->
userService.getWithNamePassword(req.query.name, req.query.password,
callback)
shop: (callback) ->
exports.getShop(callback)
}, (err, data) ->
res.send(oneK)
)
# ab -n 1000 -c 20 "http://localhost:3000/user/loginI?name=a&password=a";
# Requests per second: 218.37 [#/sec] (mean)
exports.loginI = (req, res) ->
async.auto({
user: (callback) ->
userService.getWithNamePassword(req.query.name, req.query.password,
callback)
shop: (callback) ->
exports.getShop(callback)
}, (err, data) ->
res.send([data.user, data.shop])
)
# ab -n 1000 -c 100 "http://localhost:3000/user/loginI?name=a&password=a";
# Requests per second: 187.07 [#/sec] (mean)
On Tuesday, April 10, 2012 4:12:40 PM UTC-4, Marak Squires wrote:
>
> 60rps for an express / mongo app seems way off. I suspect a possible flaw
> in application code itself.
>
> Perhaps there is bottleneck in application where logic is being fired on
> every incoming request which should not be. Maybe connection for mongodb is
> not being pooled. Perhaps there is performance flaw in one of the user-land
> you are using. I am not sure of what issue is, but I'm fairly certain there
> is room for improvement.
>
>
>
> On Tue, Apr 10, 2012 at 10:15 AM, timp <[email protected]> wrote:
>
>> Greetings,
>>
>> I was wondering if someone who has experience with working with scaling
>> problems could give me some insight on an issue.
>>
>> Long story:
>>
>> A)
>>
>> So- I'm making an iPhone/iPad app, which basically is a spiffy view of
>> json data provided by a server. The app makes modifications to the json,
>> the server stores them in mongodb, etc. Model view in the form of a game.
>>
>> I set one of my goals as to have the server run as cheaply as possible,
>> so that even if the app was not popular, I could keep it up with minimal
>> cost/overhead.
>>
>>
>> B)
>>
>> At first I wrote the server in symfony2. But, I'm weary of php,
>> especially when I read the documentation for the "apns" (apple push
>> notification) plugin, which basically says, "well, I wouldn't use what I
>> wrote if you have a lot of users, but you can if you want."
>>
>> C)
>>
>> So then I said, let's do "grails." Promises to be fast. I like
>> java/groovy whatever. I like that I can refactor java without wondering
>> what I'm breaking. But then the throughput was *very* bad. Let's say, 20
>> connections per second, in production war mode, to do trivial (but not
>> hello world, json blahblah blah) things.
>>
>> I've done some some work on games. 1 seconds on my modern computer is
>> like a whole day on a computer 10 years ago. It is unacceptable to have
>> only 10-20 pages a second, even if I've messed up the configuration.
>>
>> D)
>>
>> So then, this programmer next to me was talking about nodeJS. About how
>> great it is. So this last friday, I ported the server (which is all of 500
>> lines or so) from grails to nodeJS. Using all the callback stuff,
>> async.auto, etc etc.. But then I'm getting on 60 connections per second.
>> Apparently Json is actually really slow or something. I did some
>> optimization (one of which is just clipping data, which is unacceptable),
>> and profiled and got it to ~120 a second. (I'm using the most intensive
>> function as a baseline).
>>
>> E)
>> So then, I thought: "this is ridiculous." (And I became compulsive.
>> Which sucks.) And on Saturday and Sunday, I wrote a non blocking web
>> server in c++, modules and everything. (I did rip and clean code from a
>> previously personal project). And I'm getting ~2000 completions a second.
>>
>>
>> So you're reading this and saying, "duh. Custom compiled c++ will of
>> course be faster than javascript running in a generic framework. And
>> you've probably messed up configuration or something somewhere."
>>
>> I know this. I feel like I just lost the weekend to some stupid
>> compulsion. But at the same time, I am truly annoyed at how slow these web
>> servers/frameworks are.
>>
>>
>> ------------------------
>> ------------------------
>> ------------------------
>>
>>
>> So my question is this:
>>
>> How do real scaling companies deal with this problem?
>> Non cacheable non static page server.
>>
>> Does scaling become a business decision?
>>
>> Are things really slow, just because they can be? Why do people think
>> "nodeJS fast" ?
>>
>> Or do real scaling companies have C/C++ services, which accept
>> connections from a front end in nodeJS. (because when nodeJS doesn't
>> actually do anything, it really is only 1/6 slower than pure C/C++
>> solution, which, I guess, is pretty fast.)?
>>
>> How close is v8 to a theoretical maximum?
>> (theoretical maximum would be, I guess, compiled code is near gcc -O3,
>> and compilation is linear with a small coefficient.)
>>
>> Why aren't web server pages through lvmm? It should be possible to
>> create a fibered (which is really what continuations come down to it seems
>> to me at this moment), lvmm, jitter, any language (which lvmm supports),
>> solution to a web page.
>>
>> ----
>>
>> So, if anyone has any thoughts, let me know. If I've said anything which
>> may be rude, I am not trying to be.
>> In previous experience, I was never concerned with speed, because I was
>> never before, directly paying for it.
>>
>> I guess I'll run the server on nodeJS, and just load balance it if I ever
>> need to.
>> Although, I would very much rather have a lvmm'd pages plugged into a
>> C/C++ server.
>>
>> ----
>>
>> Thoughts? I this all academic: if things are popular, you will make
>> money, and scaling will becoming a business issue?
>>
>>
>> -tim
>>
>> --
>> Job Board: http://jobs.nodejs.org/
>> Posting guidelines:
>> https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines
>> You received this message because you are subscribed to the Google
>> Groups "nodejs" group.
>> To post to this group, send email to [email protected]
>> To unsubscribe from this group, send email to
>> [email protected]
>> For more options, visit this group at
>> http://groups.google.com/group/nodejs?hl=en?hl=en
>>
>
>
>
> --
> --
> Marak Squires
> Co-founder and Chief Evangelist
> Nodejitsu, Inc.
> [email protected]
>
>
On Tuesday, April 10, 2012 4:12:40 PM UTC-4, Marak Squires wrote:
>
> 60rps for an express / mongo app seems way off. I suspect a possible flaw
> in application code itself.
>
> Perhaps there is bottleneck in application where logic is being fired on
> every incoming request which should not be. Maybe connection for mongodb is
> not being pooled. Perhaps there is performance flaw in one of the user-land
> you are using. I am not sure of what issue is, but I'm fairly certain there
> is room for improvement.
>
>
>
> On Tue, Apr 10, 2012 at 10:15 AM, timp <[email protected]> wrote:
>
>> Greetings,
>>
>> I was wondering if someone who has experience with working with scaling
>> problems could give me some insight on an issue.
>>
>> Long story:
>>
>> A)
>>
>> So- I'm making an iPhone/iPad app, which basically is a spiffy view of
>> json data provided by a server. The app makes modifications to the json,
>> the server stores them in mongodb, etc. Model view in the form of a game.
>>
>> I set one of my goals as to have the server run as cheaply as possible,
>> so that even if the app was not popular, I could keep it up with minimal
>> cost/overhead.
>>
>>
>> B)
>>
>> At first I wrote the server in symfony2. But, I'm weary of php,
>> especially when I read the documentation for the "apns" (apple push
>> notification) plugin, which basically says, "well, I wouldn't use what I
>> wrote if you have a lot of users, but you can if you want."
>>
>> C)
>>
>> So then I said, let's do "grails." Promises to be fast. I like
>> java/groovy whatever. I like that I can refactor java without wondering
>> what I'm breaking. But then the throughput was *very* bad. Let's say, 20
>> connections per second, in production war mode, to do trivial (but not
>> hello world, json blahblah blah) things.
>>
>> I've done some some work on games. 1 seconds on my modern computer is
>> like a whole day on a computer 10 years ago. It is unacceptable to have
>> only 10-20 pages a second, even if I've messed up the configuration.
>>
>> D)
>>
>> So then, this programmer next to me was talking about nodeJS. About how
>> great it is. So this last friday, I ported the server (which is all of 500
>> lines or so) from grails to nodeJS. Using all the callback stuff,
>> async.auto, etc etc.. But then I'm getting on 60 connections per second.
>> Apparently Json is actually really slow or something. I did some
>> optimization (one of which is just clipping data, which is unacceptable),
>> and profiled and got it to ~120 a second. (I'm using the most intensive
>> function as a baseline).
>>
>> E)
>> So then, I thought: "this is ridiculous." (And I became compulsive.
>> Which sucks.) And on Saturday and Sunday, I wrote a non blocking web
>> server in c++, modules and everything. (I did rip and clean code from a
>> previously personal project). And I'm getting ~2000 completions a second.
>>
>>
>> So you're reading this and saying, "duh. Custom compiled c++ will of
>> course be faster than javascript running in a generic framework. And
>> you've probably messed up configuration or something somewhere."
>>
>> I know this. I feel like I just lost the weekend to some stupid
>> compulsion. But at the same time, I am truly annoyed at how slow these web
>> servers/frameworks are.
>>
>>
>> ------------------------
>> ------------------------
>> ------------------------
>>
>>
>> So my question is this:
>>
>> How do real scaling companies deal with this problem?
>> Non cacheable non static page server.
>>
>> Does scaling become a business decision?
>>
>> Are things really slow, just because they can be? Why do people think
>> "nodeJS fast" ?
>>
>> Or do real scaling companies have C/C++ services, which accept
>> connections from a front end in nodeJS. (because when nodeJS doesn't
>> actually do anything, it really is only 1/6 slower than pure C/C++
>> solution, which, I guess, is pretty fast.)?
>>
>> How close is v8 to a theoretical maximum?
>> (theoretical maximum would be, I guess, compiled code is near gcc -O3,
>> and compilation is linear with a small coefficient.)
>>
>> Why aren't web server pages through lvmm? It should be possible to
>> create a fibered (which is really what continuations come down to it seems
>> to me at this moment), lvmm, jitter, any language (which lvmm supports),
>> solution to a web page.
>>
>> ----
>>
>> So, if anyone has any thoughts, let me know. If I've said anything which
>> may be rude, I am not trying to be.
>> In previous experience, I was never concerned with speed, because I was
>> never before, directly paying for it.
>>
>> I guess I'll run the server on nodeJS, and just load balance it if I ever
>> need to.
>> Although, I would very much rather have a lvmm'd pages plugged into a
>> C/C++ server.
>>
>> ----
>>
>> Thoughts? I this all academic: if things are popular, you will make
>> money, and scaling will becoming a business issue?
>>
>>
>> -tim
>>
>> --
>> Job Board: http://jobs.nodejs.org/
>> Posting guidelines:
>> https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines
>> You received this message because you are subscribed to the Google
>> Groups "nodejs" group.
>> To post to this group, send email to [email protected]
>> To unsubscribe from this group, send email to
>> [email protected]
>> For more options, visit this group at
>> http://groups.google.com/group/nodejs?hl=en?hl=en
>>
>
>
>
> --
> --
> Marak Squires
> Co-founder and Chief Evangelist
> Nodejitsu, Inc.
> [email protected]
>
>
--
Job Board: http://jobs.nodejs.org/
Posting guidelines:
https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines
You received this message because you are subscribed to the Google
Groups "nodejs" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to
[email protected]
For more options, visit this group at
http://groups.google.com/group/nodejs?hl=en?hl=en
