or BSON, sorry. I assume it must parse the db results, and then it must JSON them into the result.
On Tuesday, April 10, 2012 5:24:45 PM UTC-4, Marak Squires wrote: > > Where in this code is JSON being parsed? > > Also, running apache bench from your local to your local node app is not > going to give you accurate results. It will place you in > an approximate ballpark, but if your application is built correctly, your > local system and bench will both cap out before node does. > > > > On Tue, Apr 10, 2012 at 2:13 PM, timp <[email protected]> wrote: > >> Ok, so I spent the last hour running tests, you can see the gradual >> deterioration: >> the biggest impact seems to be json parsing. >> >> >> >> # setup >> ms = require('mongoskin') >> m = { db : ms.db('localhost:27017/standard') } >> >> # setup one K out >> oneK = ''; >> ppp = 0; >> for ppp in [0..1024] >> oneK = oneK + 'x'; >> ######################## >> # app.get('/user/loginA', userRoute.loginA); >> # app.get('/user/loginB', userRoute.loginB); >> # app.get('/user/loginC', userRoute.loginC); >> # app.get('/user/loginD', userRoute.loginD); >> # app.get('/user/loginE', userRoute.loginE); >> # app.get('/user/loginF', userRoute.loginF); >> # etc >> >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginA?name=a&password=a"; >> # 2205.04 [#/sec] (mean) >> exports.loginA = (req, res) -> >> res.send(oneK); >> >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginB?name=a&password=a"; >> # 439.16 [#/sec] (mean) >> # when I run db in verbose mode: >> # Tue Apr 10 16:56:55 [conn1] query standard.users ntoreturn:1 >> nreturned:1 reslen:18048 0ms >> exports.loginB = (req, res) -> >> m.db.collection('users').findOne( >> {name:req.query.name}, >> (err, data) -> >> res.send(oneK) >> ) >> >> exports.getWithName = (name, callback) -> >> m.db.collection('users').findOne( >> {name:name}, >> callback >> ) >> >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginC?name=a&password=a"; >> # Requests per second: 403.78 [#/sec] (mean) >> exports.loginC = (req, res) -> >> exports.getWithName(req.query.name, (err, data) -> >> res.send(oneK) >> ) >> >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginD?name=a&password=a"; >> # Requests per second: 331.83 [#/sec] (mean) >> exports.loginD = (req, res) -> >> exports.getWithName(req.query.name, (err, data) -> >> res.send(data) >> ) >> >> exports.getWithNamePassword = (name, password, callback) -> >> exports.getWithName( >> name, >> (err, user) -> >> if (user.password != password) >> callback("validation failed", null) >> else >> callback(null, user) >> ) >> >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginE?name=a&password=a"; >> # Requests per second: 331.35 [#/sec] (mean) >> exports.loginE = (req, res) -> >> exports.getWithNamePassword(req.query.name, req.query.password, (err, >> data) -> >> res.send(data) >> ) >> >> # using external service >> userService = require('../../service/userService') >> >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginF?name=a&password=a"; >> # Requests per second: 324.92 [#/sec] (mean) >> exports.loginF = (req, res) -> >> exports.getWithNamePassword(req.query.name, req.query.password, (err, >> data) -> >> res.send(data) >> ) >> >> >> exports.getShop = (callback) -> >> m.db.collection('items').find({}).toArray( >> (err, data) -> >> pruned = (item.shop for item in data) >> log.debug("getShop has " + pruned.length + " items") >> callback(err, pruned) >> ) >> >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginG?name=a&password=a"; >> # Requests per second: 512.91 [#/sec] (mean) >> exports.loginG = (req, res) -> >> exports.getShop( (err, data) -> >> res.send(data) >> ) >> >> async = require('async') >> >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginH?name=a&password=a"; >> # Requests per second: 279.72 [#/sec] (mean) >> exports.loginH = (req, res) -> >> async.auto({ >> user: (callback) -> >> userService.getWithNamePassword(req.query.name, req.query.password, >> callback) >> shop: (callback) -> >> exports.getShop(callback) >> }, (err, data) -> >> res.send(oneK) >> ) >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginI?name=a&password=a"; >> # Requests per second: 218.37 [#/sec] (mean) >> exports.loginI = (req, res) -> >> async.auto({ >> user: (callback) -> >> userService.getWithNamePassword(req.query.name, req.query.password, >> callback) >> shop: (callback) -> >> exports.getShop(callback) >> }, (err, data) -> >> res.send([data.user, data.shop]) >> ) >> # ab -n 1000 -c 100 "http://localhost:3000/user/loginI?name=a&password=a"; >> # Requests per second: 187.07 [#/sec] (mean) >> >> >> >> >> >> On Tuesday, April 10, 2012 4:12:40 PM UTC-4, Marak Squires wrote: >>> >>> 60rps for an express / mongo app seems way off. I suspect a possible >>> flaw in application code itself. >>> >>> Perhaps there is bottleneck in application where logic is being fired on >>> every incoming request which should not be. Maybe connection for mongodb is >>> not being pooled. Perhaps there is performance flaw in one of the user-land >>> you are using. I am not sure of what issue is, but I'm fairly certain there >>> is room for improvement. >>> >>> >>> >>> On Tue, Apr 10, 2012 at 10:15 AM, timp <[email protected]> wrote: >>> >>>> Greetings, >>>> >>>> I was wondering if someone who has experience with working with scaling >>>> problems could give me some insight on an issue. >>>> >>>> Long story: >>>> >>>> A) >>>> >>>> So- I'm making an iPhone/iPad app, which basically is a spiffy view of >>>> json data provided by a server. The app makes modifications to the json, >>>> the server stores them in mongodb, etc. Model view in the form of a game. >>>> >>>> I set one of my goals as to have the server run as cheaply as possible, >>>> so that even if the app was not popular, I could keep it up with minimal >>>> cost/overhead. >>>> >>>> >>>> B) >>>> >>>> At first I wrote the server in symfony2. But, I'm weary of php, >>>> especially when I read the documentation for the "apns" (apple push >>>> notification) plugin, which basically says, "well, I wouldn't use what I >>>> wrote if you have a lot of users, but you can if you want." >>>> >>>> C) >>>> >>>> So then I said, let's do "grails." Promises to be fast. I like >>>> java/groovy whatever. I like that I can refactor java without wondering >>>> what I'm breaking. But then the throughput was *very* bad. Let's say, >>>> 20 >>>> connections per second, in production war mode, to do trivial (but not >>>> hello world, json blahblah blah) things. >>>> >>>> I've done some some work on games. 1 seconds on my modern computer is >>>> like a whole day on a computer 10 years ago. It is unacceptable to have >>>> only 10-20 pages a second, even if I've messed up the configuration. >>>> >>>> D) >>>> >>>> So then, this programmer next to me was talking about nodeJS. About >>>> how great it is. So this last friday, I ported the server (which is all >>>> of >>>> 500 lines or so) from grails to nodeJS. Using all the callback stuff, >>>> async.auto, etc etc.. But then I'm getting on 60 connections per >>>> second. >>>> Apparently Json is actually really slow or something. I did some >>>> optimization (one of which is just clipping data, which is unacceptable), >>>> and profiled and got it to ~120 a second. (I'm using the most intensive >>>> function as a baseline). >>>> >>>> E) >>>> So then, I thought: "this is ridiculous." (And I became compulsive. >>>> Which sucks.) And on Saturday and Sunday, I wrote a non blocking web >>>> server in c++, modules and everything. (I did rip and clean code from a >>>> previously personal project). And I'm getting ~2000 completions a second. >>>> >>>> >>>> So you're reading this and saying, "duh. Custom compiled c++ will of >>>> course be faster than javascript running in a generic framework. And >>>> you've probably messed up configuration or something somewhere." >>>> >>>> I know this. I feel like I just lost the weekend to some stupid >>>> compulsion. But at the same time, I am truly annoyed at how slow these web >>>> servers/frameworks are. >>>> >>>> >>>> ------------------------ >>>> ------------------------ >>>> ------------------------ >>>> >>>> >>>> So my question is this: >>>> >>>> How do real scaling companies deal with this problem? >>>> Non cacheable non static page server. >>>> >>>> Does scaling become a business decision? >>>> >>>> Are things really slow, just because they can be? Why do people think >>>> "nodeJS fast" ? >>>> >>>> Or do real scaling companies have C/C++ services, which accept >>>> connections from a front end in nodeJS. (because when nodeJS doesn't >>>> actually do anything, it really is only 1/6 slower than pure C/C++ >>>> solution, which, I guess, is pretty fast.)? >>>> >>>> How close is v8 to a theoretical maximum? >>>> (theoretical maximum would be, I guess, compiled code is near gcc -O3, >>>> and compilation is linear with a small coefficient.) >>>> >>>> Why aren't web server pages through lvmm? It should be possible to >>>> create a fibered (which is really what continuations come down to it seems >>>> to me at this moment), lvmm, jitter, any language (which lvmm supports), >>>> solution to a web page. >>>> >>>> ---- >>>> >>>> So, if anyone has any thoughts, let me know. If I've said anything >>>> which may be rude, I am not trying to be. >>>> In previous experience, I was never concerned with speed, because I was >>>> never before, directly paying for it. >>>> >>>> I guess I'll run the server on nodeJS, and just load balance it if I >>>> ever need to. >>>> Although, I would very much rather have a lvmm'd pages plugged into a >>>> C/C++ server. >>>> >>>> ---- >>>> >>>> Thoughts? I this all academic: if things are popular, you will make >>>> money, and scaling will becoming a business issue? >>>> >>>> >>>> -tim >>>> >>>> -- >>>> Job Board: http://jobs.nodejs.org/ >>>> Posting guidelines: https://github.com/joyent/**node/wiki/Mailing-List- >>>> **Posting-Guidelines<https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines> >>>> You received this message because you are subscribed to the Google >>>> Groups "nodejs" group. >>>> To post to this group, send email to [email protected] >>>> To unsubscribe from this group, send email to >>>> nodejs+unsubscribe@**googlegroups.com<nodejs%[email protected]> >>>> For more options, visit this group at >>>> http://groups.google.com/**group/nodejs?hl=en?hl=en<http://groups.google.com/group/nodejs?hl=en?hl=en> >>>> >>> >>> >>> >>> -- >>> -- >>> Marak Squires >>> Co-founder and Chief Evangelist >>> Nodejitsu, Inc. >>> [email protected] >>> >>> >> On Tuesday, April 10, 2012 4:12:40 PM UTC-4, Marak Squires wrote: >>> >>> 60rps for an express / mongo app seems way off. I suspect a possible >>> flaw in application code itself. >>> >>> Perhaps there is bottleneck in application where logic is being fired on >>> every incoming request which should not be. Maybe connection for mongodb is >>> not being pooled. Perhaps there is performance flaw in one of the user-land >>> you are using. I am not sure of what issue is, but I'm fairly certain there >>> is room for improvement. >>> >>> >>> >>> On Tue, Apr 10, 2012 at 10:15 AM, timp <[email protected]> wrote: >>> >>>> Greetings, >>>> >>>> I was wondering if someone who has experience with working with scaling >>>> problems could give me some insight on an issue. >>>> >>>> Long story: >>>> >>>> A) >>>> >>>> So- I'm making an iPhone/iPad app, which basically is a spiffy view of >>>> json data provided by a server. The app makes modifications to the json, >>>> the server stores them in mongodb, etc. Model view in the form of a game. >>>> >>>> I set one of my goals as to have the server run as cheaply as possible, >>>> so that even if the app was not popular, I could keep it up with minimal >>>> cost/overhead. >>>> >>>> >>>> B) >>>> >>>> At first I wrote the server in symfony2. But, I'm weary of php, >>>> especially when I read the documentation for the "apns" (apple push >>>> notification) plugin, which basically says, "well, I wouldn't use what I >>>> wrote if you have a lot of users, but you can if you want." >>>> >>>> C) >>>> >>>> So then I said, let's do "grails." Promises to be fast. I like >>>> java/groovy whatever. I like that I can refactor java without wondering >>>> what I'm breaking. But then the throughput was *very* bad. Let's say, >>>> 20 >>>> connections per second, in production war mode, to do trivial (but not >>>> hello world, json blahblah blah) things. >>>> >>>> I've done some some work on games. 1 seconds on my modern computer is >>>> like a whole day on a computer 10 years ago. It is unacceptable to have >>>> only 10-20 pages a second, even if I've messed up the configuration. >>>> >>>> D) >>>> >>>> So then, this programmer next to me was talking about nodeJS. About >>>> how great it is. So this last friday, I ported the server (which is all >>>> of >>>> 500 lines or so) from grails to nodeJS. Using all the callback stuff, >>>> async.auto, etc etc.. But then I'm getting on 60 connections per >>>> second. >>>> Apparently Json is actually really slow or something. I did some >>>> optimization (one of which is just clipping data, which is unacceptable), >>>> and profiled and got it to ~120 a second. (I'm using the most intensive >>>> function as a baseline). >>>> >>>> E) >>>> So then, I thought: "this is ridiculous." (And I became compulsive. >>>> Which sucks.) And on Saturday and Sunday, I wrote a non blocking web >>>> server in c++, modules and everything. (I did rip and clean code from a >>>> previously personal project). And I'm getting ~2000 completions a second. >>>> >>>> >>>> So you're reading this and saying, "duh. Custom compiled c++ will of >>>> course be faster than javascript running in a generic framework. And >>>> you've probably messed up configuration or something somewhere." >>>> >>>> I know this. I feel like I just lost the weekend to some stupid >>>> compulsion. But at the same time, I am truly annoyed at how slow these web >>>> servers/frameworks are. >>>> >>>> >>>> ------------------------ >>>> ------------------------ >>>> ------------------------ >>>> >>>> >>>> So my question is this: >>>> >>>> How do real scaling companies deal with this problem? >>>> Non cacheable non static page server. >>>> >>>> Does scaling become a business decision? >>>> >>>> Are things really slow, just because they can be? Why do people think >>>> "nodeJS fast" ? >>>> >>>> Or do real scaling companies have C/C++ services, which accept >>>> connections from a front end in nodeJS. (because when nodeJS doesn't >>>> actually do anything, it really is only 1/6 slower than pure C/C++ >>>> solution, which, I guess, is pretty fast.)? >>>> >>>> How close is v8 to a theoretical maximum? >>>> (theoretical maximum would be, I guess, compiled code is near gcc -O3, >>>> and compilation is linear with a small coefficient.) >>>> >>>> Why aren't web server pages through lvmm? It should be possible to >>>> create a fibered (which is really what continuations come down to it seems >>>> to me at this moment), lvmm, jitter, any language (which lvmm supports), >>>> solution to a web page. >>>> >>>> ---- >>>> >>>> So, if anyone has any thoughts, let me know. If I've said anything >>>> which may be rude, I am not trying to be. >>>> In previous experience, I was never concerned with speed, because I was >>>> never before, directly paying for it. >>>> >>>> I guess I'll run the server on nodeJS, and just load balance it if I >>>> ever need to. >>>> Although, I would very much rather have a lvmm'd pages plugged into a >>>> C/C++ server. >>>> >>>> ---- >>>> >>>> Thoughts? I this all academic: if things are popular, you will make >>>> money, and scaling will becoming a business issue? >>>> >>>> >>>> -tim >>>> >>>> -- >>>> Job Board: http://jobs.nodejs.org/ >>>> Posting guidelines: https://github.com/joyent/**node/wiki/Mailing-List- >>>> **Posting-Guidelines<https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines> >>>> You received this message because you are subscribed to the Google >>>> Groups "nodejs" group. >>>> To post to this group, send email to [email protected] >>>> To unsubscribe from this group, send email to >>>> nodejs+unsubscribe@**googlegroups.com<nodejs%[email protected]> >>>> For more options, visit this group at >>>> http://groups.google.com/**group/nodejs?hl=en?hl=en<http://groups.google.com/group/nodejs?hl=en?hl=en> >>>> >>> >>> >>> >>> -- >>> -- >>> Marak Squires >>> Co-founder and Chief Evangelist >>> Nodejitsu, Inc. >>> [email protected] >>> >>> -- >> Job Board: http://jobs.nodejs.org/ >> Posting guidelines: >> https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines >> You received this message because you are subscribed to the Google >> Groups "nodejs" group. >> To post to this group, send email to [email protected] >> To unsubscribe from this group, send email to >> [email protected] >> For more options, visit this group at >> http://groups.google.com/group/nodejs?hl=en?hl=en >> > > > > -- > -- > Marak Squires > Co-founder and Chief Evangelist > Nodejitsu, Inc. > [email protected] > > -- Job Board: http://jobs.nodejs.org/ Posting guidelines: https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines You received this message because you are subscribed to the Google Groups "nodejs" group. To post to this group, send email to [email protected] To unsubscribe from this group, send email to [email protected] For more options, visit this group at http://groups.google.com/group/nodejs?hl=en?hl=en
