by "parse the db results", I mean, the db has a communications format, and this must be transformed into whatever javascript is doing. by "JSON them into the result", I mean, convert the javascript objects into a textual representation
On Tuesday, April 10, 2012 5:28:48 PM UTC-4, timp wrote: > > or BSON, sorry. > > I assume it must parse the db results, > and then it must JSON them into the result. > > > > On Tuesday, April 10, 2012 5:24:45 PM UTC-4, Marak Squires wrote: >> >> Where in this code is JSON being parsed? >> >> Also, running apache bench from your local to your local node app is not >> going to give you accurate results. It will place you in >> an approximate ballpark, but if your application is built correctly, your >> local system and bench will both cap out before node does. >> >> >> >> On Tue, Apr 10, 2012 at 2:13 PM, timp <[email protected]> wrote: >> >>> Ok, so I spent the last hour running tests, you can see the gradual >>> deterioration: >>> the biggest impact seems to be json parsing. >>> >>> >>> >>> # setup >>> ms = require('mongoskin') >>> m = { db : ms.db('localhost:27017/standard') } >>> >>> # setup one K out >>> oneK = ''; >>> ppp = 0; >>> for ppp in [0..1024] >>> oneK = oneK + 'x'; >>> ######################## >>> # app.get('/user/loginA', userRoute.loginA); >>> # app.get('/user/loginB', userRoute.loginB); >>> # app.get('/user/loginC', userRoute.loginC); >>> # app.get('/user/loginD', userRoute.loginD); >>> # app.get('/user/loginE', userRoute.loginE); >>> # app.get('/user/loginF', userRoute.loginF); >>> # etc >>> >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginA?name=a&password=a"; >>> # 2205.04 [#/sec] (mean) >>> exports.loginA = (req, res) -> >>> res.send(oneK); >>> >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginB?name=a&password=a"; >>> # 439.16 [#/sec] (mean) >>> # when I run db in verbose mode: >>> # Tue Apr 10 16:56:55 [conn1] query standard.users ntoreturn:1 >>> nreturned:1 reslen:18048 0ms >>> exports.loginB = (req, res) -> >>> m.db.collection('users').findOne( >>> {name:req.query.name}, >>> (err, data) -> >>> res.send(oneK) >>> ) >>> >>> exports.getWithName = (name, callback) -> >>> m.db.collection('users').findOne( >>> {name:name}, >>> callback >>> ) >>> >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginC?name=a&password=a"; >>> # Requests per second: 403.78 [#/sec] (mean) >>> exports.loginC = (req, res) -> >>> exports.getWithName(req.query.name, (err, data) -> >>> res.send(oneK) >>> ) >>> >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginD?name=a&password=a"; >>> # Requests per second: 331.83 [#/sec] (mean) >>> exports.loginD = (req, res) -> >>> exports.getWithName(req.query.name, (err, data) -> >>> res.send(data) >>> ) >>> >>> exports.getWithNamePassword = (name, password, callback) -> >>> exports.getWithName( >>> name, >>> (err, user) -> >>> if (user.password != password) >>> callback("validation failed", null) >>> else >>> callback(null, user) >>> ) >>> >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginE?name=a&password=a"; >>> # Requests per second: 331.35 [#/sec] (mean) >>> exports.loginE = (req, res) -> >>> exports.getWithNamePassword(req.query.name, req.query.password, (err, >>> data) -> >>> res.send(data) >>> ) >>> >>> # using external service >>> userService = require('../../service/userService') >>> >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginF?name=a&password=a"; >>> # Requests per second: 324.92 [#/sec] (mean) >>> exports.loginF = (req, res) -> >>> exports.getWithNamePassword(req.query.name, req.query.password, (err, >>> data) -> >>> res.send(data) >>> ) >>> >>> >>> exports.getShop = (callback) -> >>> m.db.collection('items').find({}).toArray( >>> (err, data) -> >>> pruned = (item.shop for item in data) >>> log.debug("getShop has " + pruned.length + " items") >>> callback(err, pruned) >>> ) >>> >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginG?name=a&password=a"; >>> # Requests per second: 512.91 [#/sec] (mean) >>> exports.loginG = (req, res) -> >>> exports.getShop( (err, data) -> >>> res.send(data) >>> ) >>> >>> async = require('async') >>> >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginH?name=a&password=a"; >>> # Requests per second: 279.72 [#/sec] (mean) >>> exports.loginH = (req, res) -> >>> async.auto({ >>> user: (callback) -> >>> userService.getWithNamePassword(req.query.name, req.query.password, >>> callback) >>> shop: (callback) -> >>> exports.getShop(callback) >>> }, (err, data) -> >>> res.send(oneK) >>> ) >>> # ab -n 1000 -c 20 "http://localhost:3000/user/loginI?name=a&password=a"; >>> # Requests per second: 218.37 [#/sec] (mean) >>> exports.loginI = (req, res) -> >>> async.auto({ >>> user: (callback) -> >>> userService.getWithNamePassword(req.query.name, req.query.password, >>> callback) >>> shop: (callback) -> >>> exports.getShop(callback) >>> }, (err, data) -> >>> res.send([data.user, data.shop]) >>> ) >>> # ab -n 1000 -c 100 "http://localhost:3000/user/loginI?name=a&password=a >>> " >>> # Requests per second: 187.07 [#/sec] (mean) >>> >>> >>> >>> >>> >>> On Tuesday, April 10, 2012 4:12:40 PM UTC-4, Marak Squires wrote: >>>> >>>> 60rps for an express / mongo app seems way off. I suspect a possible >>>> flaw in application code itself. >>>> >>>> Perhaps there is bottleneck in application where logic is being fired >>>> on every incoming request which should not be. Maybe connection for >>>> mongodb >>>> is not being pooled. Perhaps there is performance flaw in one of the >>>> user-land you are using. I am not sure of what issue is, but I'm fairly >>>> certain there is room for improvement. >>>> >>>> >>>> >>>> On Tue, Apr 10, 2012 at 10:15 AM, timp <[email protected]> wrote: >>>> >>>>> Greetings, >>>>> >>>>> I was wondering if someone who has experience with working with >>>>> scaling problems could give me some insight on an issue. >>>>> >>>>> Long story: >>>>> >>>>> A) >>>>> >>>>> So- I'm making an iPhone/iPad app, which basically is a spiffy view of >>>>> json data provided by a server. The app makes modifications to the json, >>>>> the server stores them in mongodb, etc. Model view in the form of a game. >>>>> >>>>> I set one of my goals as to have the server run as cheaply as >>>>> possible, so that even if the app was not popular, I could keep it up >>>>> with >>>>> minimal cost/overhead. >>>>> >>>>> >>>>> B) >>>>> >>>>> At first I wrote the server in symfony2. But, I'm weary of php, >>>>> especially when I read the documentation for the "apns" (apple push >>>>> notification) plugin, which basically says, "well, I wouldn't use what I >>>>> wrote if you have a lot of users, but you can if you want." >>>>> >>>>> C) >>>>> >>>>> So then I said, let's do "grails." Promises to be fast. I like >>>>> java/groovy whatever. I like that I can refactor java without wondering >>>>> what I'm breaking. But then the throughput was *very* bad. Let's say, >>>>> 20 >>>>> connections per second, in production war mode, to do trivial (but not >>>>> hello world, json blahblah blah) things. >>>>> >>>>> I've done some some work on games. 1 seconds on my modern computer is >>>>> like a whole day on a computer 10 years ago. It is unacceptable to have >>>>> only 10-20 pages a second, even if I've messed up the configuration. >>>>> >>>>> D) >>>>> >>>>> So then, this programmer next to me was talking about nodeJS. About >>>>> how great it is. So this last friday, I ported the server (which is all >>>>> of >>>>> 500 lines or so) from grails to nodeJS. Using all the callback stuff, >>>>> async.auto, etc etc.. But then I'm getting on 60 connections per >>>>> second. >>>>> Apparently Json is actually really slow or something. I did some >>>>> optimization (one of which is just clipping data, which is unacceptable), >>>>> and profiled and got it to ~120 a second. (I'm using the most intensive >>>>> function as a baseline). >>>>> >>>>> E) >>>>> So then, I thought: "this is ridiculous." (And I became compulsive. >>>>> Which sucks.) And on Saturday and Sunday, I wrote a non blocking web >>>>> server in c++, modules and everything. (I did rip and clean code from a >>>>> previously personal project). And I'm getting ~2000 completions a >>>>> second. >>>>> >>>>> >>>>> So you're reading this and saying, "duh. Custom compiled c++ will of >>>>> course be faster than javascript running in a generic framework. And >>>>> you've probably messed up configuration or something somewhere." >>>>> >>>>> I know this. I feel like I just lost the weekend to some stupid >>>>> compulsion. But at the same time, I am truly annoyed at how slow these >>>>> web >>>>> servers/frameworks are. >>>>> >>>>> >>>>> ------------------------ >>>>> ------------------------ >>>>> ------------------------ >>>>> >>>>> >>>>> So my question is this: >>>>> >>>>> How do real scaling companies deal with this problem? >>>>> Non cacheable non static page server. >>>>> >>>>> Does scaling become a business decision? >>>>> >>>>> Are things really slow, just because they can be? Why do people think >>>>> "nodeJS fast" ? >>>>> >>>>> Or do real scaling companies have C/C++ services, which accept >>>>> connections from a front end in nodeJS. (because when nodeJS doesn't >>>>> actually do anything, it really is only 1/6 slower than pure C/C++ >>>>> solution, which, I guess, is pretty fast.)? >>>>> >>>>> How close is v8 to a theoretical maximum? >>>>> (theoretical maximum would be, I guess, compiled code is near gcc -O3, >>>>> and compilation is linear with a small coefficient.) >>>>> >>>>> Why aren't web server pages through lvmm? It should be possible to >>>>> create a fibered (which is really what continuations come down to it >>>>> seems >>>>> to me at this moment), lvmm, jitter, any language (which lvmm supports), >>>>> solution to a web page. >>>>> >>>>> ---- >>>>> >>>>> So, if anyone has any thoughts, let me know. If I've said anything >>>>> which may be rude, I am not trying to be. >>>>> In previous experience, I was never concerned with speed, because I >>>>> was never before, directly paying for it. >>>>> >>>>> I guess I'll run the server on nodeJS, and just load balance it if I >>>>> ever need to. >>>>> Although, I would very much rather have a lvmm'd pages plugged into a >>>>> C/C++ server. >>>>> >>>>> ---- >>>>> >>>>> Thoughts? I this all academic: if things are popular, you will make >>>>> money, and scaling will becoming a business issue? >>>>> >>>>> >>>>> -tim >>>>> >>>>> -- >>>>> Job Board: http://jobs.nodejs.org/ >>>>> Posting guidelines: https://github.com/joyent/** >>>>> node/wiki/Mailing-List-**Posting-Guidelines<https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines> >>>>> You received this message because you are subscribed to the Google >>>>> Groups "nodejs" group. >>>>> To post to this group, send email to [email protected] >>>>> To unsubscribe from this group, send email to >>>>> nodejs+unsubscribe@**googlegroups.com<nodejs%[email protected]> >>>>> For more options, visit this group at >>>>> http://groups.google.com/**group/nodejs?hl=en?hl=en<http://groups.google.com/group/nodejs?hl=en?hl=en> >>>>> >>>> >>>> >>>> >>>> -- >>>> -- >>>> Marak Squires >>>> Co-founder and Chief Evangelist >>>> Nodejitsu, Inc. >>>> [email protected] >>>> >>>> >>> On Tuesday, April 10, 2012 4:12:40 PM UTC-4, Marak Squires wrote: >>>> >>>> 60rps for an express / mongo app seems way off. I suspect a possible >>>> flaw in application code itself. >>>> >>>> Perhaps there is bottleneck in application where logic is being fired >>>> on every incoming request which should not be. Maybe connection for >>>> mongodb >>>> is not being pooled. Perhaps there is performance flaw in one of the >>>> user-land you are using. I am not sure of what issue is, but I'm fairly >>>> certain there is room for improvement. >>>> >>>> >>>> >>>> On Tue, Apr 10, 2012 at 10:15 AM, timp <[email protected]> wrote: >>>> >>>>> Greetings, >>>>> >>>>> I was wondering if someone who has experience with working with >>>>> scaling problems could give me some insight on an issue. >>>>> >>>>> Long story: >>>>> >>>>> A) >>>>> >>>>> So- I'm making an iPhone/iPad app, which basically is a spiffy view of >>>>> json data provided by a server. The app makes modifications to the json, >>>>> the server stores them in mongodb, etc. Model view in the form of a game. >>>>> >>>>> I set one of my goals as to have the server run as cheaply as >>>>> possible, so that even if the app was not popular, I could keep it up >>>>> with >>>>> minimal cost/overhead. >>>>> >>>>> >>>>> B) >>>>> >>>>> At first I wrote the server in symfony2. But, I'm weary of php, >>>>> especially when I read the documentation for the "apns" (apple push >>>>> notification) plugin, which basically says, "well, I wouldn't use what I >>>>> wrote if you have a lot of users, but you can if you want." >>>>> >>>>> C) >>>>> >>>>> So then I said, let's do "grails." Promises to be fast. I like >>>>> java/groovy whatever. I like that I can refactor java without wondering >>>>> what I'm breaking. But then the throughput was *very* bad. Let's say, >>>>> 20 >>>>> connections per second, in production war mode, to do trivial (but not >>>>> hello world, json blahblah blah) things. >>>>> >>>>> I've done some some work on games. 1 seconds on my modern computer is >>>>> like a whole day on a computer 10 years ago. It is unacceptable to have >>>>> only 10-20 pages a second, even if I've messed up the configuration. >>>>> >>>>> D) >>>>> >>>>> So then, this programmer next to me was talking about nodeJS. About >>>>> how great it is. So this last friday, I ported the server (which is all >>>>> of >>>>> 500 lines or so) from grails to nodeJS. Using all the callback stuff, >>>>> async.auto, etc etc.. But then I'm getting on 60 connections per >>>>> second. >>>>> Apparently Json is actually really slow or something. I did some >>>>> optimization (one of which is just clipping data, which is unacceptable), >>>>> and profiled and got it to ~120 a second. (I'm using the most intensive >>>>> function as a baseline). >>>>> >>>>> E) >>>>> So then, I thought: "this is ridiculous." (And I became compulsive. >>>>> Which sucks.) And on Saturday and Sunday, I wrote a non blocking web >>>>> server in c++, modules and everything. (I did rip and clean code from a >>>>> previously personal project). And I'm getting ~2000 completions a >>>>> second. >>>>> >>>>> >>>>> So you're reading this and saying, "duh. Custom compiled c++ will of >>>>> course be faster than javascript running in a generic framework. And >>>>> you've probably messed up configuration or something somewhere." >>>>> >>>>> I know this. I feel like I just lost the weekend to some stupid >>>>> compulsion. But at the same time, I am truly annoyed at how slow these >>>>> web >>>>> servers/frameworks are. >>>>> >>>>> >>>>> ------------------------ >>>>> ------------------------ >>>>> ------------------------ >>>>> >>>>> >>>>> So my question is this: >>>>> >>>>> How do real scaling companies deal with this problem? >>>>> Non cacheable non static page server. >>>>> >>>>> Does scaling become a business decision? >>>>> >>>>> Are things really slow, just because they can be? Why do people think >>>>> "nodeJS fast" ? >>>>> >>>>> Or do real scaling companies have C/C++ services, which accept >>>>> connections from a front end in nodeJS. (because when nodeJS doesn't >>>>> actually do anything, it really is only 1/6 slower than pure C/C++ >>>>> solution, which, I guess, is pretty fast.)? >>>>> >>>>> How close is v8 to a theoretical maximum? >>>>> (theoretical maximum would be, I guess, compiled code is near gcc -O3, >>>>> and compilation is linear with a small coefficient.) >>>>> >>>>> Why aren't web server pages through lvmm? It should be possible to >>>>> create a fibered (which is really what continuations come down to it >>>>> seems >>>>> to me at this moment), lvmm, jitter, any language (which lvmm supports), >>>>> solution to a web page. >>>>> >>>>> ---- >>>>> >>>>> So, if anyone has any thoughts, let me know. If I've said anything >>>>> which may be rude, I am not trying to be. >>>>> In previous experience, I was never concerned with speed, because I >>>>> was never before, directly paying for it. >>>>> >>>>> I guess I'll run the server on nodeJS, and just load balance it if I >>>>> ever need to. >>>>> Although, I would very much rather have a lvmm'd pages plugged into a >>>>> C/C++ server. >>>>> >>>>> ---- >>>>> >>>>> Thoughts? I this all academic: if things are popular, you will make >>>>> money, and scaling will becoming a business issue? >>>>> >>>>> >>>>> -tim >>>>> >>>>> -- >>>>> Job Board: http://jobs.nodejs.org/ >>>>> Posting guidelines: https://github.com/joyent/** >>>>> node/wiki/Mailing-List-**Posting-Guidelines<https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines> >>>>> You received this message because you are subscribed to the Google >>>>> Groups "nodejs" group. >>>>> To post to this group, send email to [email protected] >>>>> To unsubscribe from this group, send email to >>>>> nodejs+unsubscribe@**googlegroups.com<nodejs%[email protected]> >>>>> For more options, visit this group at >>>>> http://groups.google.com/**group/nodejs?hl=en?hl=en<http://groups.google.com/group/nodejs?hl=en?hl=en> >>>>> >>>> >>>> >>>> >>>> -- >>>> -- >>>> Marak Squires >>>> Co-founder and Chief Evangelist >>>> Nodejitsu, Inc. >>>> [email protected] >>>> >>>> -- >>> Job Board: http://jobs.nodejs.org/ >>> Posting guidelines: >>> https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines >>> You received this message because you are subscribed to the Google >>> Groups "nodejs" group. >>> To post to this group, send email to [email protected] >>> To unsubscribe from this group, send email to >>> [email protected] >>> For more options, visit this group at >>> http://groups.google.com/group/nodejs?hl=en?hl=en >>> >> >> >> >> -- >> -- >> Marak Squires >> Co-founder and Chief Evangelist >> Nodejitsu, Inc. >> [email protected] >> >> -- Job Board: http://jobs.nodejs.org/ Posting guidelines: https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines You received this message because you are subscribed to the Google Groups "nodejs" group. To post to this group, send email to [email protected] To unsubscribe from this group, send email to [email protected] For more options, visit this group at http://groups.google.com/group/nodejs?hl=en?hl=en
