Use the raw mongodb driver and the c++ bson parser I suggest trying native_parser set to true to avoid any gc overhead On Apr 10, 10:28 pm, timp <[email protected]> wrote: > or BSON, sorry. > > I assume it must parse the db results, > and then it must JSON them into the result. > > > > > > > > On Tuesday, April 10, 2012 5:24:45 PM UTC-4, Marak Squires wrote: > > > Where in this code is JSON being parsed? > > > Also, running apache bench from your local to your local node app is not > > going to give you accurate results. It will place you in > > an approximate ballpark, but if your application is built correctly, your > > local system and bench will both cap out before node does. > > > On Tue, Apr 10, 2012 at 2:13 PM, timp <[email protected]> wrote: > > >> Ok, so I spent the last hour running tests, you can see the gradual > >> deterioration: > >> the biggest impact seems to be json parsing. > > >> # setup > >> ms = require('mongoskin') > >> m = { db : ms.db('localhost:27017/standard') } > > >> # setup one K out > >> oneK = ''; > >> ppp = 0; > >> for ppp in [0..1024] > >> oneK = oneK + 'x'; > >> ######################## > >> # app.get('/user/loginA', userRoute.loginA); > >> # app.get('/user/loginB', userRoute.loginB); > >> # app.get('/user/loginC', userRoute.loginC); > >> # app.get('/user/loginD', userRoute.loginD); > >> # app.get('/user/loginE', userRoute.loginE); > >> # app.get('/user/loginF', userRoute.loginF); > >> # etc > > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginA?name=a&password=a"; > >> # 2205.04 [#/sec] (mean) > >> exports.loginA = (req, res) -> > >> res.send(oneK); > > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginB?name=a&password=a"; > >> # 439.16 [#/sec] (mean) > >> # when I run db in verbose mode: > >> # Tue Apr 10 16:56:55 [conn1] query standard.users ntoreturn:1 > >> nreturned:1 reslen:18048 0ms > >> exports.loginB = (req, res) -> > >> m.db.collection('users').findOne( > >> {name:req.query.name}, > >> (err, data) -> > >> res.send(oneK) > >> ) > > >> exports.getWithName = (name, callback) -> > >> m.db.collection('users').findOne( > >> {name:name}, > >> callback > >> ) > > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginC?name=a&password=a"; > >> # Requests per second: 403.78 [#/sec] (mean) > >> exports.loginC = (req, res) -> > >> exports.getWithName(req.query.name, (err, data) -> > >> res.send(oneK) > >> ) > > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginD?name=a&password=a"; > >> # Requests per second: 331.83 [#/sec] (mean) > >> exports.loginD = (req, res) -> > >> exports.getWithName(req.query.name, (err, data) -> > >> res.send(data) > >> ) > > >> exports.getWithNamePassword = (name, password, callback) -> > >> exports.getWithName( > >> name, > >> (err, user) -> > >> if (user.password != password) > >> callback("validation failed", null) > >> else > >> callback(null, user) > >> ) > > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginE?name=a&password=a"; > >> # Requests per second: 331.35 [#/sec] (mean) > >> exports.loginE = (req, res) -> > >> exports.getWithNamePassword(req.query.name, req.query.password, (err, > >> data) -> > >> res.send(data) > >> ) > > >> # using external service > >> userService = require('../../service/userService') > > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginF?name=a&password=a"; > >> # Requests per second: 324.92 [#/sec] (mean) > >> exports.loginF = (req, res) -> > >> exports.getWithNamePassword(req.query.name, req.query.password, (err, > >> data) -> > >> res.send(data) > >> ) > > >> exports.getShop = (callback) -> > >> m.db.collection('items').find({}).toArray( > >> (err, data) -> > >> pruned = (item.shop for item in data) > >> log.debug("getShop has " + pruned.length + " items") > >> callback(err, pruned) > >> ) > > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginG?name=a&password=a"; > >> # Requests per second: 512.91 [#/sec] (mean) > >> exports.loginG = (req, res) -> > >> exports.getShop( (err, data) -> > >> res.send(data) > >> ) > > >> async = require('async') > > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginH?name=a&password=a"; > >> # Requests per second: 279.72 [#/sec] (mean) > >> exports.loginH = (req, res) -> > >> async.auto({ > >> user: (callback) -> > >> userService.getWithNamePassword(req.query.name, req.query.password, > >> callback) > >> shop: (callback) -> > >> exports.getShop(callback) > >> }, (err, data) -> > >> res.send(oneK) > >> ) > >> # ab -n 1000 -c 20 "http://localhost:3000/user/loginI?name=a&password=a"; > >> # Requests per second: 218.37 [#/sec] (mean) > >> exports.loginI = (req, res) -> > >> async.auto({ > >> user: (callback) -> > >> userService.getWithNamePassword(req.query.name, req.query.password, > >> callback) > >> shop: (callback) -> > >> exports.getShop(callback) > >> }, (err, data) -> > >> res.send([data.user, data.shop]) > >> ) > >> # ab -n 1000 -c 100 "http://localhost:3000/user/loginI?name=a&password=a"; > >> # Requests per second: 187.07 [#/sec] (mean) > > >> On Tuesday, April 10, 2012 4:12:40 PM UTC-4, Marak Squires wrote: > > >>> 60rps for an express / mongo app seems way off. I suspect a possible > >>> flaw in application code itself. > > >>> Perhaps there is bottleneck in application where logic is being fired on > >>> every incoming request which should not be. Maybe connection for mongodb > >>> is > >>> not being pooled. Perhaps there is performance flaw in one of the > >>> user-land > >>> you are using. I am not sure of what issue is, but I'm fairly certain > >>> there > >>> is room for improvement. > > >>> On Tue, Apr 10, 2012 at 10:15 AM, timp <[email protected]> wrote: > > >>>> Greetings, > > >>>> I was wondering if someone who has experience with working with scaling > >>>> problems could give me some insight on an issue. > > >>>> Long story: > > >>>> A) > > >>>> So- I'm making an iPhone/iPad app, which basically is a spiffy view of > >>>> json data provided by a server. The app makes modifications to the json, > >>>> the server stores them in mongodb, etc. Model view in the form of a > >>>> game. > > >>>> I set one of my goals as to have the server run as cheaply as possible, > >>>> so that even if the app was not popular, I could keep it up with minimal > >>>> cost/overhead. > > >>>> B) > > >>>> At first I wrote the server in symfony2. But, I'm weary of php, > >>>> especially when I read the documentation for the "apns" (apple push > >>>> notification) plugin, which basically says, "well, I wouldn't use what I > >>>> wrote if you have a lot of users, but you can if you want." > > >>>> C) > > >>>> So then I said, let's do "grails." Promises to be fast. I like > >>>> java/groovy whatever. I like that I can refactor java without wondering > >>>> what I'm breaking. But then the throughput was *very* bad. Let's say, > >>>> 20 > >>>> connections per second, in production war mode, to do trivial (but not > >>>> hello world, json blahblah blah) things. > > >>>> I've done some some work on games. 1 seconds on my modern computer is > >>>> like a whole day on a computer 10 years ago. It is unacceptable to have > >>>> only 10-20 pages a second, even if I've messed up the configuration. > > >>>> D) > > >>>> So then, this programmer next to me was talking about nodeJS. About > >>>> how great it is. So this last friday, I ported the server (which is all > >>>> of > >>>> 500 lines or so) from grails to nodeJS. Using all the callback stuff, > >>>> async.auto, etc etc.. But then I'm getting on 60 connections per > >>>> second. > >>>> Apparently Json is actually really slow or something. I did some > >>>> optimization (one of which is just clipping data, which is unacceptable), > >>>> and profiled and got it to ~120 a second. (I'm using the most intensive > >>>> function as a baseline). > > >>>> E) > >>>> So then, I thought: "this is ridiculous." (And I became compulsive. > >>>> Which sucks.) And on Saturday and Sunday, I wrote a non blocking web > >>>> server in c++, modules and everything. (I did rip and clean code from a > >>>> previously personal project). And I'm getting ~2000 completions a > >>>> second. > > >>>> So you're reading this and saying, "duh. Custom compiled c++ will of > >>>> course be faster than javascript running in a generic framework. And > >>>> you've probably messed up configuration or something somewhere." > > >>>> I know this. I feel like I just lost the weekend to some stupid > >>>> compulsion. But at the same time, I am truly annoyed at how slow these > >>>> web > >>>> servers/frameworks are. > > >>>> ------------------------ > >>>> ------------------------ > >>>> ------------------------ > > >>>> So my question is this: > > >>>> How do real scaling companies deal with this problem? > >>>> Non cacheable non static page server. > > >>>> Does scaling become a business decision? > > >>>> Are things really slow, just because they can be? Why do people think > >>>> "nodeJS fast" ? > > >>>> Or do real scaling companies have C/C++ services, which accept > >>>> connections from a front end in nodeJS. (because when nodeJS doesn't > >>>> actually do anything, it really is only 1/6 slower than pure C/C++ > >>>> solution, which, I guess, is pretty fast.)? > > >>>> How close is v8 to a theoretical maximum? > >>>> (theoretical maximum would be, I guess, compiled code is near gcc -O3, > >>>> and compilation is linear with a small coefficient.) > > >>>> Why aren't web server pages through lvmm? It should be possible to > >>>> create a fibered (which is really what continuations come down to it > >>>> seems > >>>> to me at this moment), lvmm, jitter, any language (which lvmm supports), > >>>> solution to a web page. > > >>>> ---- > > >>>> So, if anyone has any thoughts, let me know. If I've said anything > >>>> which may be rude, I am not trying to be. > >>>> In previous experience, I was never concerned with speed, because I was > >>>> never before, directly paying for it. > > >>>> I guess I'll run the server on nodeJS, and just load balance it if I > >>>> ever need to. > >>>> Although, I would very much rather have a lvmm'd pages plugged into a > >>>> C/C++ server. > > >>>> ---- > > >>>> Thoughts? I this all academic: if things are popular, you will make > >>>> money, and scaling will becoming a business issue? > > >>>> -tim > > >>>> -- > >>>> Job Board:http://jobs.nodejs.org/ > >>>> Posting > > ... > > read more »
-- Job Board: http://jobs.nodejs.org/ Posting guidelines: https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines You received this message because you are subscribed to the Google Groups "nodejs" group. To post to this group, send email to [email protected] To unsubscribe from this group, send email to [email protected] For more options, visit this group at http://groups.google.com/group/nodejs?hl=en?hl=en
