Fwd: svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES docs/manual/mod/mod_lua.xml docs/manual/style/scripts/prettify.js modules/lua/lua_vmprep.h modules/lua/mod_lua.c modules/lua/mod_lua.h

2012-08-27 Thread Rüdiger Plüm



 Original Message 
Subject:svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES 
docs/manual/mod/mod_lua.xml
docs/manual/style/scripts/prettify.js modules/lua/lua_vmprep.h 
modules/lua/mod_lua.c modules/lua/mod_lua.h
Date:   Sun, 26 Aug 2012 18:39:59 GMT
From:   humbed...@apache.org



Author: humbedooh
Date: Sun Aug 26 18:39:58 2012
New Revision: 1377475

URL: http://svn.apache.org/viewvc?rev=1377475view=rev
Log:
Add new directives, LuaInputFilter/LuaOutputFilter for creating content filters 
using Lua.

Modified:
 httpd/httpd/trunk/CHANGES
 httpd/httpd/trunk/docs/manual/mod/mod_lua.xml
 httpd/httpd/trunk/docs/manual/style/scripts/prettify.js
 httpd/httpd/trunk/modules/lua/lua_vmprep.h
 httpd/httpd/trunk/modules/lua/mod_lua.c
 httpd/httpd/trunk/modules/lua/mod_lua.h



Modified: httpd/httpd/trunk/modules/lua/mod_lua.c
URL: 
http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/lua/mod_lua.c?rev=1377475r1=1377474r2=1377475view=diff
==
--- httpd/httpd/trunk/modules/lua/mod_lua.c (original)
+++ httpd/httpd/trunk/modules/lua/mod_lua.c Sun Aug 26 18:39:58 2012
@@ -55,6 +55,14 @@ typedef struct {

  apr_hash_t *lua_authz_providers;

+typedef struct
+{
+apr_bucket_brigade *tmpBucket;
+lua_State *L;
+ap_lua_vm_spec *spec;
+int broken;
+} lua_filter_ctx;
+

  /**
   * error reporting if lua has an error.
@@ -290,6 +298,281 @@ static int lua_handler(request_rec *r)
  }


+/* --- Input/output content filters --- */
+
+
+static apr_status_t lua_setup_filter_ctx(ap_filter_t* f, request_rec* r, 
lua_filter_ctx**
c) {
+apr_pool_t *pool;
+ap_lua_vm_spec *spec;
+int n, rc;
+lua_State *L;
+lua_filter_ctx *ctx;
+ap_lua_server_cfg *server_cfg = 
ap_get_module_config(r-server-module_config,
+lua_module);
+const ap_lua_dir_cfg *cfg = ap_get_module_config(r-per_dir_config,
+lua_module);
+
+ctx = apr_pcalloc(r-pool, sizeof(lua_filter_ctx));
+ctx-broken = 0;
+*c = ctx;
+/* Find the filter that was called */
+for (n = 0; n  cfg-mapped_filters-nelts; n++) {
+ap_lua_filter_handler_spec *hook_spec =
+((ap_lua_filter_handler_spec **) cfg-mapped_filters-elts)[n];
+
+if (hook_spec == NULL) {
+continue;
+}
+if (!strcasecmp(hook_spec-filter_name, f-frec-name)) {
+spec = create_vm_spec(pool, r, cfg, server_cfg,
+hook_spec-file_name,
+NULL,
+0,
+hook_spec-function_name,
+filter);
+L = ap_lua_get_lua_state(pool, spec, r);
+if (L) {
+L = lua_newthread(L);
+}
+
+if (!L) {
+ap_log_rerror(APLOG_MARK, APLOG_CRIT, 0, r, APLOGNO(01477)
+lua: Failed to obtain lua interpreter for %s 
%s,
+hook_spec-function_name, 
hook_spec-file_name);
+ap_lua_release_state(L, spec, r);
+return APR_EGENERAL;
+}
+if (hook_spec-function_name != NULL) {
+lua_getglobal(L, hook_spec-function_name);
+if (!lua_isfunction(L, -1)) {
+ap_log_rerror(APLOG_MARK, APLOG_CRIT, 0, r, APLOGNO(01478)
+lua: Unable to find function %s in %s,
+hook_spec-function_name,
+hook_spec-file_name);
+ap_lua_release_state(L, spec, r);
+return APR_EGENERAL;
+}
+
+ap_lua_run_lua_request(L, r);
+}
+else {
+int t;
+ap_lua_run_lua_request(L, r);
+
+t = lua_gettop(L);
+lua_setglobal(L, r);
+lua_settop(L, t);
+}
+ctx-L = L;
+ctx-spec = spec;
+
+/* If a Lua filter is interested in filtering a request, it must 
first do a yield,

+ * otherwise we'll assume that it's not interested and pretend we 
didn't find
it.
+ */
+rc = lua_resume(L, 1);
+if (rc == LUA_YIELD) {
+return OK;
+}
+else {
+ap_lua_release_state(L, spec, r);
+return APR_ENOENT;
+}
+}
+}
+return APR_ENOENT;
+}
+
+static apr_status_t lua_output_filter_handle(ap_filter_t *f, 
apr_bucket_brigade *pbbIn) {
+apr_bucket *e;
+request_rec *r = f-r;
+int rc;
+lua_State *L;
+lua_filter_ctx* ctx;
+conn_rec *c = r-connection;
+apr_bucket *pbktIn;
+apr_bucket_brigade *pbbOut = NULL;
+
+/* Set up the initial filter context and acquire the 

Re: Fwd: svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES docs/manual/mod/mod_lua.xml docs/manual/style/scripts/prettify.js modules/lua/lua_vmprep.h modules/lua/mod_lua.c modules/lua/mod_lua.h

2012-08-27 Thread Daniel Gruno
On 08/27/2012 09:41 AM, Rüdiger Plüm wrote:

 +/* Clean up and pass on the brigade to the next filter in the chain */
 
 
 Where do we do a cleanup here?
 
 +return APR_SUCCESS;
 +}
 +
snip
 
 Regards
 
 Rüdiger
 
Heh, I believe that's what is called a copypasto. Both cleanups and
passing on content is done earlier in the chain, the specific comment is
simply a remnant of creating the input filter based on the code used for
the output filter (you'll see the same comment in the output filter
code, where it actually makes sense). I'll remove the comment.

With regards and thanks,
Daniel.


RE: Fwd: svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES docs/manual/mod/mod_lua.xml docs/manual/style/scripts/prettify.js modules/lua/lua_vmprep.h modules/lua/mod_lua.c modules/lua/mod_lua.h

2012-08-27 Thread Plüm , Rüdiger , Vodafone Group


 -Original Message-
 From: Daniel Gruno [mailto:rum...@cord.dk]
 Sent: Montag, 27. August 2012 10:45
 To: dev@httpd.apache.org
 Subject: Re: Fwd: svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES
 docs/manual/mod/mod_lua.xml docs/manual/style/scripts/prettify.js
 modules/lua/lua_vmprep.h modules/lua/mod_lua.c modules/lua/mod_lua.h
 
 On 08/27/2012 09:41 AM, Rüdiger Plüm wrote:
 
  +/* Clean up and pass on the brigade to the next filter in the
 chain */
 
 
  Where do we do a cleanup here?
 
  +return APR_SUCCESS;
  +}
  +
 snip
 
  Regards
 
  Rüdiger
 
 Heh, I believe that's what is called a copypasto. Both cleanups and
 passing on content is done earlier in the chain, the specific comment is
 simply a remnant of creating the input filter based on the code used for
 the output filter (you'll see the same comment in the output filter
 code, where it actually makes sense). I'll remove the comment.

Thanks. I guess you noticed that I found more important issues with the code 
than this comment :-)

Regards

Rüdiger



Re: Fwd: svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES docs/manual/mod/mod_lua.xml docs/manual/style/scripts/prettify.js modules/lua/lua_vmprep.h modules/lua/mod_lua.c modules/lua/mod_lua.h

2012-08-27 Thread Daniel Gruno
On 08/27/2012 11:59 AM, Plüm, Rüdiger, Vodafone Group wrote:
 
 Thanks. I guess you noticed that I found more important issues with the code 
 than this comment :-)
 
 Regards
 
 Rüdiger
 
Haha, yeah, but that's a big mess to clean up (I'm not a programmer by
trade, so I'm a bit slow in that regard ;) ), so it will take me a while
to get through all that, but I will get through it...eventually.

With regards,
Daniel.


Re: Fwd: svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES docs/manual/mod/mod_lua.xml docs/manual/style/scripts/prettify.js modules/lua/lua_vmprep.h modules/lua/mod_lua.c modules/lua/mod_lua.h

2012-08-27 Thread Daniel Gruno
On 08/27/2012 11:59 AM, Plüm, Rüdiger, Vodafone Group wrote:
 
 Thanks. I guess you noticed that I found more important issues with the code 
 than this comment :-)
 
 Regards
 
 Rüdiger
 
I've tried my best to correct the errors you mentioned, but I'm having
some trouble figuring out how the input filter should pass along data to
the next in the chain. What I have currently is this:

/* Get the output from Lua's yield() as a string */
const char* output = lua_tolstring(L, 1, olen);
/* Make a bucket for it */
pbktOut = apr_bucket_heap_create(output, olen, 0, c-bucket_alloc);
/* Add it to the brigade */
APR_BRIGADE_INSERT_TAIL(pbbOut, pbktOut);
/* Do something more here involving cleanups or..?? */

But then what? Am I supposed to make a call to some ap_*_brigade
function? I've checked some of the example filter modules, but I can't
seem to find any suitable way of doing this other than finishing up the
brigade and returning APR_SUCCESS in the end.

Any pointers (or some code) would be much appreciated. I have plenty
imagination, but my httpd C API knowledge can be somewhat lacking at
certain points.

With regards,
Daniel.


RE: Fwd: svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES docs/manual/mod/mod_lua.xml docs/manual/style/scripts/prettify.js modules/lua/lua_vmprep.h modules/lua/mod_lua.c modules/lua/mod_lua.h

2012-08-27 Thread Plüm , Rüdiger , Vodafone Group


 -Original Message-
 From: Daniel Gruno [mailto:rum...@cord.dk]
 Sent: Montag, 27. August 2012 14:06
 To: dev@httpd.apache.org
 Subject: Re: Fwd: svn commit: r1377475 - in /httpd/httpd/trunk: CHANGES
 docs/manual/mod/mod_lua.xml docs/manual/style/scripts/prettify.js
 modules/lua/lua_vmprep.h modules/lua/mod_lua.c modules/lua/mod_lua.h
 
 On 08/27/2012 11:59 AM, Plüm, Rüdiger, Vodafone Group wrote:
 
  Thanks. I guess you noticed that I found more important issues with
 the code than this comment :-)
 
  Regards
 
  Rüdiger
 
 I've tried my best to correct the errors you mentioned, but I'm having

You should try to leave the loop if ap_pass_brigade does not return APR_SUCCESS 
and should return this (!= APR_SUCCESS) to
the caller. It is always a bad sign if the return code of ap_pass_brigade gets 
lost unhandled or unpassed.

 some trouble figuring out how the input filter should pass along data to
 the next in the chain. What I have currently is this:
 
 /* Get the output from Lua's yield() as a string */
 const char* output = lua_tolstring(L, 1, olen);
 /* Make a bucket for it */
 pbktOut = apr_bucket_heap_create(output, olen, 0, c-bucket_alloc);
 /* Add it to the brigade */
 APR_BRIGADE_INSERT_TAIL(pbbOut, pbktOut);
 /* Do something more here involving cleanups or..?? */
 
 But then what? Am I supposed to make a call to some ap_*_brigade
 function? I've checked some of the example filter modules, but I can't
 seem to find any suitable way of doing this other than finishing up the
 brigade and returning APR_SUCCESS in the end.

Keep the input brigade you fetched in the context and return to the caller.
You will be called again if more data is needed. Then you can continue 
processing the
data in your context input brigade first and fetch further data on your own if 
you run
out of data.

Regards

Rüdiger


Re: Re: Re: mod_fcgid concurrency bottleneck, issue#53693

2012-08-27 Thread Lazy
2012/8/16 pqf p...@mailtech.cn:
 How about this:
 1. procmgr_post_spawn_cmd() now return a status code from PM, so process
 handler now know the spawn request is denyed or not.
 2. if a new process is created, no sleep is needed.
 3. if no process is created, sleep a while

sorry for the late reply,

in the old code there ware no sleep() between procmgr_post_spawn_cmd()
and apply_free_procnode()

sleep() was invoked only if there ware no free procnode.

This happened only if we ware denied spawning new process or in some
cases if some other thread managed to use that procnode before us.

Your change adresses cases if some other thread stole our newly
spawned fcgi process, old code was waiting 1s before trying to spawn
another/recheck, new code doesn't, I guess this is the orginal issue
in stress tests when total number of simultaneous connections doesn't
exceed max fcgi processes. But when spawning is denied recovery time
is still long 1s.


I was refering to cases when spawn is denied.

If a vhost is overloaded or someone added sleep(60) in the code,
mod_fcgid blocks on all request to that vhost
for over a minute and it is possible to occupy 1000 threads using
under 20 new connections to slow vhost
per second. This can be mitingated by adding avaiability which will
impact time spend on waiting for free process. Overloaded vhost will
start to drop connections faster preventing the web-server reaching
MaxClients
limit.

Another question. Is it necessary to call procmgr_init_spawn_cmd()
from inside the for loop ?



 2012-08-16
 
 pqf
 
 发件人:Lazy
 发送时间:2012-08-16 16:47
 主题:Re: Re: mod_fcgid concurrency bottleneck, issue#53693
 收件人:devdev@httpd.apache.org
 抄送:

 2012/8/16 pqf p...@mailtech.cn:
 Hi, Michal
 My solution do add availability to each class, which is the
 procmgr_post_spawn_cmd() call in each loop do.
 The sleep() call is intrudused for a stress test without warm up time, in
 this case, mod_fcgid will create more processes than a slow start one(each
 process handler can't apply a free slot on the very begining, so send a
 request to process manager to create one, it's easy to reach the max # of
 process limit while httpd startup, but the idle process will be killed
 later), the sleep() call is a little like a server side warm up delay.
 But since someone said remove this sleep(), the server work fine without
 bottleneck(Maybe he didn't notise the warm up issue?), so I thought remove
 the sleep() is a good idea. But reduce the time of sleep() is fine to me
 too.

 I was referring to the case where all processes are busy, without
 sleep(), handle_request() wil quickly send spawn requsts, whith will
 be denyed by process menager, with sleep() handle_request() will
 always wait quite a long time,
 occupying slots

 --
 Michal Grzedzicki




Re: [Vote] httpd 2.2.23 release

2012-08-27 Thread Michael Felt
FYI: I get an error message from configure that I do not understand.

+ ./configure
--enable-layout=AIX
--with-apr=/opt/bin/apr-1-config
--with-apr-util=/opt/bin/apu-1-config
--with-mpm=worker
--enable-ssl
--enable-mods-shared=all  build/aix/configure.out
Package openssl was not found in the pkg-config search path.
Perhaps you should add the directory containing `openssl.pc'
to the PKG_CONFIG_PATH environment variable
No package 'openssl' found

However, the packaging proceeds fine. And I get an
It Works message.

Will add beta in name in http://dl.aixtools.net/httpd

p.s. If it is not appropriate to add a URL by non-ASF - please slap gently
:)


Re: [Vote] httpd 2.2.23 release

2012-08-27 Thread Guenter Knauf

Hi Michael,
Am 27.08.2012 15:42, schrieb Michael Felt:

No package 'openssl' found

that means that most likely your build doesnt support https protocol.


However, the packaging proceeds fine. And I get an
It Works message.

yes, but have you checked https:// too??

Gün.




Re: [Vote] httpd 2.2.23 release

2012-08-27 Thread Stefan Fritsch
[+1]  Release httpd 2.2.23 as GA

Tested on Debian sid. I see the same (non-regression) test failures as 
Rainer.

Cheers,
Stefan


回复: Re: Re: Re: mod_fcgid concurrency bottleneck, issue#53693

2012-08-27 Thread pqf
So what can mod_fcgid do in this overloaded?
1. mod_fcgid get a request
2. mod_fcgid can't apply a free slot of FCGI handler
3. mod_fcgid send a spawn request to PM
4. PM deny the request(for too much process already)
5. Now
   for( i=1; i64; i++)
  {
 a) mod_fcgid delay a while, and then send another spawn request to PM and 
try apply free slot again.
 b) mod_fcgid send another spawn request at once, even the last request is 
denied.
 c) ??
(now is a, b maybe not a good idea, any new idea?)
  }

I think the bottleneck is too much request, too less FCGI handler. httpd(or 
mod_fcgid) either drop client connections or delay a while, there is no other 
way out?


Another question. Is it necessary to call procmgr_init_spawn_cmd() from inside 
the for loop ? 
I took a brief look, it seems not necessary. I will move it out of loop and 
test.

2012-08-28



pqf



发件人:Lazy
发送时间:2012-08-27 21:47
主题:Re: Re: Re: mod_fcgid concurrency bottleneck, issue#53693
收件人:devdev@httpd.apache.org
抄送:

2012/8/16 pqf p...@mailtech.cn: 
 How about this: 
 1. procmgr_post_spawn_cmd() now return a status code from PM, so process 
 handler now know the spawn request is denyed or not. 
 2. if a new process is created, no sleep is needed. 
 3. if no process is created, sleep a while 

sorry for the late reply, 

in the old code there ware no sleep() between procmgr_post_spawn_cmd() 
and apply_free_procnode() 

sleep() was invoked only if there ware no free procnode. 

This happened only if we ware denied spawning new process or in some 
cases if some other thread managed to use that procnode before us. 

Your change adresses cases if some other thread stole our newly 
spawned fcgi process, old code was waiting 1s before trying to spawn 
another/recheck, new code doesn't, I guess this is the orginal issue 
in stress tests when total number of simultaneous connections doesn't 
exceed max fcgi processes. But when spawning is denied recovery time 
is still long 1s. 


I was refering to cases when spawn is denied. 

If a vhost is overloaded or someone added sleep(60) in the code, 
mod_fcgid blocks on all request to that vhost 
for over a minute and it is possible to occupy 1000 threads using 
under 20 new connections to slow vhost 
per second. This can be mitingated by adding avaiability which will 
impact time spend on waiting for free process. Overloaded vhost will 
start to drop connections faster preventing the web-server reaching 
MaxClients 
limit. 

Another question. Is it necessary to call procmgr_init_spawn_cmd() 
from inside the for loop ? 


 
 2012-08-16 
  
 pqf 
  
 发件人:Lazy 
 发送时间:2012-08-16 16:47 
 主题:Re: Re: mod_fcgid concurrency bottleneck, issue#53693 
 收件人:devdev@httpd.apache.org 
 抄送: 
 
 2012/8/16 pqf p...@mailtech.cn: 
 Hi, Michal 
 My solution do add availability to each class, which is the 
 procmgr_post_spawn_cmd() call in each loop do. 
 The sleep() call is intrudused for a stress test without warm up time, in 
 this case, mod_fcgid will create more processes than a slow start one(each 
 process handler can't apply a free slot on the very begining, so send a 
 request to process manager to create one, it's easy to reach the max # of 
 process limit while httpd startup, but the idle process will be killed 
 later), the sleep() call is a little like a server side warm up delay. 
 But since someone said remove this sleep(), the server work fine without 
 bottleneck(Maybe he didn't notise the warm up issue?), so I thought remove 
 the sleep() is a good idea. But reduce the time of sleep() is fine to me 
 too. 
 
 I was referring to the case where all processes are busy, without 
 sleep(), handle_request() wil quickly send spawn requsts, whith will 
 be denyed by process menager, with sleep() handle_request() will 
 always wait quite a long time, 
 occupying slots 
 
 -- 
 Michal Grzedzicki