Author: rjung
Date: Tue Mar 10 10:37:33 2015
New Revision: 1665454
URL: http://svn.apache.org/r1665454
Log:
Reduce lock contention during maintenance function.
This was observable when using a big number of AJP13
and LB workers.
Modified:
tomcat/jk/trunk/native/common/jk_ajp_common.c
tomcat/jk/trunk/native/common/jk_ajp_common.h
tomcat/jk/trunk/native/common/jk_lb_worker.c
tomcat/jk/trunk/native/common/jk_lb_worker.h
tomcat/jk/trunk/native/common/jk_service.h
tomcat/jk/trunk/native/common/jk_shm.c
tomcat/jk/trunk/native/common/jk_shm.h
tomcat/jk/trunk/native/common/jk_status.c
tomcat/jk/trunk/native/common/jk_worker.c
tomcat/jk/trunk/xdocs/miscellaneous/changelog.xml
Modified: tomcat/jk/trunk/native/common/jk_ajp_common.c
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_ajp_common.c?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_ajp_common.c (original)
+++ tomcat/jk/trunk/native/common/jk_ajp_common.c Tue Mar 10 10:37:33 2015
@@ -3376,38 +3376,17 @@ int ajp_get_endpoint(jk_worker_t *pThis,
return JK_FALSE;
}
-int JK_METHOD ajp_maintain(jk_worker_t *pThis, time_t mstarted, jk_logger_t *l)
+int JK_METHOD ajp_maintain(jk_worker_t *pThis, time_t mstarted, int global,
jk_logger_t *l)
{
JK_TRACE_ENTER(l);
if (pThis && pThis->worker_private) {
ajp_worker_t *aw = pThis->worker_private;
int i;
- long delta;
unsigned int n = 0, k = 0, cnt = 0;
unsigned int m, m_count = 0;
jk_sock_t *m_sock;
- jk_shm_lock();
-
- /* Now we check for global maintenance (once for all processes).
- * Checking workers for idleness.
- * Therefore we globally sync and we use a global timestamp.
- * Since it's possible that we come here a few milliseconds
- * before the interval has passed, we allow a little tolerance.
- */
- delta = (long)difftime(mstarted, aw->s->last_maintain_time) +
- JK_AJP_MAINTAIN_TOLERANCE;
- if (delta >= aw->maintain_time) {
- aw->s->last_maintain_time = mstarted;
- if (aw->s->state == JK_AJP_STATE_OK &&
- aw->s->used == aw->s->used_snapshot)
- aw->s->state = JK_AJP_STATE_IDLE;
- aw->s->used_snapshot = aw->s->used;
- }
-
- jk_shm_unlock();
-
/* Do connection pool maintenance only if timeouts or keepalives are
set
*/
if (aw->cache_timeout <= 0 &&
Modified: tomcat/jk/trunk/native/common/jk_ajp_common.h
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_ajp_common.h?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_ajp_common.h (original)
+++ tomcat/jk/trunk/native/common/jk_ajp_common.h Tue Mar 10 10:37:33 2015
@@ -47,10 +47,6 @@ extern "C"
#define JK_AJP_STATE_TEXT_MAX (JK_AJP_STATE_PROBE)
#define JK_AJP_STATE_TEXT_DEF (JK_AJP_STATE_TEXT_IDLE)
-/* We accept doing global maintenance if we are */
-/* JK_AJP_MAINTAIN_TOLERANCE seconds early. */
-#define JK_AJP_MAINTAIN_TOLERANCE (2)
-
/*
* Conditional request attributes
*
@@ -475,7 +471,7 @@ int ajp_connection_tcp_send_message(ajp_
int ajp_connection_tcp_get_message(ajp_endpoint_t * ae,
jk_msg_buf_t *msg, jk_logger_t *l);
-int JK_METHOD ajp_maintain(jk_worker_t *pThis, time_t now, jk_logger_t *l);
+int JK_METHOD ajp_maintain(jk_worker_t *pThis, time_t now, int global,
jk_logger_t *l);
int JK_METHOD ajp_shutdown(jk_worker_t *pThis, jk_logger_t *l);
Modified: tomcat/jk/trunk/native/common/jk_lb_worker.c
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_lb_worker.c?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_lb_worker.c (original)
+++ tomcat/jk/trunk/native/common/jk_lb_worker.c Tue Mar 10 10:37:33 2015
@@ -728,24 +728,15 @@ static jk_uint64_t decay_load(lb_worker_
return curmax;
}
-static int JK_METHOD maintain_workers(jk_worker_t *p, time_t now, jk_logger_t
*l)
+static int JK_METHOD maintain_workers(jk_worker_t *p, time_t now, int global,
jk_logger_t *l)
{
unsigned int i = 0;
jk_uint64_t curmax = 0;
- long delta;
JK_TRACE_ENTER(l);
if (p && p->worker_private) {
lb_worker_t *lb = (lb_worker_t *)p->worker_private;
- for (i = 0; i < lb->num_of_workers; i++) {
- if (lb->lb_workers[i].worker->maintain) {
- lb->lb_workers[i].worker->maintain(lb->lb_workers[i].worker,
now, l);
- }
- }
-
- jk_shm_lock();
-
/* Now we check for global maintenance (once for all processes).
* Checking workers for recovery and applying decay to the
* load values should not be done by each process individually.
@@ -753,20 +744,42 @@ static int JK_METHOD maintain_workers(jk
* Since it's possible that we come here a few milliseconds
* before the interval has passed, we allow a little tolerance.
*/
- delta = (long)difftime(now, lb->s->last_maintain_time) +
JK_LB_MAINTAIN_TOLERANCE;
- if (delta >= lb->maintain_time) {
+ if (global == JK_TRUE) {
+ time_t exponent = JK_LB_DECAY_MULT * difftime(now,
lb->s->last_maintain_time) / lb->maintain_time;
lb->s->last_maintain_time = now;
+
if (JK_IS_DEBUG_LEVEL(l))
jk_log(l, JK_LOG_DEBUG,
"decay with 2^%d",
- JK_LB_DECAY_MULT * delta / lb->maintain_time);
- curmax = decay_load(lb, JK_LB_DECAY_MULT * delta /
lb->maintain_time, l);
+ exponent);
+
+ jk_shm_lock();
+
+ curmax = decay_load(lb, exponent, l);
+
if (!recover_workers(lb, curmax, now, l)) {
force_recovery(lb, NULL, l);
}
+
+ /*
+ * Checking workers for idleness.
+ */
+ for (i = 0; i < lb->num_of_workers; i++) {
+ ajp_worker_t *aw = lb->lb_workers[i].worker->worker_private;
+ if (aw->s->state == JK_AJP_STATE_OK &&
+ aw->s->used == aw->s->used_snapshot)
+ aw->s->state = JK_AJP_STATE_IDLE;
+ aw->s->used_snapshot = aw->s->used;
+ }
+
+ jk_shm_unlock();
}
- jk_shm_unlock();
+ for (i = 0; i < lb->num_of_workers; i++) {
+ if (lb->lb_workers[i].worker->maintain) {
+ lb->lb_workers[i].worker->maintain(lb->lb_workers[i].worker,
now, global, l);
+ }
+ }
}
else {
Modified: tomcat/jk/trunk/native/common/jk_lb_worker.h
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_lb_worker.h?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_lb_worker.h (original)
+++ tomcat/jk/trunk/native/common/jk_lb_worker.h Tue Mar 10 10:37:33 2015
@@ -127,9 +127,6 @@ extern "C"
/* Time to wait before retry. */
#define WAIT_BEFORE_RECOVER (60)
-/* We accept doing global maintenance if we are */
-/* JK_LB_MAINTAIN_TOLERANCE seconds early. */
-#define JK_LB_MAINTAIN_TOLERANCE (2)
/* We divide load values by 2^x during global maintenance. */
/* The exponent x is JK_LB_DECAY_MULT*#MAINT_INTV_SINCE_LAST_MAINT */
#define JK_LB_DECAY_MULT (1)
Modified: tomcat/jk/trunk/native/common/jk_service.h
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_service.h?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_service.h (original)
+++ tomcat/jk/trunk/native/common/jk_service.h Tue Mar 10 10:37:33 2015
@@ -546,7 +546,7 @@ struct jk_worker
/*
* Maintain this worker.
*/
- int (JK_METHOD * maintain) (jk_worker_t *w, time_t now, jk_logger_t *l);
+ int (JK_METHOD * maintain) (jk_worker_t *w, time_t now, int global,
jk_logger_t *l);
/*
* Shut this worker down.
Modified: tomcat/jk/trunk/native/common/jk_shm.c
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_shm.c?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_shm.c (original)
+++ tomcat/jk/trunk/native/common/jk_shm.c Tue Mar 10 10:37:33 2015
@@ -42,6 +42,8 @@ struct jk_shm_header_data
unsigned int pos;
unsigned int childs;
unsigned int workers;
+ unsigned int maintain_checking;
+ volatile time_t maintain_time;
};
typedef struct jk_shm_header_data jk_shm_header_data_t;
@@ -324,6 +326,8 @@ int jk_shm_open(const char *fname, int s
JK_SHM_MAGIC_SIZ);
jk_shmem.hdr->h.data.size = sz;
jk_shmem.hdr->h.data.childs = 1;
+ jk_shmem.hdr->h.data.maintain_checking = 0;
+ jk_shmem.hdr->h.data.maintain_time = time(NULL);
}
else {
jk_shmem.hdr->h.data.childs++;
@@ -659,6 +663,8 @@ static int do_shm_open(const char *fname
memcpy(jk_shmem.hdr->h.data.magic, shm_signature, JK_SHM_MAGIC_SIZ);
jk_shmem.hdr->h.data.size = sz;
jk_shmem.hdr->h.data.childs = 1;
+ jk_shmem.hdr->h.data.maintain_checking = 0;
+ jk_shmem.hdr->h.data.maintain_time = time(NULL);
if (JK_IS_DEBUG_LEVEL(l))
jk_log(l, JK_LOG_DEBUG,
"Initialized shared memory %s size=%u free=%u addr=%#lx",
@@ -863,6 +869,23 @@ const char *jk_shm_name()
return jk_shmem.filename;
}
+int jk_shm_check_maintain(time_t trigger)
+{
+ int rv = JK_FALSE;
+ int maintain_checking =
JK_ATOMIC_INCREMENT(&(jk_shmem.hdr->h.data.maintain_checking));
+ /* Another process (or thread) is already checking */
+ if (maintain_checking > 1) {
+ JK_ATOMIC_DECREMENT(&(jk_shmem.hdr->h.data.maintain_checking));
+ return rv;
+ }
+ if (jk_shmem.hdr->h.data.maintain_time < trigger) {
+ jk_shmem.hdr->h.data.maintain_time = time(NULL);
+ rv = JK_TRUE;
+ }
+ JK_ATOMIC_DECREMENT(&(jk_shmem.hdr->h.data.maintain_checking));
+ return rv;
+}
+
int jk_shm_lock()
{
int rc = JK_TRUE;
Modified: tomcat/jk/trunk/native/common/jk_shm.h
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_shm.h?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_shm.h (original)
+++ tomcat/jk/trunk/native/common/jk_shm.h Tue Mar 10 10:37:33 2015
@@ -223,6 +223,8 @@ jk_shm_lb_sub_worker_t *jk_shm_alloc_lb_
jk_shm_lb_worker_t *jk_shm_alloc_lb_worker(jk_pool_t *p, const char *name,
jk_logger_t *l);
+int jk_shm_check_maintain(time_t trigger);
+
/* Lock shared memory for thread safe access */
int jk_shm_lock(void);
Modified: tomcat/jk/trunk/native/common/jk_status.c
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_status.c?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_status.c (original)
+++ tomcat/jk/trunk/native/common/jk_status.c Tue Mar 10 10:37:33 2015
@@ -2231,7 +2231,6 @@ static void display_worker_lb(jk_ws_serv
map_count = count_maps(s, name, l);
ms_min = lb->maintain_time - (int)difftime(now, lb->s->last_maintain_time);
ms_max = ms_min + lb->maintain_time;
- ms_min -= JK_LB_MAINTAIN_TOLERANCE;
if (ms_min < 0) {
ms_min = 0;
}
Modified: tomcat/jk/trunk/native/common/jk_worker.c
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/native/common/jk_worker.c?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/native/common/jk_worker.c (original)
+++ tomcat/jk/trunk/native/common/jk_worker.c Tue Mar 10 10:37:33 2015
@@ -25,6 +25,7 @@
#define _PLACE_WORKER_LIST_HERE
#include "jk_worker_list.h"
#include "jk_worker.h"
+#include "jk_shm.h"
#include "jk_util.h"
#include "jk_mt.h"
@@ -315,6 +316,7 @@ const char *wc_get_name_for_type(int typ
void wc_maintain(jk_logger_t *l)
{
static time_t last_maintain = 0;
+ int needs_global_maintenance;
int sz = jk_map_size(worker_map);
JK_TRACE_ENTER(l);
@@ -325,7 +327,8 @@ void wc_maintain(jk_logger_t *l)
* - time since last maintenance is big enough
*/
if (sz > 0 && worker_maintain_time > 0 &&
- difftime(time(NULL), last_maintain) >= worker_maintain_time) {
+ difftime(time(NULL), last_maintain) >= worker_maintain_time &&
+ !running_maintain) {
int i;
JK_ENTER_CS(&worker_lock);
if (running_maintain ||
@@ -339,8 +342,10 @@ void wc_maintain(jk_logger_t *l)
* the maintain until we are finished.
*/
running_maintain = 1;
+ last_maintain = time(NULL);
JK_LEAVE_CS(&worker_lock);
+ needs_global_maintenance = jk_shm_check_maintain(last_maintain -
worker_maintain_time);
for (i = 0; i < sz; i++) {
jk_worker_t *w = jk_map_value_at(worker_map, i);
if (w && w->maintain) {
@@ -348,11 +353,10 @@ void wc_maintain(jk_logger_t *l)
jk_log(l, JK_LOG_DEBUG,
"Maintaining worker %s",
jk_map_name_at(worker_map, i));
- w->maintain(w, time(NULL), l);
+ w->maintain(w, time(NULL), needs_global_maintenance, l);
}
}
JK_ENTER_CS(&worker_lock);
- last_maintain = time(NULL);
running_maintain = 0;
JK_LEAVE_CS(&worker_lock);
}
Modified: tomcat/jk/trunk/xdocs/miscellaneous/changelog.xml
URL:
http://svn.apache.org/viewvc/tomcat/jk/trunk/xdocs/miscellaneous/changelog.xml?rev=1665454&r1=1665453&r2=1665454&view=diff
==============================================================================
--- tomcat/jk/trunk/xdocs/miscellaneous/changelog.xml (original)
+++ tomcat/jk/trunk/xdocs/miscellaneous/changelog.xml Tue Mar 10 10:37:33 2015
@@ -45,6 +45,11 @@
<subsection name="Native">
<changelog>
<fix>
+ AJP, LB: Reduce lock contention during maintenance function.
+ This was observable when using a big number of AJP13 and LB
+ workers. (rjung)
+ </fix>
+ <fix>
<bug>57060</bug>: Allow building from outside of source tree.
Patch contributed by Petr Sumbera. (rjung)
</fix>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@tomcat.apache.org
For additional commands, e-mail: dev-h...@tomcat.apache.org
