Hi all, There has been some serious discussion for building load balancing support for mod_proxy. Here is the first in series of patches that will enable this.
The patch adds lb support in scoreboard, so that statuses for each lb worker can be calculated for each child process. It uses optional function to retrieve the number of lb workers and calculates the scoreboard size accordingly. The other option is to rewrite the scoreboard functionality and use it inside mod_proxy, but that will cause a strange shm locking observed on some platforms using jk2 shared memory, cause we cannot create the shm in parent process. Right now the lb_score only defines 1024 byte space, that can later be replaced with real struct members once when the lb functionality gets frozen. For now we will simply cast that to the desired structure. Regards, MT.
Index: scoreboard.h
===================================================================
RCS file: /home/cvspublic/httpd-2.0/include/scoreboard.h,v
retrieving revision 1.51
diff -u -r1.51 scoreboard.h
--- scoreboard.h 9 Feb 2004 20:38:21 -0000 1.51
+++ scoreboard.h 26 Jul 2004 09:33:26 -0000
@@ -32,6 +32,7 @@
#include "apr_thread_proc.h"
#include "apr_portable.h"
#include "apr_shm.h"
+#include "apr_optional.h"
/* Scoreboard file, if there is one */
#ifndef DEFAULT_SCOREBOARD
@@ -118,6 +119,7 @@
typedef struct {
int server_limit;
int thread_limit;
+ int lb_limit;
ap_scoreboard_e sb_type;
ap_generation_t running_generation; /* the generation of children which
* should still be serving requests. */
@@ -135,6 +137,13 @@
*/
};
+/* stuff which is lb specific */
+typedef struct lb_score lb_score;
+struct lb_score{
+ /* TODO: make a real stuct from this */
+ unsigned char data[1024];
+};
+
/* Scoreboard is now in 'local' memory, since it isn't updated once created,
* even in forked architectures. Child created-processes (non-fork) will
* set up these indicies into the (possibly relocated) shmem records.
@@ -143,6 +152,7 @@
global_score *global;
process_score *parent;
worker_score **servers;
+ lb_score **balancers;
} scoreboard;
typedef struct ap_sb_handle_t ap_sb_handle_t;
@@ -168,6 +178,7 @@
AP_DECLARE(worker_score *) ap_get_scoreboard_worker(int x, int y);
AP_DECLARE(process_score *) ap_get_scoreboard_process(int x);
AP_DECLARE(global_score *) ap_get_scoreboard_global(void);
+AP_DECLARE(lb_score *) ap_get_scoreboard_lb(int child_num, int lb_num);
AP_DECLARE_DATA extern scoreboard *ap_scoreboard_image;
AP_DECLARE_DATA extern const char *ap_scoreboard_fname;
@@ -184,6 +195,13 @@
* @return OK or DECLINE on success; anything else is a error
*/
AP_DECLARE_HOOK(int, pre_mpm, (apr_pool_t *p, ap_scoreboard_e sb_type))
+
+/**
+ * proxy load balancer
+ * @return the number of load balancer workers.
+ */
+APR_DECLARE_OPTIONAL_FN(int, ap_proxy_lb_workers,
+ (void));
/* for time_process_request() in http_main.c */
#define START_PREQUEST 1
Index: scoreboard.c
===================================================================
RCS file: /home/cvspublic/httpd-2.0/server/scoreboard.c,v
retrieving revision 1.74
diff -u -r1.74 scoreboard.c
--- scoreboard.c 9 Feb 2004 20:40:49 -0000 1.74
+++ scoreboard.c 26 Jul 2004 10:02:36 -0000
@@ -59,12 +59,15 @@
(apr_pool_t *p, ap_scoreboard_e sb_type),
(p, sb_type),OK,DECLINED)
+static APR_OPTIONAL_FN_TYPE(ap_proxy_lb_workers)
+ *proxy_lb_workers;
+
struct ap_sb_handle_t {
int child_num;
int thread_num;
};
-static int server_limit, thread_limit;
+static int server_limit, thread_limit, lb_limit;
static apr_size_t scoreboard_size;
/*
@@ -89,9 +92,20 @@
{
ap_mpm_query(AP_MPMQ_HARD_LIMIT_THREADS, &thread_limit);
ap_mpm_query(AP_MPMQ_HARD_LIMIT_DAEMONS, &server_limit);
+
+ if (!proxy_lb_workers)
+ proxy_lb_workers = APR_RETRIEVE_OPTIONAL_FN(ap_proxy_lb_workers);
+ if (proxy_lb_workers)
+ lb_limit = proxy_lb_workers();
+ else
+ lb_limit = 0;
+
scoreboard_size = sizeof(global_score);
scoreboard_size += sizeof(process_score) * server_limit;
scoreboard_size += sizeof(worker_score) * server_limit * thread_limit;
+ if (lb_limit)
+ scoreboard_size += sizeof(lb_score) * server_limit * lb_limit;
+
return scoreboard_size;
}
@@ -102,7 +116,8 @@
ap_calc_scoreboard_size();
ap_scoreboard_image =
- calloc(1, sizeof(scoreboard) + server_limit * sizeof(worker_score *));
+ calloc(1, sizeof(scoreboard) + server_limit * sizeof(worker_score *) +
+ server_limit * lb_limit * sizeof(lb_score *));
more_storage = shared_score;
ap_scoreboard_image->global = (global_score *)more_storage;
more_storage += sizeof(global_score);
@@ -114,9 +129,19 @@
ap_scoreboard_image->servers[i] = (worker_score *)more_storage;
more_storage += thread_limit * sizeof(worker_score);
}
+ if (lb_limit) {
+ ap_scoreboard_image->balancers =
+ (lb_score **)((char*)ap_scoreboard_image + sizeof(scoreboard) +
+ server_limit * sizeof(worker_score *));
+ for (i = 0; i < server_limit; i++) {
+ ap_scoreboard_image->balancers[i] = (lb_score *)more_storage;
+ more_storage += lb_limit * sizeof(lb_score);
+ }
+ }
ap_assert(more_storage == (char*)shared_score + scoreboard_size);
ap_scoreboard_image->global->server_limit = server_limit;
ap_scoreboard_image->global->thread_limit = thread_limit;
+ ap_scoreboard_image->global->lb_limit = lb_limit;
}
/**
@@ -260,6 +285,13 @@
memset(ap_scoreboard_image->servers[i], 0,
sizeof(worker_score) * thread_limit);
}
+ /* Clean up the lb workers data */
+ if (lb_limit) {
+ for (i = 0; i < server_limit; i++) {
+ memset(ap_scoreboard_image->balancers[i], 0,
+ sizeof(lb_score) * lb_limit);
+ }
+ }
return OK;
}
@@ -461,4 +493,13 @@
AP_DECLARE(global_score *) ap_get_scoreboard_global()
{
return ap_scoreboard_image->global;
+}
+
+AP_DECLARE(lb_score *) ap_get_scoreboard_lb(int child_num, int lb_num)
+{
+ if (((child_num < 0) || (server_limit < child_num)) ||
+ ((lb_num < 0) || (lb_limit < lb_num))) {
+ return(NULL); /* Out of range */
+ }
+ return &ap_scoreboard_image->balancers[child_num][lb_num];
}
smime.p7s
Description: S/MIME cryptographic signature
