This is a draft patch which essentially allows a high priority request
to replace a low priority request when the queue is full.
Currently if a high priority request comes along, and the destination
queue is full, the request gets rejected (or redispatched if possible).
This change allows inserting into the queue if there are lower priority
requests present. When doing so, the lowest priority request is evicted
to make room.
The intent is for that evicted request to be handled as if it just came
in. Meaning if there were a brand new request coming in, and the
configuration were such that the request should be rejected, then the
evicted request should be rejected. If redispatch is enabled a new
request would be sent to a different server, and so the evicted request
should be as well.
Now as for why this is tagged RFC: The implementation for the handling
of the evicted request is bad. If you notice inside pendconn_displace(),
the section of code that handles this (lines 498-505) has no business
being there. Those operations should be handled higher up. It also
doesn't work as intended, as instead of responding to the client with a
503, it just kills the connection.
However I'm having difficulty finding the appropriate way to do this,
and could use some guidance. I've tried several different things, and
none of the behaviors are quite right.
The simplest solution of just performing __pendconn_unlink() and then
waking the task results in the request being redispatched. The higher
level code assumes that if the request was in a queue, and is now no
longer in a queue, then redispatch is the appropriate action.
Thanks
-Patrick
From a3c8ba92a05ec877662359f963ece0cfa82051f8 Mon Sep 17 00:00:00 2001
From: Patrick Hemmer <[email protected]>
Date: Thu, 12 Sep 2019 22:56:51 -0400
Subject: [PATCH] MINOR: attempt to insert into priority queue when full
This makes it so that when a queue (server or proxy) is full, that we try to
insert into the queue and evict a request of lower priority.
The evicted request will either be redispatched if `option redispatch` is
enabled, or rejected if not.
---
include/proto/queue.h | 1 +
src/backend.c | 3 +-
src/queue.c | 119 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 122 insertions(+), 1 deletion(-)
diff --git a/include/proto/queue.h b/include/proto/queue.h
index a7ab63b35..156bf3964 100644
--- a/include/proto/queue.h
+++ b/include/proto/queue.h
@@ -37,6 +37,7 @@
extern struct pool_head *pool_head_pendconn;
struct pendconn *pendconn_add(struct stream *strm);
+int pendconn_displace(struct stream *strm);
int pendconn_dequeue(struct stream *strm);
void process_srv_queue(struct server *s);
unsigned int srv_dynamic_maxconn(const struct server *s);
diff --git a/src/backend.c b/src/backend.c
index 1b01536c1..56340d2de 100644
--- a/src/backend.c
+++ b/src/backend.c
@@ -976,7 +976,8 @@ int assign_server_and_queue(struct stream *s)
(srv->nbpend || srv->served >= srv_dynamic_maxconn(srv))) {
if (srv->maxqueue > 0 && srv->nbpend >= srv->maxqueue)
- return SRV_STATUS_FULL;
+ // queue is full. see if priority allows us to
insert
+ return pendconn_displace(s);
p = pendconn_add(s);
if (p)
diff --git a/src/queue.c b/src/queue.c
index 30b7ef056..ea015272a 100644
--- a/src/queue.c
+++ b/src/queue.c
@@ -185,6 +185,33 @@ void pendconn_unlink(struct pendconn *p)
pendconn_queue_unlock(p);
}
+/* Comapres 2 pendconn queue priority keys.
+ *
+ * Returns :
+ * -1 if k1 < k2
+ * 0 if k1 == k2
+ * 1 if k1 > k2
+ */
+static int key_cmp(u32 k1, u32 k2)
+{
+ if (KEY_CLASS(k1) < KEY_CLASS(k2))
+ return -1;
+ if (KEY_CLASS(k1) > KEY_CLASS(k2))
+ return 1;
+
+ if (k1 < NOW_OFFSET_BOUNDARY())
+ k1 += 0x100000; // key in the future
+ if (k2 < NOW_OFFSET_BOUNDARY())
+ k2 += 0x100000; // key in the future
+
+ if (k1 < k2)
+ return -1;
+ if (k1 > k2)
+ return 1;
+
+ return 0;
+}
+
/* Retrieve the first pendconn from tree <pendconns>. Classes are always
* considered first, then the time offset. The time does wrap, so the
* lookup is performed twice, one to retrieve the first class and a second
@@ -212,6 +239,31 @@ static struct pendconn *pendconn_first(struct eb_root
*pendconns)
return eb32_entry(node2, struct pendconn, node);
}
+/* Retrieve the last pendconn from tree <pendconns>.
+ * Follows the same semantics as pendconn_first.
+ */
+static struct pendconn *pendconn_last(struct eb_root *pendconns)
+{
+ struct eb32_node *node, *node2 = NULL;
+ u32 key;
+
+ node = eb32_last(pendconns);
+ if (!node)
+ return NULL;
+
+ key = KEY_CLASS_OFFSET_BOUNDARY(node->key);
+ // We don't have eb32_lookup_lt, so we simulate it
+ if (key > 0)
+ node2 = eb32_lookup_le(pendconns, key - 1);
+
+ if (!node2 ||
+ KEY_CLASS(node2->key) != KEY_CLASS(node->key)) {
+ return eb32_entry(node, struct pendconn, node);
+ }
+
+ return eb32_entry(node2, struct pendconn, node);
+}
+
/* Process the next pending connection from either a server or a proxy, and
* returns a strictly positive value on success (see below). If no pending
* connection is found, 0 is returned. Note that neither <srv> nor <px> may be
@@ -390,6 +442,73 @@ struct pendconn *pendconn_add(struct stream *strm)
return p;
}
+/* Adds the stream <strm> to the pending connection queue of server <strm>->srv
+ * or to the one of <strm>->proxy if srv is NULL, and removes the lowest
+ * priority stream from the queue. The removed stream's task is woken for
processing.
+ *
+ * Otherwise, follows all the same semantics and restrictions as pendconn_add.
+ *
+ * Returns :
+ *
+ * SRV_STATUS_QUEUED
+ * SRV_STATUS_FULL
+ * SRV_STATUS_INTERNAL
+ */
+int pendconn_displace(struct stream *strm)
+{
+ struct pendconn *p, *pd;
+ struct proxy *px;
+ struct server *srv;
+ u32 strm_key;
+
+ if (strm->flags & SF_ASSIGNED)
+ srv = objt_server(strm->target);
+ else
+ srv = NULL;
+
+ px = strm->be;
+
+ strm_key = MAKE_KEY(strm->priority_class, strm->priority_offset);
+
+ if (srv) {
+ HA_SPIN_LOCK(SERVER_LOCK, &srv->lock);
+ pd = pendconn_last(&srv->pendconns);
+ if (!pd || key_cmp(pd->node.key, strm_key) <= 0) {
+ HA_SPIN_UNLOCK(SERVER_LOCK, &srv->lock);
+ return SRV_STATUS_FULL;
+ }
+ }
+ else {
+ HA_SPIN_LOCK(PROXY_LOCK, &px->lock);
+ pd = pendconn_last(&px->pendconns);
+ if (!pd || key_cmp(pd->node.key, strm_key) <= 0) {
+ HA_SPIN_UNLOCK(PROXY_LOCK, &px->lock);
+ return SRV_STATUS_FULL;
+ }
+ }
+
+ __pendconn_unlink(pd);
+ p = pendconn_add(strm);
+
+ if (srv)
+ HA_SPIN_UNLOCK(SERVER_LOCK, &srv->lock);
+ else
+ HA_SPIN_UNLOCK(PROXY_LOCK, &px->lock);
+
+ if (((pd->strm_flags & (SF_DIRECT | SF_FORCE_PRST)) == SF_DIRECT) &&
+ (strm->be->options & PR_O_REDISP)) {
+ pd->strm_flags &= ~(SF_DIRECT | SF_ASSIGNED | SF_ADDR_SET);
+ task_wakeup(pd->strm->task, TASK_WOKEN_OTHER);
+ } else {
+ stream_shutdown(pd->strm, SF_ERR_PRXCOND);
+ pd->strm->flags |= SF_FINST_Q;
+ }
+
+ if (p)
+ return SRV_STATUS_QUEUED;
+ return SRV_STATUS_INTERNAL;
+}
+
/* Redistribute pending connections when a server goes down. The number of
* connections redistributed is returned. It must be called with the server
* lock held.
--
2.22.0
