Connect conntrack to the offload framework: connections are submitted for offload when created, providers are notified when the peers become established, and a DEL is submitted when a connection is destroyed.
The ADD is submitted after ct->ct_lock is dropped, so a concurrent flush can reclaim the connection in between; ct_offload_op_mark() therefore re-checks the reclaimed flag after marking and submits the DEL itself, so providers never keep entries for connections software has already destroyed. The expiration sweeper asks hardware for the last-used time of tracked connections and slides the deadline forward only by activity newer than what was already accounted for; connections whose hardware entries are stale or gone then expire per their timeout policy instead of being kept alive forever. Providers that report eviction (ENOENT) demote the connection back to software-only tracking. Signed-off-by: Aaron Conole <[email protected]> --- lib/conntrack-private.h | 4 +- lib/conntrack-tcp.c | 2 +- lib/conntrack.c | 87 ++++++++++++++++++++++++++++++-- lib/ct-offload.c | 107 ++++++++++++++++++++++++++++++++++++++++ lib/ct-offload.h | 24 ++++++++- 5 files changed, 217 insertions(+), 7 deletions(-) diff --git a/lib/conntrack-private.h b/lib/conntrack-private.h index f5bcd5d828..a4e253c7d8 100644 --- a/lib/conntrack-private.h +++ b/lib/conntrack-private.h @@ -97,7 +97,7 @@ struct conn { struct conn_key parent_key; /* Only used for orig_tuple support. */ uint16_t nat_action; char *alg; - atomic_flag reclaimed; /* False during the lifetime of the connection, + atomic_bool reclaimed; /* False during the lifetime of the connection, * True as soon as a thread has started freeing * its memory. */ @@ -137,7 +137,7 @@ static inline void conn_init(struct conn *conn) { ovs_mutex_init_adaptive(&conn->lock); - atomic_flag_clear(&conn->reclaimed); + atomic_store_relaxed(&conn->reclaimed, false); } enum ct_update_res { diff --git a/lib/conntrack-tcp.c b/lib/conntrack-tcp.c index 4788b5a4e6..d988563684 100644 --- a/lib/conntrack-tcp.c +++ b/lib/conntrack-tcp.c @@ -140,7 +140,7 @@ tcp_bypass_seq_chk(struct conntrack *ct, struct conn *conn) /* While hardware-confirmed, not every segment is visible to * software, so validation is skipped. Segments that do reach * software still update the baselines, so validation resumes - * correctly after demotion. */ + * correctly after demotion (ct_offload_conn_demote()). */ if (!conntrack_get_tcp_seq_chk(ct) || ct_offload_conn_is_hw_confirmed(conn)) { COVERAGE_INC(conntrack_tcp_seq_chk_bypass); diff --git a/lib/conntrack.c b/lib/conntrack.c index 1d2eaf5e9e..ccc7764370 100644 --- a/lib/conntrack.c +++ b/lib/conntrack.c @@ -27,6 +27,7 @@ #include "conntrack-tcp.h" #include "conntrack-tp.h" #include "coverage.h" +#include "ct-offload.h" #include "crc32c.h" #include "csum.h" #include "ct-dpif.h" @@ -512,7 +513,9 @@ static void conn_clean(struct conntrack *ct, struct conn *conn) OVS_EXCLUDED(conn->lock, ct->ct_lock) { - if (atomic_flag_test_and_set(&conn->reclaimed)) { + bool reclaimed = false; + if (!atomic_compare_exchange_strong(&conn->reclaimed, &reclaimed, + true)) { return; } @@ -525,6 +528,14 @@ conn_clean(struct conntrack *ct, struct conn *conn) atomic_count_dec(&zl->czl.count); } + struct ct_offload_ctx offload_ctx = { + .conn = conn, + .netdev_in = NULL, + .input_port_id = ODPP_NONE, + .dir = CT_DIR_FWD, + }; + ct_offload_conn_del(&offload_ctx); + ovsrcu_postpone(delete_conn, conn); atomic_count_dec(&ct->n_conn); } @@ -1370,6 +1381,29 @@ process_one(struct conntrack *ct, struct dp_packet *pkt, helper, alg_exp, ct_alg_ctl, tp_id); } ovs_mutex_unlock(&ct->ct_lock); + + if (conn) { + struct ct_offload_ctx offload_ctx = { + .conn = conn, + .netdev_in = NULL, + .input_port_id = pkt->md.orig_in_port, + .dir = CT_DIR_FWD, + }; + ct_offload_conn_add(&offload_ctx); + } + } + + if (!create_new_conn && conn && ctx->reply && + (pkt->md.ct_state & CS_ESTABLISHED)) { + /* EST notify: the reply packet carries the reverse-direction + * input port. */ + struct ct_offload_ctx offload_ctx = { + .conn = conn, + .netdev_in = NULL, + .input_port_id = pkt->md.orig_in_port, + .dir = CT_DIR_REV, + }; + ct_offload_conn_notify_established(&offload_ctx); } write_ct_md(pkt, zone, conn, &ctx->key, alg_exp); @@ -1515,19 +1549,66 @@ ct_sweep(struct conntrack *ct, struct rculist *list, long long now, size_t *cleaned_count) OVS_NO_THREAD_SAFETY_ANALYSIS { + struct ct_offload_op_batch batch; + struct ct_offload_op *op; struct conn *conn; size_t cleaned = 0; size_t count = 0; + ct_offload_op_batch_init(&batch); RCULIST_FOR_EACH (conn, node, list) { if (conn_expired(conn, now)) { - conn_clean(ct, conn); - cleaned++; + /* Conns and their offload data are RCU-freed and the sweeper + * stays non-quiescent for the whole scan, so the raw pointers + * are safe; the offload state itself is read under conn->lock. */ + bool tracked; + + ovs_mutex_lock(&conn->lock); + tracked = ct_offload_conn_is_tracked(conn); + ovs_mutex_unlock(&conn->lock); + + if (!tracked) { + conn_clean(ct, conn); + cleaned++; + } else { + struct ct_offload_ctx offload_ctx = { + .conn = conn, + .netdev_in = NULL, + .input_port_id = ODPP_NONE, + .dir = CT_DIR_FWD, + }; + ct_offload_op_batch_add(&batch, CT_OFFLOAD_OP_UPD, + &offload_ctx); + } } count++; } + /* The batch holds raw conn pointers; see the lifetime note above. */ + ct_offload_op_batch_submit(&batch); + + CT_OFFLOAD_BATCH_OP_FOR_EACH (&batch, idx, op) { + struct conn *c = op->ctx.conn; + + if (op->error == ENOENT) { + /* Provider evicted the entry: demote to software-only + * tracking. If idle, it still expires on a later sweep. */ + ct_offload_conn_demote(c); + continue; + } + + /* Slide the deadline only by hardware activity newer than what + * was already accounted for; connections whose hardware entry is + * gone or idle past its timeout are cleaned now. */ + if (!ct_offload_conn_extend(c, op, now)) { + conn_clean(ct, c); + cleaned++; + } + } + + ct_offload_op_batch_destroy(&batch); + if (cleaned_count) { *cleaned_count = cleaned; } diff --git a/lib/ct-offload.c b/lib/ct-offload.c index ca682f329f..d6c73c3452 100644 --- a/lib/ct-offload.c +++ b/lib/ct-offload.c @@ -290,6 +290,7 @@ ct_offload_op_mark(struct ct_offload_op *op) { struct conn *conn = op->ctx.conn; struct ct_offload_data *data; + bool finally_delete = false; if (ct_offload_private_id == CT_PRIVATE_ID_INVALID) { return; @@ -301,9 +302,17 @@ ct_offload_op_mark(struct ct_offload_op *op) switch (op->type) { case CT_OFFLOAD_OP_ADD: if (!op->error && !data) { + bool reclaimed = false; + data = xzalloc(sizeof *data); data->state = CT_OFFLOAD_STATE_ADDED; conn_private_set(conn, ct_offload_private_id, data); + + /* The conn may have been reclaimed between process_one() + * dropping ct->ct_lock and this mark, in which case + * conn_clean() skipped the DEL; send it ourselves. */ + atomic_read_relaxed(&conn->reclaimed, &reclaimed); + finally_delete = reclaimed; } if (data && op->hw_confirmed) { ct_offload_data_hw_confirm(data); @@ -329,6 +338,10 @@ ct_offload_op_mark(struct ct_offload_op *op) break; } ovs_mutex_unlock(&conn->lock); + + if (finally_delete) { + ct_offload_conn_del(&op->ctx); + } } /* ct_offload_op_batch_submit() - submit 'batch' to every open provider. @@ -386,6 +399,16 @@ ct_offload_conn_add(const struct ct_offload_ctx *ctx) void ct_offload_conn_del(const struct ct_offload_ctx *ctx) { + bool tracked; + + ovs_mutex_lock(&ctx->conn->lock); + tracked = ct_offload_conn_is_tracked(ctx->conn); + ovs_mutex_unlock(&ctx->conn->lock); + + if (!tracked) { + return; + } + struct ct_offload_op op = { .type = CT_OFFLOAD_OP_DEL, .ctx = *ctx }; struct ct_offload_op_batch batch = { .ops = &op, .n_ops = 1, .allocated = 1 @@ -413,6 +436,17 @@ ct_offload_conn_update(const struct ct_offload_ctx *ctx) void ct_offload_conn_notify_established(const struct ct_offload_ctx *ctx) { + bool pending; + + ovs_mutex_lock(&ctx->conn->lock); + pending = ct_offload_conn_is_tracked(ctx->conn) + && !ct_offload_conn_is_hw_confirmed(ctx->conn); + ovs_mutex_unlock(&ctx->conn->lock); + + if (!pending) { + return; + } + struct ct_offload_op op = { .type = CT_OFFLOAD_OP_ESTABLISHED, .ctx = *ctx }; @@ -484,3 +518,76 @@ ct_offload_conn_is_hw_confirmed(const struct conn *conn) return data && data->state == CT_OFFLOAD_STATE_HW_CONFIRMED; } + +bool +ct_offload_conn_extend(struct conn *conn, const struct ct_offload_op *op, + long long now) + OVS_EXCLUDED(conn->lock) +{ + struct ct_offload_data *data; + long long new_exp; + bool extended = false; + + if (op->error || !op->last_used + || ct_offload_private_id == CT_PRIVATE_ID_INVALID) { + return false; + } + + ovs_mutex_lock(&conn->lock); + data = conn_private_get(conn, ct_offload_private_id); + if (data) { + long long base = data->last_extend_ts ? data->last_extend_ts : now; + + if (op->last_used <= base) { + /* No hardware activity newer than what was already + * accounted for. */ + } else { + data->last_extend_ts = op->last_used; + + long long cur; + atomic_read_relaxed(&conn->expiration, &cur); + for (;;) { + new_exp = cur + (op->last_used - base); + if (new_exp <= now) { + break; + } + if (atomic_compare_exchange_strong_relaxed( + &conn->expiration, &cur, new_exp)) { + extended = true; + break; + } + /* The compare-exchange failed: 'cur' has been reloaded + * with the current value (e.g. a PMD refreshed the + * expiration for a packet); the deadline is recomputed + * from it, so a concurrently written larger value is + * never lost. */ + } + } + } + ovs_mutex_unlock(&conn->lock); + + return extended; +} + +void +ct_offload_conn_demote(struct conn *conn) +{ + struct ct_offload_data *data; + + if (ct_offload_private_id == CT_PRIVATE_ID_INVALID) { + return; + } + + ovs_mutex_lock(&conn->lock); + data = conn_private_get(conn, ct_offload_private_id); + if (data) { + conn_private_set(conn, ct_offload_private_id, NULL); + } + ovs_mutex_unlock(&conn->lock); + + /* RCU-deferred: readers of the private slot hold conn->lock, so + * the postponement is belt-and-suspenders. */ + ovsrcu_postpone(free, data); + + VLOG_DBG("conn demoted to software tracking: conn=%p", conn); +} diff --git a/lib/ct-offload.h b/lib/ct-offload.h index 1157b7edfe..ac50d9023d 100644 --- a/lib/ct-offload.h +++ b/lib/ct-offload.h @@ -66,7 +66,11 @@ struct ct_offload_ctx { * * CT_OFFLOAD_OP_UPD (uses conn, dir): * Sets op->last_used to the hardware last-used time in ms since - * epoch (0 if none). Sets op->error to 0. May set + * epoch (0 if none), and op->error to 0, or ENOENT if the + * provider previously accepted the connection and has since + * evicted it -- the framework then demotes it to software-only + * tracking. A provider that never accepted the connection must + * leave the result fields untouched. May set * op->hw_confirmed if the provider noticed the connection active * on its own. Leave the result fields untouched when not * holding the connection. @@ -209,10 +213,28 @@ void ct_offload_module_init(void); bool ct_offload_conn_is_tracked(const struct conn *); bool ct_offload_conn_is_hw_confirmed(const struct conn *); +/* ct_offload_conn_extend() - slide 'conn' expiration forward by + * hardware activity reported in 'op' (a CT_OFFLOAD_OP_UPD result). + * + * The deadline advances only by hardware activity newer than what was + * accounted for at the previous extension, so idle or evicted entries + * expire normally instead of being extended forever. Returns true if + * the expiration was extended past 'now'. */ +bool ct_offload_conn_extend(struct conn *, const struct ct_offload_op *op, + long long now); + +/* ct_offload_conn_demote() - drop the offload tracking for 'conn' + * without destroying it. Used by the sweeper when a provider reports + * ENOENT (eviction); no DEL op, as the provider already dropped its + * state. */ +void ct_offload_conn_demote(struct conn *); + /* Single-operation convenience wrappers around ct_offload_op_batch_*(), * each submitting a batch of one op to every open provider. Callers * must not hold conn->lock; see the locking notes in ct-offload.c. */ void ct_offload_conn_add(const struct ct_offload_ctx *); + +/* Submits a DEL op unless the connection was never offloaded. */ void ct_offload_conn_del(const struct ct_offload_ctx *); /* Returns the hardware last-used time in ms since epoch, or 0. */ -- 2.56.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
