Aaron Conole via dev <[email protected]> writes:

> Add the per-connection operation API on top of a batch abstraction:
> operations are accumulated into a struct ct_offload_op_batch and
> executed all at once through the provider's single batch_submit()
> entry point.  The library-level single-connection calls are
> implemented as batches of one.
>
> Batching lets providers apply their own optimizations, and will be
> used by the ct expiration logic in an upcoming commit.
>
> Signed-off-by: Aaron Conole <[email protected]>
> ---

FYI - the FreeBSD image is a bit unstable and causing failures here.  In
this instance:

  + qemu-system-x86_64 -enable-kvm -cpu host -m 4096 -smp 4 -nographic -netdev 
user,id=net0,hostfwd=tcp::2222-:22 -device virtio-net-pci,netdev=net0 -drive 
file=freebsd-run.qcow2,if=virtio,format=qcow2,cache=unsafe -device 
virtio-rng-pci -pidfile /tmp/freebsd-vm.pid -device ahci,id=ahci0 -drive 
if=none,id=seed,file=/tmp/freebsd-seed.iso,format=raw,media=cdrom,readonly=on 
-device ide-cd,bus=ahci0.0,drive=seed -drive 
if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd -drive 
if=pflash,format=raw,file=/tmp/freebsd-ovmf-vars.fd
  + trap cleanup EXIT
  + freebsd_wait_ssh 20 10
  + local max=20 delay=10 i
  + echo 'Waiting for SSH on port 2222 ...'
  ...
  + '[' 20 '!=' 20 ']'
  + echo 'ERROR: SSH not available after 200s.'
  ERROR: SSH not available after 200s.
    attempt 20/20 ...

>  lib/ct-offload.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++
>  lib/ct-offload.h | 116 ++++++++++++++++++++++++++++++++++++++++++++++-
>  2 files changed, 230 insertions(+), 1 deletion(-)

Let's try this out

Recheck-request: github-robot

> diff --git a/lib/ct-offload.c b/lib/ct-offload.c
> index 833e9078b1..111bf37f06 100644
> --- a/lib/ct-offload.c
> +++ b/lib/ct-offload.c
> @@ -78,6 +78,7 @@ ct_offload_register(const struct ct_offload_class *class)
>  
>      ovs_assert(class);
>      ovs_assert(class->name);
> +    ovs_assert(class->batch_submit);
>      ovs_assert(class->flow_might_offload);
>  
>      ovs_rwlock_wrlock(&ct_offload_rwlock);
> @@ -209,6 +210,120 @@ ct_offload_module_init(void)
>       * directly from their own module-init routines. */
>  }
>  
> +/* ct_offload_op_batch_add() - append one operation to 'batch'.
> + *
> + * The context is copied by value; no references are taken on ctx.conn or
> + * ctx.netdev_in, so they must remain valid until the batch is submitted
> + * (i.e. until ct_offload_op_batch_submit() returns: the framework
> + * accesses the ops after the provider callbacks have run). */
> +void
> +ct_offload_op_batch_add(struct ct_offload_op_batch *batch,
> +                        enum ct_offload_op_type type,
> +                        const struct ct_offload_ctx *ctx)
> +{
> +    if (batch->n_ops == batch->allocated) {
> +        size_t new_allocated = 2 * batch->allocated;
> +
> +        if (batch->ops == batch->inline_ops) {
> +            batch->ops = xmalloc(new_allocated * sizeof *batch->ops);
> +            memcpy(batch->ops, batch->inline_ops,
> +                   sizeof batch->inline_ops);
> +        } else {
> +            batch->ops = xrealloc(batch->ops,
> +                                  new_allocated * sizeof *batch->ops);
> +        }
> +        batch->allocated = new_allocated;
> +    }
> +
> +    struct ct_offload_op *op = &batch->ops[batch->n_ops++];
> +
> +    op->type = type;
> +    op->ctx = *ctx;
> +    op->error = 0;
> +    op->last_used = 0;
> +}
> +
> +/* ct_offload_op_batch_submit() - submit 'batch' to every open provider.
> + *
> + * Result fields are pre-filled (EOPNOTSUPP = "no provider claimed this
> + * op"); providers must leave them untouched when not handling a
> + * connection (see ct-offload.h). */
> +void
> +ct_offload_op_batch_submit(struct ct_offload_op_batch *batch)
> +{
> +    struct ct_offload_class_node *node;
> +
> +    /* EOPNOTSUPP = "no provider claimed this op". */
> +    for (size_t i = 0; i < batch->n_ops; i++) {
> +        batch->ops[i].error = EOPNOTSUPP;
> +        batch->ops[i].last_used = 0;
> +    }
> +
> +    ovs_rwlock_rdlock(&ct_offload_rwlock);
> +    LIST_FOR_EACH (node, list_node, &ct_offload_classes) {
> +        if (node->refcnt == 0) {
> +            continue;
> +        }
> +        node->class->batch_submit(batch);
> +    }
> +    ovs_rwlock_unlock(&ct_offload_rwlock);
> +}
> +
> +void
> +ct_offload_conn_add(const struct ct_offload_ctx *ctx)
> +{
> +    struct ct_offload_op op = { .type = CT_OFFLOAD_OP_ADD, .ctx = *ctx };
> +    struct ct_offload_op_batch batch = {
> +        .ops = &op, .n_ops = 1, .allocated = 1
> +    };
> +
> +    ct_offload_op_batch_submit(&batch);
> +
> +    if (op.error && op.error != EOPNOTSUPP) {
> +        VLOG_DBG("ct offload failed to add connection: %s",
> +                 ovs_strerror(op.error));
> +    }
> +}
> +
> +void
> +ct_offload_conn_del(const struct ct_offload_ctx *ctx)
> +{
> +    struct ct_offload_op op = { .type = CT_OFFLOAD_OP_DEL, .ctx = *ctx };
> +    struct ct_offload_op_batch batch = {
> +        .ops = &op, .n_ops = 1, .allocated = 1
> +    };
> +
> +    ct_offload_op_batch_submit(&batch);
> +}
> +
> +long long
> +ct_offload_conn_update(const struct ct_offload_ctx *ctx)
> +{
> +    struct ct_offload_op op = { .type = CT_OFFLOAD_OP_UPD, .ctx = *ctx };
> +    struct ct_offload_op_batch batch = {
> +        .ops = &op, .n_ops = 1, .allocated = 1
> +    };
> +
> +    ct_offload_op_batch_submit(&batch);
> +
> +    return op.last_used;
> +}
> +
> +/* Notifies providers that software observed the connection transition
> + * to established (reply-direction packet).  A signal only. */
> +void
> +ct_offload_conn_notify_established(const struct ct_offload_ctx *ctx)
> +{
> +    struct ct_offload_op op = {
> +        .type = CT_OFFLOAD_OP_ESTABLISHED, .ctx = *ctx
> +    };
> +    struct ct_offload_op_batch batch = {
> +        .ops = &op, .n_ops = 1, .allocated = 1
> +    };
> +
> +    ct_offload_op_batch_submit(&batch);
> +}
> +
>  /* ct_offload_flow_offloadable() - may a flow containing the ct()
>   * action 'ct_action' with flow key 'match' on 'netdev_in' be fully
>   * offloaded?  Returns true if any open provider claims it. */
> diff --git a/lib/ct-offload.h b/lib/ct-offload.h
> index c8eddb2f3a..7b35b35cef 100644
> --- a/lib/ct-offload.h
> +++ b/lib/ct-offload.h
> @@ -47,6 +47,99 @@ struct ct_offload_ctx {
>                                       * packet that triggered the op. */
>  };
>  
> +/* Operations delivered to a provider's batch_submit() callback.  Each op
> + * carries its own 'ctx'; the fields each type uses are noted below.  The
> + * provider fills in the op result fields ('error', 'last_used').
> + * At most one provider handles a given connection -- the one whose
> + * ports the connection flows through; every other provider leaves
> + * the result fields untouched:
> + *
> + *   CT_OFFLOAD_OP_ADD (uses conn, netdev_in, input_port_id, dir):
> + *       Sets op->error to 0 if accepted, another positive errno on
> + *       failure; leave the result fields untouched when not handling
> + *       the connection.
> + *
> + *   CT_OFFLOAD_OP_DEL (uses conn, dir):
> + *       Removes any offloaded state; providers must tolerate unknown
> + *       connections.  Sets op->error to 0.
> + *
> + *   CT_OFFLOAD_OP_UPD (uses conn, dir):
> + *       Sets op->last_used to the hardware last-used time in ms since
> + *       epoch (0 if none).  Sets op->error to 0.  Leave the result
> + *       fields untouched when not holding the connection.
> + *
> + *   CT_OFFLOAD_OP_ESTABLISHED (uses conn, input_port_id, dir):
> + *       Software observed the connection transition to established
> + *       (reply-direction packet); carries the reverse-direction input
> + *       port.  A signal only: providers may act on it or ignore it;
> + *       it requests nothing.  op->error may report a failure acting
> + *       on the signal (diagnostics only).  Leave the result fields
> + *       untouched otherwise. */
> +enum ct_offload_op_type {
> +    CT_OFFLOAD_OP_ADD,              /* Add operation. */
> +    CT_OFFLOAD_OP_DEL,              /* Del operation. */
> +    CT_OFFLOAD_OP_UPD,              /* Update (timestamp) operation. */
> +    CT_OFFLOAD_OP_ESTABLISHED,      /* SW observed the established
> +                                     * transition. */
> +};
> +
> +/* A single offload operation.  The 'ctx' is embedded by value, but no
> + * references are taken on ctx.conn or ctx.netdev_in; those pointers must
> + * remain valid until ct_offload_op_batch_submit() returns. */
> +struct ct_offload_op {
> +    enum ct_offload_op_type type;
> +    struct ct_offload_ctx ctx;
> +
> +    /* Result fields, filled in by the provider as documented above. */
> +    int error;
> +    long long last_used;
> +};
> +
> +/* Ops embedded directly in struct ct_offload_op_batch; larger batches
> + * fall back to a heap array. */
> +enum { CT_OFFLOAD_OP_BATCH_INLINE = 8 };
> +
> +/* Batched set of offload operations. */
> +struct ct_offload_op_batch {
> +    struct ct_offload_op *ops;
> +    size_t n_ops;
> +    size_t allocated;
> +    struct ct_offload_op inline_ops[CT_OFFLOAD_OP_BATCH_INLINE];
> +};
> +
> +#define CT_OFFLOAD_BATCH_OP_FOR_EACH(BATCH, IDX, OP)                  \
> +    for (size_t IDX = 0;                                              \
> +         (IDX) < (BATCH)->n_ops && ((OP) = &(BATCH)->ops[IDX], true); \
> +         (IDX)++)
> +
> +static inline void
> +ct_offload_op_batch_init(struct ct_offload_op_batch *batch)
> +{
> +    batch->ops = batch->inline_ops;
> +    batch->n_ops = 0;
> +    batch->allocated = CT_OFFLOAD_OP_BATCH_INLINE;
> +}
> +
> +static inline void
> +ct_offload_op_batch_destroy(struct ct_offload_op_batch *batch)
> +{
> +    if (batch->ops != batch->inline_ops) {
> +        free(batch->ops);
> +    }
> +    ct_offload_op_batch_init(batch);
> +}
> +
> +static inline size_t
> +ct_offload_op_batch_len(const struct ct_offload_op_batch *batch)
> +{
> +    return batch->n_ops;
> +}
> +
> +void ct_offload_op_batch_add(struct ct_offload_op_batch *,
> +                             enum ct_offload_op_type type,
> +                             const struct ct_offload_ctx *);
> +void ct_offload_op_batch_submit(struct ct_offload_op_batch *);
> +
>  /* CT offload class describes a conntrack offload provider implementation.
>   *
>   * Providers register globally at module init and become active when
> @@ -70,6 +163,14 @@ struct ct_offload_class {
>       * open().  May be NULL if open() is. */
>      void (*close)(void);
>  
> +    /* Processes the ops in 'batch'; the only per-connection entry
> +     * point.  Fill in each op's result fields per enum
> +     * ct_offload_op_type; ops in a batch are independent.  Called with
> +     * only ct_offload_rwlock (read side) held -- no conn->lock -- and
> +     * must not block for extended periods; take conn->lock yourself to
> +     * access conn state.  Mandatory. */
> +    void (*batch_submit)(const struct ct_offload_op_batch *batch);
> +
>      /* May a flow containing the ct() action 'ct_action' with flow key
>       * 'match' on this input netdev be fully offloaded?  Called at
>       * flow-put time through ct_offload_flow_offloadable(); there is no
> @@ -81,7 +182,7 @@ struct ct_offload_class {
>  
>  /* Register/unregister a provider class.  Must be called at module init,
>   * before any connections are created; registration alone does not activate
> - * the provider.  flow_might_offload must be non-NULL. */
> + * the provider.  batch_submit and flow_might_offload must be non-NULL. */
>  int  ct_offload_register(const struct ct_offload_class *);
>  void ct_offload_unregister(const struct ct_offload_class *);
>  
> @@ -93,6 +194,19 @@ void ct_offload_close(const char *name);
>  /* Module initialization (register built-in providers). */
>  void ct_offload_module_init(void);
>  
> +/* Single-operation convenience wrappers around ct_offload_op_batch_*(),
> + * each submitting a batch of one op to every open provider.  Callers
> + * must not hold conn->lock; see the locking notes in ct-offload.c. */
> +void      ct_offload_conn_add(const struct ct_offload_ctx *);
> +void      ct_offload_conn_del(const struct ct_offload_ctx *);
> +
> +/* Returns the hardware last-used time in ms since epoch, or 0. */
> +long long ct_offload_conn_update(const struct ct_offload_ctx *);
> +
> +/* Notifies providers of the established transition (reply-direction
> + * packet); a signal only. */
> +void       ct_offload_conn_notify_established(const struct ct_offload_ctx *);
> +
>  /* May a flow containing the ct() action 'ct_action' with flow key
>   * 'match' on 'netdev_in' be fully offloaded?  Returns true if any open
>   * provider claims it.  Called by dpif offload providers at flow-put
_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev

Reply via email to