On Thu, Nov 10, 2016 at 5:07 AM, Petri Savolainen < [email protected]> wrote:
> Added support for multi-segmented packets. The first segments > is the packet descriptor, which contains all metadata and > pointers to other segments. > > Signed-off-by: Petri Savolainen <[email protected]> > --- > .../include/odp/api/plat/packet_types.h | 6 +- > .../linux-generic/include/odp_buffer_inlines.h | 11 - > .../linux-generic/include/odp_buffer_internal.h | 23 +- > .../linux-generic/include/odp_config_internal.h | 39 +- > .../linux-generic/include/odp_packet_internal.h | 80 +-- > platform/linux-generic/include/odp_pool_internal.h | 3 - > platform/linux-generic/odp_buffer.c | 8 +- > platform/linux-generic/odp_crypto.c | 8 +- > platform/linux-generic/odp_packet.c | 712 > +++++++++++++++++---- > platform/linux-generic/odp_pool.c | 123 ++-- > platform/linux-generic/pktio/netmap.c | 4 +- > platform/linux-generic/pktio/socket.c | 3 +- > 12 files changed, 692 insertions(+), 328 deletions(-) > > diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h > b/platform/linux-generic/include/odp/api/plat/packet_types.h > index b5345ed..864494d 100644 > --- a/platform/linux-generic/include/odp/api/plat/packet_types.h > +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h > @@ -32,9 +32,11 @@ typedef ODP_HANDLE_T(odp_packet_t); > > #define ODP_PACKET_OFFSET_INVALID (0x0fffffff) > > -typedef ODP_HANDLE_T(odp_packet_seg_t); > +/* A packet segment handle stores a small index. Strong type handles are > + * pointers, which would be wasteful in this case. */ > +typedef uint8_t odp_packet_seg_t; > > -#define ODP_PACKET_SEG_INVALID _odp_cast_scalar(odp_packet_seg_t, > 0xffffffff) > +#define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)-1) > > /** odp_packet_color_t assigns names to the various pkt "colors" */ > typedef enum { > diff --git a/platform/linux-generic/include/odp_buffer_inlines.h > b/platform/linux-generic/include/odp_buffer_inlines.h > index f8688f6..cf817d9 100644 > --- a/platform/linux-generic/include/odp_buffer_inlines.h > +++ b/platform/linux-generic/include/odp_buffer_inlines.h > @@ -23,22 +23,11 @@ odp_event_type_t _odp_buffer_event_type(odp_buffer_t > buf); > void _odp_buffer_event_type_set(odp_buffer_t buf, int ev); > int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); > > -void *buffer_map(odp_buffer_hdr_t *buf, uint32_t offset, uint32_t *seglen, > - uint32_t limit); > - > static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) > { > return hdr->handle.handle; > } > > -static inline uint32_t pool_id_from_buf(odp_buffer_t buf) > -{ > - odp_buffer_bits_t handle; > - > - handle.handle = buf; > - return handle.pool_id; > -} > - > #ifdef __cplusplus > } > #endif > diff --git a/platform/linux-generic/include/odp_buffer_internal.h > b/platform/linux-generic/include/odp_buffer_internal.h > index 0ca13f8..4e75908 100644 > --- a/platform/linux-generic/include/odp_buffer_internal.h > +++ b/platform/linux-generic/include/odp_buffer_internal.h > @@ -33,10 +33,6 @@ extern "C" { > #include <odp_schedule_if.h> > #include <stddef.h> > > -ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_SEG_LEN_MIN >= 256, > - "ODP Segment size must be a minimum of 256 bytes"); > - > - > typedef union odp_buffer_bits_t { > odp_buffer_t handle; > > @@ -65,6 +61,20 @@ struct odp_buffer_hdr_t { > int burst_first; > struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; > > + struct { > + void *hdr; > + uint8_t *data; > + uint32_t len; > + } seg[CONFIG_PACKET_MAX_SEGS]; > + > + /* max data size */ > + uint32_t size; > + > + /* Initial buffer data pointer and length */ > + void *base_data; > + uint32_t base_len; > + uint8_t *buf_end; > + > union { > uint32_t all; > struct { > @@ -75,7 +85,6 @@ struct odp_buffer_hdr_t { > > int8_t type; /* buffer type */ > odp_event_type_t event_type; /* for reuse as event */ > - uint32_t size; /* max data size */ > odp_pool_t pool_hdl; /* buffer pool handle */ > union { > uint64_t buf_u64; /* user u64 */ > @@ -86,8 +95,6 @@ struct odp_buffer_hdr_t { > uint32_t uarea_size; /* size of user area */ > uint32_t segcount; /* segment count */ > uint32_t segsize; /* segment size */ > - /* block addrs */ > - void *addr[ODP_CONFIG_PACKET_MAX_SEGS]; > uint64_t order; /* sequence for ordered > queues */ > queue_entry_t *origin_qe; /* ordered queue origin */ > union { > @@ -105,8 +112,6 @@ struct odp_buffer_hdr_t { > }; > > /* Forward declarations */ > -int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount); > -void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount); > int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount); > void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount); > > diff --git a/platform/linux-generic/include/odp_config_internal.h > b/platform/linux-generic/include/odp_config_internal.h > index e24d5ab..9a4e6eb 100644 > --- a/platform/linux-generic/include/odp_config_internal.h > +++ b/platform/linux-generic/include/odp_config_internal.h > @@ -54,7 +54,7 @@ extern "C" { > * The default value (66) allows a 1500-byte packet to be received into a > single > * segment with Ethernet offset alignment and room for some header > expansion. > */ > -#define ODP_CONFIG_PACKET_HEADROOM 66 > +#define CONFIG_PACKET_HEADROOM 66 > > /* > * Default packet tailroom > @@ -65,21 +65,26 @@ extern "C" { > * without restriction. Note that most implementations will automatically > * consider any unused portion of the last segment of a packet as tailroom > */ > -#define ODP_CONFIG_PACKET_TAILROOM 0 > +#define CONFIG_PACKET_TAILROOM 0 > > /* > * Maximum number of segments per packet > */ > -#define ODP_CONFIG_PACKET_MAX_SEGS 1 > +#define CONFIG_PACKET_MAX_SEGS 1 > > /* > - * Maximum packet segment length > - * > - * This defines the maximum packet segment buffer length in bytes. The > user > - * defined segment length (seg_len in odp_pool_param_t) must not be > larger than > - * this. > + * Maximum packet segment size including head- and tailrooms > */ > -#define ODP_CONFIG_PACKET_SEG_LEN_MAX (64 * 1024) > +#define CONFIG_PACKET_SEG_SIZE (64 * 1024) > + > +/* Maximum data length in a segment > + * > + * The user defined segment length (seg_len in odp_pool_param_t) must not > + * be larger than this. > +*/ > +#define CONFIG_PACKET_MAX_SEG_LEN (CONFIG_PACKET_SEG_SIZE - \ > + CONFIG_PACKET_HEADROOM - \ > + CONFIG_PACKET_TAILROOM) > > /* > * Minimum packet segment length > @@ -88,21 +93,7 @@ extern "C" { > * defined segment length (seg_len in odp_pool_param_t) will be rounded > up into > * this value. > */ > -#define ODP_CONFIG_PACKET_SEG_LEN_MIN ODP_CONFIG_PACKET_SEG_LEN_MAX > - > -/* > - * Maximum packet buffer length > - * > - * This defines the maximum number of bytes that can be stored into a > packet > - * (maximum return value of odp_packet_buf_len(void)). Attempts to > allocate > - * (including default head- and tailrooms) or extend packets to sizes > larger > - * than this limit will fail. > - * > - * @internal In odp-linux implementation: > - * - The value MUST be an integral number of segments > - * - The value SHOULD be large enough to accommodate jumbo packets (9K) > - */ > -#define ODP_CONFIG_PACKET_BUF_LEN_MAX ODP_CONFIG_PACKET_SEG_LEN_MAX > +#define CONFIG_PACKET_SEG_LEN_MIN CONFIG_PACKET_MAX_SEG_LEN > > /* Maximum number of shared memory blocks. > * > diff --git a/platform/linux-generic/include/odp_packet_internal.h > b/platform/linux-generic/include/odp_packet_internal.h > index 0cdd5ca..d09231e 100644 > --- a/platform/linux-generic/include/odp_packet_internal.h > +++ b/platform/linux-generic/include/odp_packet_internal.h > @@ -27,8 +27,6 @@ extern "C" { > #include <odp/api/crypto.h> > #include <odp_crypto_internal.h> > > -#define PACKET_JUMBO_LEN (9 * 1024) > - > /** Minimum segment length expected by packet_parse_common() */ > #define PACKET_PARSE_SEG_LEN 96 > > @@ -218,85 +216,13 @@ static inline void > copy_packet_cls_metadata(odp_packet_hdr_t > *src_hdr, > dst_hdr->op_result = src_hdr->op_result; > } > > -static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, > - uint32_t offset, uint32_t *seglen) > -{ > - if (offset > pkt_hdr->frame_len) > - return NULL; > - > - return buffer_map(&pkt_hdr->buf_hdr, > - pkt_hdr->headroom + offset, seglen, > - pkt_hdr->headroom + pkt_hdr->frame_len); > -} > - > -static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len) > -{ > - pkt_hdr->headroom -= len; > - pkt_hdr->frame_len += len; > -} > - > -static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len) > -{ > - pkt_hdr->headroom += len; > - pkt_hdr->frame_len -= len; > -} > - > -static inline int push_head_seg(odp_packet_hdr_t *pkt_hdr, size_t len) > -{ > - uint32_t extrasegs = > - (len - pkt_hdr->headroom + pkt_hdr->buf_hdr.segsize - 1) / > - pkt_hdr->buf_hdr.segsize; > - > - if (pkt_hdr->buf_hdr.segcount + extrasegs > > - ODP_CONFIG_PACKET_MAX_SEGS || > - seg_alloc_head(&pkt_hdr->buf_hdr, extrasegs)) > - return -1; > - > - pkt_hdr->headroom += extrasegs * pkt_hdr->buf_hdr.segsize; > - return 0; > -} > - > -static inline void pull_head_seg(odp_packet_hdr_t *pkt_hdr) > -{ > - uint32_t extrasegs = (pkt_hdr->headroom - 1) / > pkt_hdr->buf_hdr.segsize; > - > - seg_free_head(&pkt_hdr->buf_hdr, extrasegs); > - pkt_hdr->headroom -= extrasegs * pkt_hdr->buf_hdr.segsize; > -} > - > -static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) > -{ > - pkt_hdr->tailroom -= len; > - pkt_hdr->frame_len += len; > -} > - > -static inline int push_tail_seg(odp_packet_hdr_t *pkt_hdr, size_t len) > +static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) > { > - uint32_t extrasegs = > - (len - pkt_hdr->tailroom + pkt_hdr->buf_hdr.segsize - 1) / > - pkt_hdr->buf_hdr.segsize; > + int last = pkt_hdr->buf_hdr.segcount - 1; > > - if (pkt_hdr->buf_hdr.segcount + extrasegs > > - ODP_CONFIG_PACKET_MAX_SEGS || > - seg_alloc_tail(&pkt_hdr->buf_hdr, extrasegs)) > - return -1; > - > - pkt_hdr->tailroom += extrasegs * pkt_hdr->buf_hdr.segsize; > - return 0; > -} > - > -static inline void pull_tail_seg(odp_packet_hdr_t *pkt_hdr) > -{ > - uint32_t extrasegs = pkt_hdr->tailroom / pkt_hdr->buf_hdr.segsize; > - > - seg_free_tail(&pkt_hdr->buf_hdr, extrasegs); > - pkt_hdr->tailroom -= extrasegs * pkt_hdr->buf_hdr.segsize; > -} > - > -static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len) > -{ > pkt_hdr->tailroom += len; > pkt_hdr->frame_len -= len; > + pkt_hdr->buf_hdr.seg[last].len -= len; > } > > static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) > diff --git a/platform/linux-generic/include/odp_pool_internal.h > b/platform/linux-generic/include/odp_pool_internal.h > index f7e951a..5d7b817 100644 > --- a/platform/linux-generic/include/odp_pool_internal.h > +++ b/platform/linux-generic/include/odp_pool_internal.h > @@ -113,9 +113,6 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t > buf[], > odp_buffer_hdr_t *buf_hdr[], int num); > void buffer_free_multi(const odp_buffer_t buf[], int num_free); > > -uint32_t pool_headroom(odp_pool_t pool); > -uint32_t pool_tailroom(odp_pool_t pool); > - > #ifdef __cplusplus > } > #endif > diff --git a/platform/linux-generic/odp_buffer.c > b/platform/linux-generic/odp_buffer.c > index eed15c0..b791039 100644 > --- a/platform/linux-generic/odp_buffer.c > +++ b/platform/linux-generic/odp_buffer.c > @@ -28,7 +28,7 @@ void *odp_buffer_addr(odp_buffer_t buf) > { > odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf); > > - return hdr->addr[0]; > + return hdr->seg[0].data; > } > > uint32_t odp_buffer_size(odp_buffer_t buf) > @@ -56,11 +56,11 @@ int odp_buffer_snprint(char *str, uint32_t n, > odp_buffer_t buf) > " pool %" PRIu64 "\n", > odp_pool_to_u64(hdr->pool_hdl)); > len += snprintf(&str[len], n-len, > - " addr %p\n", hdr->addr); > + " addr %p\n", hdr->seg[0].data); > len += snprintf(&str[len], n-len, > - " size %" PRIu32 "\n", hdr->size); > + " size %" PRIu32 "\n", hdr->size); > len += snprintf(&str[len], n-len, > - " type %i\n", hdr->type); > + " type %i\n", hdr->type); > > return len; > } > diff --git a/platform/linux-generic/odp_crypto.c > b/platform/linux-generic/odp_crypto.c > index 3ebabb7..7e686ff 100644 > --- a/platform/linux-generic/odp_crypto.c > +++ b/platform/linux-generic/odp_crypto.c > @@ -754,9 +754,13 @@ odp_crypto_operation(odp_crypto_op_params_t *params, > ODP_POOL_INVALID != session->output_pool) > params->out_pkt = odp_packet_alloc(session->output_pool, > odp_packet_len(params->pkt)); > + > + if (odp_unlikely(ODP_PACKET_INVALID == params->out_pkt)) { > + ODP_DBG("Alloc failed.\n"); > + return -1; > + } > + > if (params->pkt != params->out_pkt) { > - if (odp_unlikely(ODP_PACKET_INVALID == params->out_pkt)) > - ODP_ABORT(); > (void)odp_packet_copy_from_pkt(params->out_pkt, > 0, > params->pkt, > diff --git a/platform/linux-generic/odp_packet.c > b/platform/linux-generic/odp_packet.c > index 2eee775..a5c6ff4 100644 > --- a/platform/linux-generic/odp_packet.c > +++ b/platform/linux-generic/odp_packet.c > @@ -20,12 +20,155 @@ > #include <stdio.h> > #include <inttypes.h> > > -/* > - * > - * Alloc and free > - * ******************************************************** > - * > - */ > +static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) > +{ > + return (odp_packet_t)pkt_hdr->buf_hdr.handle.handle; > +} > + > +static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) > +{ > + return pkt_hdr->buf_hdr.handle.handle; > +} > + > +static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, > + uint32_t seg_idx) > +{ > + return pkt_hdr->buf_hdr.seg[seg_idx].len; > +} > + > +static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t > seg_idx) > This is more usefully typed as returning uint8_t * rather than void * to permit easy arithmetic on the result. The uint8_t * converts automatically to void * returns for the external API calls that use this. > +{ > + return pkt_hdr->buf_hdr.seg[seg_idx].data; > +} > + > +static inline int packet_last_seg(odp_packet_hdr_t *pkt_hdr) > +{ > + if (CONFIG_PACKET_MAX_SEGS == 1) > + return 0; > + else > + return pkt_hdr->buf_hdr.segcount - 1; > +} > + > +static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) > +{ > + return packet_seg_len(pkt_hdr, 0); > +} > + > +static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) > +{ > + int last = packet_last_seg(pkt_hdr); > + > + return packet_seg_len(pkt_hdr, last); > +} > + > +static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) > +{ > + return pkt_hdr->buf_hdr.seg[0].data; > Given the earlier definition of the packet_seg_data helper this would more consistently be: return packet_seg_data(pkt_hdr, 0); > +} > + > +static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) > +{ > + int last = packet_last_seg(pkt_hdr); > + uint32_t seg_len = pkt_hdr->buf_hdr.seg[last].len; > + > + return pkt_hdr->buf_hdr.seg[last].data + seg_len; > +} > + > +static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) > +{ > + pkt_hdr->headroom -= len; > + pkt_hdr->frame_len += len; > + pkt_hdr->buf_hdr.seg[0].data -= len; > + pkt_hdr->buf_hdr.seg[0].len += len; > +} > + > +static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) > +{ > + pkt_hdr->headroom += len; > + pkt_hdr->frame_len -= len; > + pkt_hdr->buf_hdr.seg[0].data += len; > + pkt_hdr->buf_hdr.seg[0].len -= len; > +} > + > +static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) > +{ > + int last = packet_last_seg(pkt_hdr); > + > + pkt_hdr->tailroom -= len; > + pkt_hdr->frame_len += len; > + pkt_hdr->buf_hdr.seg[last].len += len; > +} > + > +/* Copy all metadata for segmentation modification. Segment data and > lengths > + * are not copied. */ > +static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, > + odp_packet_hdr_t *src) > +{ > + dst->p = src->p; > + > + /* lengths are not copied: > + * .frame_len > + * .headroom > + * .tailroom > + */ > + > + dst->input = src->input; > + dst->dst_queue = src->dst_queue; > + dst->flow_hash = src->flow_hash; > + dst->timestamp = src->timestamp; > + dst->op_result = src->op_result; > + > + /* buffer header side packet metadata */ > + dst->buf_hdr.buf_u64 = src->buf_hdr.buf_u64; > + dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; > + dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; > + > + /* segmentation data is not copied: > + * buf_hdr.seg[] > + * buf_hdr.segcount > + */ > +} > + > +static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, > + uint32_t offset, uint32_t *seg_len, int > *seg_idx) > +{ > + void *addr; > + uint32_t len; > + int seg = 0; > + int seg_count = pkt_hdr->buf_hdr.segcount; > + > + if (odp_unlikely(offset >= pkt_hdr->frame_len)) > + return NULL; > + > + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { > + addr = pkt_hdr->buf_hdr.seg[0].data + offset; > + len = pkt_hdr->buf_hdr.seg[0].len - offset; > + } else { > + int i; > + uint32_t seg_start = 0, seg_end = 0; > + > + for (i = 0; i < seg_count; i++) { > + seg_end += pkt_hdr->buf_hdr.seg[i].len; > + > + if (odp_likely(offset < seg_end)) > + break; > + > + seg_start = seg_end; > + } > + > + addr = pkt_hdr->buf_hdr.seg[i].data + (offset - seg_start); > + len = pkt_hdr->buf_hdr.seg[i].len - (offset - seg_start); > + seg = i; > + } > + > + if (seg_len) > + *seg_len = len; > + > + if (seg_idx) > + *seg_idx = seg; > + > + return addr; > +} > > static inline void packet_parse_disable(odp_packet_hdr_t *pkt_hdr) > { > @@ -48,11 +191,23 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) > /** > * Initialize packet > */ > -static void packet_init(pool_t *pool, odp_packet_hdr_t *pkt_hdr, > - size_t size, int parse) > +static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, > + int parse) > { > - pkt_hdr->p.parsed_layers = LAYER_NONE; > + uint32_t seg_len; > + int num = pkt_hdr->buf_hdr.segcount; > + > + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num == 1)) { > + seg_len = len; > + pkt_hdr->buf_hdr.seg[0].len = len; > + } else { > + seg_len = len - ((num - 1) * CONFIG_PACKET_MAX_SEG_LEN); > + > + /* Last segment data length */ > + pkt_hdr->buf_hdr.seg[num - 1].len = seg_len; > + } > > + pkt_hdr->p.parsed_layers = LAYER_NONE; > pkt_hdr->p.input_flags.all = 0; > pkt_hdr->p.output_flags.all = 0; > pkt_hdr->p.error_flags.all = 0; > @@ -70,42 +225,260 @@ static void packet_init(pool_t *pool, > odp_packet_hdr_t *pkt_hdr, > * Packet tailroom is rounded up to fill the last > * segment occupied by the allocated length. > */ > - pkt_hdr->frame_len = size; > - pkt_hdr->headroom = pool->headroom; > - pkt_hdr->tailroom = pool->data_size - size + pool->tailroom; > + pkt_hdr->frame_len = len; > + pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; > + pkt_hdr->tailroom = CONFIG_PACKET_MAX_SEG_LEN - seg_len + > + CONFIG_PACKET_TAILROOM; > > pkt_hdr->input = ODP_PKTIO_INVALID; > } > > -int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, > - odp_packet_t pkt[], int max_num) > +static inline odp_packet_hdr_t *init_segments(odp_buffer_t buf[], int num) > { > - pool_t *pool = pool_entry_from_hdl(pool_hdl); > - int num, i; > - odp_packet_hdr_t *pkt_hdrs[max_num]; > + odp_packet_hdr_t *pkt_hdr; > + int i; > + > + /* First buffer is the packet descriptor */ > + pkt_hdr = odp_packet_hdr((odp_packet_t)buf[0]); > + > + pkt_hdr->buf_hdr.seg[0].data = pkt_hdr->buf_hdr.base_data; > + pkt_hdr->buf_hdr.seg[0].len = pkt_hdr->buf_hdr.base_len; > + > + /* Link segments */ > + if (odp_unlikely(CONFIG_PACKET_MAX_SEGS != 1)) { > + pkt_hdr->buf_hdr.segcount = num; > + > + if (odp_unlikely(num > 1)) { > + for (i = 1; i < num; i++) { > + odp_packet_hdr_t *hdr; > + odp_buffer_hdr_t *b_hdr; > + > + hdr = odp_packet_hdr((odp_packet_t) > buf[i]); > + b_hdr = &hdr->buf_hdr; > + > + pkt_hdr->buf_hdr.seg[i].hdr = hdr; > + pkt_hdr->buf_hdr.seg[i].data = > b_hdr->base_data; > + pkt_hdr->buf_hdr.seg[i].len = > b_hdr->base_len; > + } > + } > + } > + > + return pkt_hdr; > +} > + > +/* Calculate the number of segments */ > +static inline int num_segments(uint32_t len) > +{ > + uint32_t max_seg_len; > + int num; > > - num = buffer_alloc_multi(pool, (odp_buffer_t *)pkt, > - (odp_buffer_hdr_t **)pkt_hdrs, max_num); > + if (CONFIG_PACKET_MAX_SEGS == 1) > + return 1; > + > + num = 1; > + max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; > + > + if (odp_unlikely(len > max_seg_len)) { > + num = len / max_seg_len; > + > + if (odp_likely((num * max_seg_len) != len)) > + num += 1; > + } > + > + return num; > +} > + > +static inline void copy_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t > *from) > +{ > + int i; > + int n = to->buf_hdr.segcount; > + int num = from->buf_hdr.segcount; > > for (i = 0; i < num; i++) { > - odp_packet_hdr_t *pkt_hdr = pkt_hdrs[i]; > + to->buf_hdr.seg[n + i].hdr = from->buf_hdr.seg[i].hdr; > + to->buf_hdr.seg[n + i].data = from->buf_hdr.seg[i].data; > + to->buf_hdr.seg[n + i].len = from->buf_hdr.seg[i].len; > + } > + > + to->buf_hdr.segcount = n + num; > +} > > - packet_init(pool, pkt_hdr, len, 1 /* do parse */); > +static inline void copy_num_segs(odp_packet_hdr_t *to, odp_packet_hdr_t > *from, > + int num) > +{ > + int i; > > - if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize) > - pull_tail_seg(pkt_hdr); > + for (i = 0; i < num; i++) { > + to->buf_hdr.seg[i].hdr = from->buf_hdr.seg[num + i].hdr; > + to->buf_hdr.seg[i].data = from->buf_hdr.seg[num + i].data; > + to->buf_hdr.seg[i].len = from->buf_hdr.seg[num + i].len; > } > > + to->buf_hdr.segcount = num; > +} > + > +static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, > + uint32_t len, int head) > +{ > + pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); > + odp_packet_hdr_t *new_hdr; > + int num, ret; > + uint32_t seg_len, offset; > + > + num = num_segments(len); > + > + if ((pkt_hdr->buf_hdr.segcount + num) > CONFIG_PACKET_MAX_SEGS) > + return NULL; > + > + { > + odp_buffer_t buf[num]; > + > + ret = buffer_alloc_multi(pool, buf, NULL, num); > + if (odp_unlikely(ret != num)) { > + if (ret > 0) > + buffer_free_multi(buf, ret); > + > + return NULL; > + } > + > + new_hdr = init_segments(buf, num); > + } > + > + seg_len = len - ((num - 1) * pool->max_seg_len); > + offset = pool->max_seg_len - seg_len; > + > + if (head) { > + /* add into the head*/ > + copy_all_segs(new_hdr, pkt_hdr); > + > + /* adjust first segment length */ > + new_hdr->buf_hdr.seg[0].data += offset; > + new_hdr->buf_hdr.seg[0].len = seg_len; > + > + packet_seg_copy_md(new_hdr, pkt_hdr); > + new_hdr->frame_len = pkt_hdr->frame_len + len; > + new_hdr->headroom = pool->headroom + offset; > + new_hdr->tailroom = pkt_hdr->tailroom; > + > + pkt_hdr = new_hdr; > + } else { > + int last; > + > + /* add into the tail */ > + copy_all_segs(pkt_hdr, new_hdr); > + > + /* adjust last segment length */ > + last = packet_last_seg(pkt_hdr); > + pkt_hdr->buf_hdr.seg[last].len = seg_len; > + > + pkt_hdr->frame_len += len; > + pkt_hdr->tailroom = pool->tailroom + offset; > + } > + > + return pkt_hdr; > +} > + > +static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > + int num, uint32_t free_len, > + uint32_t pull_len, int head) > +{ > + int i; > + odp_packet_hdr_t *new_hdr; > + odp_buffer_t buf[num]; > + int n = pkt_hdr->buf_hdr.segcount - num; > + > + if (head) { > + for (i = 0; i < num; i++) > + buf[i] = buffer_handle(pkt_hdr->buf_ > hdr.seg[i].hdr); > + > + /* First remaining segment is the new packet descriptor */ > + new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; > + copy_num_segs(new_hdr, pkt_hdr, n); > + packet_seg_copy_md(new_hdr, pkt_hdr); > + > + /* Tailroom not changed */ > + new_hdr->tailroom = pkt_hdr->tailroom; > + /* No headroom in non-first segments */ > + new_hdr->headroom = 0; > + new_hdr->frame_len = pkt_hdr->frame_len - free_len; > + > + pull_head(new_hdr, pull_len); > + > + pkt_hdr = new_hdr; > + } else { > + for (i = 0; i < num; i++) > + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[n + > i].hdr); > + > + /* Head segment remains, no need to copy or update majority > + * of the metadata. */ > + pkt_hdr->buf_hdr.segcount = n; > + pkt_hdr->frame_len -= free_len; > + pkt_hdr->tailroom = pkt_hdr->buf_hdr.buf_end - > + (uint8_t *)packet_tail(pkt_hdr); > + > + pull_tail(pkt_hdr, pull_len); > + } > + > + buffer_free_multi(buf, num); > + > + return pkt_hdr; > +} > + > +static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, > + int num_seg, odp_packet_t *pkt, int parse) > +{ > + int num_buf, i; > + int num = max_pkt; > + int max_buf = max_pkt * num_seg; > + odp_buffer_t buf[max_buf]; > + > + num_buf = buffer_alloc_multi(pool, buf, NULL, max_buf); > + > + /* Failed to allocate all segments */ > + if (odp_unlikely(num_buf != max_buf)) { > + int num_free; > + > + num = num_buf / num_seg; > + num_free = num_buf - (num * num_seg); > + > + if (num_free > 0) > + buffer_free_multi(&buf[num_buf - num_free], > num_free); > + > + if (num == 0) > + return 0; > + } > + > + for (i = 0; i < num; i++) { > + odp_packet_hdr_t *pkt_hdr; > + > + /* First buffer is the packet descriptor */ > + pkt[i] = (odp_packet_t)buf[i * num_seg]; > + pkt_hdr = init_segments(&buf[i * num_seg], num_seg); > + > + packet_init(pkt_hdr, len, parse); > + } > + > + return num; > +} > + > +int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, > + odp_packet_t pkt[], int max_num) > +{ > + pool_t *pool = pool_entry_from_hdl(pool_hdl); > + int num, num_seg; > + > + num_seg = num_segments(len); > + num = packet_alloc(pool, len, max_num, num_seg, pkt, 1); > + > return num; > } > > odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) > { > pool_t *pool = pool_entry_from_hdl(pool_hdl); > - size_t pkt_size = len ? len : pool->data_size; > odp_packet_t pkt; > - odp_packet_hdr_t *pkt_hdr; > - int ret; > + int num, num_seg; > + int zero_len = 0; > > if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { > __odp_errno = EINVAL; > @@ -115,28 +488,32 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, > uint32_t len) > if (odp_unlikely(len > pool->max_len)) > return ODP_PACKET_INVALID; > > - ret = buffer_alloc_multi(pool, (odp_buffer_t *)&pkt, NULL, 1); > - if (ret != 1) > + if (odp_unlikely(len == 0)) { > + len = pool->data_size; > + zero_len = 1; > + } > + > + num_seg = num_segments(len); > + num = packet_alloc(pool, len, 1, num_seg, &pkt, 0); > + > + if (odp_unlikely(num == 0)) > return ODP_PACKET_INVALID; > > - pkt_hdr = odp_packet_hdr(pkt); > - packet_init(pool, pkt_hdr, pkt_size, 0 /* do not parse */); > - if (len == 0) > - pull_tail(pkt_hdr, pkt_size); > + if (odp_unlikely(zero_len)) { > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize) > - pull_tail_seg(pkt_hdr); > + pull_tail(pkt_hdr, len); > + } > > return pkt; > } > > int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, > - odp_packet_t pkt[], int num) > + odp_packet_t pkt[], int max_num) > { > pool_t *pool = pool_entry_from_hdl(pool_hdl); > - size_t pkt_size = len ? len : pool->data_size; > - int count, i; > - odp_packet_hdr_t *pkt_hdrs[num]; > + int num, num_seg; > + int zero_len = 0; > > if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { > __odp_errno = EINVAL; > @@ -146,31 +523,75 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, > uint32_t len, > if (odp_unlikely(len > pool->max_len)) > return -1; > > - count = buffer_alloc_multi(pool, (odp_buffer_t *)pkt, > - (odp_buffer_hdr_t **)pkt_hdrs, num); > + if (odp_unlikely(len == 0)) { > + len = pool->data_size; > + zero_len = 1; > + } > + > + num_seg = num_segments(len); > + num = packet_alloc(pool, len, max_num, num_seg, pkt, 0); > > - for (i = 0; i < count; ++i) { > - odp_packet_hdr_t *pkt_hdr = pkt_hdrs[i]; > + if (odp_unlikely(zero_len)) { > + int i; > > - packet_init(pool, pkt_hdr, pkt_size, 0 /* do not parse */); > - if (len == 0) > - pull_tail(pkt_hdr, pkt_size); > + for (i = 0; i < num; i++) { > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]); > > - if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize) > - pull_tail_seg(pkt_hdr); > + pull_tail(pkt_hdr, len); > + } > } > > - return count; > + return num; > } > > void odp_packet_free(odp_packet_t pkt) > { > - buffer_free_multi((odp_buffer_t *)&pkt, 1); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + int num_seg = pkt_hdr->buf_hdr.segcount; > + > + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) { > + buffer_free_multi((odp_buffer_t *)&pkt, 1); > + } else { > + odp_buffer_t buf[num_seg]; > + int i; > + > + buf[0] = (odp_buffer_t)pkt; > + > + for (i = 1; i < num_seg; i++) > + buf[i] = buffer_handle(pkt_hdr->buf_ > hdr.seg[i].hdr); > + > + buffer_free_multi(buf, num_seg); > + } > } > > void odp_packet_free_multi(const odp_packet_t pkt[], int num) > { > - buffer_free_multi((const odp_buffer_t * const)pkt, num); > + if (CONFIG_PACKET_MAX_SEGS == 1) { > + buffer_free_multi((const odp_buffer_t * const)pkt, num); > + } else { > + odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; > + int i, j; > + int bufs = 0; > + > + for (i = 0; i < num; i++) { > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]); > + int num_seg = pkt_hdr->buf_hdr.segcount; > + odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr; > + > + buf[bufs] = (odp_buffer_t)pkt[i]; > + bufs++; > + > + if (odp_likely(num_seg == 1)) > + continue; > + > + for (j = 1; j < num_seg; j++) { > + buf[bufs] = buffer_handle(buf_hdr->seg[j]. > hdr); > + bufs++; > + } > + } > + > + buffer_free_multi(buf, bufs); > + } > } > > int odp_packet_reset(odp_packet_t pkt, uint32_t len) > @@ -181,7 +602,7 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) > if (len > pool->headroom + pool->data_size + pool->tailroom) > return -1; > > - packet_init(pool, pkt_hdr, len, 0); > + packet_init(pkt_hdr, len, 0); > > return 0; > } > @@ -217,7 +638,7 @@ void *odp_packet_head(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0); > + return pkt_hdr->buf_hdr.seg[0].data - pkt_hdr->headroom; > } > > uint32_t odp_packet_buf_len(odp_packet_t pkt) > @@ -231,17 +652,14 @@ void *odp_packet_data(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - return packet_map(pkt_hdr, 0, NULL); > + return packet_data(pkt_hdr); > } > > uint32_t odp_packet_seg_len(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t seglen; > > - /* Call returns length of 1st data segment */ > - packet_map(pkt_hdr, 0, &seglen); > - return seglen; > + return packet_first_seg_len(pkt_hdr); > } > > uint32_t odp_packet_len(odp_packet_t pkt) > @@ -263,7 +681,7 @@ void *odp_packet_tail(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); > + return packet_tail(pkt_hdr); > } > > void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) > @@ -274,21 +692,38 @@ void *odp_packet_push_head(odp_packet_t pkt, > uint32_t len) > return NULL; > > push_head(pkt_hdr, len); > - return packet_map(pkt_hdr, 0, NULL); > + return packet_data(pkt_hdr); > } > > int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > void **data_ptr, uint32_t *seg_len) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > + odp_packet_hdr_t *new_hdr; > + uint32_t headroom = pkt_hdr->headroom; > > - if (len > pkt_hdr->headroom && push_head_seg(pkt_hdr, len)) > - return -1; > + if (len > headroom) { > + push_head(pkt_hdr, headroom); > + new_hdr = add_segments(pkt_hdr, len - headroom, 1); > > - push_head(pkt_hdr, len); > + if (new_hdr == NULL) { > + /* segment alloc failed, rollback changes */ > + pull_head(pkt_hdr, headroom); > + return -1; > + } > + > + *pkt = packet_handle(new_hdr); > + pkt_hdr = new_hdr; > + } else { > + push_head(pkt_hdr, len); > + } > > if (data_ptr) > - *data_ptr = packet_map(pkt_hdr, 0, seg_len); > + *data_ptr = packet_data(pkt_hdr); > + > + if (seg_len) > + *seg_len = packet_first_seg_len(pkt_hdr); > + > return 0; > } > > @@ -300,51 +735,82 @@ void *odp_packet_pull_head(odp_packet_t pkt, > uint32_t len) > return NULL; > > pull_head(pkt_hdr, len); > - return packet_map(pkt_hdr, 0, NULL); > + return packet_data(pkt_hdr); > } > > int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, > - void **data_ptr, uint32_t *seg_len) > + void **data_ptr, uint32_t *seg_len_out) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > + uint32_t seg_len = packet_first_seg_len(pkt_hdr); > > if (len > pkt_hdr->frame_len) > return -1; > > - pull_head(pkt_hdr, len); > - if (pkt_hdr->headroom >= pkt_hdr->buf_hdr.segsize) > - pull_head_seg(pkt_hdr); > + if (len < seg_len) { > + pull_head(pkt_hdr, len); > + } else if (CONFIG_PACKET_MAX_SEGS != 1) { > + int num = 0; > + uint32_t pull_len = 0; > + > + while (seg_len <= len) { > + pull_len = len - seg_len; > + num++; > + seg_len += packet_seg_len(pkt_hdr, num); > + } > + > + pkt_hdr = free_segments(pkt_hdr, num, len - pull_len, > + pull_len, 1); > + *pkt = packet_handle(pkt_hdr); > + } > > if (data_ptr) > - *data_ptr = packet_map(pkt_hdr, 0, seg_len); > + *data_ptr = packet_data(pkt_hdr); > + > + if (seg_len_out) > + *seg_len_out = packet_first_seg_len(pkt_hdr); > + > return 0; > } > > void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t origin = pkt_hdr->frame_len; > + void *old_tail; > > if (len > pkt_hdr->tailroom) > return NULL; > > + old_tail = packet_tail(pkt_hdr); > push_tail(pkt_hdr, len); > - return packet_map(pkt_hdr, origin, NULL); > + > + return old_tail; > } > > int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, > void **data_ptr, uint32_t *seg_len) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > - uint32_t origin = pkt_hdr->frame_len; > + void *ret; > + uint32_t tailroom = pkt_hdr->tailroom; > + uint32_t tail_off = pkt_hdr->frame_len; > > - if (len > pkt_hdr->tailroom && push_tail_seg(pkt_hdr, len)) > - return -1; > + if (len > tailroom) { > + push_tail(pkt_hdr, tailroom); > + ret = add_segments(pkt_hdr, len - tailroom, 0); > > - push_tail(pkt_hdr, len); > + if (ret == NULL) { > + /* segment alloc failed, rollback changes */ > + pull_tail(pkt_hdr, tailroom); > + return -1; > + } > + } else { > + push_tail(pkt_hdr, len); > + } > > if (data_ptr) > - *data_ptr = packet_map(pkt_hdr, origin, seg_len); > + *data_ptr = packet_map(pkt_hdr, tail_off, seg_len, NULL); > + > return 0; > } > > @@ -352,27 +818,45 @@ void *odp_packet_pull_tail(odp_packet_t pkt, > uint32_t len) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (len > pkt_hdr->frame_len) > + if (len > packet_last_seg_len(pkt_hdr)) > return NULL; > > pull_tail(pkt_hdr, len); > - return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); > + > + return packet_tail(pkt_hdr); > } > > int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, > void **tail_ptr, uint32_t *tailroom) > { > + int last; > + uint32_t seg_len; > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > > if (len > pkt_hdr->frame_len) > return -1; > > - pull_tail(pkt_hdr, len); > - if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize) > - pull_tail_seg(pkt_hdr); > + last = packet_last_seg(pkt_hdr); > + seg_len = packet_seg_len(pkt_hdr, last); > + > + if (len < seg_len) { > + pull_tail(pkt_hdr, len); > + } else if (CONFIG_PACKET_MAX_SEGS != 1) { > + int num = 0; > + uint32_t pull_len = 0; > + > + while (seg_len <= len) { > + pull_len = len - seg_len; > + num++; > + seg_len += packet_seg_len(pkt_hdr, last - num); > + } > + > + free_segments(pkt_hdr, num, len - pull_len, pull_len, 0); > + } > > if (tail_ptr) > - *tail_ptr = packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); > + *tail_ptr = packet_tail(pkt_hdr); > + > if (tailroom) > *tailroom = pkt_hdr->tailroom; > return 0; > @@ -381,11 +865,12 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, > uint32_t len, > void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, > odp_packet_seg_t *seg) > { > + int seg_idx; > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - void *addr = packet_map(pkt_hdr, offset, len); > + void *addr = packet_map(pkt_hdr, offset, len, &seg_idx); > > if (addr != NULL && seg != NULL) > - *seg = (odp_packet_seg_t)pkt; > + *seg = seg_idx; > > return addr; > } > @@ -445,7 +930,7 @@ void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t > *len) > > if (!packet_hdr_has_l2(pkt_hdr)) > return NULL; > - return packet_map(pkt_hdr, pkt_hdr->p.l2_offset, len); > + return packet_map(pkt_hdr, pkt_hdr->p.l2_offset, len, NULL); > } > > uint32_t odp_packet_l2_offset(odp_packet_t pkt) > @@ -475,7 +960,7 @@ void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t > *len) > > if (pkt_hdr->p.parsed_layers < LAYER_L3) > packet_parse_layer(pkt_hdr, LAYER_L3); > - return packet_map(pkt_hdr, pkt_hdr->p.l3_offset, len); > + return packet_map(pkt_hdr, pkt_hdr->p.l3_offset, len, NULL); > } > > uint32_t odp_packet_l3_offset(odp_packet_t pkt) > @@ -506,7 +991,7 @@ void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t > *len) > > if (pkt_hdr->p.parsed_layers < LAYER_L4) > packet_parse_layer(pkt_hdr, LAYER_L4); > - return packet_map(pkt_hdr, pkt_hdr->p.l4_offset, len); > + return packet_map(pkt_hdr, pkt_hdr->p.l4_offset, len, NULL); > } > > uint32_t odp_packet_l4_offset(odp_packet_t pkt) > @@ -568,29 +1053,33 @@ int odp_packet_is_segmented(odp_packet_t pkt) > > int odp_packet_num_segs(odp_packet_t pkt) > { > - return odp_packet_hdr(pkt)->buf_hdr.segcount; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + return pkt_hdr->buf_hdr.segcount; > } > > odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) > { > - return (odp_packet_seg_t)pkt; > + (void)pkt; > + > + return 0; > } > This should be written as: odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt ODP_UNUSED) { return 0; } > > odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) > { > - (void)pkt; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - /* Only one segment */ > - return (odp_packet_seg_t)pkt; > + return packet_last_seg(pkt_hdr); > } > > odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t > seg) > { > - (void)pkt; > - (void)seg; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - /* Only one segment */ > - return ODP_PACKET_SEG_INVALID; > + if (odp_unlikely(seg >= (odp_packet_seg_t)packet_last_ > seg(pkt_hdr))) > + return ODP_PACKET_SEG_INVALID; > + > + return seg + 1; > } > > /* > @@ -602,18 +1091,22 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t > pkt, odp_packet_seg_t seg) > > void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) > { > - (void)seg; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - /* Only one segment */ > - return odp_packet_data(pkt); > + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) > + return NULL; > + > + return packet_seg_data(pkt_hdr, seg); > } > > uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) > { > - (void)seg; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - /* Only one segment */ > - return odp_packet_seg_len(pkt); > + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) > + return 0; > + > + return packet_seg_len(pkt_hdr, seg); > } > > /* > @@ -688,7 +1181,7 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t > offset, uint32_t len, > uint32_t shift; > uint32_t seglen = 0; /* GCC */ > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > - void *addr = packet_map(pkt_hdr, offset, &seglen); > + void *addr = packet_map(pkt_hdr, offset, &seglen, NULL); > uint64_t uaddr = (uint64_t)(uintptr_t)addr; > uint64_t misalign; > > @@ -733,6 +1226,7 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t > src) > src, 0, src_len); > if (src != *dst) > odp_packet_free(src); > + > return 0; > } > > @@ -808,7 +1302,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, > uint32_t offset, > return -1; > > while (len > 0) { > - mapaddr = packet_map(pkt_hdr, offset, &seglen); > + mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); > cpylen = len > seglen ? seglen : len; > memcpy(dstaddr, mapaddr, cpylen); > offset += cpylen; > @@ -832,7 +1326,7 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, > uint32_t offset, > return -1; > > while (len > 0) { > - mapaddr = packet_map(pkt_hdr, offset, &seglen); > + mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); > cpylen = len > seglen ? seglen : len; > memcpy(mapaddr, srcaddr, cpylen); > offset += cpylen; > @@ -878,8 +1372,8 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, > uint32_t dst_offset, > } > > while (len > 0) { > - dst_map = packet_map(dst_hdr, dst_offset, &dst_seglen); > - src_map = packet_map(src_hdr, src_offset, &src_seglen); > + dst_map = packet_map(dst_hdr, dst_offset, &dst_seglen, > NULL); > + src_map = packet_map(src_hdr, src_offset, &src_seglen, > NULL); > > minseg = dst_seglen > src_seglen ? src_seglen : dst_seglen; > cpylen = len > minseg ? minseg : len; > @@ -1364,8 +1858,8 @@ parse_exit: > */ > int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer) > { > - uint32_t seg_len; > - void *base = packet_map(pkt_hdr, 0, &seg_len); > + uint32_t seg_len = packet_first_seg_len(pkt_hdr); > + void *base = packet_data(pkt_hdr); > > return packet_parse_common(&pkt_hdr->p, base, pkt_hdr->frame_len, > seg_len, layer); > diff --git a/platform/linux-generic/odp_pool.c > b/platform/linux-generic/odp_pool.c > index 364df97..7c462e5 100644 > --- a/platform/linux-generic/odp_pool.c > +++ b/platform/linux-generic/odp_pool.c > @@ -32,6 +32,9 @@ > ODP_STATIC_ASSERT(CONFIG_POOL_CACHE_SIZE > (2 * CACHE_BURST), > "cache_burst_size_too_large_compared_to_cache_size"); > > +ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256, > + "ODP Segment size must be a minimum of 256 bytes"); > + > /* Thread local variables */ > typedef struct pool_local_t { > pool_cache_t *cache[ODP_CONFIG_POOLS]; > @@ -46,6 +49,14 @@ static inline odp_pool_t pool_index_to_handle(uint32_t > pool_idx) > return _odp_cast_scalar(odp_pool_t, pool_idx); > } > > +static inline uint32_t pool_id_from_buf(odp_buffer_t buf) > +{ > + odp_buffer_bits_t handle; > + > + handle.handle = buf; > + return handle.pool_id; > +} > + > int odp_pool_init_global(void) > { > uint32_t i; > @@ -198,7 +209,7 @@ static void init_buffers(pool_t *pool) > ring_t *ring; > uint32_t mask; > int type; > - uint32_t size; > + uint32_t seg_size; > > ring = &pool->ring.hdr; > mask = pool->ring_mask; > @@ -223,12 +234,12 @@ static void init_buffers(pool_t *pool) > while (((uintptr_t)&data[offset]) % pool->align != 0) > offset++; > > - memset(buf_hdr, 0, sizeof(odp_buffer_hdr_t)); > + memset(buf_hdr, 0, (uintptr_t)data - (uintptr_t)buf_hdr); > > - size = pool->headroom + pool->data_size + pool->tailroom; > + seg_size = pool->headroom + pool->data_size + > pool->tailroom; > > /* Initialize buffer metadata */ > - buf_hdr->size = size; > + buf_hdr->size = seg_size; > buf_hdr->type = type; > buf_hdr->event_type = type; > buf_hdr->pool_hdl = pool->pool_hdl; > @@ -236,10 +247,18 @@ static void init_buffers(pool_t *pool) > /* Show user requested size through API */ > buf_hdr->uarea_size = pool->params.pkt.uarea_size; > buf_hdr->segcount = 1; > - buf_hdr->segsize = size; > + buf_hdr->segsize = seg_size; > > /* Pointer to data start (of the first segment) */ > - buf_hdr->addr[0] = &data[offset]; > + buf_hdr->seg[0].hdr = buf_hdr; > + buf_hdr->seg[0].data = &data[offset]; > + buf_hdr->seg[0].len = pool->data_size; > + > + /* Store base values for fast init */ > + buf_hdr->base_data = buf_hdr->seg[0].data; > + buf_hdr->base_len = buf_hdr->seg[0].len; > + buf_hdr->buf_end = &data[offset + pool->data_size + > + pool->tailroom]; > > buf_hdl = form_buffer_handle(pool->pool_idx, i); > buf_hdr->handle.handle = buf_hdl; > @@ -296,25 +315,13 @@ static odp_pool_t pool_create(const char *name, > odp_pool_param_t *params, > break; > > case ODP_POOL_PACKET: > - headroom = ODP_CONFIG_PACKET_HEADROOM; > - tailroom = ODP_CONFIG_PACKET_TAILROOM; > - num = params->pkt.num; > - uarea_size = params->pkt.uarea_size; > - > - data_size = ODP_CONFIG_PACKET_SEG_LEN_MAX; > - > - if (data_size < ODP_CONFIG_PACKET_SEG_LEN_MIN) > - data_size = ODP_CONFIG_PACKET_SEG_LEN_MIN; > - > - if (data_size > ODP_CONFIG_PACKET_SEG_LEN_MAX) { > - ODP_ERR("Too large seg len requirement"); > - return ODP_POOL_INVALID; > - } > - > - max_seg_len = ODP_CONFIG_PACKET_SEG_LEN_MAX - > - ODP_CONFIG_PACKET_HEADROOM - > - ODP_CONFIG_PACKET_TAILROOM; > - max_len = ODP_CONFIG_PACKET_MAX_SEGS * max_seg_len; > + headroom = CONFIG_PACKET_HEADROOM; > + tailroom = CONFIG_PACKET_TAILROOM; > + num = params->pkt.num; > + uarea_size = params->pkt.uarea_size; > + data_size = CONFIG_PACKET_MAX_SEG_LEN; > + max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; > + max_len = CONFIG_PACKET_MAX_SEGS * max_seg_len; > break; > > case ODP_POOL_TIMEOUT: > @@ -470,31 +477,6 @@ void _odp_buffer_event_type_set(odp_buffer_t buf, > int ev) > buf_hdl_to_hdr(buf)->event_type = ev; > } > > -void *buffer_map(odp_buffer_hdr_t *buf, > - uint32_t offset, > - uint32_t *seglen, > - uint32_t limit) > -{ > - int seg_index; > - int seg_offset; > - > - if (odp_likely(offset < buf->segsize)) { > - seg_index = 0; > - seg_offset = offset; > - } else { > - ODP_ERR("\nSEGMENTS NOT SUPPORTED\n"); > - return NULL; > - } > - > - if (seglen != NULL) { > - uint32_t buf_left = limit - offset; > - *seglen = seg_offset + buf_left <= buf->segsize ? > - buf_left : buf->segsize - seg_offset; > - } > - > - return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]); > -} > - > odp_pool_t odp_pool_lookup(const char *name) > { > uint32_t i; > @@ -727,9 +709,7 @@ void odp_buffer_free_multi(const odp_buffer_t buf[], > int num) > > int odp_pool_capability(odp_pool_capability_t *capa) > { > - uint32_t max_len = ODP_CONFIG_PACKET_SEG_LEN_MAX - > - ODP_CONFIG_PACKET_HEADROOM - > - ODP_CONFIG_PACKET_TAILROOM; > + uint32_t max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; > > memset(capa, 0, sizeof(odp_pool_capability_t)); > > @@ -743,13 +723,13 @@ int odp_pool_capability(odp_pool_capability_t *capa) > > /* Packet pools */ > capa->pkt.max_pools = ODP_CONFIG_POOLS; > - capa->pkt.max_len = ODP_CONFIG_PACKET_MAX_SEGS * max_len; > + capa->pkt.max_len = CONFIG_PACKET_MAX_SEGS * max_seg_len; > capa->pkt.max_num = CONFIG_POOL_MAX_NUM; > - capa->pkt.min_headroom = ODP_CONFIG_PACKET_HEADROOM; > - capa->pkt.min_tailroom = ODP_CONFIG_PACKET_TAILROOM; > - capa->pkt.max_segs_per_pkt = ODP_CONFIG_PACKET_MAX_SEGS; > - capa->pkt.min_seg_len = max_len; > - capa->pkt.max_seg_len = max_len; > + capa->pkt.min_headroom = CONFIG_PACKET_HEADROOM; > + capa->pkt.min_tailroom = CONFIG_PACKET_TAILROOM; > + capa->pkt.max_segs_per_pkt = CONFIG_PACKET_MAX_SEGS; > + capa->pkt.min_seg_len = max_seg_len; > + capa->pkt.max_seg_len = max_seg_len; > capa->pkt.max_uarea_size = 0; > > /* Timeout pools */ > @@ -765,7 +745,7 @@ void odp_pool_print(odp_pool_t pool_hdl) > > pool = pool_entry_from_hdl(pool_hdl); > > - printf("Pool info\n"); > + printf("\nPool info\n"); > printf("---------\n"); > printf(" pool %" PRIu64 "\n", > odp_pool_to_u64(pool->pool_hdl)); > @@ -812,19 +792,6 @@ uint64_t odp_pool_to_u64(odp_pool_t hdl) > return _odp_pri(hdl); > } > > -int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount) > -{ > - (void)buf_hdr; > - (void)segcount; > - return 0; > -} > - > -void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount) > -{ > - (void)buf_hdr; > - (void)segcount; > -} > - > int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount) > { > (void)buf_hdr; > @@ -855,13 +822,3 @@ int odp_buffer_is_valid(odp_buffer_t buf) > > return 1; > } > - > -uint32_t pool_headroom(odp_pool_t pool) > -{ > - return pool_entry_from_hdl(pool)->headroom; > -} > - > -uint32_t pool_tailroom(odp_pool_t pool) > -{ > - return pool_entry_from_hdl(pool)->tailroom; > -} > diff --git a/platform/linux-generic/pktio/netmap.c > b/platform/linux-generic/pktio/netmap.c > index cf67741..8eb8145 100644 > --- a/platform/linux-generic/pktio/netmap.c > +++ b/platform/linux-generic/pktio/netmap.c > @@ -345,9 +345,7 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, > pktio_entry_t *pktio_entry, > pkt_nm->pool = pool; > > /* max frame len taking into account the l2-offset */ > - pkt_nm->max_frame_len = ODP_CONFIG_PACKET_BUF_LEN_MAX - > - pool_headroom(pool) - > - pool_tailroom(pool); > + pkt_nm->max_frame_len = CONFIG_PACKET_MAX_SEG_LEN; > > /* allow interface to be opened with or without the 'netmap:' > prefix */ > prefix = "netmap:"; > diff --git a/platform/linux-generic/pktio/socket.c > b/platform/linux-generic/pktio/socket.c > index ab25aab..9fe4a7e 100644 > --- a/platform/linux-generic/pktio/socket.c > +++ b/platform/linux-generic/pktio/socket.c > @@ -46,7 +46,8 @@ > #include <protocols/eth.h> > #include <protocols/ip.h> > > -#define MAX_SEGS ODP_CONFIG_PACKET_MAX_SEGS > +#define MAX_SEGS CONFIG_PACKET_MAX_SEGS > +#define PACKET_JUMBO_LEN (9 * 1024) > > static int disable_pktio; /** !0 this pktio disabled, 0 enabled */ > > -- > 2.8.1 > >
