From: Brian Brooks <brian.bro...@arm.com> Use a single pointer to next segment instead of an array of pointers. This reduces buffer metadata by ~140 bytes.
Signed-off-by: Brian Brooks <brian.bro...@arm.com> Signed-off-by: Honnappa Nagarahalli <honnappa.nagaraha...@arm.com> --- /** Email created from pull request 354 (brbrooks:caterpillar) ** https://github.com/Linaro/odp/pull/354 ** Patch: https://github.com/Linaro/odp/pull/354.patch ** Base sha: 4d17f8ae64aba0e6f24877be30f86ae5880cef7e ** Merge commit sha: 223d05c5b9ecf4986ec07a833a2630eafbbc3a4d **/ platform/linux-generic/buffer/generic.c | 4 +- .../linux-generic/include/odp_buffer_internal.h | 20 +- .../linux-generic/include/odp_packet_internal.h | 91 +- platform/linux-generic/odp_packet.c | 1452 ++++++-------------- platform/linux-generic/pktio/dpdk.c | 8 +- platform/linux-generic/pktio/ipc.c | 2 +- platform/linux-generic/pool/generic.c | 25 +- test/validation/api/packet/packet.c | 2 +- 8 files changed, 500 insertions(+), 1104 deletions(-) diff --git a/platform/linux-generic/buffer/generic.c b/platform/linux-generic/buffer/generic.c index cf99407cc..4406a10e9 100644 --- a/platform/linux-generic/buffer/generic.c +++ b/platform/linux-generic/buffer/generic.c @@ -31,7 +31,7 @@ static void *generic_buffer_addr(odp_buffer_t buf) { odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf); - return hdr->seg[0].data; + return hdr->base_data; } static uint32_t generic_buffer_size(odp_buffer_t buf) @@ -61,7 +61,7 @@ int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) " pool %" PRIu64 "\n", odp_pool_to_u64(pool->pool_hdl)); len += snprintf(&str[len], n - len, - " addr %p\n", hdr->seg[0].data); + " addr %p\n", hdr->base_data); len += snprintf(&str[len], n - len, " size %" PRIu32 "\n", hdr->size); len += snprintf(&str[len], n - len, diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 514e24798..2fd00c694 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -35,12 +35,6 @@ extern "C" { #define BUFFER_BURST_SIZE CONFIG_BURST_SIZE -typedef struct seg_entry_t { - void *hdr; - uint8_t *data; - uint32_t len; -} seg_entry_t; - /* Common buffer header */ struct odp_buffer_hdr_t { @@ -53,26 +47,18 @@ struct odp_buffer_hdr_t { /* Pool type */ int8_t type; - /* Number of seg[] entries used */ - uint8_t num_seg; + /* Offset used to restore base_data */ + uint8_t pristine_offset; /* Next header which continues the segment list */ void *next_seg; - /* Last header of the segment list */ - void *last_seg; - - /* Initial buffer data pointer */ + /* Pointer to start of segment */ uint8_t *base_data; /* Pool pointer */ void *pool_ptr; - /* --- 40 bytes --- */ - - /* Segments */ - seg_entry_t seg[CONFIG_PACKET_SEGS_PER_HDR]; - #ifndef ODP_SCHEDULE_SCALABLE /* Burst counts */ uint8_t burst_num; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 98d364a8d..8f381f097 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -20,6 +20,7 @@ extern "C" { #include <odp/api/align.h> #include <odp/api/debug.h> #include <odp_buffer_internal.h> +#include <odp_debug_internal.h> #include <odp_pool_internal.h> #include <odp_buffer_inlines.h> #include <odp/api/packet.h> @@ -159,9 +160,6 @@ typedef struct { uint8_t data[0]; } odp_packet_hdr_t; -/** - * Return the packet header - */ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) { return (odp_packet_hdr_t *)(uintptr_t)pkt; @@ -182,58 +180,76 @@ static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; } -static inline seg_entry_t *seg_entry_last(odp_packet_hdr_t *hdr) +static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *pkt_hdr) { - odp_packet_hdr_t *last; - uint8_t last_seg; + int segcount = pkt_hdr->buf_hdr.segcount; + + if (odp_unlikely(segcount > 1)) { + while (--segcount) + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } - last = hdr->buf_hdr.last_seg; - last_seg = last->buf_hdr.num_seg - 1; - return &last->buf_hdr.seg[last_seg]; + ODP_ASSERT(pkt_hdr->buf_hdr.next_seg == NULL); + + return pkt_hdr; } -/** - * Initialize packet - */ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); - uint32_t seg_len; - int num = pkt_hdr->buf_hdr.segcount; - - if (odp_likely(CONFIG_PACKET_SEG_DISABLED || num == 1)) { - seg_len = len; - pkt_hdr->buf_hdr.seg[0].len = len; - } else { - seg_entry_t *last; + odp_packet_hdr_t *old_pkt_hdr = pkt_hdr; - seg_len = len - ((num - 1) * pool->seg_len); + pkt_hdr->frame_len = len; - /* Last segment data length */ - last = seg_entry_last(pkt_hdr); - last->len = seg_len; - } + pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; + pkt_hdr->tailroom = CONFIG_PACKET_TAILROOM; + pkt_hdr->input = ODP_PKTIO_INVALID; pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.output_flags.all = 0; pkt_hdr->p.error_flags.all = 0; - pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - /* - * Packet headroom is set from the pool's headroom - * Packet tailroom is rounded up to fill the last - * segment occupied by the allocated length. - */ - pkt_hdr->frame_len = len; - pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; - pkt_hdr->tailroom = pool->seg_len - seg_len + CONFIG_PACKET_TAILROOM; - - pkt_hdr->input = ODP_PKTIO_INVALID; pkt_hdr->buf_hdr.event_subtype = ODP_EVENT_PACKET_BASIC; + /* Restore each segment's base_data and size fields */ + while (len > pool->seg_len) { + pkt_hdr->buf_hdr.base_data = + &pkt_hdr->data[pkt_hdr->buf_hdr.pristine_offset]; + pkt_hdr->buf_hdr.size = pool->seg_len; + + len -= pool->seg_len; + + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } + pkt_hdr->buf_hdr.base_data = + &pkt_hdr->data[pkt_hdr->buf_hdr.pristine_offset]; + pkt_hdr->buf_hdr.size = len; + + old_pkt_hdr->tailroom += pool->seg_len - len; +} + +static inline void packet_init_segs(odp_packet_hdr_t *pkt_hdrs[], int num_seg) +{ + odp_packet_hdr_t *pkt_hdr = pkt_hdrs[0]; + + pkt_hdr->buf_hdr.segcount = num_seg; + + if (odp_unlikely(num_seg > 1)) { + for (int i = 1; i < num_seg; i++) { + pkt_hdr->buf_hdr.next_seg = pkt_hdrs[i]; + pkt_hdr = pkt_hdrs[i]; + } + } + + pkt_hdr->buf_hdr.next_seg = NULL; +} + +static inline uint32_t packet_buf_len(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->buf_hdr.size + pkt_hdr->headroom + pkt_hdr->tailroom; } static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, @@ -253,11 +269,8 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr, static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - seg_entry_t *last = seg_entry_last(pkt_hdr); - pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - last->len -= len; } static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index e0fc01b29..a9edbb9e1 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -27,8 +27,8 @@ /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { - .data = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].data), - .seg_len = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].len), + .data = offsetof(odp_packet_hdr_t, buf_hdr.base_data), + .seg_len = offsetof(odp_packet_hdr_t, buf_hdr.size), .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), @@ -40,275 +40,15 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .flow_hash = offsetof(odp_packet_hdr_t, flow_hash), .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags) - }; #include <odp/visibility_end.h> -static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) -{ - return (odp_buffer_t)pkt_hdr; -} - -static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) +static odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) { return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); } -static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, - uint32_t seg_idx) -{ - uint32_t idx = 0; - uint8_t num_seg = hdr->buf_hdr.num_seg; - - while (odp_unlikely(idx + num_seg - 1 < seg_idx)) { - idx += num_seg; - hdr = hdr->buf_hdr.next_seg; - num_seg = hdr->buf_hdr.num_seg; - } - - idx = seg_idx - idx; - - return &hdr->buf_hdr.seg[idx]; -} - -static inline void seg_entry_find_idx(odp_packet_hdr_t **p_hdr, - uint8_t *p_idx, - uint32_t find_idx) -{ - odp_packet_hdr_t *hdr = *p_hdr; - uint32_t idx = 0; - uint8_t num_seg = hdr->buf_hdr.num_seg; - - while (odp_unlikely(idx + num_seg - 1 < find_idx)) { - idx += num_seg; - hdr = hdr->buf_hdr.next_seg; - num_seg = hdr->buf_hdr.num_seg; - } - - idx = find_idx - idx; - *p_hdr = hdr; - *p_idx = idx; -} - -/* Return pointer to the current segment entry and step cur_hdr / cur_idx - * forward. - */ -static inline seg_entry_t *seg_entry_next(odp_packet_hdr_t **cur_hdr, - uint8_t *cur_idx) -{ - odp_packet_hdr_t *hdr = *cur_hdr; - uint8_t idx = *cur_idx; - uint8_t num_seg = hdr->buf_hdr.num_seg; - - if (idx == num_seg - 1) { - *cur_hdr = hdr->buf_hdr.next_seg; - *cur_idx = 0; - } else { - *cur_idx = idx + 1; - } - - return &hdr->buf_hdr.seg[idx]; -} - -static inline void seg_entry_find_offset(odp_packet_hdr_t **p_hdr, - uint8_t *p_idx, - uint32_t *seg_offset, - uint32_t *seg_idx, - uint32_t offset) -{ - int i; - odp_packet_hdr_t *hdr, *cur_hdr; - uint8_t idx, cur_idx; - seg_entry_t *seg = NULL; - uint32_t seg_start = 0, seg_end = 0; - int seg_count; - - hdr = *p_hdr; - cur_hdr = hdr; - idx = 0; - cur_idx = 0; - seg_count = hdr->buf_hdr.segcount; - - for (i = 0; i < seg_count; i++) { - cur_hdr = hdr; - cur_idx = idx; - seg = seg_entry_next(&hdr, &idx); - seg_end += seg->len; - - if (odp_likely(offset < seg_end)) - break; - - seg_start = seg_end; - } - - *p_hdr = cur_hdr; - *p_idx = cur_idx; - *seg_offset = offset - seg_start; - *seg_idx = i; -} - -static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - return seg->len; -} - -static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - return seg->data; -} - -static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) -{ - if (CONFIG_PACKET_SEG_DISABLED) - return 0; - else - return pkt_hdr->buf_hdr.segcount - 1; -} - -static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->buf_hdr.seg[0].len; -} - -static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->buf_hdr.seg[0].data; -} - -static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) -{ - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); - - return last_seg->data + last_seg->len; -} - -static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - odp_buffer_hdr_t *hdr = seg->hdr; - uint8_t *base = hdr->base_data; - uint8_t *head = seg->data; - - return CONFIG_PACKET_HEADROOM + (head - base); -} - -static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - odp_buffer_hdr_t *hdr = seg->hdr; - uint8_t *tail = seg->data + seg->len; - - return hdr->buf_end - tail; -} - -static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - pkt_hdr->headroom -= len; - pkt_hdr->frame_len += len; - pkt_hdr->buf_hdr.seg[0].data -= len; - pkt_hdr->buf_hdr.seg[0].len += len; -} - -static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - pkt_hdr->headroom += len; - pkt_hdr->frame_len -= len; - pkt_hdr->buf_hdr.seg[0].data += len; - pkt_hdr->buf_hdr.seg[0].len -= len; -} - -static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); - - pkt_hdr->tailroom -= len; - pkt_hdr->frame_len += len; - last_seg->len += len; -} - -/* Copy all metadata for segmentation modification. Segment data and lengths - * are not copied. */ -static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, - odp_packet_hdr_t *src) -{ - dst->p = src->p; - - /* lengths are not copied: - * .frame_len - * .headroom - * .tailroom - */ - - dst->input = src->input; - dst->dst_queue = src->dst_queue; - dst->flow_hash = src->flow_hash; - dst->timestamp = src->timestamp; - - /* buffer header side packet metadata */ - dst->buf_hdr.buf_u64 = src->buf_hdr.buf_u64; - dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; - - /* segmentation data is not copied: - * buf_hdr.seg[] - * buf_hdr.segcount - * buf_hdr.num_seg - * buf_hdr.next_seg - * buf_hdr.last_seg - */ -} - -static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, uint32_t *seg_len, int *seg_idx) -{ - void *addr; - uint32_t len; - int seg_id = 0; - int seg_count = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely(offset >= pkt_hdr->frame_len)) - return NULL; - - if (odp_likely(CONFIG_PACKET_SEG_DISABLED || seg_count == 1)) { - addr = pkt_hdr->buf_hdr.seg[0].data + offset; - len = pkt_hdr->buf_hdr.seg[0].len - offset; - } else { - int i; - seg_entry_t *seg = NULL; - uint32_t seg_start = 0, seg_end = 0; - odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0; - - for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - seg_end += seg->len; - - if (odp_likely(offset < seg_end)) - break; - - seg_start = seg_end; - } - - addr = seg->data + (offset - seg_start); - len = seg->len - (offset - seg_start); - seg_id = i; - } - - if (seg_len) - *seg_len = len; - - if (seg_idx) - *seg_idx = seg_id; - - return addr; -} - void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) { /* Reset parser metadata before new parse */ @@ -320,103 +60,7 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; } -static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) -{ - int cur, i; - odp_packet_hdr_t *hdr; - odp_packet_hdr_t *head = pkt_hdr[0]; - uint32_t seg_len = pool_entry_from_hdl(head->buf_hdr.pool_hdl)->seg_len; - - cur = 0; - - while (1) { - hdr = pkt_hdr[cur]; - - for (i = 0; i < CONFIG_PACKET_SEGS_PER_HDR; i++) { - odp_buffer_hdr_t *buf_hdr; - - buf_hdr = &pkt_hdr[cur]->buf_hdr; - hdr->buf_hdr.seg[i].hdr = buf_hdr; - hdr->buf_hdr.seg[i].data = buf_hdr->base_data; - hdr->buf_hdr.seg[i].len = seg_len; - - /* init_segments() handles first seg ref_cnt init */ - if (ODP_DEBUG == 1 && cur > 0) { - uint32_t prev_ref = - odp_atomic_fetch_inc_u32( - &pkt_hdr[cur]->buf_hdr.ref_cnt); - - ODP_ASSERT(prev_ref == 0); - } - - cur++; - - if (cur == num) { - /* Last segment */ - hdr->buf_hdr.num_seg = i + 1; - hdr->buf_hdr.next_seg = NULL; - head->buf_hdr.last_seg = &hdr->buf_hdr; - return; - } - } - - hdr->buf_hdr.num_seg = CONFIG_PACKET_SEGS_PER_HDR; - hdr->buf_hdr.next_seg = pkt_hdr[cur]; - } -} - -static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) -{ - odp_packet_hdr_t *hdr; - uint32_t seg_len; - - /* First segment is the packet descriptor */ - hdr = pkt_hdr[0]; - seg_len = pool_entry_from_hdl(hdr->buf_hdr.pool_hdl)->seg_len; - - /* Defaults for single segment packet */ - hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; - hdr->buf_hdr.seg[0].len = seg_len; - - if (ODP_DEBUG == 1) { - uint32_t prev_ref = - odp_atomic_fetch_inc_u32(&hdr->buf_hdr.ref_cnt); - - ODP_ASSERT(prev_ref == 0); - } - - if (!CONFIG_PACKET_SEG_DISABLED) { - hdr->buf_hdr.segcount = num; - hdr->buf_hdr.num_seg = 1; - hdr->buf_hdr.next_seg = NULL; - hdr->buf_hdr.last_seg = &hdr->buf_hdr; - - /* Link segments */ - if (odp_unlikely(num > 1)) - link_segments(pkt_hdr, num); - } -} - -static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) -{ - odp_packet_hdr_t *hdr = pkt_hdr; - void *base; - int i; - seg_entry_t *seg; - uint32_t seg_len = pool_entry_from_hdl(hdr->buf_hdr.pool_hdl)->seg_len; - uint8_t idx; - - seg_entry_find_idx(&hdr, &idx, first); - - for (i = 0; i < num; i++) { - base = hdr->buf_hdr.base_data; - seg = seg_entry_next(&hdr, &idx); - seg->len = seg_len; - seg->data = base; - } -} - -/* Calculate the number of segments */ +/* Calculate number of segments required for a packet of len 'seg_len' bytes. */ static inline int num_segments(uint32_t len, uint32_t seg_len) { int num; @@ -436,375 +80,255 @@ static inline int num_segments(uint32_t len, uint32_t seg_len) return num; } -static inline void add_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from) +/* Returns a pointer to the Nth (0-based) segment */ +static odp_packet_hdr_t *get_seg(odp_packet_hdr_t *pkt_hdr, uint32_t n) { - odp_packet_hdr_t *last = to->buf_hdr.last_seg; + uint16_t segcount = pkt_hdr->buf_hdr.segcount; + + ODP_ASSERT(n < segcount); + + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || segcount == 1)) + return pkt_hdr; - last->buf_hdr.next_seg = from; - to->buf_hdr.last_seg = from->buf_hdr.last_seg; - to->buf_hdr.segcount += from->buf_hdr.segcount; + while (n--) { + ODP_ASSERT(pkt_hdr->buf_hdr.next_seg); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } + + return pkt_hdr; } -static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) +/* + * Returns a pointer to the segment containing byte 'offset' as well as + * the number of segments and bytes skipped to get to the segment. + */ +static odp_packet_hdr_t *get_seg_at_offset(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, + uint32_t *bytes_skipped, + uint32_t *segs_skipped) { - odp_packet_hdr_t *pkt_hdr[num]; - int ret; + uint32_t skipped_bytes = 0; + uint32_t skipped_segs = 0; - ret = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num); + ODP_ASSERT(offset < pkt_hdr->frame_len); - if (odp_unlikely(ret != num)) { - if (ret > 0) - buffer_free_multi((odp_buffer_hdr_t **)pkt_hdr, ret); + if (odp_unlikely(pkt_hdr->buf_hdr.segcount > 1)) { + while (offset >= pkt_hdr->buf_hdr.size) { + skipped_bytes += pkt_hdr->buf_hdr.size; + skipped_segs++; - return NULL; + offset -= pkt_hdr->buf_hdr.size; + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + ODP_ASSERT(pkt_hdr); + } } - init_segments(pkt_hdr, num); + if (bytes_skipped) + *bytes_skipped = skipped_bytes; + if (segs_skipped) + *segs_skipped = skipped_segs; - return pkt_hdr[0]; + return pkt_hdr; } -static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, - pool_t *pool, uint32_t len, - int num, int head) +/* Link two segment chains together. Adjusts segcounts. */ +static void concat_seg(odp_packet_hdr_t *seg_a, odp_packet_hdr_t *seg_b) { - odp_packet_hdr_t *new_hdr; - uint32_t seg_len, offset; - - new_hdr = alloc_segments(pool, num); - - if (new_hdr == NULL) - return NULL; - - seg_len = len - ((num - 1) * pool->seg_len); - offset = pool->seg_len - seg_len; + odp_packet_hdr_t *seg_a_last = packet_last_seg(seg_a); - if (head) { - /* add into the head*/ - add_all_segs(new_hdr, pkt_hdr); + ODP_ASSERT(seg_a_last->buf_hdr.next_seg == NULL); + seg_a_last->buf_hdr.next_seg = seg_b; - /* adjust first segment length */ - new_hdr->buf_hdr.seg[0].data += offset; - new_hdr->buf_hdr.seg[0].len = seg_len; + ODP_ASSERT(seg_a->buf_hdr.segcount > 0); + ODP_ASSERT(seg_b->buf_hdr.segcount > 0); - packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + seg_a->buf_hdr.segcount += seg_b->buf_hdr.segcount; + seg_b->buf_hdr.segcount = 0; +} - pkt_hdr = new_hdr; - } else { - seg_entry_t *last_seg; +/* + * Returns a pointer to start of packet + 'offset' bytes and stores + * the remaining length of the resulting segment in 'seg_len' and the + * segment index in 'seg_idx'. + */ +static void *packet_map(odp_packet_hdr_t *pkt_hdr, uint32_t offset, + uint32_t *seg_len, int *seg_idx) +{ + uint32_t skipped_segs = 0; - /* add into the tail */ - add_all_segs(pkt_hdr, new_hdr); + if (odp_unlikely(offset >= pkt_hdr->frame_len)) + return NULL; - /* adjust last segment length */ - last_seg = seg_entry_last(pkt_hdr); - last_seg->len = seg_len; + if (odp_unlikely(pkt_hdr->buf_hdr.segcount > 1)) { + while (offset >= pkt_hdr->buf_hdr.size) { + skipped_segs++; - pkt_hdr->frame_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + offset -= pkt_hdr->buf_hdr.size; + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } } - return pkt_hdr; -} - -static inline int seg_is_link(void *hdr) -{ - odp_packet_hdr_t *pkt_hdr = hdr; + if (seg_len) + *seg_len = pkt_hdr->buf_hdr.size - offset; + if (seg_idx) + *seg_idx = skipped_segs; - return pkt_hdr != pkt_hdr->buf_hdr.seg[0].hdr; + return pkt_hdr->buf_hdr.base_data + offset; } -static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) +static void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) { - uint32_t ref_cnt = odp_atomic_load_u32(&buf_hdr->ref_cnt); - - /* First count increment after alloc */ - if (odp_likely(ref_cnt) == 0) - odp_atomic_store_u32(&buf_hdr->ref_cnt, 2); - else - odp_atomic_inc_u32(&buf_hdr->ref_cnt); + odp_atomic_inc_u32(&buf_hdr->ref_cnt); } -static inline uint32_t buffer_ref_dec(odp_buffer_hdr_t *buf_hdr) +static uint32_t buffer_ref_dec(odp_buffer_hdr_t *buf_hdr) { return odp_atomic_fetch_dec_u32(&buf_hdr->ref_cnt); } -static inline uint32_t buffer_ref(odp_buffer_hdr_t *buf_hdr) +static uint32_t buffer_ref(odp_buffer_hdr_t *buf_hdr) { return odp_atomic_load_u32(&buf_hdr->ref_cnt); } -static inline int is_multi_ref(uint32_t ref_cnt) +static int is_multi_ref(uint32_t ref_cnt) { - return (ref_cnt > 1); + return ref_cnt > 0; } -static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) +static void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) { - seg_entry_t *seg; - int i; - int seg_count = pkt_hdr->buf_hdr.segcount; - odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0; - - for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - buffer_ref_inc(seg->hdr); + while (pkt_hdr) { + buffer_ref_inc(&pkt_hdr->buf_hdr); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; } } -static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) +/* Allocate 'num_pkt' packets of length 'len' bytes */ +static int packet_alloc(pool_t *pool, uint32_t len, int num_pkt, + odp_packet_t *pkts) { - int i; - uint32_t ref_cnt; - int num_ref = 0; + int segs_per_pkt = num_segments(len, pool->seg_len); + int num_buf = num_pkt * segs_per_pkt; + odp_packet_hdr_t *pkt_hdr[num_buf]; + int npkt = num_pkt; + int nbuf; - for (i = 0; i < num; i++) { - /* Zero when reference API has not been used */ - ref_cnt = buffer_ref(hdr[i]); + ODP_ASSERT(segs_per_pkt <= CONFIG_PACKET_MAX_SEGS); - if (odp_unlikely(ref_cnt)) { - ref_cnt = buffer_ref_dec(hdr[i]); + nbuf = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num_buf); - if (is_multi_ref(ref_cnt)) { - num_ref++; - continue; - } - } + /* If we did not get the total number of buffers we asked for, free any + * buffers near the end of the list that cannot be used to make a + * whole packet. */ + if (odp_unlikely(nbuf != num_buf)) { + int nfree; - /* Reset link header back to normal header */ - if (odp_unlikely(seg_is_link(hdr[i]))) - hdr[i]->seg[0].hdr = hdr[i]; + npkt = nbuf / segs_per_pkt; + nfree = nbuf - (npkt * segs_per_pkt); - /* Skip references and pack to be freed headers to array head */ - if (odp_unlikely(num_ref)) - hdr[i - num_ref] = hdr[i]; + if (nfree > 0) { + odp_buffer_hdr_t **p = + (odp_buffer_hdr_t **)&pkt_hdr[nbuf - nfree]; + buffer_free_multi(p, nfree); + } + } + + for (int i = 0; i < npkt; i++) { + packet_init_segs(&pkt_hdr[i * segs_per_pkt], segs_per_pkt); + packet_init(pkt_hdr[i * segs_per_pkt], len); + pkts[i] = (odp_packet_t)pkt_hdr[i * segs_per_pkt]; } - num -= num_ref; + for (int i = 0; i < nbuf; i++) { + ODP_ASSERT(((intptr_t)pkt_hdr[i]->buf_hdr.base_data - + (intptr_t)pkt_hdr[i]) <= + (int)(sizeof(odp_packet_hdr_t) + + pool->headroom + pool->align)); + } - if (odp_likely(num)) - buffer_free_multi(hdr, num); + return npkt; } -static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) +int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], + int max_num) { - int i; - odp_buffer_hdr_t *buf_hdr[num + 1]; - - if (odp_likely(pkt_hdr->buf_hdr.num_seg == num)) { - for (i = 0; i < num; i++) - buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; + return packet_alloc(pool_entry_from_hdl(pool_hdl), + len, max_num, pkt); +} - if (odp_unlikely(seg_is_link(pkt_hdr))) { - buf_hdr[num] = &pkt_hdr->buf_hdr; - num++; - } - } else { - seg_entry_t *seg; - odp_buffer_hdr_t *link_hdr[num]; - uint8_t idx = 0; - int links = 0; - - for (i = 0; i < num; i++) { - /* Free also link headers */ - if (odp_unlikely(idx == 0 && seg_is_link(pkt_hdr))) { - link_hdr[links] = &pkt_hdr->buf_hdr; - links++; - } +/* Free 'num' segments starting from segment 'n'. */ +static void packet_free_segs(odp_packet_hdr_t *pkt, uint32_t num, uint32_t n) +{ + odp_packet_hdr_t *pkt_hdr[num]; - seg = seg_entry_next(&pkt_hdr, &idx); - buf_hdr[i] = seg->hdr; - } + ODP_ASSERT(n < pkt->buf_hdr.segcount); + ODP_ASSERT(n + num <= pkt->buf_hdr.segcount); - if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); + pkt = get_seg(pkt, n); + for (unsigned i = 0; i < num; i++) { + pkt_hdr[i] = pkt; + pkt = pkt->buf_hdr.next_seg; } - packet_free_multi(buf_hdr, num); -} - -static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, - int num, uint32_t free_len, - uint32_t pull_len, int head) -{ - seg_entry_t *seg; - int i; - int num_remain = pkt_hdr->buf_hdr.segcount - num; - odp_packet_hdr_t *hdr = pkt_hdr; - odp_packet_hdr_t *last_hdr = pkt_hdr->buf_hdr.last_seg; - uint8_t idx; - uint8_t num_seg; - odp_buffer_hdr_t *buf_hdr[num]; - odp_buffer_hdr_t *link_hdr[num]; - odp_packet_hdr_t *tmp_hdr; - int links = 0; - - if (head) { - odp_packet_hdr_t *new_hdr; - - idx = 0; - for (i = 0; i < num; i++) { - tmp_hdr = hdr; - seg = seg_entry_next(&hdr, &idx); - buf_hdr[i] = seg->hdr; - - /* Free link headers, if those become empty */ - if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { - link_hdr[links] = &tmp_hdr->buf_hdr; - links++; - } - } + buffer_free_multi((odp_buffer_hdr_t **)&pkt_hdr, num); +} - /* The first remaining header is the new packet descriptor. - * Copy remaining segments from the last to-be-removed header - * to the new header. */ - new_hdr = hdr->buf_hdr.seg[idx].hdr; - num_seg = hdr->buf_hdr.num_seg - idx; +static void packet_free(odp_packet_hdr_t *pkt_hdr) +{ + int segcount = pkt_hdr->buf_hdr.segcount; + uint32_t ref_cnt; - new_hdr->buf_hdr.next_seg = hdr->buf_hdr.next_seg; + ODP_ASSERT(segcount > 0); - if (hdr == last_hdr) - new_hdr->buf_hdr.last_seg = new_hdr; - else - new_hdr->buf_hdr.last_seg = last_hdr; + if (odp_likely(segcount == 1)) { + ref_cnt = buffer_ref((odp_buffer_hdr_t *)pkt_hdr); - new_hdr->buf_hdr.num_seg = num_seg; - new_hdr->buf_hdr.segcount = num_remain; + if (odp_unlikely(ref_cnt)) { + ref_cnt = buffer_ref_dec((odp_buffer_hdr_t *)pkt_hdr); - for (i = 0; i < num_seg; i++) { - seg = seg_entry_next(&hdr, &idx); - new_hdr->buf_hdr.seg[i] = *seg; + if (is_multi_ref(ref_cnt)) + return; } - packet_seg_copy_md(new_hdr, pkt_hdr); - - /* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - - /* Link header does not have headroom */ - if (seg_is_link(new_hdr)) - new_hdr->headroom = 0; - else - new_hdr->headroom = seg_headroom(new_hdr, 0); - - new_hdr->frame_len = pkt_hdr->frame_len - free_len; - - pull_head(new_hdr, pull_len); - - pkt_hdr = new_hdr; - - if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); - - packet_free_multi(buf_hdr, num); + buffer_free_multi((odp_buffer_hdr_t **)&pkt_hdr, 1); } else { - /* Free last 'num' bufs. - * First, find the last remaining header. */ - seg_entry_find_idx(&hdr, &idx, num_remain - 1); - last_hdr = hdr; - num_seg = idx + 1; - - seg_entry_next(&hdr, &idx); - - for (i = 0; i < num; i++) { - tmp_hdr = hdr; - seg = seg_entry_next(&hdr, &idx); - buf_hdr[i] = seg->hdr; - - /* Free link headers, if those become empty */ - if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { - link_hdr[links] = &tmp_hdr->buf_hdr; - links++; - } - } - - if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); - - packet_free_multi(buf_hdr, num); - - /* Head segment remains, no need to copy or update majority - * of the metadata. */ - last_hdr->buf_hdr.num_seg = num_seg; - last_hdr->buf_hdr.next_seg = NULL; - - pkt_hdr->buf_hdr.last_seg = last_hdr; - pkt_hdr->buf_hdr.segcount = num_remain; - pkt_hdr->frame_len -= free_len; - pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); - - pull_tail(pkt_hdr, pull_len); - } + odp_packet_hdr_t *hdr[segcount]; + int num_ref = 0; - return pkt_hdr; -} - -static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, - int num_seg, odp_packet_t *pkt) -{ - int num_buf, i; - int num = max_pkt; - int max_buf = max_pkt * num_seg; - odp_packet_hdr_t *pkt_hdr[max_buf]; + for (int i = 0; i < segcount; i++) { + ref_cnt = buffer_ref((odp_buffer_hdr_t *)pkt_hdr); - num_buf = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, - max_buf); + if (odp_unlikely(ref_cnt)) { + ref_cnt = buffer_ref_dec( + (odp_buffer_hdr_t *)pkt_hdr); - /* Failed to allocate all segments */ - if (odp_unlikely(num_buf != max_buf)) { - int num_free; + if (is_multi_ref(ref_cnt)) { + num_ref++; - num = num_buf / num_seg; - num_free = num_buf - (num * num_seg); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + continue; + } + } - if (num_free > 0) { - odp_buffer_hdr_t **p; + hdr[i - num_ref] = pkt_hdr; - p = (odp_buffer_hdr_t **)&pkt_hdr[num_buf - num_free]; - buffer_free_multi(p, num_free); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; } + ODP_ASSERT(pkt_hdr == NULL); - if (num == 0) - return 0; - } - - for (i = 0; i < num; i++) { - odp_packet_hdr_t *hdr; - - /* First buffer is the packet descriptor */ - hdr = pkt_hdr[i * num_seg]; - pkt[i] = packet_handle(hdr); - init_segments(&pkt_hdr[i * num_seg], num_seg); - - packet_init(hdr, len); + if (segcount - num_ref) + buffer_free_multi((odp_buffer_hdr_t **)&hdr, + segcount - num_ref); } - - return num; -} - -int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, - odp_packet_t pkt[], int max_num) -{ - pool_t *pool = pool_entry_from_hdl(pool_hdl); - int num, num_seg; - - num_seg = num_segments(len, pool->seg_len); - num = packet_alloc(pool, len, max_num, num_seg, pkt); - - return num; } odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) { pool_t *pool = pool_entry_from_hdl(pool_hdl); odp_packet_t pkt; - int num, num_seg; + int num; if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { __odp_errno = EINVAL; @@ -814,8 +338,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) if (odp_unlikely(len > pool->max_len)) return ODP_PACKET_INVALID; - num_seg = num_segments(len, pool->seg_len); - num = packet_alloc(pool, len, 1, num_seg, &pkt); + num = packet_alloc(pool, len, 1, &pkt); if (odp_unlikely(num == 0)) return ODP_PACKET_INVALID; @@ -827,7 +350,6 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], int max_num) { pool_t *pool = pool_entry_from_hdl(pool_hdl); - int num, num_seg; if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { __odp_errno = EINVAL; @@ -837,69 +359,18 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, if (odp_unlikely(len > pool->max_len)) return -1; - num_seg = num_segments(len, pool->seg_len); - num = packet_alloc(pool, len, max_num, num_seg, pkt); - - return num; + return packet_alloc(pool, len, max_num, pkt); } void odp_packet_free(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - int num_seg = pkt_hdr->buf_hdr.segcount; - - ODP_ASSERT(buffer_ref(&pkt_hdr->buf_hdr) > 0); - - if (odp_likely(CONFIG_PACKET_SEG_DISABLED || num_seg == 1)) { - odp_buffer_hdr_t *buf_hdr[2]; - int num = 1; - - buf_hdr[0] = &pkt_hdr->buf_hdr; - - if (odp_unlikely(seg_is_link(pkt_hdr))) { - num = 2; - buf_hdr[1] = pkt_hdr->buf_hdr.seg[0].hdr; - } - - packet_free_multi(buf_hdr, num); - } else { - free_all_segments(pkt_hdr, num_seg); - } + packet_free(packet_hdr(pkt)); } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - odp_buffer_hdr_t *buf_hdr[num]; - odp_buffer_hdr_t *buf_hdr2[num]; - int i; - int links = 0; - int num_freed = 0; - - for (i = 0; i < num; i++) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); - int num_seg = pkt_hdr->buf_hdr.segcount; - - ODP_ASSERT(buffer_ref(&pkt_hdr->buf_hdr) > 0); - - if (odp_unlikely(num_seg > 1)) { - free_all_segments(pkt_hdr, num_seg); - num_freed++; - continue; - } - - if (odp_unlikely(seg_is_link(pkt_hdr))) { - buf_hdr2[links] = pkt_hdr->buf_hdr.seg[0].hdr; - links++; - } - - buf_hdr[i - num_freed] = &pkt_hdr->buf_hdr; - } - - if (odp_unlikely(links)) - packet_free_multi(buf_hdr2, links); - - if (odp_likely(num - num_freed)) - packet_free_multi(buf_hdr, num - num_freed); + for (int i = 0; i < num; i++) + packet_free(packet_hdr(pkt[i])); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -911,8 +382,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (odp_unlikely(len > (pool->seg_len * num))) return -1; - reset_seg(pkt_hdr, 0, num); - packet_init(pkt_hdr, len); return 0; @@ -931,28 +400,35 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) if (odp_unlikely(pkt == ODP_PACKET_INVALID)) return ODP_EVENT_INVALID; - return (odp_event_t)buffer_handle(packet_hdr(pkt)); + return (odp_event_t)packet_hdr(pkt); } -/* - * - * Pointers and lengths - * ******************************************************** - * - */ - uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; + return packet_buf_len(pkt_hdr) * pkt_hdr->buf_hdr.segcount; } void *odp_packet_tail(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *seg = packet_last_seg(pkt_hdr); + + return seg->buf_hdr.buf_end - pkt_hdr->tailroom; +} + +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, + odp_packet_seg_t *seg) +{ + int seg_idx; + void *addr = packet_map( + (odp_packet_hdr_t *)(uintptr_t)pkt, offset, len, &seg_idx); + + if (addr != NULL && seg != NULL) + *seg = _odp_packet_seg_from_ndx(seg_idx); - return packet_tail(pkt_hdr); + return addr; } void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) @@ -962,225 +438,261 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->headroom) return NULL; - push_head(pkt_hdr, len); - return packet_data(pkt_hdr); + pkt_hdr->buf_hdr.base_data -= len; + pkt_hdr->buf_hdr.size += len; + + pkt_hdr->headroom -= len; + pkt_hdr->frame_len += len; + + return pkt_hdr->buf_hdr.base_data; } -int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, - void **data_ptr, uint32_t *seg_len) +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - uint32_t frame_len = pkt_hdr->frame_len; - uint32_t headroom = pkt_hdr->headroom; - int ret = 0; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (len > headroom) { - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; - int num; - void *ptr; + if (len > pkt_hdr->frame_len) + return NULL; - if (odp_unlikely((frame_len + len) > pool->max_len)) - return -1; + pkt_hdr->buf_hdr.base_data += len; + pkt_hdr->buf_hdr.size -= len; - num = num_segments(len - headroom, pool->seg_len); - push_head(pkt_hdr, headroom); - ptr = add_segments(pkt_hdr, pool, len - headroom, num, 1); + pkt_hdr->headroom += len; + pkt_hdr->frame_len -= len; - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_head(pkt_hdr, headroom); - return -1; - } + return pkt_hdr->buf_hdr.base_data; +} - *pkt = packet_handle(ptr); - pkt_hdr = ptr; - } else { - push_head(pkt_hdr, len); - } +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *seg = packet_last_seg(pkt_hdr); + void *old_tail; - if (data_ptr) - *data_ptr = packet_data(pkt_hdr); + if (len > pkt_hdr->tailroom) + return NULL; - if (seg_len) - *seg_len = packet_first_seg_len(pkt_hdr); + old_tail = seg->buf_hdr.buf_end - pkt_hdr->tailroom; + + seg->buf_hdr.size += len; - return ret; + pkt_hdr->tailroom -= len; + pkt_hdr->frame_len += len; + + return old_tail; } -void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *seg = packet_last_seg(pkt_hdr); - if (len > pkt_hdr->frame_len) + if (len > seg->buf_hdr.size) return NULL; - pull_head(pkt_hdr, len); - return packet_data(pkt_hdr); + seg->buf_hdr.size -= len; + + pkt_hdr->tailroom += len; + pkt_hdr->frame_len -= len; + + return seg->buf_hdr.buf_end - pkt_hdr->tailroom; } -int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, - void **data_ptr, uint32_t *seg_len_out) +/* Copy a subset of metadata fields from one packet to another. */ +static void packet_copy_md(odp_packet_hdr_t *dst, odp_packet_hdr_t *src) +{ + dst->p = src->p; + dst->input = src->input; + dst->dst_queue = src->dst_queue; + dst->flow_hash = src->flow_hash; + dst->timestamp = src->timestamp; + dst->buf_hdr.buf_u64 = src->buf_hdr.buf_u64; + dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; +} + +int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, + void **data_ptr, uint32_t *seg_len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - uint32_t seg_len = packet_first_seg_len(pkt_hdr); + int rv; - if (len > pkt_hdr->frame_len) - return -1; + if (len <= pkt_hdr->headroom) { + if (odp_packet_push_head(*pkt, len) == NULL) + return -1; - if (len < seg_len) { - pull_head(pkt_hdr, len); - } else if (!CONFIG_PACKET_SEG_DISABLED) { - int num = 0; - uint32_t pull_len = 0; + if (data_ptr) + *data_ptr = pkt_hdr->buf_hdr.base_data; + if (seg_len) + *seg_len = pkt_hdr->buf_hdr.size; - while (seg_len <= len) { - pull_len = len - seg_len; - num++; - seg_len += packet_seg_len(pkt_hdr, num); - } + rv = 0; + } else { + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + odp_packet_t head; - pkt_hdr = free_segments(pkt_hdr, num, len - pull_len, - pull_len, 1); - *pkt = packet_handle(pkt_hdr); - } + if (odp_unlikely(pkt_hdr->frame_len + len > pool->max_len)) + return -1; - if (data_ptr) - *data_ptr = packet_data(pkt_hdr); + pkt_hdr->frame_len += pkt_hdr->headroom; + pkt_hdr->buf_hdr.base_data -= pkt_hdr->headroom; + pkt_hdr->buf_hdr.size += pkt_hdr->headroom; - if (seg_len_out) - *seg_len_out = packet_first_seg_len(pkt_hdr); + if (packet_alloc(pool, len - pkt_hdr->headroom, 1, &head) != 1) + return -1; - return 0; + concat_seg((odp_packet_hdr_t *)(uintptr_t)head, pkt_hdr); + + packet_hdr(head)->frame_len += pkt_hdr->frame_len; + packet_hdr(head)->tailroom = pkt_hdr->tailroom; + + packet_copy_md(packet_hdr(head), pkt_hdr); + + if (data_ptr) + *data_ptr = packet_hdr(head)->buf_hdr.base_data; + if (seg_len) + *seg_len = packet_hdr(head)->buf_hdr.size; + + *pkt = head; + + rv = 1; + } + return rv; } -void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) +int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, + void **data_ptr, uint32_t *seg_len) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - void *old_tail; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); + int rv = -1; - if (len > pkt_hdr->tailroom) - return NULL; + if (len > pkt_hdr->frame_len) + return rv; - ODP_ASSERT(odp_packet_has_ref(pkt) == 0); + if (len < pkt_hdr->buf_hdr.size) { + if (odp_packet_pull_head(*pkt, len) == NULL) + return -1; - old_tail = packet_tail(pkt_hdr); - push_tail(pkt_hdr, len); + if (data_ptr) + *data_ptr = pkt_hdr->buf_hdr.base_data; + if (seg_len) + *seg_len = pkt_hdr->buf_hdr.size; - return old_tail; + rv = 0; + } else { + uint32_t bytes_skipped, segs_skipped; + odp_packet_hdr_t *head = get_seg_at_offset(pkt_hdr, len, + &bytes_skipped, + &segs_skipped); + ODP_ASSERT(bytes_skipped > 0); + ODP_ASSERT(segs_skipped > 0); + + packet_copy_md(head, pkt_hdr); + + head->buf_hdr.segcount = + pkt_hdr->buf_hdr.segcount - segs_skipped; + head->frame_len = pkt_hdr->frame_len - bytes_skipped; + head->headroom = 0; + head->tailroom = pkt_hdr->tailroom; + + packet_free_segs(pkt_hdr, segs_skipped, 0); + + if (odp_packet_pull_head( + (odp_packet_t)head, len - bytes_skipped) == NULL) + return -1; + + if (data_ptr) + *data_ptr = head->buf_hdr.base_data; + if (seg_len) + *seg_len = head->buf_hdr.size; + + *pkt = (odp_packet_t)head; + + rv = 1; + } + return rv; } int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - uint32_t frame_len = pkt_hdr->frame_len; - uint32_t tailroom = pkt_hdr->tailroom; - uint32_t tail_off = frame_len; - int ret = 0; - - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + uint32_t old_frame_len = pkt_hdr->frame_len; + uint32_t seg_len = 0; + void *offset; - if (len > tailroom) { + if (len <= pkt_hdr->tailroom) { + if (odp_packet_push_tail(*pkt, len) == NULL) + return -1; + } else { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; - int num; - void *ptr; + odp_packet_hdr_t *seg; + odp_packet_t tail; - if (odp_unlikely((frame_len + len) > pool->max_len)) + if (odp_unlikely(pkt_hdr->frame_len + len > pool->max_len)) return -1; - num = num_segments(len - tailroom, pool->seg_len); - push_tail(pkt_hdr, tailroom); - ptr = add_segments(pkt_hdr, pool, len - tailroom, num, 0); + seg = packet_last_seg(pkt_hdr); + seg->buf_hdr.size += pkt_hdr->tailroom; - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_tail(pkt_hdr, tailroom); + if (packet_alloc(pool, len - pkt_hdr->tailroom, 1, &tail) != 1) return -1; - } - } else { - push_tail(pkt_hdr, len); - } - - if (data_ptr) - *data_ptr = packet_map(pkt_hdr, tail_off, seg_len_out, NULL); - - return ret; -} -void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); + concat_seg(pkt_hdr, packet_hdr(tail)); - ODP_ASSERT(odp_packet_has_ref(pkt) == 0); + pkt_hdr->frame_len += len; + pkt_hdr->tailroom = packet_hdr(tail)->tailroom; + } - if (len > last_seg->len) - return NULL; + if (data_ptr || seg_len_out) + offset = packet_map(pkt_hdr, old_frame_len, &seg_len, NULL); - pull_tail(pkt_hdr, len); + if (data_ptr) + *data_ptr = offset; + if (seg_len_out) + *seg_len_out = seg_len; - return packet_tail(pkt_hdr); + return 0; } int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void **tail_ptr, uint32_t *tailroom) { - int last; - uint32_t seg_len; - seg_entry_t *last_seg; odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); + odp_packet_hdr_t *pkt_hdr_last = packet_last_seg(pkt_hdr); - if (len > pkt_hdr->frame_len) - return -1; + if (len <= pkt_hdr_last->buf_hdr.size) { + if (odp_packet_pull_tail(*pkt, len) == NULL) + return -1; + } else { + uint32_t bytes_skipped, segs_skipped; + odp_packet_hdr_t *last; + int last_size; + int num; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + last = get_seg_at_offset(pkt_hdr, pkt_hdr->frame_len - len, + &bytes_skipped, &segs_skipped); - last = packet_last_seg(pkt_hdr); - last_seg = seg_entry_last(pkt_hdr); - seg_len = last_seg->len; + num = pkt_hdr->buf_hdr.segcount - (segs_skipped + 1); + packet_free_segs(pkt_hdr, num, segs_skipped + 1); - if (len < seg_len) { - pull_tail(pkt_hdr, len); - } else if (!CONFIG_PACKET_SEG_DISABLED) { - int num = 0; - uint32_t pull_len = 0; + pkt_hdr->buf_hdr.segcount -= num; + last->buf_hdr.next_seg = NULL; - while (seg_len <= len) { - pull_len = len - seg_len; - num++; - seg_len += packet_seg_len(pkt_hdr, last - num); - } + pkt_hdr->frame_len -= len; - free_segments(pkt_hdr, num, len - pull_len, pull_len, 0); + last_size = pkt_hdr->frame_len - bytes_skipped; + pkt_hdr->tailroom = last->buf_hdr.size - last_size; + last->buf_hdr.size = last_size; } if (tail_ptr) - *tail_ptr = packet_tail(pkt_hdr); - + *tail_ptr = odp_packet_tail(*pkt); if (tailroom) *tailroom = pkt_hdr->tailroom; - return 0; -} - -void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, - odp_packet_seg_t *seg) -{ - int seg_idx; - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - void *addr = packet_map(pkt_hdr, offset, len, &seg_idx); - - if (addr != NULL && seg != NULL) - *seg = _odp_packet_seg_from_ndx(seg_idx); - return addr; + return 0; } -/* - * - * Meta-data - * ******************************************************** - * - */ uint32_t odp_packet_user_area_size(odp_packet_t pkt) { pool_t *pool = pool_entry_from_hdl(odp_packet_pool(pkt)); @@ -1294,13 +806,6 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) pkt_hdr->p.input_flags.timestamp = 1; } -/* - * - * Segment level - * ******************************************************** - * - */ - void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -1309,26 +814,21 @@ void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) pkt_hdr->buf_hdr.segcount)) return NULL; - return packet_seg_data(pkt_hdr, _odp_packet_seg_to_ndx(seg)); + pkt_hdr = get_seg(pkt_hdr, _odp_packet_seg_to_ndx(seg)); + + return pkt_hdr->buf_hdr.base_data; } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - pkt_hdr->buf_hdr.segcount)) - return 0; + ODP_ASSERT(_odp_packet_seg_to_ndx(seg) < pkt_hdr->buf_hdr.segcount); - return packet_seg_len(pkt_hdr, _odp_packet_seg_to_ndx(seg)); -} + pkt_hdr = get_seg(pkt_hdr, _odp_packet_seg_to_ndx(seg)); -/* - * - * Manipulation - * ******************************************************** - * - */ + return pkt_hdr->buf_hdr.size; +} int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { @@ -1402,8 +902,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1441,8 +939,6 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - ODP_ASSERT(odp_packet_has_ref(*dst) == 0); - /* Do a copy if packets are from different pools. */ if (odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { @@ -1457,7 +953,7 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) return -1; } - add_all_segs(dst_hdr, src_hdr); + concat_seg(dst_hdr, src_hdr); dst_hdr->frame_len = dst_len + src_len; dst_hdr->tailroom = src_hdr->tailroom; @@ -1473,8 +969,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1484,13 +978,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) return odp_packet_trunc_tail(pkt, pktlen - len, NULL, NULL); } -/* - * - * Copy - * ******************************************************** - * - */ - odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { odp_packet_hdr_t *srchdr = packet_hdr(pkt); @@ -1688,19 +1175,9 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, return 0; } -/* - * - * Debugging - * ******************************************************** - * - */ void odp_packet_print(odp_packet_t pkt) { - odp_packet_seg_t seg; - seg_entry_t *seg_entry; - odp_packet_hdr_t *seg_hdr; - uint8_t idx; - int max_len = 1024; + int max_len = 2048; char str[max_len]; int len = 0; int n = max_len - 1; @@ -1716,17 +1193,20 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " output_flags 0x%" PRIx32 "\n", hdr->p.output_flags.all); + len += snprintf(&str[len], n - len, + " uarea_addr 0x%p\n", + hdr->buf_hdr.uarea_addr); len += snprintf(&str[len], n - len, " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); len += snprintf(&str[len], n - len, " l3_offset %" PRIu32 "\n", hdr->p.l3_offset); len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); - len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); + len += snprintf(&str[len], n - len, + " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " headroom %" PRIu32 "\n", odp_packet_headroom(pkt)); @@ -1736,37 +1216,23 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " num_segs %i\n", odp_packet_num_segs(pkt)); - seg_hdr = hdr; - idx = 0; - seg = odp_packet_first_seg(pkt); - - while (seg != ODP_PACKET_SEG_INVALID) { - odp_buffer_hdr_t *buf_hdr; - odp_packet_hdr_t *tmp_hdr; - - tmp_hdr = seg_hdr; - seg_entry = seg_entry_next(&seg_hdr, &idx); - buf_hdr = seg_entry->hdr; - + do { len += snprintf(&str[len], n - len, - " seg_len %-4" PRIu32 " seg_data %p ", - odp_packet_seg_data_len(pkt, seg), - odp_packet_seg_data(pkt, seg)); - len += snprintf(&str[len], n - len, "ref_cnt %u", - buffer_ref(buf_hdr)); - if (seg_is_link(tmp_hdr)) { - uint32_t ref; - - ref = buffer_ref(&tmp_hdr->buf_hdr); - len += snprintf(&str[len], n - len, "L(%u)\n", ref); - } else { - len += snprintf(&str[len], n - len, "\n"); - } + " %p ref_cnt=%u size=%-5u " + "base_data=%p buf_end=%p next_seg=%p\n", + hdr, + buffer_ref(&hdr->buf_hdr), + hdr->buf_hdr.size, + hdr->buf_hdr.base_data, + hdr->buf_hdr.buf_end, + hdr->buf_hdr.next_seg); - seg = odp_packet_next_seg(pkt, seg); - } + hdr = hdr->buf_hdr.next_seg; + } while (hdr); - ODP_PRINT("%s\n", str); + str[len] = '\0'; + + ODP_PRINT("\n%s\n", str); } void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, @@ -1841,13 +1307,6 @@ int odp_packet_is_valid(odp_packet_t pkt) return 1; } -/* - * - * Internal Use Routines - * ******************************************************** - * - */ - int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) { odp_packet_hdr_t *srchdr = packet_hdr(srcpkt); @@ -2220,11 +1679,11 @@ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, odp_pktio_parser_layer_t layer) { - uint32_t seg_len = packet_first_seg_len(pkt_hdr); - void *base = packet_data(pkt_hdr); - - return packet_parse_common(&pkt_hdr->p, base, pkt_hdr->frame_len, - seg_len, layer); + return packet_parse_common(&pkt_hdr->p, + pkt_hdr->buf_hdr.base_data, + pkt_hdr->frame_len, + pkt_hdr->buf_hdr.size, + layer); } int packet_parse_l3_l4(odp_packet_hdr_t *pkt_hdr, @@ -2264,16 +1723,11 @@ odp_packet_t odp_packet_ref_static(odp_packet_t pkt) odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) { - odp_packet_t ref; - odp_packet_hdr_t *link_hdr; - odp_packet_hdr_t *next_hdr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - odp_packet_hdr_t *hdr = pkt_hdr; - seg_entry_t *seg; - uint32_t seg_idx = 0; - uint8_t idx = 0; - uint32_t seg_offset = 0; - int i, num_copy, segcount; + odp_packet_hdr_t *seg; + odp_packet_hdr_t *ref_hdr; + odp_packet_t ref; + uint32_t bytes_skipped, segs_skipped; uint32_t len; if (offset >= pkt_hdr->frame_len) { @@ -2281,73 +1735,33 @@ odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) return ODP_PACKET_INVALID; } - /* Allocate link segment */ - if (packet_alloc(pkt_hdr->buf_hdr.pool_ptr, 0, 1, 1, &ref) != 1) { + seg = get_seg_at_offset(pkt_hdr, offset, &bytes_skipped, &segs_skipped); + + if (packet_alloc(pkt_hdr->buf_hdr.pool_ptr, 0, 1, &ref) != 1) { ODP_DBG("segment alloc failed\n"); return ODP_PACKET_INVALID; } + ref_hdr = packet_hdr(ref); - link_hdr = packet_hdr(ref); - - seg_entry_find_offset(&hdr, &idx, &seg_offset, &seg_idx, offset); - num_copy = hdr->buf_hdr.num_seg - idx; - segcount = pkt_hdr->buf_hdr.segcount; - - /* In addition to segments, update reference count of - * an existing link header. */ - if (seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); - - seg = seg_entry_next(&hdr, &idx); - link_hdr->buf_hdr.num_seg = 1; - link_hdr->buf_hdr.seg[0].hdr = seg->hdr; - link_hdr->buf_hdr.seg[0].data = seg->data + seg_offset; - link_hdr->buf_hdr.seg[0].len = seg->len - seg_offset; - buffer_ref_inc(seg->hdr); - - /* The 'CONFIG_PACKET_SEGS_PER_HDR > 1' condition is required to fix an - * invalid error ('array subscript is above array bounds') thrown by - * gcc (5.4.0). */ - for (i = 1; CONFIG_PACKET_SEGS_PER_HDR > 1 && i < num_copy; i++) { - /* Update link header reference count */ - if (idx == 0 && seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); - - seg = seg_entry_next(&hdr, &idx); - - link_hdr->buf_hdr.num_seg++; - link_hdr->buf_hdr.seg[i].hdr = seg->hdr; - link_hdr->buf_hdr.seg[i].data = seg->data; - link_hdr->buf_hdr.seg[i].len = seg->len; - buffer_ref_inc(seg->hdr); - } + len = pkt_hdr->frame_len - offset; - next_hdr = hdr; + ref_hdr->buf_hdr.segcount = + 1 + (pkt_hdr->buf_hdr.segcount - segs_skipped); + ref_hdr->buf_hdr.next_seg = seg; - /* Increment ref count for remaining segments */ - for (i = seg_idx + num_copy; i < segcount; i++) { - /* Update link header reference count */ - if (idx == 0 && seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); + ref_hdr->frame_len = len; - seg = seg_entry_next(&hdr, &idx); - buffer_ref_inc(seg->hdr); - } + ref_hdr->tailroom = pkt_hdr->tailroom; + ref_hdr->headroom = 0; - len = pkt_hdr->frame_len - offset; - link_hdr->buf_hdr.next_seg = next_hdr; - link_hdr->buf_hdr.last_seg = pkt_hdr->buf_hdr.last_seg; - link_hdr->buf_hdr.segcount = segcount - seg_idx; - link_hdr->frame_len = len; - link_hdr->tailroom = pkt_hdr->tailroom; + /* Bump refcnt of trailing segments. */ + do { + buffer_ref_inc(&seg->buf_hdr); - /* Link header does not have headroom, it just points to other - * buffers. Zero length headroom ensures that head of the other buffer - * is not pushed through a reference. */ - link_hdr->headroom = 0; + seg = seg->buf_hdr.next_seg; + } while (seg); return ref; - } odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, @@ -2372,26 +1786,20 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, } return hdr; + } int odp_packet_has_ref(odp_packet_t pkt) { - odp_buffer_hdr_t *buf_hdr; - seg_entry_t *seg; - int i; - uint32_t ref_cnt; - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - int seg_count = pkt_hdr->buf_hdr.segcount; - odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0; + odp_packet_hdr_t *pkt_hdr = (odp_packet_hdr_t *)(uintptr_t)pkt; - for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - buf_hdr = seg->hdr; - ref_cnt = buffer_ref(buf_hdr); + while (pkt_hdr) { + uint32_t ref_cnt = buffer_ref(&pkt_hdr->buf_hdr); if (is_multi_ref(ref_cnt)) return 1; + + pkt_hdr = pkt_hdr->buf_hdr.next_seg; } return 0; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index bdd12292f..42df14ddd 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -106,8 +106,7 @@ static unsigned cache_size(uint32_t num) static inline uint16_t mbuf_data_off(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr) { - return (uint64_t)pkt_hdr->buf_hdr.seg[0].data - - (uint64_t)mbuf->buf_addr; + return (uint64_t)pkt_hdr->buf_hdr.base_data - (uint64_t)mbuf->buf_addr; } /** @@ -122,8 +121,7 @@ static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, mbuf->pkt_len = pkt_len; mbuf->refcnt = 1; - if (odp_unlikely(pkt_hdr->buf_hdr.base_data != - pkt_hdr->buf_hdr.seg[0].data)) + if (odp_unlikely(pkt_hdr->headroom != CONFIG_PACKET_HEADROOM)) mbuf->data_off = mbuf_data_off(mbuf, pkt_hdr); } @@ -647,7 +645,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, /* Init buffer segments. Currently, only single segment packets * are supported. */ - pkt_hdr->buf_hdr.seg[0].data = data; + pkt_hdr->buf_hdr.base_data = data; packet_init(pkt_hdr, pkt_len); pkt_hdr->input = pktio_entry->s.handle; diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 64c7606df..66c3dfad6 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -658,7 +658,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, offsets[i] = (uint8_t *)pkt_hdr - (uint8_t *)odp_shm_addr(pool->shm); - data_pool_off = (uint8_t *)pkt_hdr->buf_hdr.seg[0].data - + data_pool_off = (uint8_t *)pkt_hdr->buf_hdr.base_data - (uint8_t *)odp_shm_addr(pool->shm); /* compile all function code even if ipc disabled with config */ diff --git a/platform/linux-generic/pool/generic.c b/platform/linux-generic/pool/generic.c index 7a068472c..b16c7380a 100644 --- a/platform/linux-generic/pool/generic.c +++ b/platform/linux-generic/pool/generic.c @@ -207,7 +207,6 @@ static void init_buffers(pool_t *pool) ring_t *ring; uint32_t mask; int type; - uint32_t seg_size; uint64_t page_size; int skipped_blocks = 0; @@ -256,34 +255,26 @@ static void init_buffers(pool_t *pool) memset(buf_hdr, 0, (uintptr_t)data - (uintptr_t)buf_hdr); - seg_size = pool->headroom + pool->seg_len + pool->tailroom; - /* Initialize buffer metadata */ buf_hdr->index = i; - buf_hdr->size = seg_size; + buf_hdr->size = pool->seg_len; buf_hdr->type = type; buf_hdr->event_type = type; buf_hdr->event_subtype = ODP_EVENT_NO_SUBTYPE; buf_hdr->pool_hdl = pool->pool_hdl; buf_hdr->pool_ptr = pool; buf_hdr->uarea_addr = uarea; - /* Show user requested size through API */ - buf_hdr->segcount = 1; - buf_hdr->num_seg = 1; + buf_hdr->segcount = 0; buf_hdr->next_seg = NULL; - buf_hdr->last_seg = buf_hdr; - - /* Pointer to data start (of the first segment) */ - buf_hdr->seg[0].hdr = buf_hdr; - buf_hdr->seg[0].data = &data[offset]; - buf_hdr->seg[0].len = pool->seg_len; odp_atomic_init_u32(&buf_hdr->ref_cnt, 0); - /* Store base values for fast init */ - buf_hdr->base_data = buf_hdr->seg[0].data; - buf_hdr->buf_end = &data[offset + pool->seg_len + - pool->tailroom]; + ODP_ASSERT(offset <= 255); + buf_hdr->pristine_offset = offset; + + buf_hdr->base_data = &data[offset]; + buf_hdr->buf_end = + &data[offset + pool->seg_len + pool->tailroom]; /* Store buffer index into the global pool */ ring_enq(ring, mask, i); diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index 37550a2f5..6f1d10abf 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -2193,7 +2193,7 @@ void packet_test_ref(void) odp_packet_pool(segmented_test_packet)); CU_ASSERT_FATAL(hdr_pkt[1] != ODP_PACKET_INVALID); hdr_len[1] = odp_packet_len(hdr_pkt[1]); - offset[1] = 5; + offset[1] = 0; hdr_pkt[2] = odp_packet_copy_part(test_packet, 0, odp_packet_len(test_packet) / 4,
