Implement the APIs:
- odp_packet_splice()
- odp_packet_ref()
- odp_packet_is_a_splice(),
- odp_packet_is_spliced() APIs.
This also involves functional upgrades to the existing packet manipulation
APIs to be able to deal correctly with spliced packets as input.
Signed-off-by: Bill Fischofer <bill.fischo...@linaro.org>
---
.../linux-generic/include/odp_packet_internal.h | 54 ++++-
platform/linux-generic/odp_packet.c | 239 ++++++++++++++++++---
2 files changed, 256 insertions(+), 37 deletions(-)
diff --git a/platform/linux-generic/include/odp_packet_internal.h
b/platform/linux-generic/include/odp_packet_internal.h
index 392d670..60c3fdd 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -154,7 +154,7 @@ typedef struct {
* packet_init(). Because of this any new fields added must be reviewed for
* initialization requirements.
*/
-typedef struct {
+typedef struct odp_packet_hdr_t {
/* common buffer header */
odp_buffer_hdr_t buf_hdr;
@@ -165,6 +165,10 @@ typedef struct {
uint32_t headroom;
uint32_t tailroom;
+ uint32_t ref_offset;
+ struct odp_packet_hdr_t *ref_hdr;
+ odp_atomic_u32_t ref_count;
+
odp_pktio_t input;
/* Members below are not initialized by packet_init() */
@@ -188,6 +192,38 @@ static inline odp_packet_hdr_t
*odp_packet_hdr(odp_packet_t pkt)
return (odp_packet_hdr_t *)odp_buf_to_hdr((odp_buffer_t)pkt);
}
+static inline odp_packet_hdr_t *odp_packet_last_hdr(odp_packet_t pkt,
+ uint32_t *offset)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ uint32_t ref_offset = 0;
+
+ while (pkt_hdr->ref_hdr) {
+ ref_offset = pkt_hdr->ref_offset;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ }
+
+ if (offset)
+ *offset = ref_offset;
+ return pkt_hdr;
+}
+
+static inline odp_packet_hdr_t *odp_packet_prev_hdr(odp_packet_hdr_t *pkt_hdr,
+ odp_packet_hdr_t *cur_hdr,
+ uint32_t *offset)
+{
+ uint32_t ref_offset = 0;
+
+ while (pkt_hdr->ref_hdr != cur_hdr) {
+ ref_offset = pkt_hdr->ref_offset;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ }
+
+ if (*offset)
+ *offset = ref_offset;
+ return pkt_hdr;
+}
+
static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr,
odp_packet_hdr_t *dst_hdr)
{
@@ -207,6 +243,11 @@ static inline void
copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr,
static inline void *packet_map(odp_packet_hdr_t *pkt_hdr,
uint32_t offset, uint32_t *seglen)
{
+ while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) {
+ offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset);
+ pkt_hdr = pkt_hdr->ref_hdr;
+ }
+
if (offset > pkt_hdr->frame_len)
return NULL;
@@ -285,7 +326,16 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr,
size_t len)
static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr)
{
- return pkt_hdr->frame_len;
+ uint32_t pkt_len = 0;
+ uint32_t offset = 0;
+
+ do {
+ pkt_len += pkt_hdr->frame_len - offset;
+ offset = pkt_hdr->ref_offset;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ } while (pkt_hdr);
+
+ return pkt_len;
}
static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len)
diff --git a/platform/linux-generic/odp_packet.c
b/platform/linux-generic/odp_packet.c
index c4cf324..93e3f51 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -73,6 +73,13 @@ static void packet_init(pool_entry_t *pool, odp_packet_hdr_t
*pkt_hdr,
(pool->s.seg_size * pkt_hdr->buf_hdr.segcount) -
(pool->s.headroom + size);
+ /* By default packet has no referencees */
+ pkt_hdr->ref_hdr = NULL;
+ pkt_hdr->ref_offset = 0;
+
+ /* Start with a ref_count of 1 since packet is allocated */
+ odp_atomic_init_u32(&pkt_hdr->ref_count, 1);
+
pkt_hdr->input = ODP_PKTIO_INVALID;
}
@@ -152,18 +159,34 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t
len,
return count;
}
-void odp_packet_free(odp_packet_t pkt)
+static inline int packet_free(odp_packet_hdr_t *pkt_hdr)
{
- uint32_t pool_id = pool_id_from_buf((odp_buffer_t)pkt);
+ uint32_t ref_count;
+
+ if (pkt_hdr->ref_hdr && packet_free(pkt_hdr->ref_hdr))
+ pkt_hdr->ref_hdr = NULL;
+
+ ref_count = odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count);
+
+ if (ref_count > 1)
+ return 0;
+
+ odp_buffer_free(pkt_hdr->buf_hdr.handle.handle);
+
+ return 1;
+}
- buffer_free(pool_id, (odp_buffer_t)pkt);
+void odp_packet_free(odp_packet_t pkt)
+{
+ (void)packet_free(odp_packet_hdr(pkt));
}
void odp_packet_free_multi(const odp_packet_t pkt[], int num)
{
- uint32_t pool_id = pool_id_from_buf((odp_buffer_t)pkt[0]);
+ int i;
- buffer_free_multi(pool_id, (const odp_buffer_t * const)pkt, num);
+ for (i = 0; i < num; i++)
+ (void)packet_free(odp_packet_hdr(pkt[i]));
}
int odp_packet_reset(odp_packet_t pkt, uint32_t len)
@@ -175,6 +198,9 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len)
if (totsize > pkt_hdr->buf_hdr.size)
return -1;
+ if (pkt_hdr->ref_hdr)
+ (void)packet_free(pkt_hdr->ref_hdr);
+
packet_init(pool, pkt_hdr, len, 0);
return 0;
@@ -217,8 +243,14 @@ void *odp_packet_head(odp_packet_t pkt)
uint32_t odp_packet_buf_len(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ uint32_t buf_len = 0;
+
+ do {
+ buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ } while (pkt_hdr);
- return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount;
+ return buf_len;
}
void *odp_packet_data(odp_packet_t pkt)
@@ -231,7 +263,7 @@ void *odp_packet_data(odp_packet_t pkt)
uint32_t odp_packet_seg_len(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- uint32_t seglen;
+ uint32_t seglen = 0;
/* Call returns length of 1st data segment */
packet_map(pkt_hdr, 0, &seglen);
@@ -240,7 +272,7 @@ uint32_t odp_packet_seg_len(odp_packet_t pkt)
uint32_t odp_packet_len(odp_packet_t pkt)
{
- return odp_packet_hdr(pkt)->frame_len;
+ return packet_len(odp_packet_hdr(pkt));
}
uint32_t odp_packet_headroom(odp_packet_t pkt)
@@ -250,12 +282,12 @@ uint32_t odp_packet_headroom(odp_packet_t pkt)
uint32_t odp_packet_tailroom(odp_packet_t pkt)
{
- return odp_packet_hdr(pkt)->tailroom;
+ return odp_packet_last_hdr(pkt, NULL)->tailroom;
}
void *odp_packet_tail(odp_packet_t pkt)
{
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL);
return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL);
}
@@ -301,22 +333,33 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len,
void **data_ptr, uint32_t *seg_len)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt);
+ odp_packet_hdr_t *nxt_hdr;
- if (len > pkt_hdr->frame_len)
+ if (len > packet_len(pkt_hdr))
return -1;
+ /* Special processing for spliced packets */
+ while (len > pkt_hdr->frame_len) {
+ len -= pkt_hdr->frame_len + pkt_hdr->ref_offset;
+ nxt_hdr = pkt_hdr->ref_hdr;
+ pkt_hdr->ref_hdr = NULL;
+ (void)packet_free(pkt_hdr);
+ pkt_hdr = nxt_hdr;
+ }
+
pull_head(pkt_hdr, len);
if (pkt_hdr->headroom >= pkt_hdr->buf_hdr.segsize)
pull_head_seg(pkt_hdr);
if (data_ptr)
*data_ptr = packet_map(pkt_hdr, 0, seg_len);
+ *pkt = (odp_packet_t)pkt_hdr->buf_hdr.handle.handle;
return 0;
}
void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len)
{
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL);
uint32_t origin = pkt_hdr->frame_len;
if (len > pkt_hdr->tailroom)
@@ -329,7 +372,7 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len)
int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len,
void **data_ptr, uint32_t *seg_len)
{
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt);
+ odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(*pkt, NULL);
uint32_t origin = pkt_hdr->frame_len;
if (len > pkt_hdr->tailroom && push_tail_seg(pkt_hdr, len))
@@ -344,9 +387,10 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len,
void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len)
{
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ uint32_t offset;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, &offset);
- if (len > pkt_hdr->frame_len)
+ if (len > pkt_hdr->frame_len - offset)
return NULL;
pull_tail(pkt_hdr, len);
@@ -356,11 +400,24 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len)
int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len,
void **tail_ptr, uint32_t *tailroom)
{
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt);
+ uint32_t offset;
+ odp_packet_hdr_t *first_hdr = odp_packet_hdr(*pkt);
+ odp_packet_hdr_t *pkt_hdr, *prev_hdr;
- if (len > pkt_hdr->frame_len)
+ if (len > packet_len(first_hdr))
return -1;
+ pkt_hdr = odp_packet_last_hdr(*pkt, &offset);
+
+ /* Special processing for spliced packets */
+ while (len > pkt_hdr->frame_len - offset) {
+ len -= pkt_hdr->frame_len + offset;
+ prev_hdr = odp_packet_prev_hdr(first_hdr, pkt_hdr, &offset);
+ prev_hdr->ref_hdr = NULL;
+ (void)packet_free(pkt_hdr);
+ pkt_hdr = prev_hdr;
+ }
+
pull_tail(pkt_hdr, len);
if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize)
pull_tail_seg(pkt_hdr);
@@ -563,12 +620,22 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t
timestamp)
int odp_packet_is_segmented(odp_packet_t pkt)
{
- return odp_packet_hdr(pkt)->buf_hdr.segcount > 1;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ return pkt_hdr->buf_hdr.segcount > 1 || pkt_hdr->ref_hdr != NULL;
}
int odp_packet_num_segs(odp_packet_t pkt)
{
- return odp_packet_hdr(pkt)->buf_hdr.segcount;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ int segcount = 0;
+
+ do {
+ segcount += pkt_hdr->buf_hdr.segcount;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ } while (pkt_hdr);
+
+ return segcount;
}
odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt)
@@ -578,7 +645,7 @@ odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt)
odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt)
{
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL);
odp_buffer_bits_t seghandle;
seghandle.handle = (odp_buffer_t)pkt;
@@ -589,9 +656,22 @@ odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt)
odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ odp_packet_seg_t next_seg;
+ odp_buffer_bits_t seghandle;
+
+ seghandle.handle = (odp_buffer_t)seg;
+
+ while (seghandle.prefix != pkt_hdr->buf_hdr.handle.prefix &&
+ pkt_hdr->ref_hdr)
+ pkt_hdr = pkt_hdr->ref_hdr;
- return (odp_packet_seg_t)segment_next(&pkt_hdr->buf_hdr,
- (odp_buffer_seg_t)seg);
+ next_seg = (odp_packet_seg_t)segment_next(&pkt_hdr->buf_hdr,
+ (odp_buffer_seg_t)seg);
+ if (next_seg == ODP_PACKET_SEG_INVALID && pkt_hdr->ref_hdr)
+ next_seg = (odp_packet_seg_t)
+ pkt_hdr->ref_hdr->buf_hdr.handle.handle;
+
+ return next_seg;
}
/*
@@ -604,18 +684,36 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt,
odp_packet_seg_t seg)
void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ void *segmap;
+ uint32_t offset = 0;
+
+ do {
+ segmap = segment_map(&pkt_hdr->buf_hdr,
+ (odp_buffer_seg_t)seg, NULL,
+ pkt_hdr->frame_len,
+ pkt_hdr->headroom + offset);
+ offset = pkt_hdr->ref_offset;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ } while (!segmap && pkt_hdr);
- return segment_map(&pkt_hdr->buf_hdr, (odp_buffer_seg_t)seg, NULL,
- pkt_hdr->frame_len, pkt_hdr->headroom);
+ return segmap;
}
uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
uint32_t seglen = 0;
-
- segment_map(&pkt_hdr->buf_hdr, (odp_buffer_seg_t)seg, &seglen,
- pkt_hdr->frame_len, pkt_hdr->headroom);
+ void *segmap;
+ uint32_t offset = 0;
+
+ do {
+ segmap = segment_map(&pkt_hdr->buf_hdr,
+ (odp_buffer_seg_t)seg, &seglen,
+ pkt_hdr->frame_len,
+ pkt_hdr->headroom + offset);
+ offset = pkt_hdr->ref_offset;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ } while (!segmap && pkt_hdr);
return seglen;
}
@@ -631,7 +729,7 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t
offset, uint32_t len)
{
odp_packet_t pkt = *pkt_ptr;
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- uint32_t pktlen = pkt_hdr->frame_len;
+ uint32_t pktlen = packet_len(pkt_hdr);
odp_packet_t newpkt;
if (offset > pktlen)
@@ -760,6 +858,77 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len,
odp_packet_t *tail)
}
/*
+ * Reference / Splice
+ */
+
+static inline odp_packet_t packet_splice(odp_packet_t refpkt,
+ odp_packet_t pkt, uint32_t offset)
+{
+ odp_packet_hdr_t *ref_hdr, *pkt_hdr;
+
+ ref_hdr = odp_packet_hdr(refpkt);
+ pkt_hdr = odp_packet_hdr(pkt);
+
+ while (ref_hdr->ref_hdr)
+ ref_hdr = ref_hdr->ref_hdr;
+
+ ref_hdr->ref_hdr = pkt_hdr;
+ ref_hdr->ref_offset = offset;
+
+ do {
+ odp_atomic_inc_u32(&pkt_hdr->ref_count);
+ pkt_hdr = pkt_hdr->ref_hdr;
+ } while (pkt_hdr);
+
+ return refpkt;
+}
+
+odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset)
+{
+ odp_packet_t refpkt;
+
+ if (pkt == ODP_PACKET_INVALID || offset >= odp_packet_len(pkt))
+ return ODP_PACKET_INVALID;
+
+ refpkt = odp_packet_alloc(odp_packet_pool(pkt), 0);
+
+ if (refpkt == ODP_PACKET_INVALID)
+ return ODP_PACKET_INVALID;
+
+ return packet_splice(refpkt, pkt, offset);
+}
+
+odp_packet_t odp_packet_splice(odp_packet_t refpkt,
+ odp_packet_t pkt, uint32_t offset)
+{
+ if (pkt == ODP_PACKET_INVALID ||
+ refpkt == ODP_PACKET_INVALID ||
+ pkt == refpkt ||
+ offset >= odp_packet_len(pkt))
+ return ODP_PACKET_INVALID;
+
+ return packet_splice(refpkt, pkt, offset);
+}
+
+int odp_packet_is_a_splice(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ uint32_t splice_count = 0;
+
+ while (pkt_hdr->ref_hdr) {
+ splice_count++;
+ pkt_hdr = pkt_hdr->ref_hdr;
+ }
+
+ return splice_count;
+}
+
+int odp_packet_is_spliced(odp_packet_t pkt)
+{
+ return odp_atomic_load_u32(&odp_packet_hdr(pkt)->ref_count) - 1;
+}
+
+/*
*
* Copy
* ********************************************************
@@ -769,7 +938,7 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len,
odp_packet_t *tail)
odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool)
{
odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt);
- uint32_t pktlen = srchdr->frame_len;
+ uint32_t pktlen = packet_len(srchdr);
odp_packet_t newpkt = odp_packet_alloc(pool, pktlen);
if (newpkt != ODP_PACKET_INVALID) {
@@ -808,7 +977,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t
offset,
uint8_t *dstaddr = (uint8_t *)dst;
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (offset + len > pkt_hdr->frame_len)
+ if (offset + len > packet_len(pkt_hdr))
return -1;
while (len > 0) {
@@ -832,7 +1001,7 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t
offset,
const uint8_t *srcaddr = (const uint8_t *)src;
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (offset + len > pkt_hdr->frame_len)
+ if (offset + len > packet_len(pkt_hdr))
return -1;
while (len > 0) {
@@ -860,8 +1029,8 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t
dst_offset,
uint32_t src_seglen = 0; /* GCC */
int overlap;
- if (dst_offset + len > dst_hdr->frame_len ||
- src_offset + len > src_hdr->frame_len)
+ if (dst_offset + len > packet_len(dst_hdr) ||
+ src_offset + len > packet_len(src_hdr))
return -1;
overlap = (dst_hdr == src_hdr &&
@@ -946,7 +1115,7 @@ void odp_packet_print(odp_packet_t pkt)
len += snprintf(&str[len], n - len,
" l4_offset %" PRIu32 "\n", hdr->p.l4_offset);
len += snprintf(&str[len], n - len,
- " frame_len %" PRIu32 "\n", hdr->frame_len);
+ " frame_len %" PRIu32 "\n", packet_len(hdr));
len += snprintf(&str[len], n - len,
" input %" PRIu64 "\n",
odp_pktio_to_u64(hdr->input));
@@ -1319,7 +1488,7 @@ parse_exit:
*/
int packet_parse_full(odp_packet_hdr_t *pkt_hdr)
{
- uint32_t seg_len;
+ uint32_t seg_len = 0;
void *base = packet_map(pkt_hdr, 0, &seg_len);
return packet_parse_common(&pkt_hdr->p, base, pkt_hdr->frame_len,
--
2.7.4
