From: Matias Elo <matias....@nokia.com> Add inlined accessors for pool_t members pool_hdl and uarea_size. This enables removing the matching members from odp_buffer_hdr_t, and thus reduces per buffer overhead.
Signed-off-by: Matias Elo <matias....@nokia.com> --- /** Email created from pull request 281 (matiaselo:dev/misc_perf_opt) ** https://github.com/Linaro/odp/pull/281 ** Patch: https://github.com/Linaro/odp/pull/281.patch ** Base sha: 9163719bd4c04321592dad9da7f26539f49c8b7a ** Merge commit sha: 788ccdfbb8df9ba8bbc49305ad08f178377a5d30 **/ .../include/odp/api/plat/packet_inlines.h | 11 +++++++++-- .../include/odp/api/plat/packet_types.h | 2 -- .../include/odp/api/plat/pool_types.h | 13 +++++++++++++ .../linux-generic/include/odp_buffer_internal.h | 7 ------- platform/linux-generic/odp_packet.c | 22 +++++++++++----------- platform/linux-generic/odp_pool.c | 13 ++++++++++--- 6 files changed, 43 insertions(+), 25 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 1804fa6f2..c808f2d6f 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -21,6 +21,9 @@ /** @internal Inline function offsets */ extern const _odp_packet_inline_offset_t _odp_packet_inline; +/** @internal Pool inline function offsets */ +extern const _odp_pool_inline_offset_t _odp_pool_inline; + #if ODP_ABI_COMPAT == 1 /** @internal Inline function @param seg @return */ static inline uint32_t _odp_packet_seg_to_ndx(odp_packet_seg_t seg) @@ -68,7 +71,9 @@ static inline uint32_t _odp_packet_tailroom(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline odp_pool_t _odp_packet_pool(odp_packet_t pkt) { - return _odp_pkt_get(pkt, odp_pool_t, pool); + void *pool = _odp_pkt_get(pkt, void *, pool); + + return _odp_pool_get(pool, odp_pool_t, pool_hdl); } /** @internal Inline function @param pkt @return */ @@ -98,7 +103,9 @@ static inline void *_odp_packet_user_area(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_user_area_size(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, user_area_size); + void *pool = _odp_pkt_get(pkt, void *, pool); + + return _odp_pool_get(pool, uint32_t, uarea_size); } /** @internal Inline function @param pkt @return */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 68c66312b..009a3aa7c 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -94,8 +94,6 @@ typedef struct _odp_packet_inline_offset_t { /** @internal field offset */ uint16_t user_area; /** @internal field offset */ - uint16_t user_area_size; - /** @internal field offset */ uint16_t flow_hash; /** @internal field offset */ uint16_t timestamp; diff --git a/platform/linux-generic/include/odp/api/plat/pool_types.h b/platform/linux-generic/include/odp/api/plat/pool_types.h index 8bc816d4e..9a26dde59 100644 --- a/platform/linux-generic/include/odp/api/plat/pool_types.h +++ b/platform/linux-generic/include/odp/api/plat/pool_types.h @@ -48,6 +48,19 @@ typedef enum odp_pool_type_t { #endif +/** @internal Pool field accessor */ +#define _odp_pool_get(pool, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)pool + _odp_pool_inline.field)) + +/** @internal Pool header field offsets for inline functions */ +typedef struct _odp_pool_inline_offset_t { + /** @internal field offset */ + uint16_t pool_hdl; + /** @internal field offset */ + uint16_t uarea_size; + +} _odp_pool_inline_offset_t; + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index ac189301f..c56c5b01b 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -104,17 +104,10 @@ struct odp_buffer_hdr_t { /* User area pointer */ void *uarea_addr; - /* User area size */ - uint32_t uarea_size; - /* ipc mapped process can not walk over pointers, * offset has to be used */ uint64_t ipc_data_offset; - /* Pool handle: will be removed, used only for odp_packet_pool() - * inlining */ - odp_pool_t pool_hdl; - /* Data or next header */ uint8_t data[0]; } ODP_ALIGNED_CACHE; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index ca706b2e5..04f1f33bb 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -32,12 +32,11 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), - .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_hdl), + .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_ptr), .input = offsetof(odp_packet_hdr_t, input), .segcount = offsetof(odp_packet_hdr_t, buf_hdr.segcount), .user_ptr = offsetof(odp_packet_hdr_t, buf_hdr.buf_ctx), .user_area = offsetof(odp_packet_hdr_t, buf_hdr.uarea_addr), - .user_area_size = offsetof(odp_packet_hdr_t, buf_hdr.uarea_size), .flow_hash = offsetof(odp_packet_hdr_t, flow_hash), .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags) @@ -266,7 +265,6 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, /* buffer header side packet metadata */ dst->buf_hdr.buf_u64 = src->buf_hdr.buf_u64; dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; - dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; /* segmentation data is not copied: * buf_hdr.seg[] @@ -1837,18 +1835,20 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) { odp_packet_hdr_t *srchdr = packet_hdr(srcpkt); odp_packet_hdr_t *dsthdr = packet_hdr(dstpkt); + pool_t *src_pool = srchdr->buf_hdr.pool_ptr; + pool_t *dst_pool = dsthdr->buf_hdr.pool_ptr; + uint32_t src_uarea_size = src_pool->params.pkt.uarea_size; + uint32_t dst_uarea_size = dst_pool->params.pkt.uarea_size; dsthdr->input = srchdr->input; dsthdr->dst_queue = srchdr->dst_queue; dsthdr->buf_hdr.buf_u64 = srchdr->buf_hdr.buf_u64; if (dsthdr->buf_hdr.uarea_addr != NULL && - srchdr->buf_hdr.uarea_addr != NULL) - memcpy(dsthdr->buf_hdr.uarea_addr, - srchdr->buf_hdr.uarea_addr, - dsthdr->buf_hdr.uarea_size <= - srchdr->buf_hdr.uarea_size ? - dsthdr->buf_hdr.uarea_size : - srchdr->buf_hdr.uarea_size); + srchdr->buf_hdr.uarea_addr != NULL) { + memcpy(dsthdr->buf_hdr.uarea_addr, srchdr->buf_hdr.uarea_addr, + dst_uarea_size <= src_uarea_size ? dst_uarea_size : + src_uarea_size); + } copy_packet_parser_metadata(srchdr, dsthdr); @@ -1856,7 +1856,7 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) * user area was truncated in the process. Note this can only * happen when copying between different pools. */ - return dsthdr->buf_hdr.uarea_size < srchdr->buf_hdr.uarea_size; + return dst_uarea_size < src_uarea_size; } /** diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 7d7a423b5..c0dd41ebf 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -57,6 +57,16 @@ typedef struct pool_local_t { pool_table_t *pool_tbl; static __thread pool_local_t local; +#include <odp/visibility_begin.h> + +/* Fill in pool header field offsets for inline functions */ +const _odp_pool_inline_offset_t _odp_pool_inline ODP_ALIGNED_CACHE = { + .pool_hdl = offsetof(pool_t, pool_hdl), + .uarea_size = offsetof(pool_t, params.pkt.uarea_size) +}; + +#include <odp/visibility_end.h> + static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx) { return _odp_cast_scalar(odp_pool_t, pool_idx); @@ -288,11 +298,8 @@ static void init_buffers(pool_t *pool) buf_hdr->index = i; buf_hdr->type = type; buf_hdr->event_type = type; - buf_hdr->pool_hdl = pool->pool_hdl; buf_hdr->pool_ptr = pool; buf_hdr->uarea_addr = uarea; - /* Show user requested size through API */ - buf_hdr->uarea_size = pool->params.pkt.uarea_size; buf_hdr->segcount = 1; buf_hdr->num_seg = 1; buf_hdr->next_seg = NULL;
