Introduce memtank, highly customizable fixed sized object allocator
for DPDK applications. It offers close to the mempool level performance
on the fast path.
Main difference with the mempool is the ability to grow/shrink at runtime
with user provided grow/shirnk threshold values, plus some extra
features for higher flexibility.
Key properties:
- relies on user to provide callbacks for actual memory reservations.
User is free to choose whatever is most suitable way for his scenario,
i.e: via malloc/rte_malloc/mmap/some custom memory allocator.
- user defined constructor callback for newly allocated objects.
- bulk alloc and free APIs.
- different alloc/free policies (specified by user via flags parameter):
* lightweight as possible, but can fail
* more robust, but heavyweight - causes call to user-provided backing
memory allocator.
- backing memory grows/shrinks on demand, special API extensions
to allow user control grow/shrink size, frequency and when/where it
is going to happen (DP, CP, both, etc.).
- ability to pre-allocate all objects at memtank creation time
(mempool like behavior).
- custom object size and alignment.
- per object runtime statistics and sanity-checks (boundary violation,
double free, etc.) can be enabled/disabled at memtank creation time.
Known limitations (subject for further improvements):
- scalability:
after 8+ lcores conventional mempool (with FIFO) starts to outperform
memtank (which uses LIFO inside).
- mempool_cache integration is not part of the library and right now
has to be implemented by used manually on top of memtank API.
Envisioned usage scenarios within DPDK-based apps:
various flow/session control structures (TCP PCB, CT, NAT sessions, etc.)
that needs to be allocated/freed at the data-path.
Also can be used by 'semi-fastpath' allocations:
TBL-8 blocks for LPM, hash buckets, etc.
Initial idea is inspired by Linux/Solaris SLAB allocators.
Also re-used some ideas from my previous work for TLDK project:
https://github.com/FDio/tldk
Signed-off-by: Konstantin Ananyev <[email protected]>
---
app/test/meson.build | 2 +
app/test/test_memtank.c | 160 +++
app/test/test_memtank_stress.c | 1075 +++++++++++++++++
doc/api/doxy-api-index.md | 1 +
doc/api/doxy-api.conf.in | 1 +
.../prog_guide/img/memtank_internal.svg | 1 +
doc/guides/prog_guide/index.rst | 1 +
doc/guides/prog_guide/memtank_lib.rst | 231 ++++
lib/memtank/memtank.c | 630 ++++++++++
lib/memtank/memtank.h | 110 ++
lib/memtank/meson.build | 18 +
lib/memtank/misc.c | 375 ++++++
lib/memtank/rte_memtank.h | 303 +++++
lib/meson.build | 1 +
14 files changed, 2909 insertions(+)
create mode 100644 app/test/test_memtank.c
create mode 100644 app/test/test_memtank_stress.c
create mode 100644 doc/guides/prog_guide/img/memtank_internal.svg
create mode 100644 doc/guides/prog_guide/memtank_lib.rst
create mode 100644 lib/memtank/memtank.c
create mode 100644 lib/memtank/memtank.h
create mode 100644 lib/memtank/meson.build
create mode 100644 lib/memtank/misc.c
create mode 100644 lib/memtank/rte_memtank.h
diff --git a/app/test/meson.build b/app/test/meson.build
index 61024125a7..d1b6203cf5 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -129,6 +129,8 @@ source_file_deps = {
'test_memory.c': [],
'test_mempool.c': [],
'test_mempool_perf.c': [],
+ 'test_memtank.c': ['memtank'],
+ 'test_memtank_stress.c': ['memtank'],
'test_memzone.c': [],
'test_meter.c': ['meter'],
'test_metrics.c': ['metrics'],
diff --git a/app/test/test_memtank.c b/app/test/test_memtank.c
new file mode 100644
index 0000000000..1cf55d65e6
--- /dev/null
+++ b/app/test/test_memtank.c
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_memory.h>
+#include <rte_memtank.h>
+#include <rte_errno.h>
+#include "test.h"
+
+/* TEST SUITE */
+
+static int
+memtank_test_setup(void)
+{
+ return 0;
+}
+
+static void
+memtank_test_teardown(void)
+{
+}
+
+
+
+static void *
+test_alloc(size_t sz, void *p)
+{
+ RTE_SET_USED(p);
+ return malloc(sz);
+}
+
+static void
+test_free(void *buf, void *p)
+{
+ RTE_SET_USED(p);
+ return free(buf);
+}
+
+static int
+test_memtank_create_invalid(void)
+{
+ struct rte_memtank_prm prm;
+ struct rte_memtank *mt;
+
+ memset(&prm, 0, sizeof(prm));
+
+ rte_errno = 0;
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_EQUAL(mt, NULL, "memtank create");
+ RTE_TEST_ASSERT_EQUAL(rte_errno, EINVAL, "errno EINVAL");
+
+ prm.alloc = test_alloc;
+ rte_errno = 0;
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_EQUAL(mt, NULL, "memtank create");
+ RTE_TEST_ASSERT_EQUAL(rte_errno, EINVAL, "errno EINVAL");
+
+ prm.free = test_free;
+ rte_errno = 0;
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_EQUAL(mt, NULL, "memtank create");
+ RTE_TEST_ASSERT_EQUAL(rte_errno, EINVAL, "errno EINVAL");
+
+ prm.obj_align = 2;
+ rte_errno = 0;
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_EQUAL(mt, NULL, "memtank create");
+ RTE_TEST_ASSERT_EQUAL(rte_errno, EINVAL, "errno EINVAL");
+
+ prm.min_free = 2;
+ rte_errno = 0;
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_EQUAL(mt, NULL, "memtank create");
+ RTE_TEST_ASSERT_EQUAL(rte_errno, EINVAL, "errno EINVAL");
+
+ prm.max_free = 2;
+ rte_errno = 0;
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_EQUAL(mt, NULL, "memtank create");
+ RTE_TEST_ASSERT_EQUAL(rte_errno, EINVAL, "errno EINVAL");
+
+ prm.max_obj = 2;
+ rte_errno = 0;
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_EQUAL(mt, NULL, "memtank create");
+ RTE_TEST_ASSERT_EQUAL(rte_errno, EINVAL, "errno EINVAL");
+
+ prm.nb_obj_chunk = 2;
+ rte_errno = 0;
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_NOT_EQUAL(mt, NULL, "memtank create");
+
+ rte_memtank_destroy(mt);
+ return TEST_SUCCESS;
+}
+
+
+static int
+test_memtank_alloc(void)
+{
+ struct rte_memtank_prm prm;
+ struct rte_memtank *mt;
+
+ memset(&prm, 0, sizeof(prm));
+ prm.alloc = test_alloc;
+ prm.free = test_free;
+ prm.obj_align = 2;
+ prm.nb_obj_chunk = 2;
+ prm.min_free = 2;
+ prm.max_free = 2;
+ prm.max_obj = 10;
+
+ mt = rte_memtank_create(&prm);
+ RTE_TEST_ASSERT_NOT_EQUAL(mt, NULL, "memtank create");
+
+ void *obj[3] = { NULL };
+ uint32_t rc;
+
+ /* min_obj is 0 so this is expected to fail */
+ rc = rte_memtank_alloc(mt, obj, 1, RTE_MTANK_ALLOC_CHUNK);
+ RTE_TEST_ASSERT_EQUAL(rc, 0, "memtank alloc chunk 0 (%u)", rc);
+
+ rc = rte_memtank_alloc(mt, obj, 1, RTE_MTANK_ALLOC_GROW);
+ RTE_TEST_ASSERT_EQUAL(rc, 1, "memtank alloc 1 (%u)", rc);
+ RTE_TEST_ASSERT_NOT_EQUAL(obj[0], NULL, "alloc obj");
+
+ rc = rte_memtank_alloc(mt, obj, 3, RTE_MTANK_ALLOC_CHUNK);
+ RTE_TEST_ASSERT_EQUAL(rc, 3, "memtank alloc 3 (%u)", rc);
+
+ /* will fail - out of free objs */
+ rc = rte_memtank_alloc(mt, obj, 1, RTE_MTANK_ALLOC_CHUNK);
+ RTE_TEST_ASSERT_EQUAL(rc, 0, "memtank alloc chunk 0 (%u)", rc);
+
+ rte_memtank_destroy(mt);
+ return TEST_SUCCESS;
+}
+
+static struct unit_test_suite memtank_testsuite = {
+ .suite_name = "memtank library test suite",
+ .setup = memtank_test_setup,
+ .teardown = memtank_test_teardown,
+ .unit_test_cases = {
+ TEST_CASE(test_memtank_alloc),
+ TEST_CASE(test_memtank_create_invalid),
+ TEST_CASES_END(), /**< NULL terminate unit test array */
+ },
+};
+
+static int
+test_memtank(void)
+{
+ return unit_test_suite_runner(&memtank_testsuite);
+}
+
+REGISTER_FAST_TEST(memtank_autotest, NOHUGE_OK, ASAN_OK, test_memtank);
diff --git a/app/test/test_memtank_stress.c b/app/test/test_memtank_stress.c
new file mode 100644
index 0000000000..67b10c6611
--- /dev/null
+++ b/app/test/test_memtank_stress.c
@@ -0,0 +1,1075 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2025 Huawei Technologies Co., Ltd
+ */
+
+#include <string.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_launch.h>
+#include <rte_cycles.h>
+#include <rte_eal.h>
+#include <rte_ring.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_random.h>
+#include <rte_hexdump.h>
+#include <rte_malloc.h>
+#include <rte_memtank.h>
+
+#include "test.h"
+
+struct memstat {
+ struct {
+ rte_atomic64_t nb_call;
+ rte_atomic64_t nb_fail;
+ rte_atomic64_t sz;
+ } alloc;
+ struct {
+ rte_atomic64_t nb_call;
+ rte_atomic64_t nb_fail;
+ } free;
+ uint64_t nb_alloc_obj;
+};
+
+struct memtank_stat {
+ uint64_t nb_cycle;
+ struct {
+ uint64_t nb_call;
+ uint64_t nb_req;
+ uint64_t nb_alloc;
+ uint64_t nb_cycle;
+ uint64_t max_cycle;
+ uint64_t min_cycle;
+ } alloc;
+ struct {
+ uint64_t nb_call;
+ uint64_t nb_free;
+ uint64_t nb_cycle;
+ uint64_t max_cycle;
+ uint64_t min_cycle;
+ } free;
+ struct {
+ uint64_t nb_call;
+ uint64_t nb_chunk;
+ uint64_t nb_cycle;
+ uint64_t max_cycle;
+ uint64_t min_cycle;
+ } grow;
+ struct {
+ uint64_t nb_call;
+ uint64_t nb_chunk;
+ uint64_t nb_cycle;
+ uint64_t max_cycle;
+ uint64_t min_cycle;
+ } shrink;
+};
+
+struct master_args {
+ uint64_t run_cycles;
+ uint32_t delay_us;
+ uint32_t flags;
+};
+
+struct worker_args {
+ uint32_t max_obj;
+ uint32_t obj_size;
+ uint32_t alloc_flags;
+ uint32_t free_flags;
+ struct rte_ring *rng;
+};
+
+struct memtank_arg {
+ struct rte_memtank *mt;
+ union {
+ struct master_args master;
+ struct worker_args worker;
+ };
+ struct memtank_stat stats;
+} __rte_cache_aligned;
+
+#define BULK_NUM 32
+
+#define OBJ_SZ_MIN 1
+#define OBJ_SZ_MAX 0x100000
+#define OBJ_SZ_DEF (4 * RTE_CACHE_LINE_SIZE + 1)
+
+#define TEST_TIME 10
+#define CLEANUP_TIME 3
+
+#define FREE_THRSH_MIN 0
+#define FREE_THRSH_MAX 100
+
+enum {
+ WRK_CMD_STOP,
+ WRK_CMD_RUN,
+};
+
+enum {
+ MASTER_FLAG_GROW = 1,
+ MASTER_FLAG_SHRINK = 2,
+};
+
+enum {
+ MEM_FUNC_SYS,
+ MEM_FUNC_RTE,
+};
+
+static uint32_t wrk_cmd __rte_cache_aligned;
+
+static struct rte_memtank_prm mtnk_prm = {
+ .min_free = 4 * BULK_NUM,
+ .max_free = 32 * BULK_NUM,
+ .obj_size = OBJ_SZ_DEF,
+ .obj_align = RTE_CACHE_LINE_SIZE,
+ .nb_obj_chunk = BULK_NUM,
+ .flags = RTE_MTANK_OBJ_DBG,
+};
+
+static struct {
+ uint32_t run_time; /* test run-time in seconds */
+ uint32_t wrk_max_obj; /* max alloced objects per worker */
+ uint32_t wrk_fill_thrsh; /* wrk fill thresh % (0-100) */
+ uint32_t wrk_free_thrsh; /* wrk free thresh % (0-100) */
+ int32_t mem_func; /* memory subsystem to use for alloc/free */
+ int32_t verbose; /* verbose: print stat for each worker */
+} global_cfg = {
+ .run_time = TEST_TIME,
+ .wrk_max_obj = 2 * BULK_NUM,
+ .wrk_fill_thrsh = FREE_THRSH_MAX,
+ .wrk_free_thrsh = FREE_THRSH_MIN,
+ .mem_func = MEM_FUNC_SYS,
+ .verbose = 0,
+};
+
+static void *
+alloc_func(size_t sz)
+{
+ switch (global_cfg.mem_func) {
+ case MEM_FUNC_SYS:
+ return malloc(sz);
+ case MEM_FUNC_RTE:
+ return rte_malloc(NULL, sz, 0);
+ }
+
+ return NULL;
+}
+
+static void
+free_func(void *p)
+{
+ switch (global_cfg.mem_func) {
+ case MEM_FUNC_SYS:
+ return free(p);
+ case MEM_FUNC_RTE:
+ return rte_free(p);
+ }
+}
+
+static void *
+test_alloc1(size_t sz, void *p)
+{
+ struct memstat *ms;
+ void *buf;
+
+ ms = p;
+ buf = alloc_func(sz);
+ rte_atomic64_inc(&ms->alloc.nb_call);
+ if (buf != NULL) {
+ memset(buf, 0, sz);
+ rte_atomic64_add(&ms->alloc.sz, sz);
+ } else
+ rte_atomic64_inc(&ms->alloc.nb_fail);
+
+ return buf;
+}
+
+static void
+test_free1(void *buf, void *p)
+{
+ struct memstat *ms;
+
+ ms = p;
+
+ free_func(buf);
+ rte_atomic64_inc(&ms->free.nb_call);
+ if (buf == NULL)
+ rte_atomic64_inc(&ms->free.nb_fail);
+}
+
+static void
+global_cfg_dump(FILE *f)
+{
+ fprintf(f, "%s={\n", __func__);
+ fprintf(f, "\t.run_time=%u\n", global_cfg.run_time);
+ fprintf(f, "\t.wrk_max_obj=%u\n", global_cfg.wrk_max_obj);
+ fprintf(f, "\t.wrk_fill_thrsh=%u\n", global_cfg.wrk_fill_thrsh);
+ fprintf(f, "\t.wrk_free_thrsh=%u\n", global_cfg.wrk_free_thrsh);
+ fprintf(f, "\t.mem_func=%d\n", global_cfg.mem_func);
+ fprintf(f, "\t.verbose=%d\n", global_cfg.verbose);
+ fprintf(f, "};\n");
+}
+
+static void
+memstat_dump(FILE *f, struct memstat *ms)
+{
+ uint64_t alloc_sz, nb_alloc;
+ long double muc, mut;
+
+ nb_alloc = rte_atomic64_read(&ms->alloc.nb_call) -
+ rte_atomic64_read(&ms->alloc.nb_fail);
+ alloc_sz = rte_atomic64_read(&ms->alloc.sz) / nb_alloc;
+ nb_alloc -= rte_atomic64_read(&ms->free.nb_call) -
+ rte_atomic64_read(&ms->free.nb_fail);
+ alloc_sz *= nb_alloc;
+ mut = (alloc_sz == 0) ? 1 :
+ (long double)ms->nb_alloc_obj * mtnk_prm.obj_size / alloc_sz;
+ muc = (alloc_sz == 0) ? 1 :
+ (long double)(ms->nb_alloc_obj + mtnk_prm.max_free) *
+ mtnk_prm.obj_size / alloc_sz;
+
+ fprintf(f, "%s(%p)={\n", __func__, ms);
+ fprintf(f, "\talloc={\n");
+ fprintf(f, "\t\tnb_call=%" PRIu64 ",\n",
+ rte_atomic64_read(&ms->alloc.nb_call));
+ fprintf(f, "\t\tnb_fail=%" PRIu64 ",\n",
+ rte_atomic64_read(&ms->alloc.nb_fail));
+ fprintf(f, "\t\tsz=%" PRIu64 ",\n",
+ rte_atomic64_read(&ms->alloc.sz));
+ fprintf(f, "\t},\n");
+ fprintf(f, "\tfree={\n");
+ fprintf(f, "\t\tnb_call=%" PRIu64 ",\n",
+ rte_atomic64_read(&ms->free.nb_call));
+ fprintf(f, "\t\tnb_fail=%" PRIu64 ",\n",
+ rte_atomic64_read(&ms->free.nb_fail));
+ fprintf(f, "\t},\n");
+ fprintf(f, "\tnb_alloc_obj=%" PRIu64 ",\n", ms->nb_alloc_obj);
+ fprintf(f, "\tnb_alloc_chunk=%" PRIu64 ",\n", nb_alloc);
+ fprintf(f, "\talloc_sz=%" PRIu64 ",\n", alloc_sz);
+ fprintf(f, "\tmem_util(total)=%.2Lf %%,\n", mut * 100);
+ fprintf(f, "\tmem_util(cached)=%.2Lf %%,\n", muc * 100);
+ fprintf(f, "};\n");
+
+}
+
+static void
+memtank_stat_reset(struct memtank_stat *ms)
+{
+ static const struct memtank_stat init_stat = {
+ .alloc.min_cycle = UINT64_MAX,
+ .free.min_cycle = UINT64_MAX,
+ .grow.min_cycle = UINT64_MAX,
+ .shrink.min_cycle = UINT64_MAX,
+ };
+
+ *ms = init_stat;
+}
+
+static void
+memtank_stat_aggr(struct memtank_stat *as, const struct memtank_stat *ms)
+{
+ if (ms->alloc.nb_call != 0) {
+ as->alloc.nb_call += ms->alloc.nb_call;
+ as->alloc.nb_req += ms->alloc.nb_req;
+ as->alloc.nb_alloc += ms->alloc.nb_alloc;
+ as->alloc.nb_cycle += ms->alloc.nb_cycle;
+ as->alloc.max_cycle = RTE_MAX(as->alloc.max_cycle,
+ ms->alloc.max_cycle);
+ as->alloc.min_cycle = RTE_MIN(as->alloc.min_cycle,
+ ms->alloc.min_cycle);
+ }
+ if (ms->free.nb_call != 0) {
+ as->free.nb_call += ms->free.nb_call;
+ as->free.nb_free += ms->free.nb_free;
+ as->free.nb_cycle += ms->free.nb_cycle;
+ as->free.max_cycle = RTE_MAX(as->free.max_cycle,
+ ms->free.max_cycle);
+ as->free.min_cycle = RTE_MIN(as->free.min_cycle,
+ ms->free.min_cycle);
+ }
+ if (ms->grow.nb_call != 0) {
+ as->grow.nb_call += ms->grow.nb_call;
+ as->grow.nb_chunk += ms->grow.nb_chunk;
+ as->grow.nb_cycle += ms->grow.nb_cycle;
+ as->grow.max_cycle = RTE_MAX(as->grow.max_cycle,
+ ms->grow.max_cycle);
+ as->grow.min_cycle = RTE_MIN(as->grow.min_cycle,
+ ms->grow.min_cycle);
+ }
+ if (ms->shrink.nb_call != 0) {
+ as->shrink.nb_call += ms->shrink.nb_call;
+ as->shrink.nb_chunk += ms->shrink.nb_chunk;
+ as->shrink.nb_cycle += ms->shrink.nb_cycle;
+ as->shrink.max_cycle = RTE_MAX(as->shrink.max_cycle,
+ ms->shrink.max_cycle);
+ as->shrink.min_cycle = RTE_MIN(as->shrink.min_cycle,
+ ms->shrink.min_cycle);
+ }
+}
+
+static void
+memtank_stat_dump(FILE *f, uint32_t lc, const struct memtank_stat *ms)
+{
+ uint64_t t;
+ long double st;
+
+ st = (long double)rte_get_timer_hz() / US_PER_S;
+
+ if (lc == UINT32_MAX)
+ fprintf(f, "%s(AGGREGATE)={\n", __func__);
+ else
+ fprintf(f, "%s(lc=%u)={\n", __func__, lc);
+
+ fprintf(f, "\tnb_cycle=%" PRIu64 ",\n", ms->nb_cycle);
+ if (ms->alloc.nb_call != 0) {
+ fprintf(f, "\talloc={\n");
+ fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->alloc.nb_call);
+ fprintf(f, "\t\tnb_req=%" PRIu64 ",\n", ms->alloc.nb_req);
+ fprintf(f, "\t\tnb_alloc=%" PRIu64 ",\n", ms->alloc.nb_alloc);
+ fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->alloc.nb_cycle);
+
+ t = ms->alloc.nb_req - ms->alloc.nb_alloc;
+ fprintf(f, "\t\tfailed req: %"PRIu64 "(%.2Lf %%)\n",
+ t, (long double)t * 100 / ms->alloc.nb_req);
+ fprintf(f, "\t\tcycles/alloc: %.2Lf\n",
+ (long double)ms->alloc.nb_cycle / ms->alloc.nb_alloc);
+ fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
+ (long double)ms->alloc.nb_alloc / ms->alloc.nb_call);
+
+ fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n",
+ ms->alloc.max_cycle,
+ (long double)ms->alloc.max_cycle / st);
+ fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n",
+ ms->alloc.min_cycle,
+ (long double)ms->alloc.min_cycle / st);
+
+ fprintf(f, "\t},\n");
+ }
+ if (ms->free.nb_call != 0) {
+ fprintf(f, "\tfree={\n");
+ fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->free.nb_call);
+ fprintf(f, "\t\tnb_free=%" PRIu64 ",\n", ms->free.nb_free);
+ fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->free.nb_cycle);
+
+ fprintf(f, "\t\tcycles/free: %.2Lf\n",
+ (long double)ms->free.nb_cycle / ms->free.nb_free);
+ fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
+ (long double)ms->free.nb_free / ms->free.nb_call);
+
+ fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n",
+ ms->free.max_cycle,
+ (long double)ms->free.max_cycle / st);
+ fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n",
+ ms->free.min_cycle,
+ (long double)ms->free.min_cycle / st);
+
+ fprintf(f, "\t},\n");
+ }
+ if (ms->grow.nb_call != 0) {
+ fprintf(f, "\tgrow={\n");
+ fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->grow.nb_call);
+ fprintf(f, "\t\tnb_chunk=%" PRIu64 ",\n", ms->grow.nb_chunk);
+ fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->grow.nb_cycle);
+
+ fprintf(f, "\t\tcycles/chunk: %.2Lf\n",
+ (long double)ms->grow.nb_cycle / ms->grow.nb_chunk);
+ fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
+ (long double)ms->grow.nb_chunk / ms->grow.nb_call);
+
+ fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n",
+ ms->grow.max_cycle,
+ (long double)ms->grow.max_cycle / st);
+ fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n",
+ ms->grow.min_cycle,
+ (long double)ms->grow.min_cycle / st);
+
+ fprintf(f, "\t},\n");
+ }
+ if (ms->shrink.nb_call != 0) {
+ fprintf(f, "\tshrink={\n");
+ fprintf(f, "\t\tnb_call=%" PRIu64 ",\n", ms->shrink.nb_call);
+ fprintf(f, "\t\tnb_chunk=%" PRIu64 ",\n", ms->shrink.nb_chunk);
+ fprintf(f, "\t\tnb_cycle=%" PRIu64 ",\n", ms->shrink.nb_cycle);
+
+ fprintf(f, "\t\tcycles/chunk: %.2Lf\n",
+ (long double)ms->shrink.nb_cycle / ms->shrink.nb_chunk);
+ fprintf(f, "\t\tobj/call(avg): %.2Lf\n",
+ (long double)ms->shrink.nb_chunk / ms->shrink.nb_call);
+
+ fprintf(f, "\t\tmax cycles/call=%" PRIu64 "(%.2Lf usec),\n",
+ ms->shrink.max_cycle,
+ (long double)ms->shrink.max_cycle / st);
+ fprintf(f, "\t\tmin cycles/call=%" PRIu64 "(%.2Lf usec),\n",
+ ms->shrink.min_cycle,
+ (long double)ms->shrink.min_cycle / st);
+
+ fprintf(f, "\t},\n");
+ }
+ fprintf(f, "};\n");
+}
+
+static int32_t
+check_fill_objs(void *obj[], uint32_t sz, uint32_t num,
+ uint8_t check, uint8_t fill)
+{
+ uint32_t i;
+ uint8_t buf[sz];
+
+ static rte_spinlock_t dump_lock;
+
+ memset(buf, check, sz);
+
+ for (i = 0; i != num; i++) {
+ if (memcmp(buf, obj[i], sz) != 0) {
+ rte_spinlock_lock(&dump_lock);
+ printf("%s(%u, %u, %hu, %hu) failed at %u-th iter, "
+ "offendig object: %p\n",
+ __func__, sz, num, check, fill, i, obj[i]);
+ rte_memdump(stdout, "expected", buf, sz);
+ rte_memdump(stdout, "result", obj[i], sz);
+ rte_spinlock_unlock(&dump_lock);
+ return -EINVAL;
+ }
+ memset(obj[i], fill, sz);
+ }
+ return 0;
+}
+
+static void
+destroy_worker_ring(struct worker_args *wa)
+{
+ free(wa->rng);
+ wa->rng = NULL;
+}
+
+static int
+create_worker_ring(struct worker_args *wa, uint32_t lc)
+{
+ int32_t rc;
+ size_t sz;
+ struct rte_ring *ring;
+
+ sz = rte_ring_get_memsize(wa->max_obj);
+ ring = aligned_alloc(alignof(typeof(*ring)), sz);
+ if (ring == NULL) {
+ printf("%s(%u): alloc(%zu) for FIFO with %u elems failed",
+ __func__, lc, sz, wa->max_obj);
+ return -ENOMEM;
+ }
+ rc = rte_ring_init(ring, "", wa->max_obj,
+ RING_F_SP_ENQ | RING_F_SC_DEQ);
+ if (rc != 0) {
+ printf("%s(%u): rte_ring_init(%p, %u) failed, error: %d(%s)\n",
+ __func__, lc, ring, wa->max_obj,
+ rc, strerror(-rc));
+ free(ring);
+ return rc;
+ }
+
+ wa->rng = ring;
+ return rc;
+}
+
+static int
+test_worker_cleanup(void *arg)
+{
+ void *obj[BULK_NUM];
+ int32_t rc;
+ uint32_t lc, n, num;
+ struct memtank_arg *ma;
+ struct rte_ring *ring;
+
+ ma = arg;
+ ring = ma->worker.rng;
+ lc = rte_lcore_id();
+
+ rc = 0;
+ for (n = rte_ring_count(ring); rc == 0 && n != 0; n -= num) {
+
+ num = rte_rand() % RTE_DIM(obj);
+ num = RTE_MIN(num, n);
+
+ if (num != 0) {
+ /* retrieve objects to free */
+ rte_ring_dequeue_bulk(ring, obj, num, NULL);
+
+ /* check and fill contents of freeing objects */
+ rc = check_fill_objs(obj, ma->worker.obj_size, num,
+ lc, 0);
+ if (rc == 0) {
+ rte_memtank_free(ma->mt, obj, num,
+ ma->worker.free_flags);
+ ma->stats.free.nb_free += num;
+ }
+ }
+ }
+
+ return rc;
+}
+
+static int
+test_memtank_worker(void *arg)
+{
+ int32_t rc;
+ uint32_t lc, n, num, tfl, tfr;
+ uint64_t cl, tm0, tm1;
+ struct memtank_arg *ma;
+ struct rte_ring *ring;
+ void *obj[BULK_NUM];
+
+ ma = arg;
+ lc = rte_lcore_id();
+
+ /* calculate fill threshold */
+ tfl = (FREE_THRSH_MAX - global_cfg.wrk_fill_thrsh) *
+ ma->worker.max_obj / FREE_THRSH_MAX;
+
+ /* calculate free threshold */
+ tfr = ma->worker.max_obj * global_cfg.wrk_free_thrsh / FREE_THRSH_MAX;
+
+ ring = ma->worker.rng;
+
+ while (wrk_cmd != WRK_CMD_RUN) {
+ rte_smp_rmb();
+ rte_pause();
+ }
+
+ cl = rte_rdtsc_precise();
+
+ do {
+ num = rte_rand() % RTE_DIM(obj);
+ n = rte_ring_free_count(ring);
+ num = (n >= tfl) ? RTE_MIN(num, n) : 0;
+
+ /* perform alloc*/
+ if (num != 0) {
+ tm0 = rte_rdtsc_precise();
+ n = rte_memtank_alloc(ma->mt, obj, num,
+ ma->worker.alloc_flags);
+ tm1 = rte_rdtsc_precise();
+
+ /* check and fill contents of allocated objects */
+ rc = check_fill_objs(obj, ma->worker.obj_size, n,
+ 0, lc);
+ if (rc != 0)
+ break;
+
+ tm1 = tm1 - tm0;
+
+ /* collect alloc stat */
+ ma->stats.alloc.nb_call++;
+ ma->stats.alloc.nb_req += num;
+ ma->stats.alloc.nb_alloc += n;
+ ma->stats.alloc.nb_cycle += tm1;
+ ma->stats.alloc.max_cycle =
+ RTE_MAX(ma->stats.alloc.max_cycle, tm1);
+ ma->stats.alloc.min_cycle =
+ RTE_MIN(ma->stats.alloc.min_cycle, tm1);
+
+ /* store allocated objects */
+ rte_ring_enqueue_bulk(ring, obj, n, NULL);
+ }
+
+ /* get some objects to free */
+ num = rte_rand() % RTE_DIM(obj);
+ n = rte_ring_count(ring);
+ num = (n >= tfr) ? RTE_MIN(num, n) : 0;
+
+ /* perform free*/
+ if (num != 0) {
+
+ /* retrieve objects to free */
+ rte_ring_dequeue_bulk(ring, obj, num, NULL);
+
+ /* check and fill contents of freeing objects */
+ rc = check_fill_objs(obj, ma->worker.obj_size, num,
+ lc, 0);
+ if (rc != 0)
+ break;
+
+ tm0 = rte_rdtsc_precise();
+ rte_memtank_free(ma->mt, obj, num,
+ ma->worker.free_flags);
+ tm1 = rte_rdtsc_precise();
+
+ tm1 = tm1 - tm0;
+
+ /* collect free stat */
+ ma->stats.free.nb_call++;
+ ma->stats.free.nb_free += num;
+ ma->stats.free.nb_cycle += tm1;
+ ma->stats.free.max_cycle =
+ RTE_MAX(ma->stats.free.max_cycle, tm1);
+ ma->stats.free.min_cycle =
+ RTE_MIN(ma->stats.free.min_cycle, tm1);
+ }
+
+ rte_smp_mb();
+ } while (wrk_cmd == WRK_CMD_RUN);
+
+ ma->stats.nb_cycle = rte_rdtsc_precise() - cl;
+
+ return rc;
+}
+
+static int
+test_memtank_master(void *arg)
+{
+ struct memtank_arg *ma;
+ uint64_t cl, tm0, tm1, tm2;
+ uint32_t i, n;
+
+ ma = (struct memtank_arg *)arg;
+
+ for (cl = 0, i = 0; cl < ma->master.run_cycles;
+ cl += tm2 - tm0, i++) {
+
+ tm0 = rte_rdtsc_precise();
+
+ if (ma->master.flags & MASTER_FLAG_SHRINK) {
+
+ n = rte_memtank_shrink(ma->mt);
+ tm1 = rte_rdtsc_precise();
+ ma->stats.shrink.nb_call++;
+ ma->stats.shrink.nb_chunk += n;
+ tm1 = tm1 - tm0;
+
+ if (n != 0) {
+ ma->stats.shrink.nb_cycle += tm1;
+ ma->stats.shrink.max_cycle =
+ RTE_MAX(ma->stats.shrink.max_cycle,
+ tm1);
+ ma->stats.shrink.min_cycle =
+ RTE_MIN(ma->stats.shrink.min_cycle,
+ tm1);
+ }
+ }
+
+ if (ma->master.flags & MASTER_FLAG_GROW) {
+
+ tm1 = rte_rdtsc_precise();
+ n = rte_memtank_grow(ma->mt);
+ tm2 = rte_rdtsc_precise();
+ ma->stats.grow.nb_call++;
+ ma->stats.grow.nb_chunk += n;
+ tm2 = tm2 - tm1;
+
+ if (n != 0) {
+ ma->stats.grow.nb_cycle += tm2;
+ ma->stats.grow.max_cycle =
+ RTE_MAX(ma->stats.grow.max_cycle,
+ tm2);
+ ma->stats.grow.min_cycle =
+ RTE_MIN(ma->stats.grow.min_cycle,
+ tm2);
+ }
+ }
+
+ wrk_cmd = WRK_CMD_RUN;
+ rte_smp_mb();
+
+ rte_delay_us(ma->master.delay_us);
+ tm2 = rte_rdtsc_precise();
+ }
+
+ ma->stats.nb_cycle = cl;
+
+ rte_smp_mb();
+ wrk_cmd = WRK_CMD_STOP;
+
+ return 0;
+}
+
+static int
+fill_worker_args(struct worker_args *wa, uint32_t alloc_flags,
+ uint32_t free_flags, uint32_t lc)
+{
+ wa->max_obj = global_cfg.wrk_max_obj;
+ wa->obj_size = mtnk_prm.obj_size;
+ wa->alloc_flags = alloc_flags;
+ wa->free_flags = free_flags;
+
+ return create_worker_ring(wa, lc);
+}
+
+static void
+fill_master_args(struct master_args *ma, uint32_t flags)
+{
+ uint64_t tm;
+
+ tm = global_cfg.run_time * rte_get_timer_hz();
+
+ ma->run_cycles = tm;
+ ma->delay_us = US_PER_S / MS_PER_S;
+ ma->flags = flags;
+}
+
+static int
+test_memtank_cleanup(struct rte_memtank *mt, struct memstat *ms,
+ struct memtank_arg arg[], const char *tname)
+{
+ int32_t rc;
+ uint32_t lc;
+
+ printf("%s(%s)\n", __func__, tname);
+
+ RTE_LCORE_FOREACH_WORKER(lc)
+ rte_eal_remote_launch(test_worker_cleanup, &arg[lc], lc);
+
+ /* launch on master */
+ lc = rte_lcore_id();
+ arg[lc].master.run_cycles = CLEANUP_TIME * rte_get_timer_hz();
+ test_memtank_master(&arg[lc]);
+
+ rc = 0;
+ ms->nb_alloc_obj = 0;
+ RTE_LCORE_FOREACH_WORKER(lc) {
+ rc |= rte_eal_wait_lcore(lc);
+ ms->nb_alloc_obj += arg[lc].stats.alloc.nb_alloc -
+ arg[lc].stats.free.nb_free;
+ }
+
+ rte_memtank_dump(stdout, mt, RTE_MTANK_DUMP_STAT);
+
+ memstat_dump(stdout, ms);
+ rc = rte_memtank_sanity_check(mt, 0);
+
+ return rc;
+}
+
+/*
+ * alloc/free by workers threads.
+ * grow/shrink by master
+ */
+static int
+test_memtank_mt(const char *tname, uint32_t alloc_flags, uint32_t free_flags)
+{
+ int32_t rc;
+ uint32_t lc;
+ struct rte_memtank *mt;
+ struct rte_memtank_prm prm;
+ struct memstat ms;
+ struct memtank_stat wrk_stats;
+ struct memtank_arg arg[RTE_MAX_LCORE];
+
+ printf("%s(%s) start\n", __func__, tname);
+
+ memset(&prm, 0, sizeof(prm));
+ memset(&ms, 0, sizeof(ms));
+
+ prm = mtnk_prm;
+ prm.alloc = test_alloc1;
+ prm.free = test_free1;
+ prm.udata = &ms;
+
+ mt = rte_memtank_create(&prm);
+ if (mt == NULL) {
+ printf("%s(%s): memtank_create() failed\n", __func__, tname);
+ return -ENOMEM;
+ }
+
+ /* dump initial memory stats */
+ memstat_dump(stdout, &ms);
+
+ rc = 0;
+ memset(arg, 0, sizeof(arg));
+
+ /* prepare args on all slaves */
+ RTE_LCORE_FOREACH_WORKER(lc) {
+ arg[lc].mt = mt;
+ rc = fill_worker_args(&arg[lc].worker, alloc_flags,
+ free_flags, lc);
+ if (rc != 0)
+ break;
+ memtank_stat_reset(&arg[lc].stats);
+ }
+
+ if (rc != 0) {
+ rte_memtank_destroy(mt);
+ return rc;
+ }
+
+ /* launch on all slaves */
+ RTE_LCORE_FOREACH_WORKER(lc)
+ rte_eal_remote_launch(test_memtank_worker, &arg[lc], lc);
+
+ /* launch on master */
+ lc = rte_lcore_id();
+ arg[lc].mt = mt;
+ fill_master_args(&arg[lc].master,
+ MASTER_FLAG_GROW | MASTER_FLAG_SHRINK);
+ test_memtank_master(&arg[lc]);
+
+ /* wait for slaves and collect stats. */
+
+ memtank_stat_reset(&wrk_stats);
+
+ rc = 0;
+ RTE_LCORE_FOREACH_WORKER(lc) {
+ rc |= rte_eal_wait_lcore(lc);
+ if (global_cfg.verbose != 0)
+ memtank_stat_dump(stdout, lc, &arg[lc].stats);
+ memtank_stat_aggr(&wrk_stats, &arg[lc].stats);
+ ms.nb_alloc_obj += arg[lc].stats.alloc.nb_alloc -
+ arg[lc].stats.free.nb_free;
+ }
+
+ memtank_stat_dump(stdout, UINT32_MAX, &wrk_stats);
+
+ lc = rte_lcore_id();
+ memtank_stat_dump(stdout, lc, &arg[lc].stats);
+ rte_memtank_dump(stdout, mt, RTE_MTANK_DUMP_STAT);
+
+ memstat_dump(stdout, &ms);
+ rc |= rte_memtank_sanity_check(mt, 0);
+
+ /* run cleanup on all slave cores */
+ if (rc == 0)
+ rc = test_memtank_cleanup(mt, &ms, arg, tname);
+
+ RTE_LCORE_FOREACH_WORKER(lc)
+ destroy_worker_ring(&arg[lc].worker);
+
+ rte_memtank_destroy(mt);
+ return rc;
+}
+
+/*
+ * alloc/free by workers threads.
+ * grow/shrink by master
+ */
+static int
+test_memtank_mt1(const char *tname)
+{
+ return test_memtank_mt(tname, 0, 0);
+}
+
+/*
+ * alloc/free with grow/shrink by worker threads.
+ * master does nothing
+ */
+static int
+test_memtank_mt2(const char *tname)
+{
+ const uint32_t alloc_flags = RTE_MTANK_ALLOC_CHUNK |
+ RTE_MTANK_ALLOC_GROW;
+ const uint32_t free_flags = RTE_MTANK_FREE_SHRINK;
+
+ return test_memtank_mt(tname, alloc_flags, free_flags);
+}
+
+static int
+parse_uint_val(const char *str, uint32_t *val, uint32_t min, uint32_t max)
+{
+ unsigned long v;
+ char *end;
+
+ errno = 0;
+ v = strtoul(str, &end, 0);
+ if (errno != 0 || end[0] != 0 || v < min || v > max)
+ return -EINVAL;
+
+ val[0] = v;
+ return 0;
+}
+
+static int
+parse_mem_str(const char *str)
+{
+ uint32_t i;
+
+ static const struct {
+ const char *name;
+ int32_t val;
+ } name2val[] = {
+ {
+ .name = "sys",
+ .val = MEM_FUNC_SYS,
+ },
+ {
+ .name = "rte",
+ .val = MEM_FUNC_RTE,
+ },
+ };
+
+ for (i = 0; i != RTE_DIM(name2val); i++) {
+ if (strcmp(str, name2val[i].name) == 0)
+ return name2val[i].val;
+ }
+ return -EINVAL;
+}
+
+/* update global values based on provided user input */
+static int
+update_global_cfg(void)
+{
+ mtnk_prm.max_obj = global_cfg.wrk_max_obj * rte_lcore_count();
+ return 0;
+}
+
+static int
+parse_opt(int argc, char * const argv[])
+{
+ int32_t opt, rc;
+ uint32_t v;
+
+ rc = 0;
+ optind = 0;
+ optarg = NULL;
+
+ while ((opt = getopt(argc, argv, "a:F:f:M:m:s:t:w:v")) != EOF) {
+ switch (opt) {
+ case 'a':
+ rc = parse_mem_str(optarg);
+ if (rc >= 0)
+ global_cfg.mem_func = rc;
+ break;
+ case 'F':
+ rc = parse_uint_val(optarg, &v, FREE_THRSH_MIN,
+ FREE_THRSH_MAX);
+ if (rc == 0)
+ global_cfg.wrk_fill_thrsh = v;
+ break;
+ case 'f':
+ rc = parse_uint_val(optarg, &v, FREE_THRSH_MIN,
+ FREE_THRSH_MAX);
+ if (rc == 0)
+ global_cfg.wrk_free_thrsh = v;
+ break;
+ case 'M':
+ rc = parse_uint_val(optarg, &v, 0, UINT32_MAX);
+ if (rc == 0)
+ mtnk_prm.max_free = v;
+ break;
+ case 'm':
+ rc = parse_uint_val(optarg, &v, 0, UINT32_MAX);
+ if (rc == 0)
+ mtnk_prm.min_free = v;
+ break;
+ case 's':
+ rc = parse_uint_val(optarg, &v, OBJ_SZ_MIN,
+ OBJ_SZ_MAX);
+ if (rc == 0)
+ mtnk_prm.obj_size = v;
+ break;
+ case 't':
+ rc = parse_uint_val(optarg, &v, 0, UINT32_MAX);
+ if (rc == 0)
+ global_cfg.run_time = v;
+ break;
+ case 'w':
+ rc = parse_uint_val(optarg, &v, 0, UINT32_MAX);
+ if (rc == 0)
+ global_cfg.wrk_max_obj = v;
+ break;
+ case 'v':
+ global_cfg.verbose = 1;
+ break;
+ default:
+ rc = -EINVAL;
+ }
+ }
+
+ if (rc < 0)
+ printf("%s: invalid value: \"%s\" for option: \'%c\'\n",
+ __func__, optarg, opt);
+
+ return rc;
+}
+
+static int
+run_memtank_stress(int argc, char *argv[])
+{
+ int32_t rc;
+ uint32_t i, k;
+
+ const struct {
+ const char *name;
+ int (*func)(const char *tname);
+ } tests[] = {
+ {
+ .name = "MT1-WRK_ALLOC_FREE-MST_GROW_SHRINK",
+ .func = test_memtank_mt1,
+ },
+ {
+ .name = "MT1-WRK_ALLOC+GROW_FREE+SHRINK",
+ .func = test_memtank_mt2,
+ },
+ };
+
+ rc = parse_opt(argc, argv);
+ if (rc < 0) {
+ printf("%s: parse_op failed with error code: %d\n",
+ __func__, rc);
+ return rc;
+ }
+
+ /* update global values based on provided user input */
+ rc = update_global_cfg();
+ if (rc < 0)
+ return rc;
+
+ global_cfg_dump(stdout);
+
+ for (i = 0, k = 0; i != RTE_DIM(tests); i++) {
+
+ printf("TEST %s START\n", tests[i].name);
+
+ rc = tests[i].func(tests[i].name);
+ k += (rc == 0);
+
+ if (rc != 0)
+ printf("TEST %s FAILED\n", tests[i].name);
+ else
+ printf("TEST %s OK\n", tests[i].name);
+ }
+
+ printf("Number of tests:\t%u\nSuccess:\t%u\nFailed:\t%u\n",
+ i, k, i - k);
+ return (k != i);
+}
+
+static int
+test_memtank_stress(void)
+{
+ int32_t rc;
+ uint32_t i;
+ const char *val;
+ char *sp, *str, *argv[16];
+ char buf[0x100];
+
+ static const char *dlm = " \t\n";
+ static const char *evar = "DPDK_MEMTANK_STRESS_TEST_PARAM";
+ static const char *eval = "";
+
+ val = getenv(evar);
+ if (val == NULL)
+ val = eval;
+
+ snprintf(buf, sizeof(buf), "%s", val);
+
+ for (i = 0, str = buf; i != RTE_DIM(argv); str = NULL, i++) {
+ argv[i] = strtok_r(str, dlm, &sp);
+ if (argv[i] == NULL)
+ break;
+ }
+
+ if (i == RTE_DIM(argv)) {
+ printf("invalid value: \"%s\" for $(%s)\n", val, evar);
+ return -EINVAL;
+ }
+
+ rc = run_memtank_stress(i, argv);
+ return rc;
+}
+
+REGISTER_STRESS_TEST(memtank_stress_autotest, test_memtank_stress);
diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 9296042119..68203b9d7b 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -71,6 +71,7 @@ The public API headers are grouped by topics:
[mempool](@ref rte_mempool.h),
[malloc](@ref rte_malloc.h),
[memcpy](@ref rte_memcpy.h)
+ [memtank](@ref rte_memtank.h)
- **timers**:
[cycles](@ref rte_cycles.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index bedd944681..042482fe16 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -59,6 +59,7 @@ INPUT = @TOPDIR@/doc/api/doxy-api-index.md \
@TOPDIR@/lib/mbuf \
@TOPDIR@/lib/member \
@TOPDIR@/lib/mempool \
+ @TOPDIR@/lib/memtank \
@TOPDIR@/lib/meter \
@TOPDIR@/lib/metrics \
@TOPDIR@/lib/mldev \
diff --git a/doc/guides/prog_guide/img/memtank_internal.svg
b/doc/guides/prog_guide/img/memtank_internal.svg
new file mode 100644
index 0000000000..0f13242ed5
--- /dev/null
+++ b/doc/guides/prog_guide/img/memtank_internal.svg
@@ -0,0 +1 @@
+<svg width="1280" height="720" xmlns="http://www.w3.org/2000/svg";
xmlns:xlink="http://www.w3.org/1999/xlink"; overflow="hidden"><defs><clipPath
id="clip0"><rect x="0" y="0" width="1280" height="720"/></clipPath></defs><g
clip-path="url(#clip0)"><rect x="0" y="0" width="1280" height="720"
fill="#FFFFFF"/><text font-family="Calibri Light,Calibri
Light_MSFontService,sans-serif" font-weight="300" font-size="59"
transform="translate(110.933 111)">memtank<tspan font-size="59" x="236.9"
y="0">(internal structure)</tspan></text><rect x="392.5" y="170.5" width="86"
height="16" stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="392.5" y="186.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="392.5" y="202.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="392.5" y="218.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="392.5" y="233.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="392.5" y="249.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="392.5" y="265.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="392.5" y="281.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><text font-family="Calibri,Calibri_MSFontService,sans-serif"
font-weight="400" font-size="16" transform="translate(205.267
215)">min_free<tspan font-size="16" x="62.62" y="0">(grow thresh)
</tspan><tspan font-family="Wingdings,Wingdings_MSFontService,sans-serif"
font-size="16" x="154" y="0"></tspan><tspan font-size="16" x="-3.66667"
y="81">max_free</tspan><tspan font-size="16" x="61.4733" y="81">(shrink thresh)
</tspan><tspan font-family="Wingdings,Wingdings_MSFontService,sans-serif"
font-size="16" x="159.593" y="81"></tspan><tspan font-style="italic"
font-size="24" x="370.867" y="-3">memtank_alloc</tspan><tspan
font-style="italic" font-size="24" x="520.973" y="-3">(</tspan><tspan
font-style="italic" font-size="24" x="528.307" y="-3">mtnk</tspan><tspan
font-style="italic" font-size="24" x="578.24" y="-3">, </tspan><tspan
font-style="italic" font-size="24" x="589.74" y="-3">obj</tspan><tspan
font-style="italic" font-size="24" x="620.073" y="-3">[], </tspan><tspan
font-style="italic" font-size="24" x="646.24" y="-3">num</tspan><tspan
font-style="italic" font-size="24" x="689.907" y="-3">, flags)
</tspan></text><path d="M488.5 207.167 561.833 207.167 561.833 207.833 488.5
207.833ZM560.5 203.5 568.5 207.5 560.5 211.5Z" fill="#203864"/><text
font-family="Calibri,Calibri_MSFontService,sans-serif" font-style="italic"
font-weight="400" font-size="24" transform="translate(577.2
274)">memtank_free(<tspan font-size="24" x="150.273" y="0">mtnk</tspan><tspan
font-size="24" x="200.207" y="0">, </tspan>obj<tspan font-size="24" x="242.04"
y="0">[], </tspan><tspan font-size="24" x="268.207" y="0">num</tspan><tspan
font-size="24" x="311.873" y="0">, flags) </tspan></text><path d="M496.167
269.167 569.5 269.167 569.5 269.833 496.167 269.833ZM497.5 273.5 489.5 269.5
497.5 265.5Z" fill="#5B9BD5"/><rect x="90" y="353" width="162" height="29"
fill="#E7E6E6"/><text font-family="Calibri,Calibri_MSFontService,sans-serif"
font-weight="400" font-size="16" transform="translate(99.9518
373)">mchunks<tspan font-size="16" x="58.1867" y="0">[USED]</tspan></text><path
d="M264.5 365.5 876.43 365.5 876.43 403.05 877.055 403.05" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M847.5 383.5 910.397 383.5" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M865.5 402.5 888.672 402.5" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M858.5 393.5 898.224 393.5" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><rect x="325.5" y="385.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="325.5" y="401.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="325.5" y="417.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="325.5" y="433.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="505.5" y="383.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="505.5" y="399.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="505.5" y="415.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="505.5" y="431.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="686.5" y="385.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="686.5" y="401.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="686.5" y="417.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><rect x="686.5" y="433.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#FFFFFF"/><path d="M368.5 365.5 368.5 385.369" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M548.5 365.5 548.5 384.022" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M728.5 365.5 728.917 385.369" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><rect x="90" y="478" width="162" height="29"
fill="#E7E6E6"/><text font-family="Calibri,Calibri_MSFontService,sans-serif"
font-weight="400" font-size="16" transform="translate(99.5345
498)">mchunks<tspan font-size="16" x="58.1867" y="0">[FULL]</tspan></text><path
d="M264.5 490.5 729.18 490.5 729.18 526.369 729.059 526.369" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M699.5 507.5 762.397 507.5" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M718.5 526.5 741.672 526.5" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M710.5 517.5 750.224 517.5" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><rect x="325.5" y="510.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="325.5" y="526.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="325.5" y="542.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="325.5" y="558.5" width="85" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="505.5" y="508.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="505.5" y="524.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="505.5" y="540.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><rect x="505.5" y="556.5" width="86" height="16"
stroke="#2F528F" stroke-width="1.33333" stroke-miterlimit="8"
fill="#D6DCE5"/><path d="M367.5 490.5 367.5 510.369" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><path d="M548.5 490.5 548.5 509.022" stroke="#000000"
stroke-width="0.666667" stroke-miterlimit="8" fill="none"
fill-rule="evenodd"/><text
font-family="Calibri,Calibri_MSFontService,sans-serif" font-style="italic"
font-weight="400" font-size="24" transform="translate(178.705
644)">memtank_grow<tspan font-size="24" x="154.32" y="0">(</tspan><tspan
font-size="24" x="161.653" y="0">mtnk</tspan><tspan font-size="24" x="211.587"
y="0">) </tspan><tspan font-size="24" x="303.585"
y="0">memtank_shrink</tspan>(<tspan font-size="24" x="473.858"
y="0">mtnk</tspan><tspan font-size="24" x="523.792" y="0">)
</tspan></text><path d="M0.333333-8.08095e-07 0.333422 36.6397-0.333245
36.6397-0.333333 8.08095e-07ZM4.00009 35.3063 0.000104987 43.3064-3.99991
35.3064Z" fill="#203864" transform="matrix(1 0 0 -1 367.5 620.806)"/><path
d="M548.833 577.5 548.833 615.486 548.167 615.486 548.167 577.5ZM552.5 614.153
548.5 622.153 544.5 614.153Z" fill="#5B9BD5"/><path d="M416.833 310.167 416.833
360.426 416.167 360.426 416.167 310.167ZM412.5 311.5 416.5 303.5 420.5 311.5Z"
fill="#203864"/><path d="M0.333346 6.66666 0.333438 56.926-0.333228
56.926-0.333321 6.66666ZM-3.99999 8.00001 0 0 4.00001 7.99999Z" fill="#5B9BD5"
transform="matrix(1 0 0 -1 450.5 360.426)"/><text fill="#203864"
font-family="Calibri,Calibri_MSFontService,sans-serif" font-style="italic"
font-weight="400" font-size="16" transform="translate(313.161 342)">ALLOC_CHUNK
<tspan font-size="16" x="-51.8536" y="264">ALLOC_GROW </tspan><tspan
fill="#4472C4" font-size="16" x="244.219" y="260">FREE_SHRINK
</tspan></text><rect x="90" y="161" width="162" height="29"
fill="#E7E6E6"/><text font-family="Calibri,Calibri_MSFontService,sans-serif"
font-weight="400" font-size="16" transform="translate(99.5345 181)">LIFO queue
</text></g></svg>
\ No newline at end of file
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index e6f24945b0..834266f8de 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -26,6 +26,7 @@ Memory Management
lcore_var
mempool_lib
+ memtank_lib
mbuf_lib
multi_proc_support
diff --git a/doc/guides/prog_guide/memtank_lib.rst
b/doc/guides/prog_guide/memtank_lib.rst
new file mode 100644
index 0000000000..51fe822572
--- /dev/null
+++ b/doc/guides/prog_guide/memtank_lib.rst
@@ -0,0 +1,231 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright(c) 2026 Huawei Technologies Co., Ltd
+
+Memtank Library
+===============
+
+The memtank library is a fixed sized object allocator for DPDK applications.
+Same a s mempool it allows to alloc/free bulk of objects of fixed size in a
+lightweight manner.
+But in addition it can grow/shrink dynamically plus provides extra
+additional API for higher flexibility:
+
+* manual grow()/shrink() functions
+
+* different alloc/free policies
+ (can be specified by user via flags parameter):
+
+ * lightweight as possible, but can fail
+
+ * more robust, but heavyweight - might cause call to user-provided backing
+ memory allocator.
+
+* user provided callbacks for actual system-wide memory reservations.
+ User is free to choose whatever is most suitable way for his/her scenario,
+ i.e: via malloc/rte_malloc/mmap/some custom allocator.
+
+* user defined constructor callback for newly allocated objects.
+
+* built-in per object runtime verify (boundary violation, double free, etc.) –
+ controlled by flags at memtank creation time.
+
+
+Memtank usage scenarios
+-----------------------
+
+Use memtank when:
+
+* Relatively small objects of the same size are allocated and freed on the
+ data path with low, predictable latency requirements.
+
+* Pre-allocation memory for maximum possible number of objects is not
+ feasible.
+
+
+Internals
+---------
+
+Internally each memtank consists of:
+
+* Fixed size LIFO queue that serves as a pool of free objects for fast
+ allocation/deallocation. It's internals and behavior are very similar
+ to current mempool LIFO driver.
+
+* Two lists (USED, FREE) of memchunks. Memchunk is an analog of SLAB:
+ For performance reasons memtank tries to allocate memory in relatively
+ big chunks (memchunks) and then split each memchunk in dozens (or hundreds)
+ of objects. Objects from memchunks are used to populate pool of free
+ objects (see above).
+
+* Each memchunk consists of some metadata plus array of free objects (LIFO)
+ that belong to that chunk. As soon as number of free objects in the chunk
+ becomes equal to the total number of objects it considered as ```FREE```
+ and can be 'shrinked' - relased back to the memory subsystem.
+
+* Each object within memtank is a properly aligned and initialized data buffer
+ that will be provided to the user followed by the metadata that is used
+ to determine which memchunk it belongs to plus some extra fields used for
+ statisitics collection and runtime verification. Total size of the metadata
+ for each object: 32B.
+
+There are two user defined thresholds that control memtank grow/shrink
+behavior:
+
+* ```min_free``` - grow threshold. That value controls two things: when it is
+ time to request for more memory from the underlying memory subsystem and how
+ many memory has to be requested/released in one go.
+
+* ```max_free``` - shrink threshold. That value determines when it is ok for
+ memtank to start to release unused memory back to the underlying memory
+ subsystem.
+
+Also user can define a grow limit: ```max_obj``` - maximimum possible number
+of objects that given memtank can contain. By setting all these three
+parameters to the same value, memtank behaves like mempool with LIFO driver.
+
+.. _figure_memtank-internals:
+
+.. figure:: img/memtank_internal.svg
+
+
+Brief API description
+---------------------
+
+* ```rte_memtank_create()```/```rte_memtank_destroy()``` are responsible for
+creation/destroying the memntank.
+
+* ```rte_memtank_alloc()```/```rte_memtank_free()``` - perform objects
+ allocation/deallocation from/to the memntank. Note that both of them
+ operate in bulks and accept extra flag parameter to allow user to specify
+ exact behavior.
+
+* ```rte_memtank_chunk_alloc()```/```rte_memtank_chunk_free()``` also perform
+ allocation/deallocation from/to the memntank. Though these functions bypass
+ pooll of free objects and allocate/free objects straight from/to the pool.
+
+* ```rte_memtank_grow()```/```rte_memtank_shrink()``` are intended to
+ explicitly reserve/release memory from/to underlying memory subsystem and
+ add/remove objects to/from the tank. Possible usage scenario - either some
+ house-keeping task, or even data-path thread during idle periods.
+
+* ```rte_memtank_dump()```/```rte_memtank_sanity_check()``` - miscelanneous
+ API for statistics/internal dumping and sanity cheking.
+
+
+Aled public API functions except ```rte_memetank_destroy()``` are MT safe and
+can be called concurrently from different threads.
+
+Object allocation
+~~~~~~~~~~~~~~~~~
+
+By default ```rte_memtank_alloc()``` first tries to get objects from the free
+objects pool. If there are not enough free objects in the pool, then behavior
+depends on the flag values user provided:
+
+* none - alloc() will simply return to the user obtained from the pool objects.
+
+* ```RTE_MTANK_ALLOC_CHUNK``` - alloc() will try to get remaining free objects
+ from already allocated memchunks.
+
+* If already allocated memchunks also don't contain enough
+ free objects and ```RTE_MTANK_ALLOC_GROW``` is specified, then it will try
+ to perform ```grow``` operation by allocating extra memory from the
+ underlying memory susbystem and creating new memchunks to satisfy user
+ request.
+
+In last two cases, it will try to refill free pool up to ```min_free```
+threshold value.
+
+Object de-allocation
+~~~~~~~~~~~~~~~~~~~~
+
+In reverse, ```ret_memtank_free()``` first tries to put objects back to
+the free pool. In case there is not enough room, it puts remaining free
+objects to the memchunks they belong to. After that, if
+```RTE_MTANK_FREE_SHRINK```` is specified it starts ```shrink``` operation
+to return unused memchunks back to the memory subsystem.
+
+
+Grow/Shrink
+~~~~~~~~~~~
+
+Apart from invoking ```grow```/```shrink``` implicitly (via alloc/free flags)
+there is an API for explicit invocation:
+
+* ```rte_memtank_grow(struct rte_memtank \*)``` - if number of objects in
+the free pool drops below ```min_free``` thershold, it requests next memory
+region from the udnerlying memory subsystem, creates new memchunks from it
+and populates the pool.
+
+* ```rte_memtank_shrink(struct memtank \*)``` - if total number of free objects
+in the tank exceeds ```max_free``` theshold it de-allocates unused memchunks
+back to the underlying memory subsystem.
+
+
+Create/Destroy
+~~~~~~~~~~~~~~
+
+.. code-block:: c
+
+ sruct user_defined_type;
+
+ /*
+ * User defined callbacks to reserve/release memory from/to backing
+ * memory subsystem.
+ */
+
+ static void *
+ user_defined_alloc(size_t sz, void *udata)
+ {
+ RTE_SET_USED(udata);
+ return rte_malloc(NULL, sz, 0);
+ }
+
+ static void
+ user_defined_free(void *buf, void *udata)
+ {
+ RTE_SET_USED(udata);
+ rte_free(buf);
+ }
+
+ /*
+ * As used needs new memtank he fills memtank param structure and calls
+ * rte_memtank_create():
+ */
+ static struct rte_memtank_prm prm = {
+ /* min number of free objs in the pool (grow threshold). */
+ .min_free = 1024,
+ /* max number of free objs (shrink threshold)a */
+ .max_free = 1024 * 1024,
+ .obj_size = sizeof(struct user_defined_type);
+ .obj_align = alignof(struct user_defined_type);
+ .nb_obj_chunk = 2 * 1024,
+ /* enable obj runtime verify and stats collection */
+ .flags = RTE_MTANK_OBJ_DBG,
+ /* user defined callbacks to reserve/release actual memory */
+ .alloc = user_defined_alloc,
+ .free = user_define_free,
+ };
+
+ struct rte_memtank *mt = rte_memtank_create(&prm);
+
+ ....
+
+ /* no more objects from the memtank are in use */
+ rte_memtank_destroy(mt);
+
+
+Known limitations (subject for further improvements):
+-----------------------------------------------------
+
+* scalability:
+ after 8+ lcores conventional mempool (with FIFO) starts to outperform
+ memtank (which by default uses LIFO inside).
+
+* mempool_cache integration is not part of the library and right now
+ has to be implemented by used manually on top of memtank API.
+
+* As pool of free objects might contain objects from different memchunks,
+ it can prevent some memchunks to get deallocated and returned back to
+ the memory subsystem.
+
diff --git a/lib/memtank/memtank.c b/lib/memtank/memtank.c
new file mode 100644
index 0000000000..2c6ec948a5
--- /dev/null
+++ b/lib/memtank/memtank.c
@@ -0,0 +1,630 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2025 Huawei Technologies Co., Ltd
+ */
+
+#include "memtank.h"
+#include <rte_bitops.h>
+#include <rte_errno.h>
+#include <eal_export.h>
+
+#define MEMTANK_OBJ_BULK 0x100
+#define MEMTANK_CHUNK_BULK 0x100
+
+#define ALIGN_MUL_CEIL(v, mul) \
+ ((typeof(v))(((uint64_t)(v) + (mul) - 1) / (mul)))
+
+
+static inline size_t
+memtank_meta_size(uint32_t nb_free)
+{
+ size_t sz;
+ static const struct rte_memtank *mt;
+
+ sz = sizeof(*mt) + nb_free * sizeof(mt->mtf.free[0]);
+ sz = RTE_ALIGN_CEIL(sz, alignof(typeof(*mt)));
+ return sz;
+}
+
+static inline size_t
+memchunk_meta_size(uint32_t nb_obj)
+{
+ size_t sz;
+ static const struct memchunk *ch;
+
+ sz = sizeof(*ch) + nb_obj * sizeof(ch->free[0]);
+ sz = RTE_ALIGN_CEIL(sz, alignof(typeof(*ch)));
+ return sz;
+}
+
+static inline size_t
+memobj_size(uint32_t obj_size, uint32_t obj_align)
+{
+ size_t sz;
+ static const struct memobj *obj;
+
+ sz = sizeof(*obj) + obj_size;
+ sz = RTE_ALIGN_CEIL(sz, obj_align);
+ return sz;
+}
+
+static inline size_t
+memchunk_size(uint32_t nb_obj, uint32_t obj_size, uint32_t obj_align)
+{
+ size_t algn, sz;
+ static const struct memchunk *ch;
+
+ algn = RTE_MAX(alignof(typeof(*ch)), obj_align);
+ sz = memchunk_meta_size(nb_obj);
+ sz += nb_obj * memobj_size(obj_size, obj_align);
+ sz = RTE_ALIGN_CEIL(sz + algn - 1, algn);
+ return sz;
+}
+
+static void
+init_chunk(struct rte_memtank *mt, struct memchunk *ch)
+{
+ uint32_t i, n, sz;
+ uintptr_t p;
+ struct memobj *obj;
+
+ const struct memobj cobj = {
+ .red_zone1 = RED_ZONE_V1,
+ .chunk = ch,
+ .red_zone2 = RED_ZONE_V2,
+ };
+
+ n = mt->prm.nb_obj_chunk;
+ sz = mt->obj_size;
+
+ /* get start of memobj array */
+ p = (uintptr_t)ch + memchunk_meta_size(n);
+ p = RTE_ALIGN_CEIL(p, mt->prm.obj_align);
+
+ for (i = 0; i != n; i++) {
+ obj = obj_pub_full(p, sz);
+ obj[0] = cobj;
+ ch->free[i] = (void *)p;
+ p += sz;
+ }
+
+ ch->nb_total = n;
+ ch->nb_free = n;
+
+ if (mt->prm.init != NULL)
+ mt->prm.init(ch->free, n, mt->prm.udata);
+}
+
+static __rte_always_inline void
+copy_objs(void *dst[], void * const src[], uint32_t num)
+{
+ memcpy(dst, src, num * sizeof(dst[0]));
+}
+
+static inline uint32_t
+get_free(struct memtank_free *t, void *obj[], uint32_t num)
+{
+ uint32_t len, n;
+
+ rte_spinlock_lock(&t->lock);
+
+ len = t->nb_free;
+ n = RTE_MIN(num, len);
+ len -= n;
+ copy_objs(obj, t->free + len, n);
+ t->nb_free = len;
+
+ rte_spinlock_unlock(&t->lock);
+ return n;
+}
+
+static inline uint32_t
+put_free(struct memtank_free *t, void * const obj[], uint32_t num)
+{
+ uint32_t len, n;
+
+ rte_spinlock_lock(&t->lock);
+
+ len = t->nb_free;
+ n = t->max_free - len;
+ n = RTE_MIN(num, n);
+ copy_objs(t->free + len, obj, n);
+ t->nb_free = len + n;
+
+ rte_spinlock_unlock(&t->lock);
+ return n;
+}
+
+static inline void
+fill_free(struct rte_memtank *mt, uint32_t num, uint32_t flags)
+{
+ uint32_t i, l, k, n;
+ void *free[MEMTANK_OBJ_BULK];
+
+ for (i = 0; i != num; i += n) {
+ /* how many objects we need to add into @free */
+ n = RTE_MIN(num - i, RTE_DIM(free));
+ k = rte_memtank_chunk_alloc(mt, free, n, flags);
+ l = put_free(&mt->mtf, free, k);
+
+ /* @free is full, return allocated objects back to chunks */
+ if (l != k)
+ rte_memtank_chunk_free(mt, free + l, k - l, 0);
+
+ /* either free is full, or chunks are empty */
+ if (l != n)
+ break;
+ }
+}
+
+static void
+put_chunk(struct rte_memtank *mt, struct memchunk *ch, void * const obj[],
+ uint32_t num)
+{
+ uint32_t k, n;
+ struct mchunk_list *ls;
+
+ /* chunk should be in the *used* list */
+ k = MC_USED;
+ ls = &mt->chl[k];
+ rte_spinlock_lock(&ls->lock);
+
+ n = ch->nb_free;
+ RTE_ASSERT(n + num <= ch->nb_total);
+
+ copy_objs(ch->free + n, obj, num);
+ ch->nb_free = n + num;
+
+ /* chunk is full now */
+ if (ch->nb_free == ch->nb_total) {
+ TAILQ_REMOVE(&ls->chunk, ch, link);
+ k = MC_FULL;
+ /* chunk is not empty anymore, move it to the head */
+ } else if (n == 0) {
+ TAILQ_REMOVE(&ls->chunk, ch, link);
+ TAILQ_INSERT_HEAD(&ls->chunk, ch, link);
+ }
+
+ rte_spinlock_unlock(&ls->lock);
+
+ /* insert this chunk into the *full* list */
+ if (k == MC_FULL) {
+ ls = &mt->chl[k];
+ rte_spinlock_lock(&ls->lock);
+ TAILQ_INSERT_HEAD(&ls->chunk, ch, link);
+ rte_spinlock_unlock(&ls->lock);
+ }
+}
+
+static inline uint32_t
+_shrink_chunk(struct rte_memtank *mt, struct memchunk *ch[MEMTANK_CHUNK_BULK],
+ uint32_t num)
+{
+ uint32_t i, k;
+ struct mchunk_list *ls;
+
+ ls = &mt->chl[MC_FULL];
+ rte_spinlock_lock(&ls->lock);
+
+ for (k = 0; k != num; k++) {
+ ch[k] = TAILQ_LAST(&ls->chunk, mchunk_head);
+ if (ch[k] == NULL)
+ break;
+ TAILQ_REMOVE(&ls->chunk, ch[k], link);
+ }
+
+ rte_spinlock_unlock(&ls->lock);
+
+ rte_atomic_fetch_sub_explicit(&mt->nb_chunks, k,
+ rte_memory_order_acq_rel);
+
+ for (i = 0; i != k; i++)
+ mt->prm.free(ch[i]->raw, mt->prm.udata);
+
+ return k;
+}
+
+
+static uint32_t
+shrink_chunk(struct rte_memtank *mt, uint32_t num)
+{
+ uint32_t i, k, n;
+ struct memchunk *ch[MEMTANK_CHUNK_BULK];
+
+ k = 0;
+ n = 0;
+ for (i = 0; i != num && n != k; i += k) {
+ n = RTE_MIN(num - i, RTE_DIM(ch));
+ k = _shrink_chunk(mt, ch, n);
+ }
+
+ return i;
+}
+
+static struct memchunk *
+alloc_chunk(struct rte_memtank *mt)
+{
+ void *p;
+ struct memchunk *ch;
+
+ p = mt->prm.alloc(mt->chunk_size, mt->prm.udata);
+ if (p == NULL)
+ return NULL;
+ ch = RTE_PTR_ALIGN_CEIL(p, alignof(typeof(*ch)));
+ ch->raw = p;
+ return ch;
+}
+
+/* Determine by how many chunks we can actually grow */
+static inline uint32_t
+grow_num(struct rte_memtank *mt, uint32_t num)
+{
+ uint32_t k, n, max;
+
+ max = mt->max_chunk;
+ n = num + rte_atomic_fetch_add_explicit(&mt->nb_chunks, num,
+ rte_memory_order_acq_rel);
+
+ if (n <= max)
+ return num;
+
+ k = n - max;
+ return (k >= num) ? 0 : num - k;
+}
+
+static uint32_t
+grow_chunk(struct rte_memtank *mt, uint32_t num)
+{
+ uint32_t k, n;
+ struct mchunk_list *fls;
+ struct mchunk_head ls;
+ struct memchunk *ch;
+
+ /* check can we grow further */
+ k = grow_num(mt, num);
+
+ TAILQ_INIT(&ls);
+
+ for (n = 0; n != k; n++) {
+ ch = alloc_chunk(mt);
+ if (ch == NULL)
+ break;
+ init_chunk(mt, ch);
+ TAILQ_INSERT_HEAD(&ls, ch, link);
+ }
+
+ if (n != 0) {
+ fls = &mt->chl[MC_FULL];
+ rte_spinlock_lock(&fls->lock);
+ TAILQ_CONCAT(&fls->chunk, &ls, link);
+ rte_spinlock_unlock(&fls->lock);
+ }
+
+ if (n != num)
+ rte_atomic_fetch_sub_explicit(&mt->nb_chunks, num - n,
+ rte_memory_order_acq_rel);
+
+ return n;
+}
+
+static void
+obj_dbg_alloc(struct rte_memtank *mt, void * const obj[], uint32_t nb_obj)
+{
+ uint32_t i, sz;
+ struct memobj *po;
+
+ sz = mt->obj_size;
+ for (i = 0; i != nb_obj; i++) {
+ po = obj_pub_full((uintptr_t)obj[i], sz);
+ RTE_VERIFY(memobj_verify(po, 0) == 0);
+ po->dbg.nb_alloc++;
+ }
+}
+
+static void
+obj_dbg_free(struct rte_memtank *mt, void * const obj[], uint32_t nb_obj)
+{
+ uint32_t i, sz;
+ struct memobj *po;
+
+ sz = mt->obj_size;
+ for (i = 0; i != nb_obj; i++) {
+ po = obj_pub_full((uintptr_t)obj[i], sz);
+ RTE_VERIFY(memobj_verify(po, 1) == 0);
+ po->dbg.nb_free++;
+ }
+}
+
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_chunk_free, 26.11)
+void
+rte_memtank_chunk_free(struct rte_memtank *mt, void * const obj[],
+ uint32_t nb_obj, uint32_t flags)
+{
+ size_t csz;
+ uint32_t i, j, k, osz;
+ struct memobj *mo;
+ struct memchunk *ch;
+
+ csz = mt->chunk_size;
+ osz = mt->obj_size;
+
+ if (mt->flags & RTE_MTANK_OBJ_DBG)
+ obj_dbg_free(mt, obj, nb_obj);
+
+ k = 0;
+ for (i = 0; i != nb_obj; i = j) {
+
+ mo = obj_pub_full((uintptr_t)obj[i], osz);
+ ch = mo->chunk;
+
+ /* find number of consequtive objs from the same chunk */
+ for (j = i + 1; j != nb_obj; j++) {
+ if (obj_check_chunk((uintptr_t)obj[j], osz,
+ (uintptr_t)ch, csz) != 0)
+ break;
+ RTE_ASSERT(ch ==
+ obj_pub_full((uintptr_t)obj[j], osz)->chunk);
+ }
+
+ put_chunk(mt, ch, obj + i, j - i);
+ k++;
+ }
+
+ if (flags & RTE_MTANK_FREE_SHRINK)
+ shrink_chunk(mt, k);
+}
+
+static uint32_t
+get_chunk(struct mchunk_list *ls, struct mchunk_head *els,
+ struct mchunk_head *uls, void *obj[], uint32_t nb_obj)
+{
+ uint32_t l, k, n;
+ struct memchunk *ch, *nch;
+
+ rte_spinlock_lock(&ls->lock);
+
+ n = 0;
+ for (ch = TAILQ_FIRST(&ls->chunk);
+ n != nb_obj && ch != NULL && ch->nb_free != 0;
+ ch = nch, n += k) {
+
+ k = RTE_MIN(nb_obj - n, ch->nb_free);
+ l = ch->nb_free - k;
+ copy_objs(obj + n, ch->free + l, k);
+ ch->nb_free = l;
+
+ nch = TAILQ_NEXT(ch, link);
+
+ /* chunk is empty now */
+ if (l == 0) {
+ TAILQ_REMOVE(&ls->chunk, ch, link);
+ TAILQ_INSERT_TAIL(els, ch, link);
+ } else if (uls != NULL) {
+ TAILQ_REMOVE(&ls->chunk, ch, link);
+ TAILQ_INSERT_HEAD(uls, ch, link);
+ }
+ }
+
+ rte_spinlock_unlock(&ls->lock);
+ return n;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_chunk_alloc, 26.11)
+uint32_t
+rte_memtank_chunk_alloc(struct rte_memtank *mt, void *obj[], uint32_t nb_obj,
+ uint32_t flags)
+{
+ uint32_t k, n;
+ struct mchunk_head els, uls;
+
+ /* walk though the *used* list first */
+ n = get_chunk(&mt->chl[MC_USED], &mt->chl[MC_USED].chunk, NULL,
+ obj, nb_obj);
+
+ if (n != nb_obj) {
+
+ TAILQ_INIT(&els);
+ TAILQ_INIT(&uls);
+
+ /* walk though the *full* list */
+ n += get_chunk(&mt->chl[MC_FULL], &els, &uls,
+ obj + n, nb_obj - n);
+
+ if (n != nb_obj && (flags & RTE_MTANK_ALLOC_GROW) != 0) {
+
+ /*
+ * try to allocate extra memchunks.
+ * note that at rare situations with really high load
+ * when number of allocated chunks is close to the
+ * max allowed limit, when multiple threads are
+ * trying to do grow_chunk() simultaneously, it
+ * can fail for some of them leading to a failure
+ * to allocate new elements.
+ */
+ k = ALIGN_MUL_CEIL(nb_obj - n,
+ mt->prm.nb_obj_chunk);
+ k = grow_chunk(mt, k);
+
+ /* walk through the *full* list again */
+ if (k != 0)
+ n += get_chunk(&mt->chl[MC_FULL], &els, &uls,
+ obj + n, nb_obj - n);
+ }
+
+ /* concatenate with *used* list our temporary lists */
+ rte_spinlock_lock(&mt->chl[MC_USED].lock);
+
+ /* put new non-emtpy elems at head of the *used* list */
+ TAILQ_CONCAT(&uls, &mt->chl[MC_USED].chunk, link);
+ TAILQ_CONCAT(&mt->chl[MC_USED].chunk, &uls, link);
+
+ /* put new emtpy elems at tail of the *used* list */
+ TAILQ_CONCAT(&mt->chl[MC_USED].chunk, &els, link);
+
+ rte_spinlock_unlock(&mt->chl[MC_USED].lock);
+ }
+
+ if (mt->flags & RTE_MTANK_OBJ_DBG)
+ obj_dbg_alloc(mt, obj, n);
+
+ return n;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_grow, 26.11)
+int
+rte_memtank_grow(struct rte_memtank *mt)
+{
+ uint32_t k, n, num;
+ struct memtank_free *t;
+
+ t = &mt->mtf;
+
+ /* how many chunks we need to grow */
+ k = t->min_free - t->nb_free;
+ if ((int32_t)k <= 0)
+ return 0;
+
+ num = ALIGN_MUL_CEIL(k, mt->prm.nb_obj_chunk);
+
+ /* try to grow and refill the *free* */
+ n = grow_chunk(mt, num);
+ if (n != 0)
+ fill_free(mt, k, 0);
+
+ return n;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_shrink, 26.11)
+int
+rte_memtank_shrink(struct rte_memtank *mt)
+{
+ uint32_t n;
+ struct memtank_free *t;
+
+ t = &mt->mtf;
+
+ /* how many chunks we need to shrink */
+ if (t->nb_free < t->max_free)
+ return 0;
+
+ /* how many chunks we need to free */
+ n = ALIGN_MUL_CEIL(t->min_free, mt->prm.nb_obj_chunk);
+
+ /* free up to *num* chunks */
+ return shrink_chunk(mt, n);
+}
+
+static int
+check_param(const struct rte_memtank_prm *prm)
+{
+ if (prm->alloc == NULL || prm->free == NULL ||
+ prm->min_free > prm->max_free ||
+ prm->max_free > prm->max_obj ||
+ rte_is_power_of_2(prm->obj_align) == 0 ||
+ prm->min_free == 0 ||
+ prm->nb_obj_chunk == 0)
+ return -EINVAL;
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_create, 26.11)
+struct rte_memtank *
+rte_memtank_create(const struct rte_memtank_prm *prm)
+{
+ int32_t rc;
+ size_t sz;
+ void *p;
+ struct rte_memtank *mt;
+
+ rc = check_param(prm);
+ if (rc != 0) {
+ rte_errno = -rc;
+ return NULL;
+ }
+
+ sz = memtank_meta_size(prm->max_free);
+ p = prm->alloc(sz, prm->udata);
+ if (p == NULL) {
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ mt = RTE_PTR_ALIGN_CEIL(p, alignof(typeof(*mt)));
+
+ memset(mt, 0, sizeof(*mt));
+ mt->prm = *prm;
+
+ mt->raw = p;
+ mt->chunk_size = memchunk_size(prm->nb_obj_chunk, prm->obj_size,
+ prm->obj_align);
+ mt->obj_size = memobj_size(prm->obj_size, prm->obj_align);
+ mt->max_chunk = ALIGN_MUL_CEIL(prm->max_obj, prm->nb_obj_chunk);
+ mt->flags = prm->flags;
+
+ mt->mtf.min_free = prm->min_free;
+ mt->mtf.max_free = prm->max_free;
+
+ TAILQ_INIT(&mt->chl[MC_FULL].chunk);
+ TAILQ_INIT(&mt->chl[MC_USED].chunk);
+
+ return mt;
+}
+
+static void
+free_mchunk_list(struct rte_memtank *mt, struct mchunk_list *ls)
+{
+ struct memchunk *ch;
+
+ for (ch = TAILQ_FIRST(&ls->chunk); ch != NULL;
+ ch = TAILQ_FIRST(&ls->chunk)) {
+ TAILQ_REMOVE(&ls->chunk, ch, link);
+ mt->prm.free(ch->raw, mt->prm.udata);
+ }
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_destroy, 26.11)
+void
+rte_memtank_destroy(struct rte_memtank *mt)
+{
+ if (mt != NULL) {
+ free_mchunk_list(mt, &mt->chl[MC_FULL]);
+ free_mchunk_list(mt, &mt->chl[MC_USED]);
+ mt->prm.free(mt->raw, mt->prm.udata);
+ }
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_alloc, 26.11)
+uint32_t
+rte_memtank_alloc(struct rte_memtank *mt, void *obj[], uint32_t num,
+ uint32_t flags)
+{
+ uint32_t n;
+ struct memtank_free *t;
+
+ t = &mt->mtf;
+ n = get_free(t, obj, num);
+
+ /* not enough free objects, try to allocate via memchunks */
+ if (n != num && flags != 0) {
+ n += rte_memtank_chunk_alloc(mt, obj + n, num - n, flags);
+
+ /* refill *free* tank */
+ if (n == num)
+ fill_free(mt, t->min_free, flags);
+ }
+
+ return n;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_free, 26.11)
+void
+rte_memtank_free(struct rte_memtank *t, void * const obj[], uint32_t num,
+ uint32_t flags)
+{
+ uint32_t n;
+
+ n = put_free(&t->mtf, obj, num);
+ if (n != num)
+ rte_memtank_chunk_free(t, obj + n, num - n, flags);
+}
diff --git a/lib/memtank/memtank.h b/lib/memtank/memtank.h
new file mode 100644
index 0000000000..872f2f7def
--- /dev/null
+++ b/lib/memtank/memtank.h
@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2025 Huawei Technologies Co., Ltd
+ */
+
+#ifndef _MEMTANK_H_
+#define _MEMTANK_H_
+
+#include <rte_memtank.h>
+#include <rte_atomic.h>
+#include <rte_spinlock.h>
+#include <rte_log.h>
+#include <stdalign.h>
+#include <errno.h>
+
+extern int memtank_logtype;
+#define RTE_LOGTYPE_MTANK memtank_logtype
+#define MTANK_LOG(level, ...) \
+ RTE_LOG_LINE(level, MTANK, "" __VA_ARGS__)
+
+struct memobj {
+ uint64_t red_zone1;
+ struct memchunk *chunk; /* ptr to the chunk it belongs to */
+ struct {
+ uint32_t nb_alloc;
+ uint32_t nb_free;
+ } dbg;
+ uint64_t red_zone2;
+};
+
+#define RED_ZONE_V1 UINT64_C(0xBADECAFEBADECAFE)
+#define RED_ZONE_V2 UINT64_C(0xDEADBEEFDEADBEEF)
+
+struct memchunk {
+ TAILQ_ENTRY(memchunk) link; /* link to the next chunk in the tank */
+ void *raw; /* un-aligned ptr returned by alloc() */
+ uint32_t nb_total; /* total number of objects in the chunk */
+ uint32_t nb_free; /* number of free object in the chunk */
+ void *free[]; /* array of free objects */
+} __rte_cache_aligned;
+
+
+TAILQ_HEAD(mchunk_head, memchunk);
+
+struct mchunk_list {
+ rte_spinlock_t lock;
+ struct mchunk_head chunk; /* list of chunks */
+} __rte_cache_aligned;
+
+enum {
+ MC_FULL, /* all memchunk objs are free */
+ MC_USED, /* some of memchunk objs are allocated */
+ MC_NUM,
+};
+
+struct memtank_free {
+ rte_spinlock_t lock;
+ uint32_t min_free;
+ uint32_t max_free;
+ uint32_t nb_free;
+ void *free[];
+} __rte_cache_aligned;
+
+struct rte_memtank {
+ /* user provided data */
+ struct rte_memtank_prm prm;
+
+ /*run-time data */
+ void *raw; /* un-aligned ptr returned by alloc() */
+ size_t chunk_size; /* full size of each memchunk */
+ uint32_t obj_size; /* full size of each memobj */
+ uint32_t max_chunk; /* max allowed number of chunks */
+ uint32_t flags; /* behavior flags */
+ RTE_ATOMIC(uint32_t) nb_chunks; /* number of allocated chunks */
+ struct mchunk_list chl[MC_NUM]; /* lists of memchunks */
+ struct memtank_free mtf; /* cached free objects */
+};
+
+/**
+ * Obtain pointer to internal memobj struct from public one
+ */
+static inline struct memobj *
+obj_pub_full(uintptr_t p, uint32_t obj_sz)
+{
+ uintptr_t v;
+
+ v = p + obj_sz - sizeof(struct memobj);
+ return (struct memobj *)v;
+}
+
+/**
+ * Fast check: does given object belongs to that memchunk.
+ * Returns zero, if object is within the chunk, non-zero value otherwise.
+ */
+static inline int
+obj_check_chunk(uintptr_t obj, size_t obj_sz, uintptr_t chn, size_t chn_sz)
+{
+ return (obj <= chn || obj + obj_sz > chn + chn_sz);
+}
+
+static inline int
+memobj_verify(const struct memobj *mo, uint32_t finc)
+{
+ if (mo->red_zone1 != RED_ZONE_V1 || mo->red_zone2 != RED_ZONE_V2 ||
+ mo->dbg.nb_alloc != mo->dbg.nb_free + finc)
+ return -EINVAL;
+ return 0;
+}
+
+#endif /* _MEMTANK_H_ */
diff --git a/lib/memtank/meson.build b/lib/memtank/meson.build
new file mode 100644
index 0000000000..a4c54c09bd
--- /dev/null
+++ b/lib/memtank/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017 Intel Corporation
+
+extra_flags = []
+
+foreach flag: extra_flags
+ if cc.has_argument(flag)
+ cflags += flag
+ endif
+endforeach
+
+sources = files('memtank.c',
+ 'misc.c',
+)
+headers = files(
+ 'rte_memtank.h',
+)
+deps += ['ring', 'telemetry']
diff --git a/lib/memtank/misc.c b/lib/memtank/misc.c
new file mode 100644
index 0000000000..526bbbbcf1
--- /dev/null
+++ b/lib/memtank/misc.c
@@ -0,0 +1,375 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2025 Huawei Technologies Co., Ltd
+ */
+
+#include "memtank.h"
+#include <inttypes.h>
+#include <stdlib.h>
+#include <eal_export.h>
+
+#define CHUNK_OBJ_LT_NUM 4
+
+struct mchunk_stat {
+ uint32_t nb_empty;
+ uint32_t nb_full;
+ struct {
+ uint32_t nb_chunk;
+ uint32_t nb_obj;
+ struct {
+ uint32_t val;
+ uint32_t num;
+ } chunk_obj_lt[CHUNK_OBJ_LT_NUM];
+ } used;
+};
+
+struct mfree_stat {
+ uint32_t nb_chunk;
+ struct mchunk_stat chunk;
+};
+
+RTE_LOG_REGISTER_DEFAULT(memtank_logtype, INFO);
+
+static void
+mchunk_stat_dump(FILE *f, const struct mchunk_stat *st)
+{
+ uint32_t i;
+
+ fprintf(f, "\t\tstat={\n");
+ fprintf(f, "\t\t\tnb_empty=%u,\n", st->nb_empty);
+ fprintf(f, "\t\t\tnb_full=%u,\n", st->nb_full);
+ fprintf(f, "\t\t\tused={\n");
+ fprintf(f, "\t\t\t\tnb_chunk=%u,\n", st->used.nb_chunk);
+ fprintf(f, "\t\t\t\tnb_obj=%u,\n", st->used.nb_obj);
+
+ for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) {
+ if (st->used.chunk_obj_lt[i].num != 0)
+ fprintf(f, "\t\t\t\tnb_chunk_obj_lt_%u=%u,\n",
+ st->used.chunk_obj_lt[i].val,
+ st->used.chunk_obj_lt[i].num);
+ }
+
+ fprintf(f, "\t\t\t},\n");
+ fprintf(f, "\t\t},\n");
+}
+
+static void
+mchunk_stat_init(struct mchunk_stat *st, uint32_t nb_obj_chunk)
+{
+ uint32_t i;
+
+ memset(st, 0, sizeof(*st));
+ for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) {
+ st->used.chunk_obj_lt[i].val = (i + 1) * nb_obj_chunk /
+ RTE_DIM(st->used.chunk_obj_lt);
+ }
+}
+
+static void
+mchunk_stat_collect(struct mchunk_stat *st, const struct memchunk *ch)
+{
+ uint32_t i, n;
+
+ n = ch->nb_total - ch->nb_free;
+
+ if (ch->nb_free == 0)
+ st->nb_empty++;
+ else if (n == 0)
+ st->nb_full++;
+ else {
+ st->used.nb_chunk++;
+ st->used.nb_obj += n;
+
+ for (i = 0; i != RTE_DIM(st->used.chunk_obj_lt); i++) {
+ if (n < st->used.chunk_obj_lt[i].val) {
+ st->used.chunk_obj_lt[i].num++;
+ break;
+ }
+ }
+ }
+}
+
+static void
+mchunk_list_dump(FILE *f, struct rte_memtank *mt, uint32_t idx, uint32_t flags)
+{
+ struct mchunk_list *ls;
+ const struct memchunk *ch;
+ struct mchunk_stat mcs;
+
+ ls = &mt->chl[idx];
+ mchunk_stat_init(&mcs, mt->prm.nb_obj_chunk);
+
+ rte_spinlock_lock(&ls->lock);
+
+ for (ch = TAILQ_FIRST(&ls->chunk); ch != NULL;
+ ch = TAILQ_NEXT(ch, link)) {
+
+ /* collect chunk stats */
+ if (flags & RTE_MTANK_DUMP_CHUNK_STAT)
+ mchunk_stat_collect(&mcs, ch);
+
+ /* dump chunk metadata */
+ if (flags & RTE_MTANK_DUMP_CHUNK) {
+ fprintf(f, "\t\tmemchunk@%p={\n", ch);
+ fprintf(f, "\t\t\traw=%p,\n", ch->raw);
+ fprintf(f, "\t\t\tnb_total=%u,\n", ch->nb_total);
+ fprintf(f, "\t\t\tnb_free=%u,\n", ch->nb_free);
+ fprintf(f, "\t\t},\n");
+ }
+ }
+
+ rte_spinlock_unlock(&ls->lock);
+
+ /* print chunk stats */
+ if (flags & RTE_MTANK_DUMP_CHUNK_STAT)
+ mchunk_stat_dump(f, &mcs);
+}
+
+static void
+mfree_stat_init(struct mfree_stat *st, uint32_t nb_obj_chunk)
+{
+ st->nb_chunk = 0;
+ mchunk_stat_init(&st->chunk, nb_obj_chunk);
+}
+
+static int
+ptr_cmp(const void *p1, const void *p2)
+{
+ uintptr_t rc, v1, v2;
+
+ v1 = *(const uintptr_t *)p1;
+ v2 = *(const uintptr_t *)p2;
+ rc = v1 - v2;
+ return (rc > v1) ? -1 : ((rc > 0) ? 1 : 0);
+}
+
+static void
+mfree_stat_collect(struct mfree_stat *st, struct rte_memtank *mt)
+{
+ uint32_t i, j, n, sz;
+ uintptr_t *p;
+ const struct memobj *mo;
+
+ sz = mt->obj_size;
+
+ p = malloc(mt->mtf.max_free * sizeof(*p));
+ if (p == NULL)
+ return;
+
+ /**
+ * grab free lock and keep it till we analyze related memchunks,
+ * to make sure none of these memchunks will be freed until
+ * we are finished.
+ */
+ rte_spinlock_lock(&mt->mtf.lock);
+
+ /* collect chunks for all objects in free[] */
+ n = mt->mtf.nb_free;
+ memcpy(p, mt->mtf.free, n * sizeof(*p));
+ for (i = 0; i != n; i++) {
+ mo = obj_pub_full(p[i], sz);
+ p[i] = (uintptr_t)mo->chunk;
+ }
+
+ /* sort chunk pointers */
+ qsort(p, n, sizeof(*p), ptr_cmp);
+
+ /* for each chunk collect stats */
+ for (i = 0; i != n; i = j) {
+
+ RTE_ASSERT(st->nb_chunk < mt->max_chunk);
+ st->nb_chunk++;
+ mchunk_stat_collect(&st->chunk, (const struct memchunk *)p[i]);
+ for (j = i + 1; j != n && p[i] == p[j]; j++)
+ ;
+ }
+
+ rte_spinlock_unlock(&mt->mtf.lock);
+ free(p);
+}
+
+static void
+mfree_stat_dump(FILE *f, const struct mfree_stat *st)
+{
+ fprintf(f, "\tfree_stat={\n");
+ fprintf(f, "\t\tnb_chunk=%u,\n", st->nb_chunk);
+ mchunk_stat_dump(f, &st->chunk);
+ fprintf(f, "\t},\n");
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_dump, 26.11)
+void
+rte_memtank_dump(FILE *f, struct rte_memtank *mt, uint32_t flags)
+{
+ uint32_t n;
+
+ if (f == NULL || mt == NULL)
+ return;
+
+ fprintf(f, "rte_memtank@%p={\n", mt);
+ fprintf(f, "\tmin_free=%u,\n", mt->mtf.min_free);
+ fprintf(f, "\tmax_free=%u,\n", mt->mtf.max_free);
+ fprintf(f, "\tnb_free=%u,\n", mt->mtf.nb_free);
+ fprintf(f, "\tchunk_size=%zu,\n", mt->chunk_size);
+ fprintf(f, "\tobj_size=%u,\n", mt->obj_size);
+ fprintf(f, "\tmax_chunk=%u,\n", mt->max_chunk);
+ fprintf(f, "\tflags=%#x,\n", mt->flags);
+ n = rte_atomic_load_explicit(&mt->nb_chunks, rte_memory_order_relaxed);
+ fprintf(f, "\tnb_chunks=%u,\n", n);
+
+ if (flags & RTE_MTANK_DUMP_FREE_STAT) {
+ struct mfree_stat mfs;
+ mfree_stat_init(&mfs, mt->prm.nb_obj_chunk);
+ mfree_stat_collect(&mfs, mt);
+ mfree_stat_dump(f, &mfs);
+ }
+
+ if (flags & (RTE_MTANK_DUMP_CHUNK | RTE_MTANK_DUMP_CHUNK_STAT)) {
+
+ fprintf(f, "\t[FULL]={\n");
+ mchunk_list_dump(f, mt, MC_FULL, flags);
+ fprintf(f, "\t},\n");
+
+ fprintf(f, "\t[USED]={,\n");
+ mchunk_list_dump(f, mt, MC_USED, flags);
+ fprintf(f, "\t},\n");
+ }
+ fprintf(f, "};\n");
+}
+
+static int
+mobj_bulk_check(const char *fname, const struct rte_memtank *mt,
+ const uintptr_t p[], uint32_t num, uint32_t fmsk)
+{
+ int32_t ret;
+ uintptr_t align;
+ uint32_t i, k, sz;
+ const struct memobj *mo;
+
+ k = ((mt->flags & RTE_MTANK_OBJ_DBG) != 0) & fmsk;
+ sz = mt->obj_size;
+ align = mt->prm.obj_align - 1;
+
+ ret = 0;
+ for (i = 0; i != num; i++) {
+
+ if (p[i] == (uintptr_t)NULL) {
+ ret--;
+ MTANK_LOG(ERR,
+ "%s(mt=%p, %p[%u]): NULL object",
+ fname, mt, p, i);
+ } else if ((p[i] & align) != 0) {
+ ret--;
+ MTANK_LOG(ERR,
+ "%s(mt=%p, %p[%u]): object %#zx violates "
+ "expected alignment %#zx",
+ fname, mt, p, i, p[i], align);
+ } else {
+ mo = obj_pub_full(p[i], sz);
+ if (memobj_verify(mo, k) != 0) {
+ ret--;
+ MTANK_LOG(ERR,
+ "%s(mt=%p, %p[%u]): "
+ "invalid object header @%#zx={"
+ "red_zone1=%#" PRIx64 ","
+ "dbg={nb_alloc=%u,nb_free=%u},"
+ "red_zone2=%#" PRIx64
+ "}",
+ fname, mt, p, i, p[i],
+ mo->red_zone1,
+ mo->dbg.nb_alloc, mo->dbg.nb_free,
+ mo->red_zone2);
+ }
+ }
+ }
+
+ return ret;
+}
+
+/* grab free lock and check objects in free[] */
+static int
+mfree_check(struct rte_memtank *mt)
+{
+ int32_t rc;
+
+ rte_spinlock_lock(&mt->mtf.lock);
+ rc = mobj_bulk_check(__func__, mt, (const uintptr_t *)mt->mtf.free,
+ mt->mtf.nb_free, 1);
+ rte_spinlock_unlock(&mt->mtf.lock);
+ return rc;
+}
+
+static int
+mchunk_check(const struct rte_memtank *mt, const struct memchunk *mc,
+ uint32_t tc)
+{
+ int32_t n, rc;
+
+ rc = 0;
+ n = mc->nb_total - mc->nb_free;
+
+ rc -= (mc->nb_total != mt->prm.nb_obj_chunk);
+ rc -= (tc == MC_FULL) ? (n != 0) : (n <= 0);
+ rc -= (RTE_PTR_ALIGN_CEIL(mc->raw, alignof(typeof(*mc))) != mc);
+
+ if (rc != 0)
+ MTANK_LOG(ERR, "%s(mt=%p, tc=%u): invalid memchunk @%p={"
+ "raw=%p, nb_total=%u, nb_free=%u}",
+ __func__, mt, tc, mc,
+ mc->raw, mc->nb_total, mc->nb_free);
+
+ rc += mobj_bulk_check(__func__, mt, (const uintptr_t *)mc->free,
+ mc->nb_free, 0);
+ return rc;
+}
+
+static int
+mchunk_list_check(struct rte_memtank *mt, uint32_t tc, uint32_t *nb_chunk)
+{
+ int32_t rc;
+ uint32_t n;
+ struct mchunk_list *ls;
+ const struct memchunk *ch;
+
+ ls = &mt->chl[tc];
+ rte_spinlock_lock(&ls->lock);
+
+ rc = 0;
+ for (n = 0, ch = TAILQ_FIRST(&ls->chunk); ch != NULL;
+ ch = TAILQ_NEXT(ch, link), n++)
+ rc += mchunk_check(mt, ch, tc);
+
+ rte_spinlock_unlock(&ls->lock);
+
+ *nb_chunk = n;
+ return rc;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_memtank_sanity_check, 26.11)
+int
+rte_memtank_sanity_check(struct rte_memtank *mt, int32_t ct)
+{
+ int32_t rc;
+ uint32_t n, nf, nu;
+
+ rc = mfree_check(mt);
+
+ nf = 0;
+ nu = 0;
+ rc += mchunk_list_check(mt, MC_FULL, &nf);
+ rc += mchunk_list_check(mt, MC_USED, &nu);
+
+ /*
+ * if some other threads concurently do alloc/free/grow/shrink
+ * these numbers can still not match.
+ */
+ n = rte_atomic_load_explicit(&mt->nb_chunks, rte_memory_order_relaxed);
+ if (nf + nu != n && ct == 0) {
+ MTANK_LOG(ERR,
+ "%s(mt=%p) nb_chunks: expected=%u, full=%u, used=%u",
+ __func__, mt, n, nf, nu);
+ rc--;
+ }
+
+ return rc;
+}
diff --git a/lib/memtank/rte_memtank.h b/lib/memtank/rte_memtank.h
new file mode 100644
index 0000000000..7359b39840
--- /dev/null
+++ b/lib/memtank/rte_memtank.h
@@ -0,0 +1,303 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2025 Huawei Technologies Co., Ltd
+ */
+
+#ifndef _RTE_MEMTANK_H_
+#define _RTE_MEMTANK_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_common.h>
+#include <rte_compat.h>
+#include <stdio.h>
+
+/**
+ * @file
+ * RTE memtank
+ *
+ * Same a s mempool it allows to alloc/free objects of fixed size
+ * in a lightweight manner (probably not as lightweight as mempool,
+ * but hopefully close enough).
+ * But in addition it can grow/shrink dynamically plus provides extra
+ * additional API for higher flexibility:
+ * - manual grow()/shrink() functions
+ * - different alloc/free policies
+ * (can be specified by user via flags parameter).
+ *
+ * Internally it consists of:
+ * - LIFO queue (fast allocator/deallocator)
+ * - lists of memchunks (USED, FREE).
+ *
+ * For performance reasons memtank tries to allocate memory in
+ * relatively big chunks (memchunks) and then split each memchunk
+ * in dozens (or hundreds) of objects.
+ * There are two thresholds:
+ * - min_free (grow threshold)
+ * - max_free (shrink threshold)
+ */
+
+struct rte_memtank;
+
+/** generic memtank behavior flags */
+enum {
+ /** Enable obj debugging */
+ RTE_MTANK_OBJ_DBG = 1,
+};
+
+struct rte_memtank_prm {
+ /** min number of free objs in the ring (grow threshold). */
+ uint32_t min_free;
+ uint32_t max_free; /**< max number of free objs (empty threshold) */
+ uint32_t max_obj; /**< max number of objs (grow limit) */
+ uint32_t obj_size; /**< size of each mem object */
+ uint32_t obj_align; /**< alignment of each mem object */
+ uint32_t nb_obj_chunk; /**< number of objects per chunk */
+ uint32_t flags; /**< behavior flags */
+ /** user provided function to alloc chunk of memory */
+ void * (*alloc)(size_t len, void *udata);
+ /** user provided function to free chunk of memory */
+ void (*free)(void *mem, void *udata);
+ /** user provided function to initialiaze an object */
+ void (*init)(void *obj[], uint32_t num, void *udata);
+ void *udata; /**< opaque user data for alloc/free/init */
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Allocate and intitialize new memtank instance, based on the
+ * parameters provided. Note that it uses user-provided *alloc()* function
+ * to allocate space for the memtank metadata.
+ * @param prm
+ * Parameters used to create and initialise new memtank.
+ * @return
+ * - Pointer to new memtank insteance created, if operation completed
+ * successfully.
+ * - NULL on error with rte_errno set appropriately.
+ */
+__rte_experimental
+struct rte_memtank *
+rte_memtank_create(const struct rte_memtank_prm *prm);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Destroy the memtank and free all memory referenced by the memtank.
+ * The objects must not be used by other cores as they will be freed.
+ *
+ * @param t
+ * A pointer to the memtank instance.
+ */
+__rte_experimental
+void
+rte_memtank_destroy(struct rte_memtank *t);
+
+
+/** alloc flags */
+enum {
+ RTE_MTANK_ALLOC_CHUNK = 1,
+ /** Allocate extra memchunks if needed */
+ RTE_MTANK_ALLOC_GROW = 2,
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Allocate up to requested number of objects from the memtank.
+ * Note that depending on *alloc* behavior (flags) some new memory chunks
+ * can be allocated from the underlying memory subsystem.
+ *
+ * @param t
+ * A pointer to the memtank instance.
+ * @param obj
+ * An array of void * pointers (objects) that will be filled.
+ * @param num
+ * Number of objects to allocate from the memtank.
+ * @param flags
+ * Flags that control allocation behavior.
+ * @return
+ * Number of allocated objects.
+ */
+__rte_experimental
+uint32_t
+rte_memtank_alloc(struct rte_memtank *t, void *obj[], uint32_t num,
+ uint32_t flags);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Allocate up to requested number of objects from the memtank.
+ * Note that this function bypasses *free* cache(s) and tries to allocate
+ * objects straight from the memory chunks.
+ * Note that depending on *alloc* behavior (flags) some new memory chunks
+ * can be allocated from the underlying memory subsystem.
+ *
+ * @param t
+ * A pointer to the memtank instance.
+ * @param obj
+ * An array of void * pointers (objects) that will be filled.
+ * @param nb_obj
+ * Number of objects to allocate from the memtank.
+ * @param flags
+ * Flags that control allocation behavior.
+ * @return
+ * Number of allocated objects.
+ */
+__rte_experimental
+uint32_t
+rte_memtank_chunk_alloc(struct rte_memtank *t, void *obj[], uint32_t nb_obj,
+ uint32_t flags);
+
+/** free flags */
+enum {
+ /** Free unneeded chunk of memory */
+ RTE_MTANK_FREE_SHRINK = 1,
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Free (put) provided objects back to the memtank.
+ * Note that depending on *free* behavior (flags) some memory chunks can be
+ * returned (freed) to the underlying memory subsystem.
+ *
+ * @param t
+ * A pointer to the memtank instance.
+ * @param obj
+ * An array of object pointers to be freed.
+ * @param num
+ * Number of objects to free.
+ * @param flags
+ * Flags that control free behavior.
+ */
+__rte_experimental
+void
+rte_memtank_free(struct rte_memtank *t, void * const obj[], uint32_t num,
+ uint32_t flags);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Free (put) provided objects back to the memtank.
+ * Note that this function bypasses *free* cache(s) and tries to put
+ * objects straight to the memory chunks.
+ * Note that depending on *free* behavior (flags) some memory chunks can be
+ * returned (freed) to the underlying memory subsystem.
+ *
+ * @param t
+ * A pointer to the memtank instance.
+ * @param obj
+ * An array of object pointers to be freed.
+ * @param nb_obj
+ * Number of objects to allocate from the memtank.
+ * @param flags
+ * Flags that control allocation behavior.
+ */
+__rte_experimental
+void
+rte_memtank_chunk_free(struct rte_memtank *t, void * const obj[],
+ uint32_t nb_obj, uint32_t flags);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Check does number of objects in *free* cache is below memtank grow
+ * threshold (min_free). If yes, then tries to allocate memory for new
+ * objects from the underlying memory subsystem.
+ *
+ * @param t
+ * A pointer to the memtank instance.
+ * @return
+ * Number of newly allocated memory chunks.
+ */
+__rte_experimental
+int
+rte_memtank_grow(struct rte_memtank *t);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Check does number of objects in *free* cache have reached memtank shrink
+ * threshold (max_free). If yes, then tries to return excessive memory to
+ * the underlying memory subsystem.
+ *
+ * @param t
+ * A pointer to the memtank instance.
+ * @return
+ * Number of freed memory chunks.
+ */
+__rte_experimental
+int
+rte_memtank_shrink(struct rte_memtank *t);
+
+/** dump flags */
+enum {
+ RTE_MTANK_DUMP_FREE_STAT = 1,
+ RTE_MTANK_DUMP_CHUNK_STAT = 2,
+ RTE_MTANK_DUMP_CHUNK = 4,
+ /* first not used power of two */
+ RTE_MTANK_DUMP_END = 8,
+
+ /** dump all stats */
+ RTE_MTANK_DUMP_STAT =
+ (RTE_MTANK_DUMP_FREE_STAT | RTE_MTANK_DUMP_CHUNK_STAT),
+ /** dump everything */
+ RTE_MTANK_DUMP_ALL = RTE_MTANK_DUMP_END - 1,
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Dump information about the memtank to the file.
+ * Note that depending of *flags* value it might cause some internal locks
+ * grabbing, and might affect performance of others threads that
+ * concurently use same memtank.
+ *
+ * @param f
+ * A pinter to the file.
+ * @param t
+ * A pointer to the memtank instance.
+ * @param flags
+ * Flags that control dump behavior.
+ */
+__rte_experimental
+void
+rte_memtank_dump(FILE *f, struct rte_memtank *t, uint32_t flags);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Check the consistency of the given memtank instance.
+ * Dumps error messages to the RTE log subsystem, if some inconsitency
+ * is detected.
+ *
+ * @param t
+ * A pointer to the memtank instance.
+ * @param ct
+ * Value greater then zero, if some other threads do concurently use
+ * that memtank.
+ * @return
+ * Zero on success, or negative value otherwise.
+ */
+__rte_experimental
+int
+rte_memtank_sanity_check(struct rte_memtank *t, int32_t ct);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MEMTANK_H_ */
diff --git a/lib/meson.build b/lib/meson.build
index af5c160cb8..3b13dfee6c 100644
--- a/lib/meson.build
+++ b/lib/meson.build
@@ -19,6 +19,7 @@ libraries = [
'ring',
'rcu', # rcu depends on ring
'mempool',
+ 'memtank',
'mbuf',
'net',
'meter',
--
2.51.0
