Rename all TVM runtime specific files and functions with tvmrt prefix. Retain mvtvm name prefix for functions and files related to mvtvm vdev component.
Signed-off-by: Srikanth Yalavarthi <[email protected]> --- drivers/ml/cnxk/cn10k_ml_model.c | 2 +- drivers/ml/cnxk/cnxk_ml_dev.h | 4 +- drivers/ml/cnxk/cnxk_ml_io.h | 6 +- drivers/ml/cnxk/cnxk_ml_model.c | 4 +- drivers/ml/cnxk/cnxk_ml_model.h | 12 +- drivers/ml/cnxk/cnxk_ml_ops.c | 24 +- drivers/ml/cnxk/cnxk_ml_ops.h | 10 +- drivers/ml/cnxk/meson.build | 6 +- drivers/ml/cnxk/mvtvm_ml_ops.c | 818 +----------------- drivers/ml/cnxk/mvtvm_ml_ops.h | 66 -- drivers/ml/cnxk/mvtvm_ml_stubs.c | 97 --- drivers/ml/cnxk/mvtvm_ml_stubs.h | 19 - .../{mvtvm_ml_model.c => tvmrt_ml_model.c} | 269 +++--- .../{mvtvm_ml_model.h => tvmrt_ml_model.h} | 44 +- drivers/ml/cnxk/tvmrt_ml_ops.c | 818 ++++++++++++++++++ drivers/ml/cnxk/tvmrt_ml_ops.h | 79 ++ drivers/ml/cnxk/tvmrt_ml_stubs.c | 106 +++ drivers/ml/cnxk/tvmrt_ml_stubs.h | 32 + 18 files changed, 1232 insertions(+), 1184 deletions(-) rename drivers/ml/cnxk/{mvtvm_ml_model.c => tvmrt_ml_model.c} (72%) rename drivers/ml/cnxk/{mvtvm_ml_model.h => tvmrt_ml_model.h} (78%) create mode 100644 drivers/ml/cnxk/tvmrt_ml_ops.c create mode 100644 drivers/ml/cnxk/tvmrt_ml_ops.h create mode 100644 drivers/ml/cnxk/tvmrt_ml_stubs.c create mode 100644 drivers/ml/cnxk/tvmrt_ml_stubs.h diff --git a/drivers/ml/cnxk/cn10k_ml_model.c b/drivers/ml/cnxk/cn10k_ml_model.c index 12a2dda800e..9cfecea4448 100644 --- a/drivers/ml/cnxk/cn10k_ml_model.c +++ b/drivers/ml/cnxk/cn10k_ml_model.c @@ -725,7 +725,7 @@ int cn10k_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, uint16_t *layer_id) { if (model->type == ML_CNXK_MODEL_TYPE_TVM) - return mvtvm_ml_model_get_layer_id(model, layer_name, layer_id); + return tvmrt_ml_model_get_layer_id(model, layer_name, layer_id); *layer_id = 0; diff --git a/drivers/ml/cnxk/cnxk_ml_dev.h b/drivers/ml/cnxk/cnxk_ml_dev.h index 9e373e65715..e93d76d1af8 100644 --- a/drivers/ml/cnxk/cnxk_ml_dev.h +++ b/drivers/ml/cnxk/cnxk_ml_dev.h @@ -9,7 +9,7 @@ #include "cn10k_ml_dev.h" -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT #include "mvtvm_ml_dev.h" #endif @@ -87,7 +87,7 @@ struct cnxk_ml_dev { /* CN10K device structure */ struct cn10k_ml_dev cn10k_mldev; -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT /* MVTVM device structure */ struct mvtvm_ml_dev mvtvm_mldev; #endif diff --git a/drivers/ml/cnxk/cnxk_ml_io.h b/drivers/ml/cnxk/cnxk_ml_io.h index 17f5b4619f0..3ca852706f5 100644 --- a/drivers/ml/cnxk/cnxk_ml_io.h +++ b/drivers/ml/cnxk/cnxk_ml_io.h @@ -5,7 +5,7 @@ #ifndef _CNXK_ML_IO_H_ #define _CNXK_ML_IO_H_ -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT #include <dlpack/dlpack.h> #endif @@ -15,7 +15,7 @@ #define ML_CNXK_MAX_MODELS 16 /* Maximum number of layers per model */ -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT #define ML_CNXK_MODEL_MAX_LAYERS 128 #else #define ML_CNXK_MODEL_MAX_LAYERS 1 @@ -59,7 +59,7 @@ struct cnxk_ml_io { /* Zero point */ int64_t zero_point; -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT /* Shape - int64_t */ int64_t shape_i64[ML_CNXK_MODEL_MAX_DIMS]; diff --git a/drivers/ml/cnxk/cnxk_ml_model.c b/drivers/ml/cnxk/cnxk_ml_model.c index ed6a1ed8665..442bc9460c0 100644 --- a/drivers/ml/cnxk/cnxk_ml_model.c +++ b/drivers/ml/cnxk/cnxk_ml_model.c @@ -16,7 +16,7 @@ cnxk_ml_model_get_type(struct rte_ml_model_params *params) uint32_t payload_crc32c; uint32_t header_crc32c; - type = mvtvm_ml_model_type_get(params); + type = tvmrt_ml_model_type_get(params); if (type == ML_CNXK_MODEL_TYPE_TVM) return ML_CNXK_MODEL_TYPE_TVM; else if (type == ML_CNXK_MODEL_TYPE_INVALID) @@ -89,6 +89,6 @@ cnxk_ml_model_dump(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, if (layer->type == ML_CNXK_LAYER_TYPE_MRVL) cn10k_ml_layer_print(cnxk_mldev, layer, fp); else - mvtvm_ml_layer_print(cnxk_mldev, layer, fp); + tvmrt_ml_layer_print(cnxk_mldev, layer, fp); } } diff --git a/drivers/ml/cnxk/cnxk_ml_model.h b/drivers/ml/cnxk/cnxk_ml_model.h index 1cd5ca1906a..59aea16f970 100644 --- a/drivers/ml/cnxk/cnxk_ml_model.h +++ b/drivers/ml/cnxk/cnxk_ml_model.h @@ -11,10 +11,10 @@ #include "cn10k_ml_model.h" -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM -#include "mvtvm_ml_model.h" +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT +#include "tvmrt_ml_model.h" #else -#include "mvtvm_ml_stubs.h" +#include "tvmrt_ml_stubs.h" #endif #include "cnxk_ml_io.h" @@ -152,9 +152,9 @@ struct cnxk_ml_model { /* Model specific data - glow */ struct cn10k_ml_model_data glow; -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM - /* Model type specific data - mvtvm */ - struct mvtvm_ml_model_data mvtvm; +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT + /* Model type specific data - tvmrt */ + struct tvmrt_ml_model_data tvmrt; #endif }; diff --git a/drivers/ml/cnxk/cnxk_ml_ops.c b/drivers/ml/cnxk/cnxk_ml_ops.c index 938982c7556..675cdaae2bd 100644 --- a/drivers/ml/cnxk/cnxk_ml_ops.c +++ b/drivers/ml/cnxk/cnxk_ml_ops.c @@ -250,7 +250,7 @@ cnxk_ml_xstats_model_name_update(struct cnxk_ml_dev *cnxk_mldev, uint16_t model_ if (model->type == ML_CNXK_MODEL_TYPE_GLOW) cn10k_ml_xstat_model_name_set(cnxk_mldev, model, stat_id, i, suffix); else - mvtvm_ml_model_xstat_name_set(cnxk_mldev, model, stat_id, i, suffix); + tvmrt_ml_model_xstat_name_set(cnxk_mldev, model, stat_id, i, suffix); stat_id++; } @@ -312,7 +312,7 @@ cnxk_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, uint16_t obj_idx, int32_ goto exit_xstats; model_xstats: - value = mvtvm_ml_model_xstat_get(cnxk_mldev, model, type); + value = tvmrt_ml_model_xstat_get(cnxk_mldev, model, type); exit_xstats: roc_clk_freq_get(&rclk_freq, &sclk_freq); @@ -1211,7 +1211,7 @@ cnxk_ml_model_load(struct rte_ml_dev *dev, struct rte_ml_model_params *params, u if (type == ML_CNXK_MODEL_TYPE_GLOW) ret = cn10k_ml_model_load(cnxk_mldev, params, model); else - ret = mvtvm_ml_model_load(cnxk_mldev, params, model); + ret = tvmrt_ml_model_load(cnxk_mldev, params, model); if (ret != 0) goto error; @@ -1225,7 +1225,7 @@ cnxk_ml_model_load(struct rte_ml_dev *dev, struct rte_ml_model_params *params, u total_wb_pages = total_wb_pages + model->layer[layer_id].glow.ocm_map.wb_pages; max_scratch_pages = PLT_MAX(max_scratch_pages, model->layer[layer_id].glow.ocm_map.scratch_pages); -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT } else { for (layer_id = 0; layer_id < model->nb_layers; layer_id++) { if (model->layer[layer_id].type == ML_CNXK_LAYER_TYPE_MRVL) { @@ -1247,7 +1247,7 @@ cnxk_ml_model_load(struct rte_ml_dev *dev, struct rte_ml_model_params *params, u plt_ml_dbg("layer_id = %u: wb_pages = %u, scratch_pages = %u", layer_id, model->layer[layer_id].glow.ocm_map.wb_pages, model->layer[layer_id].glow.ocm_map.scratch_pages); -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT } else { for (layer_id = 0; layer_id < model->nb_layers; layer_id++) { if (model->layer[layer_id].type == ML_CNXK_LAYER_TYPE_MRVL) { @@ -1263,9 +1263,9 @@ cnxk_ml_model_load(struct rte_ml_dev *dev, struct rte_ml_model_params *params, u if (model->type == ML_CNXK_MODEL_TYPE_GLOW) cn10k_ml_model_unload(cnxk_mldev, model); -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT else { - mvtvm_ml_model_unload(cnxk_mldev, model); + tvmrt_ml_model_unload(cnxk_mldev, model); return -ENOMEM; } #endif @@ -1312,7 +1312,7 @@ cnxk_ml_model_unload(struct rte_ml_dev *dev, uint16_t model_id) if (model->type == ML_CNXK_MODEL_TYPE_GLOW) ret = cn10k_ml_model_unload(cnxk_mldev, model); else - ret = mvtvm_ml_model_unload(cnxk_mldev, model); + ret = tvmrt_ml_model_unload(cnxk_mldev, model); if (ret != 0) return ret; @@ -1343,7 +1343,7 @@ cnxk_ml_model_start(struct rte_ml_dev *dev, uint16_t model_id) if (model->type == ML_CNXK_MODEL_TYPE_GLOW) return cn10k_ml_model_start(cnxk_mldev, model); else - return mvtvm_ml_model_start(cnxk_mldev, model); + return tvmrt_ml_model_start(cnxk_mldev, model); return 0; } @@ -1368,7 +1368,7 @@ cnxk_ml_model_stop(struct rte_ml_dev *dev, uint16_t model_id) if (model->type == ML_CNXK_MODEL_TYPE_GLOW) return cn10k_ml_model_stop(cnxk_mldev, model); else - return mvtvm_ml_model_stop(cnxk_mldev, model); + return tvmrt_ml_model_stop(cnxk_mldev, model); return 0; } @@ -1444,7 +1444,7 @@ cnxk_ml_io_quantize(struct rte_ml_dev *dev, uint16_t model_id, struct rte_ml_buf if (model->type == ML_CNXK_MODEL_TYPE_GLOW) info = cn10k_ml_model_io_info_get(model, 0); else - info = mvtvm_ml_model_io_info_get(model, 0); + info = tvmrt_ml_model_io_info_get(model, 0); if (info == NULL) return -EINVAL; @@ -1500,7 +1500,7 @@ cnxk_ml_io_dequantize(struct rte_ml_dev *dev, uint16_t model_id, struct rte_ml_b if (model->type == ML_CNXK_MODEL_TYPE_GLOW) info = cn10k_ml_model_io_info_get(model, model->nb_layers - 1); else - info = mvtvm_ml_model_io_info_get(model, model->nb_layers - 1); + info = tvmrt_ml_model_io_info_get(model, model->nb_layers - 1); if (info == NULL) return -EINVAL; diff --git a/drivers/ml/cnxk/cnxk_ml_ops.h b/drivers/ml/cnxk/cnxk_ml_ops.h index 7a79fec412e..ad10c3d45dc 100644 --- a/drivers/ml/cnxk/cnxk_ml_ops.h +++ b/drivers/ml/cnxk/cnxk_ml_ops.h @@ -12,10 +12,12 @@ #include "cn10k_ml_ops.h" -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT #include "mvtvm_ml_ops.h" +#include "tvmrt_ml_ops.h" #else #include "mvtvm_ml_stubs.h" +#include "tvmrt_ml_stubs.h" #endif /* Request structure */ @@ -25,9 +27,9 @@ struct __rte_aligned(ROC_ALIGN) cnxk_ml_req { /* CN10K */ struct cn10k_ml_req cn10k_req; -#ifdef RTE_MLDEV_CNXK_ENABLE_MVTVM - /* MVTVM */ - struct mvtvm_ml_req mvtvm_req; +#ifdef RTE_MLDEV_CNXK_ENABLE_TVMRT + /* TVMRT */ + struct tvmrt_ml_req tvmrt_req; #endif }; diff --git a/drivers/ml/cnxk/meson.build b/drivers/ml/cnxk/meson.build index 5f078bd4fd7..da3cb7a2967 100644 --- a/drivers/ml/cnxk/meson.build +++ b/drivers/ml/cnxk/meson.build @@ -53,12 +53,13 @@ deps += ['mldev', 'common_cnxk', 'kvargs', 'hash'] if enable_mvtvm -dpdk_conf.set('RTE_MLDEV_CNXK_ENABLE_MVTVM', 1) +dpdk_conf.set('RTE_MLDEV_CNXK_ENABLE_TVMRT', 1) sources += files( 'mvtvm_ml_dev.c', 'mvtvm_ml_ops.c', - 'mvtvm_ml_model.c', + 'tvmrt_ml_ops.c', + 'tvmrt_ml_model.c', ) ext_deps += jansson_dep @@ -80,6 +81,7 @@ message('drivers/ml/cnxk: Disabled TVM model support') sources += files( 'mvtvm_ml_stubs.c', + 'tvmrt_ml_stubs.c', ) endif diff --git a/drivers/ml/cnxk/mvtvm_ml_ops.c b/drivers/ml/cnxk/mvtvm_ml_ops.c index bc47a4bbd75..69b800c8421 100644 --- a/drivers/ml/cnxk/mvtvm_ml_ops.c +++ b/drivers/ml/cnxk/mvtvm_ml_ops.c @@ -2,189 +2,19 @@ * Copyright (c) 2023 Marvell. */ -#include <errno.h> -#include <linux/limits.h> -#include <stdbool.h> #include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/mman.h> -#include <unistd.h> - -#include <dlpack/dlpack.h> -#include <jansson.h> - -#include <rte_common.h> -#include <rte_cycles.h> -#include <rte_mldev.h> -#include <rte_mldev_pmd.h> - -#include <mldev_utils.h> #include "cnxk_ml_dev.h" -#include "cnxk_ml_model.h" -#include "cnxk_ml_ops.h" -#include "cnxk_ml_xstats.h" - -/* ML model macros */ -#define MVTVM_ML_MODEL_MEMZONE_NAME "ml_mvtvm_model_mz" - -/* Shared memory file descriptor name */ -#define ML_MODEL_SHMFD_NAME "mvtvm_shmfd" - -/* Shared memory file descriptor path */ -#define ML_MODEL_SHMFD_PATH "/proc/%d/fd/%d" - -static int -mvtvm_ml_tvm_func_get(struct cnxk_ml_model *model, TVMModuleHandle module, const char *name, - TVMFunctionHandle *func) -{ - int ret; - - ret = TVMModGetFunction(module, name, 0, func); - if (ret != 0) { - plt_err("Model load failed, model_id = %u, ret = %d, msg = %s", model->model_id, - ret, TVMGetLastError()); - return ret; - } - - if (*func == NULL) { - ret = -ENOENT; - plt_err("Model load failed, model_id = %u, function '%s' not found", - model->model_id, name); - } - - return ret; -} - -static int -mvtvm_ml_tvm_func_call(struct cnxk_ml_model *model, TVMFunctionHandle func, const char *name, - TVMValue *values, int *types, int num_args, TVMValue *ret_val, int *ret_type, - int ret_type_code) -{ - int ret; - - ret = TVMFuncCall(func, values, types, num_args, ret_val, ret_type); - if (ret != 0) { - plt_err("Error calling TVM function '%s', model_id = %u, ret = %d, msg = %s", name, - model->model_id, ret, TVMGetLastError()); - return ret; - } - - if (*ret_type != ret_type_code) { - ret = -EINVAL; - plt_err("TVM function '%s' returned unexpected type, model_id = %u, expected = %d, " - "actual = %d", - name, model->model_id, ret_type_code, *ret_type); - } - - return ret; -} - -static void -mvtvm_ml_tvm_func_free(TVMFunctionHandle *func) -{ - if ((func != NULL) && (*func != NULL)) { - TVMFuncFree(*func); - *func = NULL; - } -} - -static void -mvtvm_ml_tvm_mod_free(TVMModuleHandle *mod) -{ - if ((mod != NULL) && (*mod != NULL)) { - TVMModFree(*mod); - *mod = NULL; - } -} - -__rte_hot static void -mvtvm_ml_set_poll_addr(struct cnxk_ml_req *req) -{ - req->status = &req->mvtvm_req.status; -} - -void -mvtvm_ml_model_xstat_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, - uint16_t stat_id, uint16_t entry, char *suffix) -{ - snprintf(cnxk_mldev->xstats.entries[stat_id].map.name, - sizeof(cnxk_mldev->xstats.entries[stat_id].map.name), "%s-%s-%s", model->name, - model_xstats[entry].name, suffix); -} - -#define ML_AVG_FOREACH_QP_MVTVM(cnxk_mldev, model, qp_id, value, count) \ - do { \ - value = 0; \ - for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \ - value += model->mvtvm.burst_xstats[qp_id].tvm_rt_latency_tot; \ - count += model->mvtvm.burst_xstats[qp_id].dequeued_count - \ - model->mvtvm.burst_xstats[qp_id].tvm_rt_reset_count; \ - } \ - if (count != 0) \ - value = value / count; \ - } while (0) - -#define ML_MIN_FOREACH_QP_MVTVM(cnxk_mldev, model, qp_id, value, count) \ - do { \ - value = UINT64_MAX; \ - for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \ - value = PLT_MIN(value, \ - model->mvtvm.burst_xstats[qp_id].tvm_rt_latency_min); \ - count += model->mvtvm.burst_xstats[qp_id].dequeued_count - \ - model->mvtvm.burst_xstats[qp_id].tvm_rt_reset_count; \ - } \ - if (count == 0) \ - value = 0; \ - } while (0) - -#define ML_MAX_FOREACH_QP_MVTVM(cnxk_mldev, model, qp_id, value, count) \ - do { \ - value = 0; \ - for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \ - value = PLT_MAX(value, \ - model->mvtvm.burst_xstats[qp_id].tvm_rt_latency_max); \ - count += model->mvtvm.burst_xstats[qp_id].dequeued_count - \ - model->mvtvm.burst_xstats[qp_id].tvm_rt_reset_count; \ - } \ - if (count == 0) \ - value = 0; \ - } while (0) - -uint64_t -mvtvm_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, - enum cnxk_ml_xstats_type type) -{ - uint64_t count = 0; - uint64_t value = 0; - uint32_t qp_id; - - switch (type) { - case avg_rt_latency: - ML_AVG_FOREACH_QP_MVTVM(cnxk_mldev, model, qp_id, value, count); - break; - case min_rt_latency: - ML_MIN_FOREACH_QP_MVTVM(cnxk_mldev, model, qp_id, value, count); - break; - case max_rt_latency: - ML_MAX_FOREACH_QP_MVTVM(cnxk_mldev, model, qp_id, value, count); - break; - default: - value = 0; - } - - return value; -} +#include "mvtvm_ml_ops.h" int mvtvm_ml_dev_info_get(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_dev_info *dev_info) { - struct mvtvm_ml_dev *mvtvm_mldev; + struct mvtvm_ml_dev *tvmrt_mldev; - mvtvm_mldev = &cnxk_mldev->mvtvm_mldev; + tvmrt_mldev = &cnxk_mldev->mvtvm_mldev; - dev_info->max_queue_pairs = mvtvm_mldev->max_nb_qpairs; + dev_info->max_queue_pairs = tvmrt_mldev->max_nb_qpairs; dev_info->max_desc = ML_MVTVM_MAX_DESC_PER_QP; dev_info->max_io = ML_CNXK_MODEL_MAX_INPUT_OUTPUT; dev_info->max_segments = ML_MVTVM_MAX_SEGMENTS; @@ -201,643 +31,3 @@ mvtvm_ml_dev_dump(struct cnxk_ml_dev *cnxk_mldev, FILE *fp) return 0; } - -int -mvtvm_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params, - struct cnxk_ml_model *model) -{ - struct mvtvm_ml_model_object object[ML_MVTVM_MODEL_OBJECT_MAX]; - struct tvmrt_glow_callback *callback; - char str[RTE_MEMZONE_NAMESIZE]; - char path[PATH_MAX]; - const struct plt_memzone *mz; - size_t model_object_size = 0; - size_t model_xstats_size = 0; - uint64_t mz_size = 0; - TVMFunctionHandle create_fn = NULL; - TVMFunctionHandle register_cb_fn = NULL; - TVMModuleHandle module_so = NULL; - TVMByteArray tvm_params; - TVMValue ret_value = {0}; - TVMValue arg_values[4] = {0}; - TVMValue tvm_arg_values[1] = {0}; - int ret_type = kTVMNullptr; - int arg_types[4] = {0}; - int tvm_arg_types[1] = {0}; - DLDevice device; - int ret; - - RTE_SET_USED(cnxk_mldev); - model->mvtvm.fd = -1; - - ret = mvtvm_ml_model_blob_parse(params, object); - if (ret != 0) - return ret; - - model_object_size = RTE_ALIGN_CEIL(object[0].size, RTE_CACHE_LINE_MIN_SIZE) + - RTE_ALIGN_CEIL(object[1].size, RTE_CACHE_LINE_MIN_SIZE) + - RTE_ALIGN_CEIL(object[2].size, RTE_CACHE_LINE_MIN_SIZE); - - model_xstats_size = - cnxk_mldev->mldev->data->nb_queue_pairs * sizeof(struct mvtvm_ml_model_xstats); - - mz_size += model_object_size + model_xstats_size; - - /* Allocate memzone for model object */ - snprintf(str, RTE_MEMZONE_NAMESIZE, "%s_%u", MVTVM_ML_MODEL_MEMZONE_NAME, model->model_id); - mz = plt_memzone_reserve_aligned(str, mz_size, 0, ML_CN10K_ALIGN_SIZE); - if (!mz) { - plt_err("plt_memzone_reserve failed : %s", str); - return -ENOMEM; - } - - /* Copy mod.so */ - model->mvtvm.so.buffer = mz->addr; - model->mvtvm.so.size = object[0].size; - rte_memcpy(model->mvtvm.so.name, object[0].name, RTE_ML_STR_MAX); - rte_memcpy(model->mvtvm.so.buffer, object[0].buffer, object[0].size); - rte_free(object[0].buffer); - - /* Copy mod.json */ - model->mvtvm.json.buffer = - RTE_PTR_ADD(model->mvtvm.so.buffer, - RTE_ALIGN_CEIL(model->mvtvm.so.size, RTE_CACHE_LINE_MIN_SIZE)); - model->mvtvm.json.size = object[1].size; - rte_memcpy(model->mvtvm.json.name, object[1].name, RTE_ML_STR_MAX); - rte_memcpy(model->mvtvm.json.buffer, object[1].buffer, object[1].size); - rte_free(object[1].buffer); - - /* Copy mod.params */ - model->mvtvm.params.buffer = - RTE_PTR_ADD(model->mvtvm.json.buffer, - RTE_ALIGN_CEIL(model->mvtvm.json.size, RTE_CACHE_LINE_MIN_SIZE)); - model->mvtvm.params.size = object[2].size; - rte_memcpy(model->mvtvm.params.name, object[2].name, RTE_ML_STR_MAX); - rte_memcpy(model->mvtvm.params.buffer, object[2].buffer, object[2].size); - rte_free(object[2].buffer); - - ret = mvtvm_ml_model_json_parse(model); - if (ret != 0) - goto error; - - if (cnxk_mldev->type == CNXK_ML_DEV_TYPE_VDEV && - model->subtype != ML_CNXK_MODEL_SUBTYPE_TVM_LLVM) { - plt_err("Unsupported model sub-type"); - ret = -ENOTSUP; - goto error; - } - - /* Set callback function array */ - if (model->subtype != ML_CNXK_MODEL_SUBTYPE_TVM_LLVM) { - callback = &model->mvtvm.cb; - callback->tvmrt_glow_layer_load = cn10k_ml_layer_load; - callback->tvmrt_glow_layer_unload = cn10k_ml_layer_unload; - callback->tvmrt_io_alloc = cn10k_ml_io_alloc; - callback->tvmrt_io_free = cn10k_ml_io_free; - callback->tvmrt_malloc = cn10k_ml_malloc; - callback->tvmrt_free = cn10k_ml_free; - callback->tvmrt_quantize = mvtvm_ml_io_quantize; - callback->tvmrt_dequantize = mvtvm_ml_io_dequantize; - callback->tvmrt_inference = cn10k_ml_inference_sync; - } else { - callback = NULL; - } - - /* Initialize model in TVM runtime */ - if (model->mvtvm.graph_module != NULL) { - ret = -EBUSY; - plt_err("TVM runtime: Model load failed, model_id = %u, error = %d", - model->model_id, ret); - goto error; - } - - snprintf(path, sizeof(path), "%s_%d_%u", ML_MODEL_SHMFD_NAME, getpid(), model->model_id); - model->mvtvm.fd = memfd_create(path, 0); - if (model->mvtvm.fd < 0) { - ret = (errno == 0) ? -EIO : -errno; - plt_err("TVM runtime: Model load failed, model_id = %u, error = %d", - model->model_id, ret); - goto error; - } - - if (write(model->mvtvm.fd, model->mvtvm.so.buffer, model->mvtvm.so.size) != - (ssize_t)model->mvtvm.so.size) { - ret = (errno == 0) ? -EIO : -errno; - plt_err("TVM runtime: Model load failed, model_id = %u, error = %d", - model->model_id, ret); - goto error; - } - - if (lseek(model->mvtvm.fd, 0, SEEK_SET) < 0) { - ret = (errno == 0) ? -EIO : -errno; - plt_err("TVM runtime: Model load failed, model_id = %u, error = %d", - model->model_id, ret); - goto error; - } - - snprintf(path, sizeof(path), ML_MODEL_SHMFD_PATH, getpid(), model->mvtvm.fd); - ret = TVMModLoadFromFile(path, "so", &module_so); - if (ret != 0) { - plt_err("TVM runtime: Model load failed, model_id = %u, ret = %d, msg = %s", - model->model_id, ret, TVMGetLastError()); - goto error; - } - - /* Set device info */ - device.device_type = kDLCPU; - device.device_id = 0; - - if (callback != NULL) { - ret = mvtvm_ml_tvm_func_get(model, module_so, "register_cb", ®ister_cb_fn); - if (ret != 0) - goto error; - - arg_values[0].v_handle = callback; - arg_types[0] = kTVMOpaqueHandle; - arg_values[1].v_handle = cnxk_mldev; - arg_types[1] = kTVMOpaqueHandle; - arg_values[2].v_int64 = model->model_id; - arg_types[2] = kDLInt; - - ret = mvtvm_ml_tvm_func_call(model, register_cb_fn, "register_cb", arg_values, - arg_types, 3, &ret_value, &ret_type, kTVMNullptr); - if (ret != 0) - goto error; - } - - ret = TVMFuncGetGlobal("tvm.graph_executor.create", &create_fn); - if (ret != 0) { - plt_err("TVM runtime: Model load failed, model_id = %u, ret = %d, msg = %s", - model->model_id, ret, TVMGetLastError()); - goto error; - } - - arg_values[0].v_str = (const char *)model->mvtvm.json.buffer; - arg_types[0] = kTVMStr; - arg_values[1].v_handle = module_so; - arg_types[1] = kTVMModuleHandle; - arg_values[2].v_int64 = device.device_type; - arg_types[2] = kDLInt; - arg_values[3].v_int64 = device.device_id; - arg_types[3] = kDLInt; - - ret = mvtvm_ml_tvm_func_call(model, create_fn, "tvm.graph_executor.create", arg_values, - arg_types, 4, &ret_value, &ret_type, kTVMModuleHandle); - if (ret != 0) - goto error; - model->mvtvm.graph_module = ret_value.v_handle; - - ret = mvtvm_ml_tvm_func_get(model, model->mvtvm.graph_module, "load_params", - &model->mvtvm.load_params); - if (ret != 0) - goto error; - ret = mvtvm_ml_tvm_func_get(model, model->mvtvm.graph_module, "set_input_zero_copy", - &model->mvtvm.set_input_zero_copy); - if (ret != 0) - goto error; - ret = mvtvm_ml_tvm_func_get(model, model->mvtvm.graph_module, "set_output_zero_copy", - &model->mvtvm.set_output_zero_copy); - if (ret != 0) - goto error; - ret = mvtvm_ml_tvm_func_get(model, model->mvtvm.graph_module, "run", &model->mvtvm.run); - if (ret != 0) - goto error; - - mvtvm_ml_tvm_func_free(®ister_cb_fn); - mvtvm_ml_tvm_mod_free(&module_so); - - /* Load model parameters into TVM runtime */ - tvm_params.data = (const char *)model->mvtvm.params.buffer; - tvm_params.size = model->mvtvm.params.size; - tvm_arg_values[0].v_handle = &tvm_params; - tvm_arg_types[0] = kTVMBytes; - - ret = mvtvm_ml_tvm_func_call(model, model->mvtvm.load_params, "load_params", tvm_arg_values, - tvm_arg_types, 1, &ret_value, &ret_type, kTVMNullptr); - if (ret != 0) - goto error; - - /* Update model I/O data */ - mvtvm_ml_model_io_info_set(model); - - /* Set model info */ - mvtvm_ml_model_info_set(cnxk_mldev, model); - - /* Update model xstats name */ - cnxk_ml_xstats_model_name_update(cnxk_mldev, model->model_id); - - model->mvtvm.burst_xstats = - RTE_PTR_ADD(model->mvtvm.params.buffer, - RTE_ALIGN_CEIL(model->mvtvm.params.size, RTE_CACHE_LINE_MIN_SIZE)); - - for (int qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { - model->mvtvm.burst_xstats[qp_id].tvm_rt_latency_tot = 0; - model->mvtvm.burst_xstats[qp_id].tvm_rt_latency = 0; - model->mvtvm.burst_xstats[qp_id].tvm_rt_latency_min = UINT64_MAX; - model->mvtvm.burst_xstats[qp_id].tvm_rt_latency_max = 0; - model->mvtvm.burst_xstats[qp_id].tvm_rt_reset_count = 0; - model->mvtvm.burst_xstats[qp_id].dequeued_count = 0; - } - - /* Set model specific fast path functions */ - if (model->subtype == ML_CNXK_MODEL_SUBTYPE_TVM_MRVL) { - model->enqueue_single = cn10k_ml_enqueue_single; - model->result_update = cn10k_ml_result_update; - model->set_error_code = cn10k_ml_set_error_code; - model->set_poll_addr = cn10k_ml_set_poll_addr; - model->op_error_get = cn10k_ml_op_error_get; - } else { - model->enqueue_single = mvtvm_ml_enqueue_single; - model->result_update = mvtvm_ml_result_update; - model->set_error_code = mvtvm_ml_set_error_code; - model->set_poll_addr = mvtvm_ml_set_poll_addr; - model->op_error_get = mvtvm_ml_op_error_get; - } - - return 0; - -error: - mvtvm_ml_tvm_func_free(®ister_cb_fn); - if (model != NULL) { - mvtvm_ml_tvm_func_free(&model->mvtvm.load_params); - mvtvm_ml_tvm_func_free(&model->mvtvm.set_input_zero_copy); - mvtvm_ml_tvm_func_free(&model->mvtvm.set_output_zero_copy); - mvtvm_ml_tvm_func_free(&model->mvtvm.run); - mvtvm_ml_tvm_mod_free(&model->mvtvm.graph_module); - if (model->mvtvm.fd >= 0) - close(model->mvtvm.fd); - memset(&model->mvtvm, 0, sizeof(model->mvtvm)); - model->mvtvm.fd = -1; - } - mvtvm_ml_tvm_mod_free(&module_so); - plt_memzone_free(mz); - - return ret; -} - -int -mvtvm_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) -{ - char str[RTE_MEMZONE_NAMESIZE]; - const struct plt_memzone *mz; - - RTE_SET_USED(cnxk_mldev); - - /* Unload model from TVM runtime */ - if (model->model_id >= cnxk_mldev->mldev->data->nb_models) - return -EINVAL; - - if (model->mvtvm.graph_module == NULL) - return -EINVAL; - - mvtvm_ml_tvm_func_free(&model->mvtvm.load_params); - mvtvm_ml_tvm_func_free(&model->mvtvm.set_input_zero_copy); - mvtvm_ml_tvm_func_free(&model->mvtvm.set_output_zero_copy); - mvtvm_ml_tvm_func_free(&model->mvtvm.run); - mvtvm_ml_tvm_mod_free(&model->mvtvm.graph_module); - if (model->mvtvm.fd >= 0) - close(model->mvtvm.fd); - memset(&model->mvtvm, 0, sizeof(model->mvtvm)); - model->mvtvm.fd = -1; - - snprintf(str, RTE_MEMZONE_NAMESIZE, "%s_%u", MVTVM_ML_MODEL_MEMZONE_NAME, model->model_id); - mz = plt_memzone_lookup(str); - if (mz == NULL) { - plt_err("Memzone lookup failed for TVM model: model_id = %u, mz = %s", - model->model_id, str); - return -EINVAL; - } - - return plt_memzone_free(mz); -} - -int -mvtvm_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) -{ - struct cnxk_ml_layer *layer; - - uint16_t layer_id = 0; - int ret = 0; - -next_layer: - layer = &model->layer[layer_id]; - if (layer->type == ML_CNXK_LAYER_TYPE_MRVL) { - ret = cn10k_ml_layer_start(cnxk_mldev, model->model_id, layer->name); - if (ret != 0) { - plt_err("Layer start failed, model_id = %u, layer_name = %s, error = %d", - model->model_id, layer->name, ret); - return ret; - } - } - layer_id++; - - if (layer_id < model->nb_layers) - goto next_layer; - - return 0; -} - -int -mvtvm_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) -{ - struct cnxk_ml_layer *layer; - - uint16_t layer_id = 0; - int ret = 0; - -next_layer: - layer = &model->layer[layer_id]; - if (layer->type == ML_CNXK_LAYER_TYPE_MRVL) { - ret = cn10k_ml_layer_stop(cnxk_mldev, model->model_id, layer->name); - if (ret != 0) { - plt_err("Layer stop failed, model_id = %u, layer_name = %s, error = %d", - model->model_id, layer->name, ret); - return ret; - } - } - layer_id++; - - if (layer_id < model->nb_layers) - goto next_layer; - - return 0; -} - -int -mvtvm_ml_io_quantize(void *device, uint16_t model_id, const char *layer_name, - const DLTensor **deq_tensor, void *qbuffer) -{ - struct cnxk_ml_io_info *info = NULL; - struct cnxk_ml_dev *cnxk_mldev; - struct cnxk_ml_model *model; - uint16_t layer_id = 0; - uint8_t *lcl_dbuffer; - uint8_t *lcl_qbuffer; - uint32_t i; - int ret; - -#ifdef CNXK_ML_DEV_DEBUG - if ((device == NULL) || (deq_tensor == NULL) || (qbuffer == NULL)) - return -EINVAL; -#endif - - cnxk_mldev = (struct cnxk_ml_dev *)device; - - model = cnxk_mldev->mldev->data->models[model_id]; -#ifdef CNXK_ML_DEV_DEBUG - if (model == NULL) { - plt_err("Invalid model_id = %u", model_id); - return -EINVAL; - } -#endif - - /* Get layer id */ - for (layer_id = 0; layer_id < model->nb_layers; layer_id++) { - if (strcmp(model->layer[layer_id].name, layer_name) == 0) - break; - } - -#ifdef CNXK_ML_DEV_DEBUG - if (layer_id == model->nb_layers) { - plt_err("Invalid layer name: %s", layer_name); - return -EINVAL; - } - - if (model->layer[layer_id].type != ML_CNXK_LAYER_TYPE_MRVL) { - plt_err("Invalid layer name / type: %s", layer_name); - return -EINVAL; - } -#endif - - info = &model->layer[layer_id].info; - lcl_qbuffer = (uint8_t *)qbuffer; - - for (i = 0; i < info->nb_inputs; i++) { - lcl_dbuffer = PLT_PTR_ADD(deq_tensor[i]->data, deq_tensor[i]->byte_offset); - - ret = cnxk_ml_io_quantize_single(&info->input[i], lcl_dbuffer, lcl_qbuffer); - if (ret < 0) - return ret; - - lcl_qbuffer += info->input[i].sz_q; - } - - return 0; -} - -int -mvtvm_ml_io_dequantize(void *device, uint16_t model_id, const char *layer_name, void *qbuffer, - const DLTensor **deq_tensor) -{ - struct cnxk_ml_io_info *info = NULL; - struct cnxk_ml_dev *cnxk_mldev; - struct cnxk_ml_model *model; - uint16_t layer_id = 0; - uint8_t *lcl_dbuffer; - uint8_t *lcl_qbuffer; - uint32_t i; - int ret; - -#ifdef CNXK_ML_DEV_DEBUG - if ((device == NULL) || (deq_tensor == NULL) || (qbuffer == NULL)) - return -EINVAL; -#endif - - cnxk_mldev = (struct cnxk_ml_dev *)device; - - model = cnxk_mldev->mldev->data->models[model_id]; -#ifdef CNXK_ML_DEV_DEBUG - if (model == NULL) { - plt_err("Invalid model_id = %u", model_id); - return -EINVAL; - } -#endif - - for (layer_id = 0; layer_id < model->nb_layers; layer_id++) { - if (strcmp(model->layer[layer_id].name, layer_name) == 0) - break; - } - -#ifdef CNXK_ML_DEV_DEBUG - if (layer_id == model->nb_layers) { - plt_err("Invalid layer name: %s", layer_name); - return -EINVAL; - } - - if (model->layer[layer_id].type != ML_CNXK_LAYER_TYPE_MRVL) { - plt_err("Invalid layer name / type: %s", layer_name); - return -EINVAL; - } -#endif - - info = &model->layer[layer_id].info; - lcl_qbuffer = (uint8_t *)qbuffer; - - for (i = 0; i < info->nb_outputs; i++) { - lcl_dbuffer = PLT_PTR_ADD(deq_tensor[i]->data, deq_tensor[i]->byte_offset); - - ret = cnxk_ml_io_dequantize_single(&info->output[i], lcl_qbuffer, lcl_dbuffer); - if (ret < 0) - return ret; - - lcl_qbuffer += info->output[i].sz_q; - } - - return 0; -} - -static int -mvtvm_ml_model_run(struct cnxk_ml_model *model, struct rte_ml_op *op, struct cnxk_ml_req *req) -{ - uint8_t i; - struct mvtvm_ml_result *run_result; - TVMValue arg_values[2] = {0}; - int arg_types[2] = {0}; - TVMValue ret_value = {0}; - int ret_type = kTVMNullptr; - int ret = 0; - - rte_memcpy(req->mvtvm_req.input_tensor, model->mvtvm.input_tensor, - model->mvtvm.info.nb_inputs * sizeof(DLTensor)); - for (i = 0; i < model->mvtvm.info.nb_inputs; i++) { - req->mvtvm_req.input_tensor[i].data = op->input[i]->addr; - req->mvtvm_req.input_tensor[i].byte_offset = 0; - } - - rte_memcpy(req->mvtvm_req.output_tensor, model->mvtvm.output_tensor, - model->mvtvm.info.nb_outputs * sizeof(DLTensor)); - for (i = 0; i < model->mvtvm.info.nb_outputs; i++) { - req->mvtvm_req.output_tensor[i].data = op->output[i]->addr; - req->mvtvm_req.output_tensor[i].byte_offset = 0; - } - - run_result = &req->mvtvm_req.result; - run_result->stats.start_ns = rte_get_tsc_cycles(); - run_result->error_code = 0; - - for (i = 0; i < model->mvtvm.info.nb_inputs; i++) { - arg_values[0].v_int64 = i; - arg_types[0] = kDLInt; - arg_values[1].v_handle = &req->mvtvm_req.input_tensor[i]; - arg_types[1] = kTVMDLTensorHandle; - ret = mvtvm_ml_tvm_func_call(model, model->mvtvm.set_input_zero_copy, - "set_input_zero_copy", arg_values, arg_types, 2, - &ret_value, &ret_type, kTVMNullptr); - if (ret != 0) - goto out; - } - - for (i = 0; i < model->mvtvm.info.nb_outputs; i++) { - arg_values[0].v_int64 = i; - arg_types[0] = kDLInt; - arg_values[1].v_handle = &req->mvtvm_req.output_tensor[i]; - arg_types[1] = kTVMDLTensorHandle; - ret = mvtvm_ml_tvm_func_call(model, model->mvtvm.set_output_zero_copy, - "set_output_zero_copy", arg_values, arg_types, 2, - &ret_value, &ret_type, kTVMNullptr); - if (ret != 0) - goto out; - } - - ret = mvtvm_ml_tvm_func_call(model, model->mvtvm.run, "run", NULL, NULL, 0, &ret_value, - &ret_type, kTVMNullptr); - if (ret != 0) - goto out; - -out: - run_result->stats.end_ns = rte_get_tsc_cycles(); - req->mvtvm_req.status = 0x1; - - plt_write64(ML_CNXK_POLL_JOB_FINISH, req->status); - - if (ret != 0) - run_result->error_code = -EIO; - - return 0; -} - -__rte_hot void -mvtvm_ml_set_error_code(struct cnxk_ml_req *req, uint64_t etype, uint64_t stype) -{ - RTE_SET_USED(stype); - - req->mvtvm_req.result.error_code = etype; -} - -__rte_hot int -mvtvm_ml_op_error_get(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, - struct rte_ml_op_error *error) -{ - RTE_SET_USED(cnxk_mldev); - RTE_SET_USED(op); - RTE_SET_USED(error); - - return 0; -} - -__rte_hot bool -mvtvm_ml_enqueue_single(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, uint16_t layer_id, - struct cnxk_ml_qp *qp, uint64_t head) -{ - struct cnxk_ml_model *model; - struct cnxk_ml_queue *queue; - struct cnxk_ml_req *req; - - RTE_SET_USED(layer_id); - - queue = &qp->queue; - req = &queue->reqs[head]; - model = cnxk_mldev->mldev->data->models[op->model_id]; - - model->set_poll_addr(req); - memset(&req->mvtvm_req.result, 0, sizeof(struct mvtvm_ml_result)); - req->mvtvm_req.result.error_code = 0x0; - req->mvtvm_req.result.user_ptr = op->user_ptr; - - cnxk_ml_set_poll_ptr(req); - mvtvm_ml_model_run(model, op, req); - req->timeout = plt_tsc_cycles() + queue->wait_cycles; - req->op = op; - - return true; -} - -__rte_hot void -mvtvm_ml_result_update(struct cnxk_ml_dev *cnxk_mldev, int qp_id, void *request) -{ - struct mvtvm_ml_model_xstats *xstats; - struct mvtvm_ml_result *result; - struct cnxk_ml_model *model; - struct cnxk_ml_req *req; - uint64_t tvm_rt_latency; - struct cnxk_ml_qp *qp; - struct rte_ml_op *op; - - req = (struct cnxk_ml_req *)request; - result = &req->mvtvm_req.result; - op = req->op; - qp = cnxk_mldev->mldev->data->queue_pairs[qp_id]; - op->impl_opaque = result->error_code; - - if (likely(result->error_code == 0)) { - qp->stats.dequeued_count++; - op->status = RTE_ML_OP_STATUS_SUCCESS; - - model = cnxk_mldev->mldev->data->models[op->model_id]; - xstats = &model->mvtvm.burst_xstats[qp_id]; - - if (unlikely(xstats->dequeued_count == xstats->tvm_rt_reset_count)) { - xstats->tvm_rt_latency_min = UINT64_MAX; - xstats->tvm_rt_latency_max = 0; - } - tvm_rt_latency = result->stats.end_ns - result->stats.start_ns; - xstats->tvm_rt_latency = tvm_rt_latency; - xstats->tvm_rt_latency_tot += tvm_rt_latency; - xstats->tvm_rt_latency_min = RTE_MIN(xstats->tvm_rt_latency_min, tvm_rt_latency); - xstats->tvm_rt_latency_max = RTE_MAX(xstats->tvm_rt_latency_max, tvm_rt_latency); - xstats->dequeued_count++; - } else { - qp->stats.dequeue_err_count++; - op->status = RTE_ML_OP_STATUS_ERROR; - } -} diff --git a/drivers/ml/cnxk/mvtvm_ml_ops.h b/drivers/ml/cnxk/mvtvm_ml_ops.h index 593f4585b3c..b83246517e7 100644 --- a/drivers/ml/cnxk/mvtvm_ml_ops.h +++ b/drivers/ml/cnxk/mvtvm_ml_ops.h @@ -5,77 +5,11 @@ #ifndef _MVTVM_ML_OPS_H_ #define _MVTVM_ML_OPS_H_ -#include <dlpack/dlpack.h> - #include <rte_mldev.h> #include "cnxk_ml_xstats.h" -#include "mvtvm_ml_model.h" - -struct cnxk_ml_dev; -struct cnxk_ml_model; -struct cnxk_ml_layer; -struct cnxk_ml_qp; -struct cnxk_ml_req; - -/* Inference stats */ -struct mvtvm_ml_stats { - /* Start ns */ - uint64_t start_ns; - - /* Start ns */ - uint64_t end_ns; -}; - -/* Result structure */ -struct mvtvm_ml_result { - /* Job error code */ - uint64_t error_code; - - /* Inference stats */ - struct mvtvm_ml_stats stats; - - /* User context pointer */ - void *user_ptr; -}; - -/* MVTVM specific request */ -struct mvtvm_ml_req { - /* Input tensors */ - DLTensor input_tensor[ML_CNXK_MODEL_MAX_INPUT_OUTPUT]; - - /* Output tensors */ - DLTensor output_tensor[ML_CNXK_MODEL_MAX_INPUT_OUTPUT]; - - /* Status field for poll mode requests */ - volatile uint64_t status; - - /* Result */ - struct mvtvm_ml_result result; -}; int mvtvm_ml_dev_info_get(struct cnxk_ml_dev *mldev, struct rte_ml_dev_info *dev_info); int mvtvm_ml_dev_dump(struct cnxk_ml_dev *cnxk_mldev, FILE *fp); -int mvtvm_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params, - struct cnxk_ml_model *model); -int mvtvm_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); -int mvtvm_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); -int mvtvm_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); -int mvtvm_ml_io_quantize(void *device, uint16_t model_id, const char *layer_name, - const DLTensor **deq_tensor, void *qbuffer); -int mvtvm_ml_io_dequantize(void *device, uint16_t model_id, const char *layer_name, void *qbuffer, - const DLTensor **deq_tensor); - -__rte_hot bool mvtvm_ml_enqueue_single(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, - uint16_t layer_id, struct cnxk_ml_qp *qp, uint64_t head); -__rte_hot int mvtvm_ml_op_error_get(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, - struct rte_ml_op_error *error); -__rte_hot void mvtvm_ml_result_update(struct cnxk_ml_dev *cnxk_mldev, int qp_id, void *request); -__rte_hot void mvtvm_ml_set_error_code(struct cnxk_ml_req *req, uint64_t etype, uint64_t stype); - -void mvtvm_ml_model_xstat_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, - uint16_t stat_id, uint16_t entry, char *suffix); -uint64_t mvtvm_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, - enum cnxk_ml_xstats_type type); #endif /* _MVTVM_ML_OPS_H_ */ diff --git a/drivers/ml/cnxk/mvtvm_ml_stubs.c b/drivers/ml/cnxk/mvtvm_ml_stubs.c index 7c13fac42d6..2341658657f 100644 --- a/drivers/ml/cnxk/mvtvm_ml_stubs.c +++ b/drivers/ml/cnxk/mvtvm_ml_stubs.c @@ -7,65 +7,6 @@ #include "mvtvm_ml_stubs.h" #include "cnxk_ml_dev.h" -#include "cnxk_ml_model.h" -#include "cnxk_ml_xstats.h" - -enum cnxk_ml_model_type -mvtvm_ml_model_type_get(struct rte_ml_model_params *params) -{ - RTE_SET_USED(params); - - return ML_CNXK_MODEL_TYPE_UNKNOWN; -} - -int -mvtvm_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, uint16_t *layer_id) -{ - RTE_SET_USED(model); - RTE_SET_USED(layer_name); - RTE_SET_USED(layer_id); - - return -EINVAL; -} - -struct cnxk_ml_io_info * -mvtvm_ml_model_io_info_get(struct cnxk_ml_model *model, uint16_t layer_id) -{ - RTE_SET_USED(model); - RTE_SET_USED(layer_id); - - return NULL; -} - -void -mvtvm_ml_layer_print(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer, FILE *fp) -{ - RTE_SET_USED(cnxk_mldev); - RTE_SET_USED(layer); - RTE_SET_USED(fp); -} - -void -mvtvm_ml_model_xstat_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, - uint16_t stat_id, uint16_t entry, char *suffix) -{ - RTE_SET_USED(cnxk_mldev); - RTE_SET_USED(model); - RTE_SET_USED(stat_id); - RTE_SET_USED(entry); - RTE_SET_USED(suffix); -} - -uint64_t -mvtvm_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, - enum cnxk_ml_xstats_type type) -{ - RTE_SET_USED(cnxk_mldev); - RTE_SET_USED(model); - RTE_SET_USED(type); - - return 0; -} int mvtvm_ml_dev_info_get(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_dev_info *dev_info) @@ -84,41 +25,3 @@ mvtvm_ml_dev_dump(struct cnxk_ml_dev *cnxk_mldev, FILE *fp) return -EINVAL; } - -int -mvtvm_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params, - struct cnxk_ml_model *model) -{ - RTE_SET_USED(cnxk_mldev); - RTE_SET_USED(params); - RTE_SET_USED(model); - - return -EINVAL; -} - -int -mvtvm_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) -{ - RTE_SET_USED(cnxk_mldev); - RTE_SET_USED(model); - - return -EINVAL; -} - -int -mvtvm_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) -{ - RTE_SET_USED(cnxk_mldev); - RTE_SET_USED(model); - - return -EINVAL; -} - -int -mvtvm_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) -{ - RTE_SET_USED(cnxk_mldev); - RTE_SET_USED(model); - - return -EINVAL; -} diff --git a/drivers/ml/cnxk/mvtvm_ml_stubs.h b/drivers/ml/cnxk/mvtvm_ml_stubs.h index 15985a75bf4..0eebdbb1ed2 100644 --- a/drivers/ml/cnxk/mvtvm_ml_stubs.h +++ b/drivers/ml/cnxk/mvtvm_ml_stubs.h @@ -7,28 +7,9 @@ #include <rte_mldev.h> -#include "cnxk_ml_xstats.h" - struct cnxk_ml_dev; -struct cnxk_ml_model; -struct cnxk_ml_layer; -enum cnxk_ml_model_type mvtvm_ml_model_type_get(struct rte_ml_model_params *params); int mvtvm_ml_dev_info_get(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_dev_info *dev_info); int mvtvm_ml_dev_dump(struct cnxk_ml_dev *cnxk_mldev, FILE *fp); -int mvtvm_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params, - struct cnxk_ml_model *model); -int mvtvm_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); -int mvtvm_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); -int mvtvm_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); - -int mvtvm_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, - uint16_t *layer_id); -struct cnxk_ml_io_info *mvtvm_ml_model_io_info_get(struct cnxk_ml_model *model, uint16_t layer_id); -void mvtvm_ml_layer_print(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer, FILE *fp); -void mvtvm_ml_model_xstat_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, - uint16_t stat_id, uint16_t entry, char *suffix); -uint64_t mvtvm_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, - enum cnxk_ml_xstats_type type); #endif /* _MVTVM_ML_STUBS_H_ */ diff --git a/drivers/ml/cnxk/mvtvm_ml_model.c b/drivers/ml/cnxk/tvmrt_ml_model.c similarity index 72% rename from drivers/ml/cnxk/mvtvm_ml_model.c rename to drivers/ml/cnxk/tvmrt_ml_model.c index e1946607629..9bff424f0d8 100644 --- a/drivers/ml/cnxk/mvtvm_ml_model.c +++ b/drivers/ml/cnxk/tvmrt_ml_model.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2023 Marvell. + * Copyright (c) 2026 Marvell. */ #include <errno.h> @@ -22,13 +22,13 @@ #include "cnxk_ml_utils.h" /* Objects list */ -char mvtvm_object_list[ML_MVTVM_MODEL_OBJECT_MAX][RTE_ML_STR_MAX] = {"mod.so", "mod.json", - "mod.params"}; +static char tvmrt_object_list[ML_TVMRT_MODEL_OBJECT_MAX][RTE_ML_STR_MAX] = {"mod.so", "mod.json", + "mod.params"}; enum cnxk_ml_model_type -mvtvm_ml_model_type_get(struct rte_ml_model_params *params) +tvmrt_ml_model_type_get(struct rte_ml_model_params *params) { - bool object_found[ML_MVTVM_MODEL_OBJECT_MAX] = {false, false, false}; + bool object_found[ML_TVMRT_MODEL_OBJECT_MAX] = {false, false, false}; enum cnxk_ml_model_type model_type; struct archive_entry *entry; struct archive *a; @@ -47,18 +47,18 @@ mvtvm_ml_model_type_get(struct rte_ml_model_params *params) /* Parse buffer for available objects */ while (archive_read_next_header(a, &entry) == ARCHIVE_OK) { - for (i = 0; i < ML_MVTVM_MODEL_OBJECT_MAX; i++) { + for (i = 0; i < ML_TVMRT_MODEL_OBJECT_MAX; i++) { if (!object_found[i] && - (strcmp(archive_entry_pathname(entry), mvtvm_object_list[i]) == 0)) + (strcmp(archive_entry_pathname(entry), tvmrt_object_list[i]) == 0)) object_found[i] = true; } archive_read_data_skip(a); } /* Check if all objects are available */ - for (i = 0; i < ML_MVTVM_MODEL_OBJECT_MAX; i++) { + for (i = 0; i < ML_TVMRT_MODEL_OBJECT_MAX; i++) { if (!object_found[i]) { - plt_err("Object %s not found in archive!", mvtvm_object_list[i]); + plt_err("Object %s not found in archive!", tvmrt_object_list[i]); model_type = ML_CNXK_MODEL_TYPE_INVALID; goto cleanup; } @@ -74,15 +74,15 @@ mvtvm_ml_model_type_get(struct rte_ml_model_params *params) } int -mvtvm_ml_model_blob_parse(struct rte_ml_model_params *params, struct mvtvm_ml_model_object *object) +tvmrt_ml_model_blob_parse(struct rte_ml_model_params *params, struct tvmrt_ml_model_object *object) { - bool object_found[ML_MVTVM_MODEL_OBJECT_MAX] = {false, false, false}; + bool object_found[ML_TVMRT_MODEL_OBJECT_MAX] = {false, false, false}; struct archive_entry *entry; struct archive *a; uint8_t i; int ret; - memset(object, 0, ML_MVTVM_MODEL_OBJECT_MAX * sizeof(*object)); + memset(object, 0, ML_TVMRT_MODEL_OBJECT_MAX * sizeof(*object)); /* Open archive */ a = archive_read_new(); @@ -97,10 +97,10 @@ mvtvm_ml_model_blob_parse(struct rte_ml_model_params *params, struct mvtvm_ml_mo /* Read archive */ while (archive_read_next_header(a, &entry) == ARCHIVE_OK) { - for (i = 0; i < ML_MVTVM_MODEL_OBJECT_MAX; i++) { + for (i = 0; i < ML_TVMRT_MODEL_OBJECT_MAX; i++) { if (!object_found[i] && - (strcmp(archive_entry_pathname(entry), mvtvm_object_list[i]) == 0)) { - memcpy(object[i].name, mvtvm_object_list[i], RTE_ML_STR_MAX); + (strcmp(archive_entry_pathname(entry), tvmrt_object_list[i]) == 0)) { + memcpy(object[i].name, tvmrt_object_list[i], RTE_ML_STR_MAX); object[i].size = archive_entry_size(entry); object[i].buffer = rte_malloc(NULL, object[i].size, 0); if (object[i].buffer == NULL) { @@ -122,9 +122,9 @@ mvtvm_ml_model_blob_parse(struct rte_ml_model_params *params, struct mvtvm_ml_mo } /* Check if all objects are parsed */ - for (i = 0; i < ML_MVTVM_MODEL_OBJECT_MAX; i++) { + for (i = 0; i < ML_TVMRT_MODEL_OBJECT_MAX; i++) { if (!object_found[i]) { - plt_err("Object %s not found in archive!", mvtvm_object_list[i]); + plt_err("Object %s not found in archive!", tvmrt_object_list[i]); ret = -EINVAL; goto error; } @@ -133,7 +133,7 @@ mvtvm_ml_model_blob_parse(struct rte_ml_model_params *params, struct mvtvm_ml_mo goto cleanup; error: - for (i = 0; i < ML_MVTVM_MODEL_OBJECT_MAX; i++) { + for (i = 0; i < ML_TVMRT_MODEL_OBJECT_MAX; i++) { if (object[i].buffer != NULL) rte_free(object[i].buffer); } @@ -146,7 +146,7 @@ mvtvm_ml_model_blob_parse(struct rte_ml_model_params *params, struct mvtvm_ml_mo } int -mvtvm_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, uint16_t *layer_id) +tvmrt_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, uint16_t *layer_id) { uint16_t i; @@ -171,7 +171,7 @@ mvtvm_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, } static void -mvtvm_ml_param_names_free(struct mvtvm_ml_param_names *param_names) +tvmrt_ml_param_names_free(struct tvmrt_ml_param_names *param_names) { size_t i; @@ -184,7 +184,7 @@ mvtvm_ml_param_names_free(struct mvtvm_ml_param_names *param_names) } static int -mvtvm_ml_blob_read_u64(const uint8_t *blob, size_t blob_size, size_t *offset, uint64_t *value) +tvmrt_ml_blob_read_u64(const uint8_t *blob, size_t blob_size, size_t *offset, uint64_t *value) { if (*offset + sizeof(*value) > blob_size) return -EINVAL; @@ -196,8 +196,8 @@ mvtvm_ml_blob_read_u64(const uint8_t *blob, size_t blob_size, size_t *offset, ui } static int -mvtvm_ml_param_names_parse(const uint8_t *blob, size_t blob_size, - struct mvtvm_ml_param_names *param_names) +tvmrt_ml_param_names_parse(const uint8_t *blob, size_t blob_size, + struct tvmrt_ml_param_names *param_names) { uint64_t magic = 0; uint64_t reserved = 0; @@ -208,18 +208,18 @@ mvtvm_ml_param_names_parse(const uint8_t *blob, size_t blob_size, memset(param_names, 0, sizeof(*param_names)); - ret = mvtvm_ml_blob_read_u64(blob, blob_size, &offset, &magic); + ret = tvmrt_ml_blob_read_u64(blob, blob_size, &offset, &magic); if (ret != 0) return ret; if (magic != TVM_NDARRAY_LIST_MAGIC) return -EINVAL; - ret = mvtvm_ml_blob_read_u64(blob, blob_size, &offset, &reserved); + ret = tvmrt_ml_blob_read_u64(blob, blob_size, &offset, &reserved); if (ret != 0) return ret; - ret = mvtvm_ml_blob_read_u64(blob, blob_size, &offset, &count); + ret = tvmrt_ml_blob_read_u64(blob, blob_size, &offset, &count); if (ret != 0) return ret; @@ -234,7 +234,7 @@ mvtvm_ml_param_names_parse(const uint8_t *blob, size_t blob_size, for (i = 0; i < count; i++) { uint64_t name_len = 0; - ret = mvtvm_ml_blob_read_u64(blob, blob_size, &offset, &name_len); + ret = tvmrt_ml_blob_read_u64(blob, blob_size, &offset, &name_len); if (ret != 0) goto error; @@ -261,12 +261,12 @@ mvtvm_ml_param_names_parse(const uint8_t *blob, size_t blob_size, return 0; error: - mvtvm_ml_param_names_free(param_names); + tvmrt_ml_param_names_free(param_names); return ret; } static int -mvtvm_ml_dtype_parse(const char *dtype_str, DLDataType *dtype) +tvmrt_ml_dtype_parse(const char *dtype_str, DLDataType *dtype) { const char *lanes_str; char base[32] = {0}; @@ -321,7 +321,7 @@ mvtvm_ml_dtype_parse(const char *dtype_str, DLDataType *dtype) } static int -mvtvm_ml_model_io_set(struct cnxk_ml_io *io, const char *name, json_t *shape, const char *dtype_str, +tvmrt_ml_model_io_set(struct cnxk_ml_io *io, const char *name, json_t *shape, const char *dtype_str, DLDevice device) { size_t i; @@ -346,7 +346,7 @@ mvtvm_ml_model_io_set(struct cnxk_ml_io *io, const char *name, json_t *shape, co io->shape_i64[i] = json_integer_value(dim); } - ret = mvtvm_ml_dtype_parse(dtype_str, &io->datatype); + ret = tvmrt_ml_dtype_parse(dtype_str, &io->datatype); if (ret != 0) return ret; @@ -357,8 +357,9 @@ mvtvm_ml_model_io_set(struct cnxk_ml_io *io, const char *name, json_t *shape, co return 0; } + static int -mvtvm_ml_json_graph_get_arrays(json_t *json_parsed, json_t **nodes, json_t **arg_nodes, +tvmrt_ml_json_graph_get_arrays(json_t *json_parsed, json_t **nodes, json_t **arg_nodes, json_t **heads, json_t **node_row_ptr, json_t **shape_values, json_t **dtype_values) { @@ -395,9 +396,9 @@ mvtvm_ml_json_graph_get_arrays(json_t *json_parsed, json_t **nodes, json_t **arg } int -mvtvm_ml_model_json_parse(struct cnxk_ml_model *model) +tvmrt_ml_model_json_parse(struct cnxk_ml_model *model) { - struct mvtvm_ml_param_names param_names; + struct tvmrt_ml_param_names param_names; json_error_t json_error; json_t *json_parsed; json_t *json_nodes; @@ -414,16 +415,16 @@ mvtvm_ml_model_json_parse(struct cnxk_ml_model *model) size_t j; /* Parse param names from params buffer */ - ret = mvtvm_ml_param_names_parse(model->mvtvm.params.buffer, model->mvtvm.params.size, + ret = tvmrt_ml_param_names_parse(model->tvmrt.params.buffer, model->tvmrt.params.size, ¶m_names); if (ret != 0) return ret; /* Load JSON graph (single load for both stages) */ - json_parsed = json_loadb((const char *)model->mvtvm.json.buffer, model->mvtvm.json.size, 0, + json_parsed = json_loadb((const char *)model->tvmrt.json.buffer, model->tvmrt.json.size, 0, &json_error); if (json_parsed == NULL) { - mvtvm_ml_param_names_free(¶m_names); + tvmrt_ml_param_names_free(¶m_names); plt_err("TVM runtime: Failed to parse JSON graph, model_id = %u", model->model_id); return -EINVAL; } @@ -522,12 +523,12 @@ mvtvm_ml_model_json_parse(struct cnxk_ml_model *model) device.device_type = kDLCPU; device.device_id = 0; - ret = mvtvm_ml_json_graph_get_arrays(json_parsed, &json_nodes, &json_arg_nodes, &json_heads, + ret = tvmrt_ml_json_graph_get_arrays(json_parsed, &json_nodes, &json_arg_nodes, &json_heads, &json_node_row_ptr, &json_shape_values, &json_dtype_values); if (ret == 0) { - model->mvtvm.info.nb_inputs = 0; - model->mvtvm.info.nb_outputs = 0; + model->tvmrt.info.nb_inputs = 0; + model->tvmrt.info.nb_outputs = 0; for (i = 0; i < json_array_size(json_arg_nodes); i++) { json_t *arg_node_idx = json_array_get(json_arg_nodes, i); @@ -561,18 +562,18 @@ mvtvm_ml_model_json_parse(struct cnxk_ml_model *model) if (j < param_names.count) continue; - if (model->mvtvm.info.nb_inputs >= ML_CNXK_MODEL_MAX_INPUT_OUTPUT) { + if (model->tvmrt.info.nb_inputs >= ML_CNXK_MODEL_MAX_INPUT_OUTPUT) { ret = -ENOTSUP; break; } - ret = mvtvm_ml_model_io_set( - &model->mvtvm.info.input[model->mvtvm.info.nb_inputs], + ret = tvmrt_ml_model_io_set( + &model->tvmrt.info.input[model->tvmrt.info.nb_inputs], json_string_value(name), shape, json_string_value(dtype), device); if (ret != 0) break; - model->mvtvm.info.nb_inputs++; + model->tvmrt.info.nb_inputs++; } for (i = 0; ret == 0 && i < json_array_size(json_heads); i++) { @@ -641,18 +642,18 @@ mvtvm_ml_model_json_parse(struct cnxk_ml_model *model) break; } - if (model->mvtvm.info.nb_outputs >= ML_CNXK_MODEL_MAX_INPUT_OUTPUT) { + if (model->tvmrt.info.nb_outputs >= ML_CNXK_MODEL_MAX_INPUT_OUTPUT) { ret = -ENOTSUP; break; } - ret = mvtvm_ml_model_io_set( - &model->mvtvm.info.output[model->mvtvm.info.nb_outputs], + ret = tvmrt_ml_model_io_set( + &model->tvmrt.info.output[model->tvmrt.info.nb_outputs], json_string_value(name), shape, json_string_value(dtype), device); if (ret != 0) break; - model->mvtvm.info.nb_outputs++; + model->tvmrt.info.nb_outputs++; } } @@ -661,17 +662,17 @@ mvtvm_ml_model_json_parse(struct cnxk_ml_model *model) model->model_id, ret); json_decref(json_parsed); - mvtvm_ml_param_names_free(¶m_names); + tvmrt_ml_param_names_free(¶m_names); return ret; error: json_decref(json_parsed); - mvtvm_ml_param_names_free(¶m_names); + tvmrt_ml_param_names_free(¶m_names); return ret; } static enum rte_ml_io_type -mvtvm_ml_io_type_map(DLDataType dltype) +tvmrt_ml_io_type_map(DLDataType dltype) { switch (dltype.code) { case kDLInt: @@ -714,7 +715,7 @@ mvtvm_ml_io_type_map(DLDataType dltype) } void -mvtvm_ml_model_io_info_set(struct cnxk_ml_model *model) +tvmrt_ml_model_io_info_set(struct cnxk_ml_model *model) { int32_t i; uint32_t j; @@ -723,97 +724,97 @@ mvtvm_ml_model_io_info_set(struct cnxk_ml_model *model) goto tvm_mrvl_model; /* Inputs, set for layer_id = 0 */ - model->mvtvm.info.total_input_sz_d = 0; - model->mvtvm.info.total_input_sz_q = 0; - for (i = 0; i < model->mvtvm.info.nb_inputs; i++) { - model->mvtvm.info.input[i].dtype = - mvtvm_ml_io_type_map(model->mvtvm.info.input[i].datatype); - model->mvtvm.info.input[i].qtype = - mvtvm_ml_io_type_map(model->mvtvm.info.input[i].model_datatype); - - model->mvtvm.info.input[i].nb_elements = 1; - for (j = 0; j < model->mvtvm.info.input[i].nb_dims; j++) { - model->mvtvm.info.input[i].shape[j] = - PLT_U32_CAST(model->mvtvm.info.input[i].shape_i64[j]); - model->mvtvm.info.input[i].nb_elements *= - model->mvtvm.info.input[i].shape[j]; + model->tvmrt.info.total_input_sz_d = 0; + model->tvmrt.info.total_input_sz_q = 0; + for (i = 0; i < model->tvmrt.info.nb_inputs; i++) { + model->tvmrt.info.input[i].dtype = + tvmrt_ml_io_type_map(model->tvmrt.info.input[i].datatype); + model->tvmrt.info.input[i].qtype = + tvmrt_ml_io_type_map(model->tvmrt.info.input[i].model_datatype); + + model->tvmrt.info.input[i].nb_elements = 1; + for (j = 0; j < model->tvmrt.info.input[i].nb_dims; j++) { + model->tvmrt.info.input[i].shape[j] = + PLT_U32_CAST(model->tvmrt.info.input[i].shape_i64[j]); + model->tvmrt.info.input[i].nb_elements *= + model->tvmrt.info.input[i].shape[j]; } - model->mvtvm.info.input[i].sz_d = - model->mvtvm.info.input[i].nb_elements * - rte_ml_io_type_size_get(model->mvtvm.info.input[i].dtype); - model->mvtvm.info.input[i].sz_q = - model->mvtvm.info.input[i].nb_elements * - rte_ml_io_type_size_get(model->mvtvm.info.input[i].qtype); + model->tvmrt.info.input[i].sz_d = + model->tvmrt.info.input[i].nb_elements * + rte_ml_io_type_size_get(model->tvmrt.info.input[i].dtype); + model->tvmrt.info.input[i].sz_q = + model->tvmrt.info.input[i].nb_elements * + rte_ml_io_type_size_get(model->tvmrt.info.input[i].qtype); - model->mvtvm.info.total_input_sz_d += model->mvtvm.info.input[i].sz_d; - model->mvtvm.info.total_input_sz_q += model->mvtvm.info.input[i].sz_q; + model->tvmrt.info.total_input_sz_d += model->tvmrt.info.input[i].sz_d; + model->tvmrt.info.total_input_sz_q += model->tvmrt.info.input[i].sz_q; - model->mvtvm.input_tensor[i].device = model->mvtvm.info.input[i].device; - model->mvtvm.input_tensor[i].ndim = model->mvtvm.info.input[i].nb_dims; - model->mvtvm.input_tensor[i].dtype = model->mvtvm.info.input[i].datatype; - model->mvtvm.input_tensor[i].shape = model->mvtvm.info.input[i].shape_i64; - model->mvtvm.input_tensor[i].strides = NULL; - model->mvtvm.input_tensor[i].byte_offset = 0; + model->tvmrt.input_tensor[i].device = model->tvmrt.info.input[i].device; + model->tvmrt.input_tensor[i].ndim = model->tvmrt.info.input[i].nb_dims; + model->tvmrt.input_tensor[i].dtype = model->tvmrt.info.input[i].datatype; + model->tvmrt.input_tensor[i].shape = model->tvmrt.info.input[i].shape_i64; + model->tvmrt.input_tensor[i].strides = NULL; + model->tvmrt.input_tensor[i].byte_offset = 0; plt_ml_dbg("model_id = %u, input[%u] - sz_d = %u sz_q = %u", model->model_id, i, - model->mvtvm.info.input[i].sz_d, model->mvtvm.info.input[i].sz_q); + model->tvmrt.info.input[i].sz_d, model->tvmrt.info.input[i].sz_q); } /* Outputs, set for nb_layers - 1 */ - model->mvtvm.info.total_output_sz_d = 0; - model->mvtvm.info.total_output_sz_q = 0; - for (i = 0; i < model->mvtvm.info.nb_outputs; i++) { - model->mvtvm.info.output[i].dtype = - mvtvm_ml_io_type_map(model->mvtvm.info.output[i].datatype); - model->mvtvm.info.output[i].qtype = - mvtvm_ml_io_type_map(model->mvtvm.info.output[i].model_datatype); - - model->mvtvm.info.output[i].nb_elements = 1; - for (j = 0; j < model->mvtvm.info.output[i].nb_dims; j++) { - model->mvtvm.info.output[i].shape[j] = - PLT_U32_CAST(model->mvtvm.info.output[i].shape_i64[j]); - model->mvtvm.info.output[i].nb_elements *= - model->mvtvm.info.output[i].shape[j]; + model->tvmrt.info.total_output_sz_d = 0; + model->tvmrt.info.total_output_sz_q = 0; + for (i = 0; i < model->tvmrt.info.nb_outputs; i++) { + model->tvmrt.info.output[i].dtype = + tvmrt_ml_io_type_map(model->tvmrt.info.output[i].datatype); + model->tvmrt.info.output[i].qtype = + tvmrt_ml_io_type_map(model->tvmrt.info.output[i].model_datatype); + + model->tvmrt.info.output[i].nb_elements = 1; + for (j = 0; j < model->tvmrt.info.output[i].nb_dims; j++) { + model->tvmrt.info.output[i].shape[j] = + PLT_U32_CAST(model->tvmrt.info.output[i].shape_i64[j]); + model->tvmrt.info.output[i].nb_elements *= + model->tvmrt.info.output[i].shape[j]; } - model->mvtvm.info.output[i].sz_d = - model->mvtvm.info.output[i].nb_elements * - rte_ml_io_type_size_get(model->mvtvm.info.output[i].dtype); - model->mvtvm.info.output[i].sz_q = - model->mvtvm.info.output[i].nb_elements * - rte_ml_io_type_size_get(model->mvtvm.info.output[i].qtype); + model->tvmrt.info.output[i].sz_d = + model->tvmrt.info.output[i].nb_elements * + rte_ml_io_type_size_get(model->tvmrt.info.output[i].dtype); + model->tvmrt.info.output[i].sz_q = + model->tvmrt.info.output[i].nb_elements * + rte_ml_io_type_size_get(model->tvmrt.info.output[i].qtype); - model->mvtvm.info.total_output_sz_d += model->mvtvm.info.output[i].sz_d; - model->mvtvm.info.total_output_sz_q += model->mvtvm.info.output[i].sz_q; + model->tvmrt.info.total_output_sz_d += model->tvmrt.info.output[i].sz_d; + model->tvmrt.info.total_output_sz_q += model->tvmrt.info.output[i].sz_q; - model->mvtvm.output_tensor[i].device = model->mvtvm.info.output[i].device; - model->mvtvm.output_tensor[i].ndim = model->mvtvm.info.output[i].nb_dims; - model->mvtvm.output_tensor[i].dtype = model->mvtvm.info.output[i].datatype; - model->mvtvm.output_tensor[i].shape = model->mvtvm.info.output[i].shape_i64; - model->mvtvm.output_tensor[i].strides = NULL; - model->mvtvm.output_tensor[i].byte_offset = 0; + model->tvmrt.output_tensor[i].device = model->tvmrt.info.output[i].device; + model->tvmrt.output_tensor[i].ndim = model->tvmrt.info.output[i].nb_dims; + model->tvmrt.output_tensor[i].dtype = model->tvmrt.info.output[i].datatype; + model->tvmrt.output_tensor[i].shape = model->tvmrt.info.output[i].shape_i64; + model->tvmrt.output_tensor[i].strides = NULL; + model->tvmrt.output_tensor[i].byte_offset = 0; plt_ml_dbg("model_id = %u, output[%u] - sz_d = %u sz_q = %u", model->model_id, i, - model->mvtvm.info.output[i].sz_d, model->mvtvm.info.output[i].sz_q); + model->tvmrt.info.output[i].sz_d, model->tvmrt.info.output[i].sz_q); } return; tvm_mrvl_model: - cn10k_ml_layer_io_info_set(&model->mvtvm.info, &model->layer[0].glow.metadata); + cn10k_ml_layer_io_info_set(&model->tvmrt.info, &model->layer[0].glow.metadata); } struct cnxk_ml_io_info * -mvtvm_ml_model_io_info_get(struct cnxk_ml_model *model, uint16_t layer_id) +tvmrt_ml_model_io_info_get(struct cnxk_ml_model *model, uint16_t layer_id) { RTE_SET_USED(layer_id); - return &model->mvtvm.info; + return &model->tvmrt.info; } void -mvtvm_ml_model_info_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) +tvmrt_ml_model_info_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) { struct rte_ml_model_info *info; struct rte_ml_io_info *output; @@ -837,43 +838,43 @@ mvtvm_ml_model_info_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *mo info->io_layout = RTE_ML_IO_LAYOUT_SPLIT; info->min_batches = model->batch_size; info->max_batches = model->batch_size; - info->nb_inputs = model->mvtvm.info.nb_inputs; + info->nb_inputs = model->tvmrt.info.nb_inputs; info->input_info = input; - info->nb_outputs = model->mvtvm.info.nb_outputs; + info->nb_outputs = model->tvmrt.info.nb_outputs; info->output_info = output; info->wb_size = 0; /* Set input info */ for (i = 0; i < info->nb_inputs; i++) { - rte_memcpy(input[i].name, model->mvtvm.info.input[i].name, MRVL_ML_INPUT_NAME_LEN); - input[i].nb_dims = model->mvtvm.info.input[i].nb_dims; - input[i].shape = &model->mvtvm.info.input[i].shape[0]; - input[i].type = model->mvtvm.info.input[i].qtype; - input[i].nb_elements = model->mvtvm.info.input[i].nb_elements; - input[i].size = model->mvtvm.info.input[i].nb_elements * - rte_ml_io_type_size_get(model->mvtvm.info.input[i].qtype); - input[i].scale = model->mvtvm.info.input[i].scale; + rte_memcpy(input[i].name, model->tvmrt.info.input[i].name, MRVL_ML_INPUT_NAME_LEN); + input[i].nb_dims = model->tvmrt.info.input[i].nb_dims; + input[i].shape = &model->tvmrt.info.input[i].shape[0]; + input[i].type = model->tvmrt.info.input[i].qtype; + input[i].nb_elements = model->tvmrt.info.input[i].nb_elements; + input[i].size = model->tvmrt.info.input[i].nb_elements * + rte_ml_io_type_size_get(model->tvmrt.info.input[i].qtype); + input[i].scale = model->tvmrt.info.input[i].scale; input[i].zero_point = 0; } /* Set output info */ for (i = 0; i < info->nb_outputs; i++) { - rte_memcpy(output[i].name, model->mvtvm.info.output[i].name, + rte_memcpy(output[i].name, model->tvmrt.info.output[i].name, MRVL_ML_OUTPUT_NAME_LEN); - output[i].nb_dims = model->mvtvm.info.output[i].nb_dims; - output[i].shape = &model->mvtvm.info.output[i].shape[0]; - output[i].type = model->mvtvm.info.output[i].qtype; - output[i].nb_elements = model->mvtvm.info.output[i].nb_elements; - output[i].size = model->mvtvm.info.output[i].nb_elements * - rte_ml_io_type_size_get(model->mvtvm.info.output[i].qtype); - output[i].scale = model->mvtvm.info.output[i].scale; + output[i].nb_dims = model->tvmrt.info.output[i].nb_dims; + output[i].shape = &model->tvmrt.info.output[i].shape[0]; + output[i].type = model->tvmrt.info.output[i].qtype; + output[i].nb_elements = model->tvmrt.info.output[i].nb_elements; + output[i].size = model->tvmrt.info.output[i].nb_elements * + rte_ml_io_type_size_get(model->tvmrt.info.output[i].qtype); + output[i].scale = model->tvmrt.info.output[i].scale; output[i].zero_point = 0; } return; tvm_mrvl_model: - cn10k_ml_model_info_set(cnxk_mldev, model, &model->mvtvm.info, + cn10k_ml_model_info_set(cnxk_mldev, model, &model->tvmrt.info, &model->layer[0].glow.metadata); strlcpy(info->name, model->name, RTE_ML_STR_MAX); @@ -882,7 +883,7 @@ mvtvm_ml_model_info_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *mo } void -mvtvm_ml_layer_print(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer, FILE *fp) +tvmrt_ml_layer_print(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer, FILE *fp) { char str[STR_LEN]; uint8_t i; diff --git a/drivers/ml/cnxk/mvtvm_ml_model.h b/drivers/ml/cnxk/tvmrt_ml_model.h similarity index 78% rename from drivers/ml/cnxk/mvtvm_ml_model.h rename to drivers/ml/cnxk/tvmrt_ml_model.h index 1151b0a39ed..bf35dadb8a6 100644 --- a/drivers/ml/cnxk/mvtvm_ml_model.h +++ b/drivers/ml/cnxk/tvmrt_ml_model.h @@ -1,9 +1,9 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2023 Marvell. + * Copyright (c) 2026 Marvell. */ -#ifndef _MVTVM_ML_MODEL_H_ -#define _MVTVM_ML_MODEL_H_ +#ifndef _TVMRT_ML_MODEL_H_ +#define _TVMRT_ML_MODEL_H_ #include <dlpack/dlpack.h> #include <jansson.h> @@ -30,19 +30,19 @@ struct cnxk_ml_model; struct cnxk_ml_layer; /* Maximum number of objects per model */ -#define ML_MVTVM_MODEL_OBJECT_MAX 3 +#define ML_TVMRT_MODEL_OBJECT_MAX 3 /* Magic number for TVM parameter blobs. */ #define TVM_NDARRAY_LIST_MAGIC 0xF7E58D4F05049CB7ULL /* TVM parameter names structure */ -struct mvtvm_ml_param_names { +struct tvmrt_ml_param_names { char **name; size_t count; }; /* TVM object / artifact info structure */ -struct mvtvm_ml_model_object { +struct tvmrt_ml_model_object { /* Name */ char name[RTE_ML_STR_MAX]; @@ -85,7 +85,7 @@ struct tvmrt_glow_callback { }; /* Model fast-path stats */ -struct mvtvm_ml_model_xstats { +struct tvmrt_ml_model_xstats { /* Total TVM runtime latency, sum of all inferences */ uint64_t tvm_rt_latency_tot; @@ -105,11 +105,11 @@ struct mvtvm_ml_model_xstats { uint64_t tvm_rt_reset_count; }; -struct mvtvm_ml_model_data { +struct tvmrt_ml_model_data { /* Model objects */ - struct mvtvm_ml_model_object so; - struct mvtvm_ml_model_object json; - struct mvtvm_ml_model_object params; + struct tvmrt_ml_model_object so; + struct tvmrt_ml_model_object json; + struct tvmrt_ml_model_object params; /* TVM runtime callbacks */ struct tvmrt_glow_callback cb; @@ -130,7 +130,7 @@ struct mvtvm_ml_model_data { struct cnxk_ml_io_info info; /* Stats for burst ops */ - struct mvtvm_ml_model_xstats *burst_xstats; + struct tvmrt_ml_model_xstats *burst_xstats; /* Input Tensor */ DLTensor input_tensor[ML_CNXK_MODEL_MAX_INPUT_OUTPUT]; @@ -139,15 +139,15 @@ struct mvtvm_ml_model_data { DLTensor output_tensor[ML_CNXK_MODEL_MAX_INPUT_OUTPUT]; }; -enum cnxk_ml_model_type mvtvm_ml_model_type_get(struct rte_ml_model_params *params); -int mvtvm_ml_model_blob_parse(struct rte_ml_model_params *params, - struct mvtvm_ml_model_object *object); -int mvtvm_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, +enum cnxk_ml_model_type tvmrt_ml_model_type_get(struct rte_ml_model_params *params); +int tvmrt_ml_model_blob_parse(struct rte_ml_model_params *params, + struct tvmrt_ml_model_object *object); +int tvmrt_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, uint16_t *layer_id); -void mvtvm_ml_model_io_info_set(struct cnxk_ml_model *model); -struct cnxk_ml_io_info *mvtvm_ml_model_io_info_get(struct cnxk_ml_model *model, uint16_t layer_id); -void mvtvm_ml_model_info_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); -void mvtvm_ml_layer_print(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer, FILE *fp); -int mvtvm_ml_model_json_parse(struct cnxk_ml_model *model); +void tvmrt_ml_model_io_info_set(struct cnxk_ml_model *model); +struct cnxk_ml_io_info *tvmrt_ml_model_io_info_get(struct cnxk_ml_model *model, uint16_t layer_id); +void tvmrt_ml_model_info_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); +void tvmrt_ml_layer_print(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer, FILE *fp); +int tvmrt_ml_model_json_parse(struct cnxk_ml_model *model); -#endif /* _MVTVM_ML_MODEL_H_ */ +#endif /* _TVMRT_ML_MODEL_H_ */ diff --git a/drivers/ml/cnxk/tvmrt_ml_ops.c b/drivers/ml/cnxk/tvmrt_ml_ops.c new file mode 100644 index 00000000000..f6b3a51c51e --- /dev/null +++ b/drivers/ml/cnxk/tvmrt_ml_ops.c @@ -0,0 +1,818 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Marvell. + */ + +#include <errno.h> +#include <linux/limits.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <unistd.h> + +#include <dlpack/dlpack.h> +#include <jansson.h> + +#include <rte_common.h> +#include <rte_cycles.h> +#include <rte_mldev.h> +#include <rte_mldev_pmd.h> + +#include <mldev_utils.h> + +#include "cnxk_ml_dev.h" +#include "cnxk_ml_model.h" +#include "cnxk_ml_ops.h" +#include "cnxk_ml_xstats.h" + +/* ML model macros */ +#define TVMRT_ML_MODEL_MEMZONE_NAME "ml_tvmrt_model_mz" + +/* Shared memory file descriptor name */ +#define ML_MODEL_SHMFD_NAME "tvmrt_shmfd" + +/* Shared memory file descriptor path */ +#define ML_MODEL_SHMFD_PATH "/proc/%d/fd/%d" + +static int +tvmrt_ml_tvm_func_get(struct cnxk_ml_model *model, TVMModuleHandle module, const char *name, + TVMFunctionHandle *func) +{ + int ret; + + ret = TVMModGetFunction(module, name, 0, func); + if (ret != 0) { + plt_err("Model load failed, model_id = %u, ret = %d, msg = %s", model->model_id, + ret, TVMGetLastError()); + return ret; + } + + if (*func == NULL) { + ret = -ENOENT; + plt_err("Model load failed, model_id = %u, function '%s' not found", + model->model_id, name); + } + + return ret; +} + +static int +tvmrt_ml_tvm_func_call(struct cnxk_ml_model *model, TVMFunctionHandle func, const char *name, + TVMValue *values, int *types, int num_args, TVMValue *ret_val, int *ret_type, + int ret_type_code) +{ + int ret; + + ret = TVMFuncCall(func, values, types, num_args, ret_val, ret_type); + if (ret != 0) { + plt_err("Error calling TVM function '%s', model_id = %u, ret = %d, msg = %s", name, + model->model_id, ret, TVMGetLastError()); + return ret; + } + + if (*ret_type != ret_type_code) { + ret = -EINVAL; + plt_err("TVM function '%s' returned unexpected type, model_id = %u, expected = %d, " + "actual = %d", + name, model->model_id, ret_type_code, *ret_type); + } + + return ret; +} + +static void +tvmrt_ml_tvm_func_free(TVMFunctionHandle *func) +{ + if ((func != NULL) && (*func != NULL)) { + TVMFuncFree(*func); + *func = NULL; + } +} + +static void +tvmrt_ml_tvm_mod_free(TVMModuleHandle *mod) +{ + if ((mod != NULL) && (*mod != NULL)) { + TVMModFree(*mod); + *mod = NULL; + } +} + +__rte_hot static void +tvmrt_ml_set_poll_addr(struct cnxk_ml_req *req) +{ + req->status = &req->tvmrt_req.status; +} + +void +tvmrt_ml_model_xstat_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, + uint16_t stat_id, uint16_t entry, char *suffix) +{ + snprintf(cnxk_mldev->xstats.entries[stat_id].map.name, + sizeof(cnxk_mldev->xstats.entries[stat_id].map.name), "%s-%s-%s", model->name, + model_xstats[entry].name, suffix); +} + +#define ML_AVG_FOREACH_QP_TVMRT(cnxk_mldev, model, qp_id, value, count) \ + do { \ + value = 0; \ + for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \ + value += model->tvmrt.burst_xstats[qp_id].tvm_rt_latency_tot; \ + count += model->tvmrt.burst_xstats[qp_id].dequeued_count - \ + model->tvmrt.burst_xstats[qp_id].tvm_rt_reset_count; \ + } \ + if (count != 0) \ + value = value / count; \ + } while (0) + +#define ML_MIN_FOREACH_QP_TVMRT(cnxk_mldev, model, qp_id, value, count) \ + do { \ + value = UINT64_MAX; \ + for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \ + value = PLT_MIN(value, \ + model->tvmrt.burst_xstats[qp_id].tvm_rt_latency_min); \ + count += model->tvmrt.burst_xstats[qp_id].dequeued_count - \ + model->tvmrt.burst_xstats[qp_id].tvm_rt_reset_count; \ + } \ + if (count == 0) \ + value = 0; \ + } while (0) + +#define ML_MAX_FOREACH_QP_TVMRT(cnxk_mldev, model, qp_id, value, count) \ + do { \ + value = 0; \ + for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \ + value = PLT_MAX(value, \ + model->tvmrt.burst_xstats[qp_id].tvm_rt_latency_max); \ + count += model->tvmrt.burst_xstats[qp_id].dequeued_count - \ + model->tvmrt.burst_xstats[qp_id].tvm_rt_reset_count; \ + } \ + if (count == 0) \ + value = 0; \ + } while (0) + +uint64_t +tvmrt_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, + enum cnxk_ml_xstats_type type) +{ + uint64_t count = 0; + uint64_t value = 0; + uint32_t qp_id; + + switch (type) { + case avg_rt_latency: + ML_AVG_FOREACH_QP_TVMRT(cnxk_mldev, model, qp_id, value, count); + break; + case min_rt_latency: + ML_MIN_FOREACH_QP_TVMRT(cnxk_mldev, model, qp_id, value, count); + break; + case max_rt_latency: + ML_MAX_FOREACH_QP_TVMRT(cnxk_mldev, model, qp_id, value, count); + break; + default: + value = 0; + } + + return value; +} + +int +tvmrt_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params, + struct cnxk_ml_model *model) +{ + struct tvmrt_ml_model_object object[ML_TVMRT_MODEL_OBJECT_MAX]; + struct tvmrt_glow_callback *callback; + char str[RTE_MEMZONE_NAMESIZE]; + char path[PATH_MAX]; + const struct plt_memzone *mz; + size_t model_object_size = 0; + size_t model_xstats_size = 0; + uint64_t mz_size = 0; + TVMFunctionHandle create_fn = NULL; + TVMFunctionHandle register_cb_fn = NULL; + TVMModuleHandle module_so = NULL; + TVMByteArray tvm_params; + TVMValue ret_value = {0}; + TVMValue arg_values[4] = {0}; + TVMValue tvm_arg_values[1] = {0}; + int ret_type = kTVMNullptr; + int arg_types[4] = {0}; + int tvm_arg_types[1] = {0}; + DLDevice device; + int ret; + + RTE_SET_USED(cnxk_mldev); + model->tvmrt.fd = -1; + + ret = tvmrt_ml_model_blob_parse(params, object); + if (ret != 0) + return ret; + + model_object_size = RTE_ALIGN_CEIL(object[0].size, RTE_CACHE_LINE_MIN_SIZE) + + RTE_ALIGN_CEIL(object[1].size, RTE_CACHE_LINE_MIN_SIZE) + + RTE_ALIGN_CEIL(object[2].size, RTE_CACHE_LINE_MIN_SIZE); + + model_xstats_size = + cnxk_mldev->mldev->data->nb_queue_pairs * sizeof(struct tvmrt_ml_model_xstats); + + mz_size += model_object_size + model_xstats_size; + + /* Allocate memzone for model object */ + snprintf(str, RTE_MEMZONE_NAMESIZE, "%s_%u", TVMRT_ML_MODEL_MEMZONE_NAME, model->model_id); + mz = plt_memzone_reserve_aligned(str, mz_size, 0, ML_CN10K_ALIGN_SIZE); + if (!mz) { + plt_err("plt_memzone_reserve failed : %s", str); + return -ENOMEM; + } + + /* Copy mod.so */ + model->tvmrt.so.buffer = mz->addr; + model->tvmrt.so.size = object[0].size; + rte_memcpy(model->tvmrt.so.name, object[0].name, RTE_ML_STR_MAX); + rte_memcpy(model->tvmrt.so.buffer, object[0].buffer, object[0].size); + rte_free(object[0].buffer); + + /* Copy mod.json */ + model->tvmrt.json.buffer = + RTE_PTR_ADD(model->tvmrt.so.buffer, + RTE_ALIGN_CEIL(model->tvmrt.so.size, RTE_CACHE_LINE_MIN_SIZE)); + model->tvmrt.json.size = object[1].size; + rte_memcpy(model->tvmrt.json.name, object[1].name, RTE_ML_STR_MAX); + rte_memcpy(model->tvmrt.json.buffer, object[1].buffer, object[1].size); + rte_free(object[1].buffer); + + /* Copy mod.params */ + model->tvmrt.params.buffer = + RTE_PTR_ADD(model->tvmrt.json.buffer, + RTE_ALIGN_CEIL(model->tvmrt.json.size, RTE_CACHE_LINE_MIN_SIZE)); + model->tvmrt.params.size = object[2].size; + rte_memcpy(model->tvmrt.params.name, object[2].name, RTE_ML_STR_MAX); + rte_memcpy(model->tvmrt.params.buffer, object[2].buffer, object[2].size); + rte_free(object[2].buffer); + + ret = tvmrt_ml_model_json_parse(model); + if (ret != 0) + goto error; + + if (cnxk_mldev->type == CNXK_ML_DEV_TYPE_VDEV && + model->subtype != ML_CNXK_MODEL_SUBTYPE_TVM_LLVM) { + plt_err("Unsupported model sub-type"); + ret = -ENOTSUP; + goto error; + } + + /* Set callback function array */ + if (model->subtype != ML_CNXK_MODEL_SUBTYPE_TVM_LLVM) { + callback = &model->tvmrt.cb; + callback->tvmrt_glow_layer_load = cn10k_ml_layer_load; + callback->tvmrt_glow_layer_unload = cn10k_ml_layer_unload; + callback->tvmrt_io_alloc = cn10k_ml_io_alloc; + callback->tvmrt_io_free = cn10k_ml_io_free; + callback->tvmrt_malloc = cn10k_ml_malloc; + callback->tvmrt_free = cn10k_ml_free; + callback->tvmrt_quantize = tvmrt_ml_io_quantize; + callback->tvmrt_dequantize = tvmrt_ml_io_dequantize; + callback->tvmrt_inference = cn10k_ml_inference_sync; + } else { + callback = NULL; + } + + /* Initialize model in TVM runtime */ + if (model->tvmrt.graph_module != NULL) { + ret = -EBUSY; + plt_err("TVM runtime: Model load failed, model_id = %u, error = %d", + model->model_id, ret); + goto error; + } + + snprintf(path, sizeof(path), "%s_%d_%u", ML_MODEL_SHMFD_NAME, getpid(), model->model_id); + model->tvmrt.fd = memfd_create(path, 0); + if (model->tvmrt.fd < 0) { + ret = (errno == 0) ? -EIO : -errno; + plt_err("TVM runtime: Model load failed, model_id = %u, error = %d", + model->model_id, ret); + goto error; + } + + if (write(model->tvmrt.fd, model->tvmrt.so.buffer, model->tvmrt.so.size) != + (ssize_t)model->tvmrt.so.size) { + ret = (errno == 0) ? -EIO : -errno; + plt_err("TVM runtime: Model load failed, model_id = %u, error = %d", + model->model_id, ret); + goto error; + } + + if (lseek(model->tvmrt.fd, 0, SEEK_SET) < 0) { + ret = (errno == 0) ? -EIO : -errno; + plt_err("TVM runtime: Model load failed, model_id = %u, error = %d", + model->model_id, ret); + goto error; + } + + snprintf(path, sizeof(path), ML_MODEL_SHMFD_PATH, getpid(), model->tvmrt.fd); + ret = TVMModLoadFromFile(path, "so", &module_so); + if (ret != 0) { + plt_err("TVM runtime: Model load failed, model_id = %u, ret = %d, msg = %s", + model->model_id, ret, TVMGetLastError()); + goto error; + } + + /* Set device info */ + device.device_type = kDLCPU; + device.device_id = 0; + + if (callback != NULL) { + ret = tvmrt_ml_tvm_func_get(model, module_so, "register_cb", ®ister_cb_fn); + if (ret != 0) + goto error; + + arg_values[0].v_handle = callback; + arg_types[0] = kTVMOpaqueHandle; + arg_values[1].v_handle = cnxk_mldev; + arg_types[1] = kTVMOpaqueHandle; + arg_values[2].v_int64 = model->model_id; + arg_types[2] = kDLInt; + + ret = tvmrt_ml_tvm_func_call(model, register_cb_fn, "register_cb", arg_values, + arg_types, 3, &ret_value, &ret_type, kTVMNullptr); + if (ret != 0) + goto error; + } + + ret = TVMFuncGetGlobal("tvm.graph_executor.create", &create_fn); + if (ret != 0) { + plt_err("TVM runtime: Model load failed, model_id = %u, ret = %d, msg = %s", + model->model_id, ret, TVMGetLastError()); + goto error; + } + + arg_values[0].v_str = (const char *)model->tvmrt.json.buffer; + arg_types[0] = kTVMStr; + arg_values[1].v_handle = module_so; + arg_types[1] = kTVMModuleHandle; + arg_values[2].v_int64 = device.device_type; + arg_types[2] = kDLInt; + arg_values[3].v_int64 = device.device_id; + arg_types[3] = kDLInt; + + ret = tvmrt_ml_tvm_func_call(model, create_fn, "tvm.graph_executor.create", arg_values, + arg_types, 4, &ret_value, &ret_type, kTVMModuleHandle); + if (ret != 0) + goto error; + model->tvmrt.graph_module = ret_value.v_handle; + + ret = tvmrt_ml_tvm_func_get(model, model->tvmrt.graph_module, "load_params", + &model->tvmrt.load_params); + if (ret != 0) + goto error; + ret = tvmrt_ml_tvm_func_get(model, model->tvmrt.graph_module, "set_input_zero_copy", + &model->tvmrt.set_input_zero_copy); + if (ret != 0) + goto error; + ret = tvmrt_ml_tvm_func_get(model, model->tvmrt.graph_module, "set_output_zero_copy", + &model->tvmrt.set_output_zero_copy); + if (ret != 0) + goto error; + ret = tvmrt_ml_tvm_func_get(model, model->tvmrt.graph_module, "run", &model->tvmrt.run); + if (ret != 0) + goto error; + + tvmrt_ml_tvm_func_free(®ister_cb_fn); + tvmrt_ml_tvm_mod_free(&module_so); + + /* Load model parameters into TVM runtime */ + tvm_params.data = (const char *)model->tvmrt.params.buffer; + tvm_params.size = model->tvmrt.params.size; + tvm_arg_values[0].v_handle = &tvm_params; + tvm_arg_types[0] = kTVMBytes; + + ret = tvmrt_ml_tvm_func_call(model, model->tvmrt.load_params, "load_params", tvm_arg_values, + tvm_arg_types, 1, &ret_value, &ret_type, kTVMNullptr); + if (ret != 0) + goto error; + + /* Update model I/O data */ + tvmrt_ml_model_io_info_set(model); + + /* Set model info */ + tvmrt_ml_model_info_set(cnxk_mldev, model); + + /* Update model xstats name */ + cnxk_ml_xstats_model_name_update(cnxk_mldev, model->model_id); + + model->tvmrt.burst_xstats = + RTE_PTR_ADD(model->tvmrt.params.buffer, + RTE_ALIGN_CEIL(model->tvmrt.params.size, RTE_CACHE_LINE_MIN_SIZE)); + + for (int qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { + model->tvmrt.burst_xstats[qp_id].tvm_rt_latency_tot = 0; + model->tvmrt.burst_xstats[qp_id].tvm_rt_latency = 0; + model->tvmrt.burst_xstats[qp_id].tvm_rt_latency_min = UINT64_MAX; + model->tvmrt.burst_xstats[qp_id].tvm_rt_latency_max = 0; + model->tvmrt.burst_xstats[qp_id].tvm_rt_reset_count = 0; + model->tvmrt.burst_xstats[qp_id].dequeued_count = 0; + } + + /* Set model specific fast path functions */ + if (model->subtype == ML_CNXK_MODEL_SUBTYPE_TVM_MRVL) { + model->enqueue_single = cn10k_ml_enqueue_single; + model->result_update = cn10k_ml_result_update; + model->set_error_code = cn10k_ml_set_error_code; + model->set_poll_addr = cn10k_ml_set_poll_addr; + model->op_error_get = cn10k_ml_op_error_get; + } else { + model->enqueue_single = tvmrt_ml_enqueue_single; + model->result_update = tvmrt_ml_result_update; + model->set_error_code = tvmrt_ml_set_error_code; + model->set_poll_addr = tvmrt_ml_set_poll_addr; + model->op_error_get = tvmrt_ml_op_error_get; + } + + return 0; + +error: + tvmrt_ml_tvm_func_free(®ister_cb_fn); + if (model != NULL) { + tvmrt_ml_tvm_func_free(&model->tvmrt.load_params); + tvmrt_ml_tvm_func_free(&model->tvmrt.set_input_zero_copy); + tvmrt_ml_tvm_func_free(&model->tvmrt.set_output_zero_copy); + tvmrt_ml_tvm_func_free(&model->tvmrt.run); + tvmrt_ml_tvm_mod_free(&model->tvmrt.graph_module); + if (model->tvmrt.fd >= 0) + close(model->tvmrt.fd); + memset(&model->tvmrt, 0, sizeof(model->tvmrt)); + model->tvmrt.fd = -1; + } + tvmrt_ml_tvm_mod_free(&module_so); + plt_memzone_free(mz); + + return ret; +} + +int +tvmrt_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) +{ + char str[RTE_MEMZONE_NAMESIZE]; + const struct plt_memzone *mz; + + RTE_SET_USED(cnxk_mldev); + + /* Unload model from TVM runtime */ + if (model->model_id >= cnxk_mldev->mldev->data->nb_models) + return -EINVAL; + + if (model->tvmrt.graph_module == NULL) + return -EINVAL; + + tvmrt_ml_tvm_func_free(&model->tvmrt.load_params); + tvmrt_ml_tvm_func_free(&model->tvmrt.set_input_zero_copy); + tvmrt_ml_tvm_func_free(&model->tvmrt.set_output_zero_copy); + tvmrt_ml_tvm_func_free(&model->tvmrt.run); + tvmrt_ml_tvm_mod_free(&model->tvmrt.graph_module); + if (model->tvmrt.fd >= 0) + close(model->tvmrt.fd); + memset(&model->tvmrt, 0, sizeof(model->tvmrt)); + model->tvmrt.fd = -1; + + snprintf(str, RTE_MEMZONE_NAMESIZE, "%s_%u", TVMRT_ML_MODEL_MEMZONE_NAME, model->model_id); + mz = plt_memzone_lookup(str); + if (mz == NULL) { + plt_err("Memzone lookup failed for TVM model: model_id = %u, mz = %s", + model->model_id, str); + return -EINVAL; + } + + return plt_memzone_free(mz); +} + +int +tvmrt_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) +{ + struct cnxk_ml_layer *layer; + + uint16_t layer_id = 0; + int ret = 0; + +next_layer: + layer = &model->layer[layer_id]; + if (layer->type == ML_CNXK_LAYER_TYPE_MRVL) { + ret = cn10k_ml_layer_start(cnxk_mldev, model->model_id, layer->name); + if (ret != 0) { + plt_err("Layer start failed, model_id = %u, layer_name = %s, error = %d", + model->model_id, layer->name, ret); + return ret; + } + } + layer_id++; + + if (layer_id < model->nb_layers) + goto next_layer; + + return 0; +} + +int +tvmrt_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) +{ + struct cnxk_ml_layer *layer; + + uint16_t layer_id = 0; + int ret = 0; + +next_layer: + layer = &model->layer[layer_id]; + if (layer->type == ML_CNXK_LAYER_TYPE_MRVL) { + ret = cn10k_ml_layer_stop(cnxk_mldev, model->model_id, layer->name); + if (ret != 0) { + plt_err("Layer stop failed, model_id = %u, layer_name = %s, error = %d", + model->model_id, layer->name, ret); + return ret; + } + } + layer_id++; + + if (layer_id < model->nb_layers) + goto next_layer; + + return 0; +} + +int +tvmrt_ml_io_quantize(void *device, uint16_t model_id, const char *layer_name, + const DLTensor **deq_tensor, void *qbuffer) +{ + struct cnxk_ml_io_info *info = NULL; + struct cnxk_ml_dev *cnxk_mldev; + struct cnxk_ml_model *model; + uint16_t layer_id = 0; + uint8_t *lcl_dbuffer; + uint8_t *lcl_qbuffer; + uint32_t i; + int ret; + +#ifdef CNXK_ML_DEV_DEBUG + if ((device == NULL) || (deq_tensor == NULL) || (qbuffer == NULL)) + return -EINVAL; +#endif + + cnxk_mldev = (struct cnxk_ml_dev *)device; + + model = cnxk_mldev->mldev->data->models[model_id]; +#ifdef CNXK_ML_DEV_DEBUG + if (model == NULL) { + plt_err("Invalid model_id = %u", model_id); + return -EINVAL; + } +#endif + + /* Get layer id */ + for (layer_id = 0; layer_id < model->nb_layers; layer_id++) { + if (strcmp(model->layer[layer_id].name, layer_name) == 0) + break; + } + +#ifdef CNXK_ML_DEV_DEBUG + if (layer_id == model->nb_layers) { + plt_err("Invalid layer name: %s", layer_name); + return -EINVAL; + } + + if (model->layer[layer_id].type != ML_CNXK_LAYER_TYPE_MRVL) { + plt_err("Invalid layer name / type: %s", layer_name); + return -EINVAL; + } +#endif + + info = &model->layer[layer_id].info; + lcl_qbuffer = (uint8_t *)qbuffer; + + for (i = 0; i < info->nb_inputs; i++) { + lcl_dbuffer = PLT_PTR_ADD(deq_tensor[i]->data, deq_tensor[i]->byte_offset); + + ret = cnxk_ml_io_quantize_single(&info->input[i], lcl_dbuffer, lcl_qbuffer); + if (ret < 0) + return ret; + + lcl_qbuffer += info->input[i].sz_q; + } + + return 0; +} + +int +tvmrt_ml_io_dequantize(void *device, uint16_t model_id, const char *layer_name, void *qbuffer, + const DLTensor **deq_tensor) +{ + struct cnxk_ml_io_info *info = NULL; + struct cnxk_ml_dev *cnxk_mldev; + struct cnxk_ml_model *model; + uint16_t layer_id = 0; + uint8_t *lcl_dbuffer; + uint8_t *lcl_qbuffer; + uint32_t i; + int ret; + +#ifdef CNXK_ML_DEV_DEBUG + if ((device == NULL) || (deq_tensor == NULL) || (qbuffer == NULL)) + return -EINVAL; +#endif + + cnxk_mldev = (struct cnxk_ml_dev *)device; + + model = cnxk_mldev->mldev->data->models[model_id]; +#ifdef CNXK_ML_DEV_DEBUG + if (model == NULL) { + plt_err("Invalid model_id = %u", model_id); + return -EINVAL; + } +#endif + + for (layer_id = 0; layer_id < model->nb_layers; layer_id++) { + if (strcmp(model->layer[layer_id].name, layer_name) == 0) + break; + } + +#ifdef CNXK_ML_DEV_DEBUG + if (layer_id == model->nb_layers) { + plt_err("Invalid layer name: %s", layer_name); + return -EINVAL; + } + + if (model->layer[layer_id].type != ML_CNXK_LAYER_TYPE_MRVL) { + plt_err("Invalid layer name / type: %s", layer_name); + return -EINVAL; + } +#endif + + info = &model->layer[layer_id].info; + lcl_qbuffer = (uint8_t *)qbuffer; + + for (i = 0; i < info->nb_outputs; i++) { + lcl_dbuffer = PLT_PTR_ADD(deq_tensor[i]->data, deq_tensor[i]->byte_offset); + + ret = cnxk_ml_io_dequantize_single(&info->output[i], lcl_qbuffer, lcl_dbuffer); + if (ret < 0) + return ret; + + lcl_qbuffer += info->output[i].sz_q; + } + + return 0; +} + +static int +tvmrt_ml_model_run(struct cnxk_ml_model *model, struct rte_ml_op *op, struct cnxk_ml_req *req) +{ + uint8_t i; + struct tvmrt_ml_result *run_result; + TVMValue arg_values[2] = {0}; + int arg_types[2] = {0}; + TVMValue ret_value = {0}; + int ret_type = kTVMNullptr; + int ret = 0; + + rte_memcpy(req->tvmrt_req.input_tensor, model->tvmrt.input_tensor, + model->tvmrt.info.nb_inputs * sizeof(DLTensor)); + for (i = 0; i < model->tvmrt.info.nb_inputs; i++) { + req->tvmrt_req.input_tensor[i].data = op->input[i]->addr; + req->tvmrt_req.input_tensor[i].byte_offset = 0; + } + + rte_memcpy(req->tvmrt_req.output_tensor, model->tvmrt.output_tensor, + model->tvmrt.info.nb_outputs * sizeof(DLTensor)); + for (i = 0; i < model->tvmrt.info.nb_outputs; i++) { + req->tvmrt_req.output_tensor[i].data = op->output[i]->addr; + req->tvmrt_req.output_tensor[i].byte_offset = 0; + } + + run_result = &req->tvmrt_req.result; + run_result->stats.start_ns = rte_get_tsc_cycles(); + run_result->error_code = 0; + + for (i = 0; i < model->tvmrt.info.nb_inputs; i++) { + arg_values[0].v_int64 = i; + arg_types[0] = kDLInt; + arg_values[1].v_handle = &req->tvmrt_req.input_tensor[i]; + arg_types[1] = kTVMDLTensorHandle; + ret = tvmrt_ml_tvm_func_call(model, model->tvmrt.set_input_zero_copy, + "set_input_zero_copy", arg_values, arg_types, 2, + &ret_value, &ret_type, kTVMNullptr); + if (ret != 0) + goto out; + } + + for (i = 0; i < model->tvmrt.info.nb_outputs; i++) { + arg_values[0].v_int64 = i; + arg_types[0] = kDLInt; + arg_values[1].v_handle = &req->tvmrt_req.output_tensor[i]; + arg_types[1] = kTVMDLTensorHandle; + ret = tvmrt_ml_tvm_func_call(model, model->tvmrt.set_output_zero_copy, + "set_output_zero_copy", arg_values, arg_types, 2, + &ret_value, &ret_type, kTVMNullptr); + if (ret != 0) + goto out; + } + + ret = tvmrt_ml_tvm_func_call(model, model->tvmrt.run, "run", NULL, NULL, 0, &ret_value, + &ret_type, kTVMNullptr); + if (ret != 0) + goto out; + +out: + run_result->stats.end_ns = rte_get_tsc_cycles(); + req->tvmrt_req.status = 0x1; + + plt_write64(ML_CNXK_POLL_JOB_FINISH, req->status); + + if (ret != 0) + run_result->error_code = -EIO; + + return 0; +} + +__rte_hot void +tvmrt_ml_set_error_code(struct cnxk_ml_req *req, uint64_t etype, uint64_t stype) +{ + RTE_SET_USED(stype); + + req->tvmrt_req.result.error_code = etype; +} + +__rte_hot int +tvmrt_ml_op_error_get(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, + struct rte_ml_op_error *error) +{ + RTE_SET_USED(cnxk_mldev); + RTE_SET_USED(op); + RTE_SET_USED(error); + + return 0; +} + +__rte_hot bool +tvmrt_ml_enqueue_single(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, uint16_t layer_id, + struct cnxk_ml_qp *qp, uint64_t head) +{ + struct cnxk_ml_model *model; + struct cnxk_ml_queue *queue; + struct cnxk_ml_req *req; + + RTE_SET_USED(layer_id); + + queue = &qp->queue; + req = &queue->reqs[head]; + model = cnxk_mldev->mldev->data->models[op->model_id]; + + model->set_poll_addr(req); + memset(&req->tvmrt_req.result, 0, sizeof(struct tvmrt_ml_result)); + req->tvmrt_req.result.error_code = 0x0; + req->tvmrt_req.result.user_ptr = op->user_ptr; + + cnxk_ml_set_poll_ptr(req); + tvmrt_ml_model_run(model, op, req); + req->timeout = plt_tsc_cycles() + queue->wait_cycles; + req->op = op; + + return true; +} + +__rte_hot void +tvmrt_ml_result_update(struct cnxk_ml_dev *cnxk_mldev, int qp_id, void *request) +{ + struct tvmrt_ml_model_xstats *xstats; + struct tvmrt_ml_result *result; + struct cnxk_ml_model *model; + struct cnxk_ml_req *req; + uint64_t tvm_rt_latency; + struct cnxk_ml_qp *qp; + struct rte_ml_op *op; + + req = (struct cnxk_ml_req *)request; + result = &req->tvmrt_req.result; + op = req->op; + qp = cnxk_mldev->mldev->data->queue_pairs[qp_id]; + op->impl_opaque = result->error_code; + + if (likely(result->error_code == 0)) { + qp->stats.dequeued_count++; + op->status = RTE_ML_OP_STATUS_SUCCESS; + + model = cnxk_mldev->mldev->data->models[op->model_id]; + xstats = &model->tvmrt.burst_xstats[qp_id]; + + if (unlikely(xstats->dequeued_count == xstats->tvm_rt_reset_count)) { + xstats->tvm_rt_latency_min = UINT64_MAX; + xstats->tvm_rt_latency_max = 0; + } + tvm_rt_latency = result->stats.end_ns - result->stats.start_ns; + xstats->tvm_rt_latency = tvm_rt_latency; + xstats->tvm_rt_latency_tot += tvm_rt_latency; + xstats->tvm_rt_latency_min = RTE_MIN(xstats->tvm_rt_latency_min, tvm_rt_latency); + xstats->tvm_rt_latency_max = RTE_MAX(xstats->tvm_rt_latency_max, tvm_rt_latency); + xstats->dequeued_count++; + } else { + qp->stats.dequeue_err_count++; + op->status = RTE_ML_OP_STATUS_ERROR; + } +} diff --git a/drivers/ml/cnxk/tvmrt_ml_ops.h b/drivers/ml/cnxk/tvmrt_ml_ops.h new file mode 100644 index 00000000000..f76f2ea000d --- /dev/null +++ b/drivers/ml/cnxk/tvmrt_ml_ops.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Marvell. + */ + +#ifndef _TVMRT_ML_OPS_H_ +#define _TVMRT_ML_OPS_H_ + +#include <dlpack/dlpack.h> + +#include <rte_mldev.h> + +#include "cnxk_ml_xstats.h" +#include "tvmrt_ml_model.h" + +struct cnxk_ml_dev; +struct cnxk_ml_model; +struct cnxk_ml_layer; +struct cnxk_ml_qp; +struct cnxk_ml_req; + +/* Inference stats */ +struct tvmrt_ml_stats { + /* Start ns */ + uint64_t start_ns; + + /* Start ns */ + uint64_t end_ns; +}; + +/* Result structure */ +struct tvmrt_ml_result { + /* Job error code */ + uint64_t error_code; + + /* Inference stats */ + struct tvmrt_ml_stats stats; + + /* User context pointer */ + void *user_ptr; +}; + +/* TVMRT specific request */ +struct tvmrt_ml_req { + /* Input tensors */ + DLTensor input_tensor[ML_CNXK_MODEL_MAX_INPUT_OUTPUT]; + + /* Output tensors */ + DLTensor output_tensor[ML_CNXK_MODEL_MAX_INPUT_OUTPUT]; + + /* Status field for poll mode requests */ + volatile uint64_t status; + + /* Result */ + struct tvmrt_ml_result result; +}; + +int tvmrt_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params, + struct cnxk_ml_model *model); +int tvmrt_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); +int tvmrt_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); +int tvmrt_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); +int tvmrt_ml_io_quantize(void *device, uint16_t model_id, const char *layer_name, + const DLTensor **deq_tensor, void *qbuffer); +int tvmrt_ml_io_dequantize(void *device, uint16_t model_id, const char *layer_name, void *qbuffer, + const DLTensor **deq_tensor); + +__rte_hot bool tvmrt_ml_enqueue_single(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, + uint16_t layer_id, struct cnxk_ml_qp *qp, uint64_t head); +__rte_hot int tvmrt_ml_op_error_get(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, + struct rte_ml_op_error *error); +__rte_hot void tvmrt_ml_result_update(struct cnxk_ml_dev *cnxk_mldev, int qp_id, void *request); +__rte_hot void tvmrt_ml_set_error_code(struct cnxk_ml_req *req, uint64_t etype, uint64_t stype); + +void tvmrt_ml_model_xstat_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, + uint16_t stat_id, uint16_t entry, char *suffix); +uint64_t tvmrt_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, + enum cnxk_ml_xstats_type type); + +#endif /* _TVMRT_ML_OPS_H_ */ diff --git a/drivers/ml/cnxk/tvmrt_ml_stubs.c b/drivers/ml/cnxk/tvmrt_ml_stubs.c new file mode 100644 index 00000000000..0b4e53b4327 --- /dev/null +++ b/drivers/ml/cnxk/tvmrt_ml_stubs.c @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Marvell. + */ + +#include <rte_mldev.h> + +#include "tvmrt_ml_stubs.h" + +#include "cnxk_ml_dev.h" +#include "cnxk_ml_model.h" +#include "cnxk_ml_xstats.h" + +enum cnxk_ml_model_type +tvmrt_ml_model_type_get(struct rte_ml_model_params *params) +{ + RTE_SET_USED(params); + + return ML_CNXK_MODEL_TYPE_UNKNOWN; +} + +int +tvmrt_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, uint16_t *layer_id) +{ + RTE_SET_USED(model); + RTE_SET_USED(layer_name); + RTE_SET_USED(layer_id); + + return -EINVAL; +} + +struct cnxk_ml_io_info * +tvmrt_ml_model_io_info_get(struct cnxk_ml_model *model, uint16_t layer_id) +{ + RTE_SET_USED(model); + RTE_SET_USED(layer_id); + + return NULL; +} + +void +tvmrt_ml_layer_print(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer, FILE *fp) +{ + RTE_SET_USED(cnxk_mldev); + RTE_SET_USED(layer); + RTE_SET_USED(fp); +} + +void +tvmrt_ml_model_xstat_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, + uint16_t stat_id, uint16_t entry, char *suffix) +{ + RTE_SET_USED(cnxk_mldev); + RTE_SET_USED(model); + RTE_SET_USED(stat_id); + RTE_SET_USED(entry); + RTE_SET_USED(suffix); +} + +uint64_t +tvmrt_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, + enum cnxk_ml_xstats_type type) +{ + RTE_SET_USED(cnxk_mldev); + RTE_SET_USED(model); + RTE_SET_USED(type); + + return 0; +} + +int +tvmrt_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params, + struct cnxk_ml_model *model) +{ + RTE_SET_USED(cnxk_mldev); + RTE_SET_USED(params); + RTE_SET_USED(model); + + return -EINVAL; +} + +int +tvmrt_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) +{ + RTE_SET_USED(cnxk_mldev); + RTE_SET_USED(model); + + return -EINVAL; +} + +int +tvmrt_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) +{ + RTE_SET_USED(cnxk_mldev); + RTE_SET_USED(model); + + return -EINVAL; +} + +int +tvmrt_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model) +{ + RTE_SET_USED(cnxk_mldev); + RTE_SET_USED(model); + + return -EINVAL; +} diff --git a/drivers/ml/cnxk/tvmrt_ml_stubs.h b/drivers/ml/cnxk/tvmrt_ml_stubs.h new file mode 100644 index 00000000000..e0260d36514 --- /dev/null +++ b/drivers/ml/cnxk/tvmrt_ml_stubs.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Marvell. + */ + +#ifndef _TVMRT_ML_STUBS_H_ +#define _TVMRT_ML_STUBS_H_ + +#include <rte_mldev.h> + +#include "cnxk_ml_xstats.h" + +struct cnxk_ml_dev; +struct cnxk_ml_model; +struct cnxk_ml_layer; + +enum cnxk_ml_model_type tvmrt_ml_model_type_get(struct rte_ml_model_params *params); +int tvmrt_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params, + struct cnxk_ml_model *model); +int tvmrt_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); +int tvmrt_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); +int tvmrt_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model); + +int tvmrt_ml_model_get_layer_id(struct cnxk_ml_model *model, const char *layer_name, + uint16_t *layer_id); +struct cnxk_ml_io_info *tvmrt_ml_model_io_info_get(struct cnxk_ml_model *model, uint16_t layer_id); +void tvmrt_ml_layer_print(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer, FILE *fp); +void tvmrt_ml_model_xstat_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, + uint16_t stat_id, uint16_t entry, char *suffix); +uint64_t tvmrt_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model, + enum cnxk_ml_xstats_type type); + +#endif /* _TVMRT_ML_STUBS_H_ */ -- 2.34.1
