> -----Original Message-----
> From: Huang, Wei <wei.hu...@intel.com>
> Sent: Thursday, May 26, 2022 11:33 AM
> To: dev@dpdk.org; tho...@monjalon.net; nipun.gu...@nxp.com;
> hemant.agra...@nxp.com
> Cc: sta...@dpdk.org; Xu, Rosen <rosen...@intel.com>; Zhang, Tianfei
> <tianfei.zh...@intel.com>; Zhang, Qi Z <qi.z.zh...@intel.com>; Huang, Wei
> <wei.hu...@intel.com>
> Subject: [PATCH v3 4/5] raw/ifpga: support ofs card probe
>
> PAC N6000 is the first OFS platform, its device id is added to ifpga device
> support
> list.
>
> Previous FPGA platform like Intel PAC N3000 and N5000, FME DFL (Device
> Feature List) starts from BAR0 by default, port DFL location is indicated in
> PORTn_OFFSET register in FME. In OFS implementation, FME DFL and port DFL
> location can be defined individually in PCIe VSEC (Vendor Specific Extented
> Capabilities). In this patch, DFL definition is searched in VSEC, the legacy
> DFL is
> used only when DFL VSEC is not present.
>
> In original DFL enumeration process, AFU is expected to locate in port DFL,
> but
> this is not the case in OFS implementation. In this patch, enumeration can
> search
> AFU in any PF/VF which has no FME and port.
>
> Signed-off-by: Wei Huang <wei.hu...@intel.com>
> ---
> v2: fix build error in UB2004-32
> ---
> v3: update commit log with Tianfei's comment, treat 7 as special BAR index.
> ---
> drivers/raw/ifpga/base/ifpga_api.c | 28 +-
> drivers/raw/ifpga/base/ifpga_defines.h | 1 +
> drivers/raw/ifpga/base/ifpga_enumerate.c | 681 ++++++++++++++++++++++--
> -----
> drivers/raw/ifpga/base/ifpga_enumerate.h | 2 +
> drivers/raw/ifpga/base/ifpga_feature_dev.c | 18 +-
> drivers/raw/ifpga/base/ifpga_hw.h | 2 +
> drivers/raw/ifpga/base/opae_hw_api.c | 2 +-
> drivers/raw/ifpga/base/opae_hw_api.h | 2 +
> drivers/raw/ifpga/ifpga_rawdev.c | 48 +-
> 9 files changed, 592 insertions(+), 192 deletions(-)
>
> diff --git a/drivers/raw/ifpga/base/ifpga_api.c
> b/drivers/raw/ifpga/base/ifpga_api.c
> index 4610ef1..f19cc26 100644
> --- a/drivers/raw/ifpga/base/ifpga_api.c
> +++ b/drivers/raw/ifpga/base/ifpga_api.c
> @@ -13,15 +13,22 @@
> static int ifpga_acc_get_uuid(struct opae_accelerator *acc,
> struct uuid *uuid)
> {
> - struct opae_bridge *br = acc->br;
> - struct ifpga_port_hw *port;
> + struct ifpga_afu_info *afu_info = acc->data;
> + struct opae_reg_region *region;
> + u64 val = 0;
>
> - if (!br || !br->data)
> - return -EINVAL;
> + if (!afu_info)
> + return -ENODEV;
>
> - port = br->data;
> + region = &afu_info->region[0];
> + if (uuid) {
> + val = readq(region->addr + sizeof(struct feature_header));
> + opae_memcpy(uuid->b, &val, sizeof(u64));
> + val = readq(region->addr + sizeof(struct feature_header) + 8);
> + opae_memcpy(uuid->b + 8, &val, sizeof(u64));
> + }
>
> - return fpga_get_afu_uuid(port, uuid);
> + return 0;
> }
>
> static int ifpga_acc_set_irq(struct opae_accelerator *acc, @@ -32,6 +39,9 @@
> static int ifpga_acc_set_irq(struct opae_accelerator *acc,
> struct ifpga_port_hw *port;
> struct fpga_uafu_irq_set irq_set;
>
> + if (!afu_info)
> + return -ENODEV;
> +
> if (!br || !br->data)
> return -EINVAL;
>
> @@ -68,7 +78,7 @@ static int ifpga_acc_get_region_info(struct
> opae_accelerator *acc,
> struct ifpga_afu_info *afu_info = acc->data;
>
> if (!afu_info)
> - return -EINVAL;
> + return -ENODEV;
>
> if (info->index >= afu_info->num_regions)
> return -EINVAL;
> @@ -89,7 +99,7 @@ static int ifpga_acc_read(struct opae_accelerator *acc,
> unsigned int region_idx,
> struct opae_reg_region *region;
>
> if (!afu_info)
> - return -EINVAL;
> + return -ENODEV;
>
> if (offset + byte <= offset)
> return -EINVAL;
> @@ -129,7 +139,7 @@ static int ifpga_acc_write(struct opae_accelerator *acc,
> struct opae_reg_region *region;
>
> if (!afu_info)
> - return -EINVAL;
> + return -ENODEV;
>
> if (offset + byte <= offset)
> return -EINVAL;
> diff --git a/drivers/raw/ifpga/base/ifpga_defines.h
> b/drivers/raw/ifpga/base/ifpga_defines.h
> index 8f62033..9a280eb 100644
> --- a/drivers/raw/ifpga/base/ifpga_defines.h
> +++ b/drivers/raw/ifpga/base/ifpga_defines.h
> @@ -73,6 +73,7 @@
> enum fpga_id_type {
> FME_ID,
> PORT_ID,
> + AFU_ID,
> FPGA_ID_MAX,
> };
>
> diff --git a/drivers/raw/ifpga/base/ifpga_enumerate.c
> b/drivers/raw/ifpga/base/ifpga_enumerate.c
> index 48b8af4..0e09904 100644
> --- a/drivers/raw/ifpga/base/ifpga_enumerate.c
> +++ b/drivers/raw/ifpga/base/ifpga_enumerate.c
> @@ -2,6 +2,10 @@
> * Copyright(c) 2010-2018 Intel Corporation
> */
>
> +#include <fcntl.h>
> +#include <inttypes.h>
> +#include <unistd.h>
> +
> #include "opae_hw_api.h"
> #include "ifpga_api.h"
>
> @@ -9,6 +13,19 @@
> #include "ifpga_enumerate.h"
> #include "ifpga_feature_dev.h"
>
> +struct dfl_fpga_enum_dfl {
> + u64 start;
> + u64 len;
> + void *addr;
> + TAILQ_ENTRY(dfl_fpga_enum_dfl) node;
> +};
> +
> +TAILQ_HEAD(dfl_fpga_enum_dfls, dfl_fpga_enum_dfl); struct
> +dfl_fpga_enum_info {
> + struct ifpga_hw *hw;
> + struct dfl_fpga_enum_dfls dfls;
> +};
> +
> struct build_feature_devs_info {
> struct opae_adapter_data_pci *pci_data;
>
> @@ -21,7 +38,6 @@ struct build_feature_devs_info {
> void *ioaddr;
> void *ioend;
> uint64_t phys_addr;
> - int current_bar;
>
> void *pfme_hdr;
>
> @@ -141,8 +157,10 @@ static u64 feature_id(void __iomem *start)
> unsigned int size, unsigned int vec_start,
> unsigned int vec_cnt)
> {
> - return build_info_add_sub_feature(binfo, start, fid, size, vec_start,
> - vec_cnt);
> + if (binfo->current_type != AFU_ID)
> + return build_info_add_sub_feature(binfo, start, fid, size,
> + vec_start, vec_cnt);
> + return 0;
> }
>
> /*
> @@ -152,13 +170,14 @@ static u64 feature_id(void __iomem *start)
> */
> static bool feature_is_UAFU(struct build_feature_devs_info *binfo) {
> - if (binfo->current_type != PORT_ID)
> - return false;
> + if ((binfo->current_type == PORT_ID) ||
> + (binfo->current_type == AFU_ID))
> + return true;
>
> - return true;
> + return false;
> }
>
> -static int parse_feature_port_uafu(struct build_feature_devs_info *binfo,
> +static int parse_feature_uafu(struct build_feature_devs_info *binfo,
> struct feature_header *hdr)
> {
> u64 id = PORT_FEATURE_ID_UAFU;
> @@ -169,9 +188,19 @@ static int parse_feature_port_uafu(struct
> build_feature_devs_info *binfo,
> int ret;
> int size;
>
> + if (binfo->acc_info) {
> + dev_info(binfo, "Sub AFU found @ %p.\n", start);
> + return 0;
> + }
> +
> capability.csr = readq(&port_hdr->capability);
>
> - size = capability.mmio_size << 10;
> + if (binfo->current_type == AFU_ID) {
> + size = AFU_REGION_SIZE;
> + } else {
> + capability.csr = readq(&port_hdr->capability);
> + size = capability.mmio_size << 10;
> + }
>
> ret = create_feature_instance(binfo, hdr, id, size, 0, 0);
> if (ret)
> @@ -185,44 +214,13 @@ static int parse_feature_port_uafu(struct
> build_feature_devs_info *binfo,
> info->region[0].phys_addr = binfo->phys_addr +
> (uint8_t *)start - (uint8_t *)binfo->ioaddr;
> info->region[0].len = size;
> - info->num_regions = 1;
> + info->num_regions = AFU_MAX_REGION;
>
> binfo->acc_info = info;
>
> return ret;
> }
>
> -static int parse_feature_afus(struct build_feature_devs_info *binfo,
> - struct feature_header *hdr)
> -{
> - int ret;
> - struct feature_afu_header *afu_hdr, header;
> - u8 __iomem *start;
> - u8 __iomem *end = binfo->ioend;
> -
> - start = (u8 __iomem *)hdr;
> - for (; start < end; start += header.next_afu) {
> - if ((unsigned int)(end - start) <
> - (unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr)))
> - return -EINVAL;
> -
> - hdr = (struct feature_header *)start;
> - afu_hdr = (struct feature_afu_header *)(hdr + 1);
> - header.csr = readq(&afu_hdr->csr);
> -
> - if (feature_is_UAFU(binfo)) {
> - ret = parse_feature_port_uafu(binfo, hdr);
> - if (ret)
> - return ret;
> - }
> -
> - if (!header.next_afu)
> - break;
> - }
> -
> - return 0;
> -}
> -
> /* create and register proper private data */ static int
> build_info_commit_dev(struct build_feature_devs_info *binfo) { @@ -235,13
> +233,9 @@ static int build_info_commit_dev(struct build_feature_devs_info
> *binfo)
> struct ifpga_fme_hw *fme;
> struct ifpga_feature *feature;
>
> - if (!binfo->fiu)
> - return 0;
> -
> if (binfo->current_type == PORT_ID) {
> - /* return error if no valid acc info data structure */
> - if (!info)
> - return -EFAULT;
> + if (!binfo->fiu)
> + return 0;
>
> br = opae_bridge_alloc(hw->adapter->name, &ifpga_br_ops,
> binfo->fiu);
> @@ -254,7 +248,7 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
> port = &hw->port[binfo->current_port_id];
> feature = get_feature_by_id(&port->feature_list,
> PORT_FEATURE_ID_UINT);
> - if (feature)
> + if (feature && info)
> info->num_irqs = feature->vec_cnt;
>
> acc = opae_accelerator_alloc(hw->adapter->name,
> @@ -264,17 +258,21 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
> return -ENOMEM;
> }
>
> + acc->adapter = hw->adapter;
> acc->br = br;
> if (hw->adapter->mgr)
> acc->mgr = hw->adapter->mgr;
> acc->index = br->id;
>
> fme = &hw->fme;
> - fme->nums_acc_region = info->num_regions;
> + fme->nums_acc_region = info ? info->num_regions : 0;
>
> opae_adapter_add_acc(hw->adapter, acc);
>
> } else if (binfo->current_type == FME_ID) {
> + if (!binfo->fiu)
> + return 0;
> +
> mgr = opae_manager_alloc(hw->adapter->name,
> &ifpga_mgr_ops,
> &ifpga_mgr_network_ops, binfo->fiu);
> if (!mgr)
> @@ -282,6 +280,22 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
>
> mgr->adapter = hw->adapter;
> hw->adapter->mgr = mgr;
> + } else if (binfo->current_type == AFU_ID) {
> + if (!info)
> + return -EFAULT;
> +
> + info->num_irqs = 0;
> + acc = opae_accelerator_alloc(hw->adapter->name,
> + &ifpga_acc_ops, info);
> + if (!acc)
> + return -ENOMEM;
> +
> + acc->adapter = hw->adapter;
> + acc->br = NULL;
> + acc->mgr = NULL;
> + acc->index = hw->num_afus++;
> +
> + opae_adapter_add_acc(hw->adapter, acc);
> }
>
> binfo->fiu = NULL;
> @@ -295,11 +309,15 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo) {
> int ret;
>
> + if ((type == AFU_ID) && (binfo->current_type == PORT_ID))
> + return 0;
> +
> ret = build_info_commit_dev(binfo);
> if (ret)
> return ret;
>
> binfo->current_type = type;
> + binfo->acc_info = NULL;
>
> if (type == FME_ID) {
> binfo->fiu = &binfo->hw->fme;
> @@ -311,6 +329,41 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
> return 0;
> }
>
> +static int parse_feature_afus(struct build_feature_devs_info *binfo,
> + struct feature_header *hdr)
> +{
> + int ret;
> + struct feature_afu_header *afu_hdr, header;
> + u8 __iomem *start;
> + u8 __iomem *end = binfo->ioend;
> +
> + ret = build_info_create_dev(binfo, AFU_ID, 0);
> + if (ret)
> + return ret;
> +
> + start = (u8 __iomem *)hdr;
> + for (; start < end; start += header.next_afu) {
> + if ((unsigned int)(end - start) <
> + (unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr)))
> + return -EINVAL;
> +
> + hdr = (struct feature_header *)start;
> + afu_hdr = (struct feature_afu_header *)(hdr + 1);
> + header.csr = readq(&afu_hdr->csr);
> +
> + if (feature_is_UAFU(binfo)) {
> + ret = parse_feature_uafu(binfo, hdr);
> + if (ret)
> + return ret;
> + }
> +
> + if (!header.next_afu)
> + break;
> + }
> +
> + return 0;
> +}
> +
> static int parse_feature_fme(struct build_feature_devs_info *binfo,
> struct feature_header *start)
> {
> @@ -405,7 +458,7 @@ static int parse_feature_fiu(struct
> build_feature_devs_info *binfo,
> if (ret)
> return ret;
> } else {
> - dev_info(binfo, "No AFUs detected on Port\n");
> + dev_info(binfo, "No AFU detected on Port\n");
> }
>
> break;
> @@ -426,7 +479,7 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
>
> id = feature_id(start);
>
> - if (id == PORT_FEATURE_ID_UINT) {
> + if ((binfo->current_type == PORT_ID) && (id ==
> PORT_FEATURE_ID_UINT))
> +{
> struct feature_port_uint *port_uint = start;
> struct feature_port_uint_cap uint_cap;
>
> @@ -437,7 +490,8 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
> } else {
> dev_debug(binfo, "UAFU doesn't support interrupt\n");
> }
> - } else if (id == PORT_FEATURE_ID_ERROR) {
> + } else if ((binfo->current_type == PORT_ID) &&
> + (id == PORT_FEATURE_ID_ERROR)) {
> struct feature_port_error *port_err = start;
> struct feature_port_err_capability port_err_cap;
>
> @@ -449,7 +503,8 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
> dev_debug(&binfo, "Port error doesn't support
> interrupt\n");
> }
>
> - } else if (id == FME_FEATURE_ID_GLOBAL_ERR) {
> + } else if ((binfo->current_type == FME_ID) &&
> + (id == FME_FEATURE_ID_GLOBAL_ERR)) {
> struct feature_fme_err *fme_err = start;
> struct feature_fme_error_capability fme_err_cap;
>
> @@ -497,9 +552,15 @@ static int parse_feature_private(struct
> build_feature_devs_info *binfo,
> return parse_feature_fme_private(binfo, hdr);
> case PORT_ID:
> return parse_feature_port_private(binfo, hdr);
> + case AFU_ID:
> + dev_err(binfo, "private feature %x belonging to AFU "
> + "is not supported yet.\n", header.id);
> + break;
> default:
> - dev_err(binfo, "private feature %x belonging to AFU %d
> (unknown_type) is not supported yet.\n",
> + dev_err(binfo, "private feature %x belonging to TYPE %d "
> + "(unknown_type) is not supported yet.\n",
> header.id, binfo->current_type);
> + break;
> }
> return 0;
> }
> @@ -530,32 +591,57 @@ static int parse_feature(struct
> build_feature_devs_info *binfo,
> return ret;
> }
>
> -static int
> -parse_feature_list(struct build_feature_devs_info *binfo, u8 __iomem *start)
> +static int build_info_prepare(struct build_feature_devs_info *binfo,
> + struct dfl_fpga_enum_dfl *dfl)
> {
> + if (!binfo || !dfl)
> + return -EINVAL;
> +
> + binfo->ioaddr = dfl->addr;
> + binfo->ioend = (u8 *)dfl->addr + dfl->len;
> + binfo->phys_addr = dfl->start;
> +
> + return 0;
> +}
> +
> +static int parse_feature_list(struct build_feature_devs_info *binfo,
> + struct dfl_fpga_enum_dfl *dfl)
> +{
> + u8 *start, *end;
> struct feature_header *hdr, header;
> - u8 __iomem *end = (u8 __iomem *)binfo->ioend;
> int ret = 0;
>
> + ret = build_info_prepare(binfo, dfl);
> + if (ret)
> + return ret;
> +
> + start = (u8 *)binfo->ioaddr;
> + end = (u8 *)binfo->ioend;
> +
> + /* walk through the device feature list via DFH's next DFH pointer. */
> for (; start < end; start += header.next_header_offset) {
> if ((unsigned int)(end - start) < (unsigned int)sizeof(*hdr)) {
> - dev_err(binfo, "The region is too small to contain a
> feature.\n");
> - ret = -EINVAL;
> + dev_err(binfo, "The region is too small to "
> + "contain a feature.\n");
> + ret = -EINVAL;
> break;
> }
>
> hdr = (struct feature_header *)start;
> - header.csr = readq(hdr);
> + header.csr = opae_readq(hdr);
>
> - dev_debug(binfo, "%s: address=0x%p, val=0x%llx,
> header.id=0x%x, header.next_offset=0x%x, header.eol=0x%x,
> header.type=0x%x\n",
> - __func__, hdr, (unsigned long long)header.csr,
> - header.id, header.next_header_offset,
> - header.end_of_list, header.type);
> + dev_debug(binfo, "%s: address=0x%p, val=0x%"PRIx64", "
> + "header.id=0x%x, header.next_offset=0x%x, "
> + "header.eol=0x%x, header.type=0x%x\n",
> + __func__, hdr, header.csr, header.id,
> + header.next_header_offset, header.end_of_list,
> + header.type);
>
> ret = parse_feature(binfo, hdr);
> if (ret)
> return ret;
>
> + /* stop parsing if EOL(End of List) is set or offset is 0 */
> if (header.end_of_list || !header.next_header_offset)
> break;
> }
> @@ -563,82 +649,9 @@ static int parse_feature(struct build_feature_devs_info
> *binfo,
> return build_info_commit_dev(binfo);
> }
>
> -/* switch the memory mapping to BAR# @bar */ -static int
> parse_switch_to(struct build_feature_devs_info *binfo, int bar) -{
> - struct opae_adapter_data_pci *pci_data = binfo->pci_data;
> -
> - if (!pci_data->region[bar].addr)
> - return -ENOMEM;
> -
> - binfo->ioaddr = pci_data->region[bar].addr;
> - binfo->ioend = (u8 __iomem *)binfo->ioaddr + pci_data-
> >region[bar].len;
> - binfo->phys_addr = pci_data->region[bar].phys_addr;
> - binfo->current_bar = bar;
> -
> - return 0;
> -}
> -
> -static int parse_ports_from_fme(struct build_feature_devs_info *binfo) -{
> - struct feature_fme_header *fme_hdr;
> - struct feature_fme_port port;
> - int i = 0, ret = 0;
> -
> - if (!binfo->pfme_hdr) {
> - dev_info(binfo, "VF is detected.\n");
> - return ret;
> - }
> -
> - fme_hdr = binfo->pfme_hdr;
> -
> - do {
> - port.csr = readq(&fme_hdr->port[i]);
> - if (!port.port_implemented)
> - break;
> -
> - /* skip port which only could be accessed via VF */
> - if (port.afu_access_control == FME_AFU_ACCESS_VF)
> - continue;
> -
> - ret = parse_switch_to(binfo, port.port_bar);
> - if (ret)
> - break;
> -
> - ret = parse_feature_list(binfo,
> - (u8 __iomem *)binfo->ioaddr +
> - port.port_offset);
> - if (ret)
> - break;
> - } while (++i < MAX_FPGA_PORT_NUM);
> -
> - return ret;
> -}
> -
> -static struct build_feature_devs_info * -build_info_alloc_and_init(struct
> ifpga_hw *hw) -{
> - struct build_feature_devs_info *binfo;
> -
> - binfo = zmalloc(sizeof(*binfo));
> - if (!binfo)
> - return binfo;
> -
> - binfo->hw = hw;
> - binfo->pci_data = hw->pci_data;
> -
> - /* fpga feature list starts from BAR 0 */
> - if (parse_switch_to(binfo, 0)) {
> - free(binfo);
> - return NULL;
> - }
> -
> - return binfo;
> -}
> -
> static void build_info_free(struct build_feature_devs_info *binfo) {
> - free(binfo);
> + opae_free(binfo);
> }
>
> static void ifpga_print_device_feature_list(struct ifpga_hw *hw) @@ -648,6
> +661,11 @@ static void ifpga_print_device_feature_list(struct ifpga_hw *hw)
> struct ifpga_feature *feature;
> int i;
>
> + if (fme->state == IFPGA_FME_UNUSED) {
> + dev_info(hw, "FME is not present\n");
> + return;
> + }
> +
> dev_info(hw, "found fme_device, is in PF: %s\n",
> is_ifpga_hw_pf(hw) ? "yes" : "no");
>
> @@ -685,40 +703,411 @@ static void ifpga_print_device_feature_list(struct
> ifpga_hw *hw)
> }
> }
>
> -int ifpga_bus_enumerate(struct ifpga_hw *hw)
> +static struct dfl_fpga_enum_info *dfl_fpga_enum_info_alloc(struct
> +ifpga_hw *hw)
> {
> - struct build_feature_devs_info *binfo;
> + struct dfl_fpga_enum_info *info;
> +
> + info = opae_zmalloc(sizeof(*info));
> + if (!info)
> + return NULL;
> +
> + info->hw = hw;
> + TAILQ_INIT(&info->dfls);
> +
> + return info;
> +}
> +
> +static void dfl_fpga_enum_info_free(struct dfl_fpga_enum_info *info) {
> + struct dfl_fpga_enum_dfl *tmp, *dfl;
> +
> + if (!info)
> + return;
> +
> + /* remove all device feature lists in the list. */
> + for (dfl = TAILQ_FIRST(&info->dfls);
> + dfl && (tmp = TAILQ_NEXT(dfl, node), 1);
> + dfl = tmp) {
> + TAILQ_REMOVE(&info->dfls, dfl, node);
> + opae_free(dfl);
> + }
> +
> + opae_free(info);
> +}
> +
> +static int dfl_fpga_enum_info_add_dfl(struct dfl_fpga_enum_info *info,
> + u64 start, u64 len, void *addr)
> +{
> + struct dfl_fpga_enum_dfl *dfl;
> +
> + dfl = opae_zmalloc(sizeof(*dfl));
> + if (!dfl)
> + return -ENOMEM;
> +
> + dfl->start = start;
> + dfl->len = len;
> + dfl->addr = addr;
> +
> + TAILQ_INSERT_TAIL(&info->dfls, dfl, node);
> +
> + return 0;
> +}
> +
> +#define PCI_CFG_SPACE_SIZE 256
> +#define PCI_CFG_SPACE_EXP_SIZE 4096
> +#define PCI_EXT_CAP_ID(header) (header & 0x0000ffff)
> +#define PCI_EXT_CAP_NEXT(header) ((header >> 20) & 0xffc)
> +
> +static int
> +pci_find_next_ecap(int fd, int start, u32 cap) {
> + u32 header;
> + int ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
> + int pos = PCI_CFG_SPACE_SIZE;
> int ret;
>
> - binfo = build_info_alloc_and_init(hw);
> + if (start > 0)
> + pos = start;
> +
> + ret = pread(fd, &header, sizeof(header), pos);
> + if (ret < 0)
> + return ret;
> +
> + /*
> + * If we have no capabilities, this is indicated by cap ID,
> + * cap version and next pointer all being 0.
> + */
> + if (header == 0)
> + return 0;
> +
> + while (ttl-- > 0) {
> + if ((PCI_EXT_CAP_ID(header) == cap) && (pos != start))
> + return pos;
> +
> + pos = PCI_EXT_CAP_NEXT(header);
> + if (pos < PCI_CFG_SPACE_SIZE)
> + break;
> + ret = pread(fd, &header, sizeof(header), pos);
> + if (ret < 0)
> + return ret;
> + }
> +
> + return 0;
> +}
> +
> +#define PCI_EXT_CAP_ID_VNDR 0x0B
> +#define PCI_VNDR_HEADER 4
> +#define PCI_VNDR_HEADER_ID(x) ((x) & 0xffff)
> +#define PCI_VENDOR_ID_INTEL 0x8086
> +#define PCI_VSEC_ID_INTEL_DFLS 0x43
> +#define PCI_VNDR_DFLS_CNT 0x8
> +#define PCI_VNDR_DFLS_RES 0xc
> +#define PCI_VNDR_DFLS_RES_BAR_MASK GENMASK(2, 0) #define
> +PCI_VNDR_DFLS_RES_OFF_MASK GENMASK(31, 3)
> +
> +static int find_dfls_by_vsec(struct dfl_fpga_enum_info *info) {
> + struct ifpga_hw *hw;
> + struct opae_adapter_data_pci *pci_data;
> + char path[64];
> + u32 bir, offset, vndr_hdr, i, dfl_cnt, dfl_res;
> + int fd, ret, dfl_res_off, voff = 0;
> + u64 start, len;
> + void *addr;
> +
> + if (!info || !info->hw)
> + return -EINVAL;
> + hw = info->hw;
> +
> + if (!hw->adapter || !hw->pci_data)
> + return -EINVAL;
> + pci_data = hw->pci_data;
> +
> + ret = snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/config",
> + hw->adapter->name);
> + if ((unsigned int)ret >= sizeof(path))
> + return -EINVAL;
> +
> + fd = open(path, O_RDWR);
> + if (fd < 0)
> + return -EIO;
> +
> + while ((voff = pci_find_next_ecap(fd, voff,
> + PCI_EXT_CAP_ID_VNDR))) {
> + vndr_hdr = 0;
> + ret = pread(fd, &vndr_hdr, sizeof(vndr_hdr),
> + voff + PCI_VNDR_HEADER);
> + if (ret < 0)
> + return -EIO;
> + if (PCI_VNDR_HEADER_ID(vndr_hdr) ==
> PCI_VSEC_ID_INTEL_DFLS &&
> + pci_data->vendor_id == PCI_VENDOR_ID_INTEL)
> + break;
> + }
> +
> + if (!voff) {
> + dev_debug(hw, "%s no DFL VSEC found\n", __func__);
> + return -ENODEV;
> + }
> +
> + dfl_cnt = 0;
> + ret = pread(fd, &dfl_cnt, sizeof(dfl_cnt), voff + PCI_VNDR_DFLS_CNT);
> + if (ret < 0)
> + return -EIO;
> +
> + dfl_res_off = voff + PCI_VNDR_DFLS_RES;
> + if (dfl_res_off + (dfl_cnt * sizeof(u32)) > PCI_CFG_SPACE_EXP_SIZE) {
> + dev_err(hw, "%s DFL VSEC too big for PCIe config space\n",
> + __func__);
> + return -EINVAL;
> + }
> +
> + for (i = 0; i < dfl_cnt; i++, dfl_res_off += sizeof(u32)) {
> + dfl_res = GENMASK(31, 0);
> + ret = pread(fd, &dfl_res, sizeof(dfl_res), dfl_res_off);
> + bir = dfl_res & PCI_VNDR_DFLS_RES_BAR_MASK;
> + if (bir >= PCI_MAX_RESOURCE) {
> + dev_err(hw, "%s bad bir number %d\n",
> + __func__, bir);
> + return -EINVAL;
> + }
> +
> + len = pci_data->region[bir].len;
> + offset = dfl_res & PCI_VNDR_DFLS_RES_OFF_MASK;
> + if (offset >= len) {
> + dev_err(hw, "%s bad offset %u >= %"PRIu64"\n",
> + __func__, offset, len);
> + return -EINVAL;
> + }
> +
> + dev_debug(hw, "%s BAR %d offset 0x%x\n", __func__, bir,
> offset);
> + len -= offset;
> + start = pci_data->region[bir].phys_addr + offset;
> + addr = pci_data->region[bir].addr + offset;
> + dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> + }
> +
> + return 0;
> +}
> +
> +/* default method of finding dfls starting at offset 0 of bar 0 */
> +static int find_dfls_by_default(struct dfl_fpga_enum_info *info) {
> + struct ifpga_hw *hw;
> + struct opae_adapter_data_pci *pci_data;
> + int port_num, bar, i, ret = 0;
> + u64 start, len;
> + void *addr;
> + u32 offset;
> + struct feature_header hdr;
> + struct feature_fme_capability cap;
> + struct feature_fme_port port;
> + struct feature_fme_header *fme_hdr;
> +
> + if (!info || !info->hw)
> + return -EINVAL;
> + hw = info->hw;
> +
> + if (!hw->pci_data)
> + return -EINVAL;
> + pci_data = hw->pci_data;
> +
> + /* start to find Device Feature List from Bar 0 */
> + addr = pci_data->region[0].addr;
> + if (!addr)
> + return -ENOMEM;
> +
> + /*
> + * PF device has FME and Ports/AFUs, and VF device only has one
> + * Port/AFU. Check them and add related "Device Feature List" info
> + * for the next step enumeration.
> + */
> + hdr.csr = opae_readq(addr);
> + if ((hdr.type == FEATURE_TYPE_FIU) && (hdr.id ==
> FEATURE_FIU_ID_FME)) {
> + start = pci_data->region[0].phys_addr;
> + len = pci_data->region[0].len;
> + addr = pci_data->region[0].addr;
> +
> + dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +
> + /*
> + * find more Device Feature Lists (e.g. Ports) per information
> + * indicated by FME module.
> + */
> + fme_hdr = (struct feature_fme_header *)addr;
> + cap.csr = opae_readq(&fme_hdr->capability);
> + port_num = (int)cap.num_ports;
> +
> + dev_info(hw, "port_num = %d\n", port_num);
> + if (port_num > MAX_FPGA_PORT_NUM)
> + port_num = MAX_FPGA_PORT_NUM;
> +
> + for (i = 0; i < port_num; i++) {
> + port.csr = opae_readq(&fme_hdr->port[i]);
> +
> + /* skip ports which are not implemented. */
> + if (!port.port_implemented)
> + continue;
> +
> + /* skip port which only could be accessed via VF */
> + if (port.afu_access_control == FME_AFU_ACCESS_VF)
> + continue;
> +
> + /*
> + * add Port's Device Feature List information for next
> + * step enumeration.
> + */
> + bar = (int)port.port_bar;
> + offset = port.port_offset;
> + if (bar == FME_PORT_OFST_BAR_SKIP) {
> + continue;
> + } else if (bar >= PCI_MAX_RESOURCE) {
> + dev_err(hw, "bad BAR %d for port %d\n", bar,
> i);
> + ret = -EINVAL;
> + break;
> + } else {
> + dev_info(hw, "BAR %d offset %u\n", bar,
> offset);
> + }
> +
> + len = pci_data->region[bar].len;
> + if (offset >= len) {
> + dev_warn(hw, "bad port offset %u >= %pa\n",
> + offset, &len);
> + continue;
> + }
> +
> + len -= offset;
> + start = pci_data->region[bar].phys_addr + offset;
> + addr = pci_data->region[bar].addr + offset;
> + dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> + }
> + } else if ((hdr.type == FEATURE_TYPE_FIU) &&
> + (hdr.id == FEATURE_FIU_ID_PORT)) {
> + start = pci_data->region[0].phys_addr;
> + len = pci_data->region[0].len;
> + addr = pci_data->region[0].addr;
> +
> + dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> + } else if (hdr.type == FEATURE_TYPE_AFU) {
> + start = pci_data->region[0].phys_addr;
> + len = pci_data->region[0].len;
> + addr = pci_data->region[0].addr;
> +
> + dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> + } else {
> + dev_info(hw, "Unknown feature type 0x%x id 0x%x\n",
> + hdr.type, hdr.id);
> + ret = -ENODEV;
> + }
> +
> + return ret;
> +}
> +
> +static int dfl_fpga_feature_devs_enumerate(struct dfl_fpga_enum_info
> +*info) {
> + struct build_feature_devs_info *binfo;
> + struct dfl_fpga_enum_dfl *dfl;
> + int ret = 0;
> +
> + if (!info || !info->hw)
> + return -EINVAL;
> +
> + /* create and init build info for enumeration */
> + binfo = opae_zmalloc(sizeof(*binfo));
> if (!binfo)
> return -ENOMEM;
>
> - ret = parse_feature_list(binfo, binfo->ioaddr);
> + binfo->hw = info->hw;
> + binfo->pci_data = info->hw->pci_data;
> +
> + /*
> + * start enumeration for all feature devices based on Device Feature
> + * Lists.
> + */
> + TAILQ_FOREACH(dfl, &info->dfls, node) {
> + ret = parse_feature_list(binfo, dfl);
> + if (ret)
> + break;
> + }
> +
> + build_info_free(binfo);
> +
> + return ret;
> +}
> +
> +int ifpga_bus_enumerate(struct ifpga_hw *hw) {
> + struct dfl_fpga_enum_info *info;
> + int ret;
> +
> + /* allocate enumeration info */
> + info = dfl_fpga_enum_info_alloc(hw);
> + if (!info)
> + return -ENOMEM;
> +
> + ret = find_dfls_by_vsec(info);
> + if (ret < 0)
> + ret = find_dfls_by_default(info);
> +
> if (ret)
> goto exit;
>
> - ret = parse_ports_from_fme(binfo);
> - if (ret)
> + /* start enumeration with prepared enumeration information */
> + ret = dfl_fpga_feature_devs_enumerate(info);
> + if (ret < 0) {
> + dev_err(hw, "Enumeration failure\n");
> goto exit;
> + }
>
> ifpga_print_device_feature_list(hw);
>
> exit:
> - build_info_free(binfo);
> + dfl_fpga_enum_info_free(info);
> +
> return ret;
> }
>
> -int ifpga_bus_init(struct ifpga_hw *hw)
> +static void ifpga_print_acc_list(struct opae_adapter *adapter)
> {
> + struct opae_accelerator *acc;
> + struct ifpga_afu_info *info;
> + struct uuid guid;
> + char buf[48];
> int i;
> +
> + opae_adapter_for_each_acc(adapter, acc) {
> + info = acc->data;
> + if (!info)
> + continue;
> + acc->ops->get_uuid(acc, &guid);
> + i = sprintf(buf, "%02x%02x%02x%02x-",
> + guid.b[15], guid.b[14], guid.b[13], guid.b[12]);
> + i += sprintf(buf+i, "%02x%02x-", guid.b[11], guid.b[10]);
> + i += sprintf(buf+i, "%02x%02x-", guid.b[9], guid.b[8]);
> + i += sprintf(buf+i, "%02x%02x-", guid.b[7], guid.b[6]);
> + sprintf(buf+i, "%02x%02x%02x%02x%02x%02x",
> + guid.b[5], guid.b[4], guid.b[3],
> + guid.b[2], guid.b[1], guid.b[0]);
> + dev_info(hw, "AFU(%s-%d)@%p: len:0x%"PRIx64", guid:%s\n",
> + acc->name, acc->index, info->region[0].addr,
> + info->region[0].len, buf);
> + }
> +}
> +
> +int ifpga_bus_init(struct ifpga_hw *hw) {
> + int i, ret = 0;
> struct ifpga_port_hw *port;
>
> - fme_hw_init(&hw->fme);
> + ret = fme_hw_init(&hw->fme);
> + if (ret)
> + return ret;
> +
> for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
> port = &hw->port[i];
> port_hw_init(port);
> }
> + ifpga_print_acc_list(hw->adapter);
>
> return 0;
> }
> diff --git a/drivers/raw/ifpga/base/ifpga_enumerate.h
> b/drivers/raw/ifpga/base/ifpga_enumerate.h
> index 95ed594..e6b04f0 100644
> --- a/drivers/raw/ifpga/base/ifpga_enumerate.h
> +++ b/drivers/raw/ifpga/base/ifpga_enumerate.h
> @@ -5,6 +5,8 @@
> #ifndef _IFPGA_ENUMERATE_H_
> #define _IFPGA_ENUMERATE_H_
>
> +#define FME_PORT_OFST_BAR_SKIP 7
> +
> int ifpga_bus_init(struct ifpga_hw *hw); int ifpga_bus_uinit(struct ifpga_hw
> *hw); int ifpga_bus_enumerate(struct ifpga_hw *hw); diff --git
> a/drivers/raw/ifpga/base/ifpga_feature_dev.c
> b/drivers/raw/ifpga/base/ifpga_feature_dev.c
> index 0813513..dbecc7b 100644
> --- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
> +++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
> @@ -59,7 +59,7 @@ int __fpga_port_disable(struct ifpga_port_hw *port)
> if (fpga_wait_register_field(port_sftrst_ack, control,
> &port_hdr->control, RST_POLL_TIMEOUT,
> RST_POLL_INVL)) {
> - dev_err(port, "timeout, fail to reset device\n");
> + dev_err(port, "timeout, fail to reset FIM port\n");
> return -ETIMEDOUT;
> }
>
> @@ -277,10 +277,11 @@ static void feature_uinit(struct ifpga_feature_list
> *list)
> struct ifpga_feature *feature;
>
> TAILQ_FOREACH(feature, list, next) {
> - if (feature->state != IFPGA_FEATURE_ATTACHED)
> + if (feature->state != IFPGA_FEATURE_INITED)
> continue;
> if (feature->ops && feature->ops->uinit)
> feature->ops->uinit(feature);
> + feature->state = IFPGA_FEATURE_ATTACHED;
> }
> }
>
> @@ -301,6 +302,9 @@ static int feature_init(struct feature_driver *drv,
> ret = feature->ops->init(feature);
> if (ret)
> goto error;
> + else
> + feature->state =
> +
> IFPGA_FEATURE_INITED;
> }
> }
> }
> @@ -315,14 +319,8 @@ static int feature_init(struct feature_driver *drv,
>
> int fme_hw_init(struct ifpga_fme_hw *fme) {
> - int ret;
> -
> - if (fme->state != IFPGA_FME_IMPLEMENTED)
> - return -ENODEV;
> -
> - ret = feature_init(fme_feature_drvs, &fme->feature_list);
> - if (ret)
> - return ret;
> + if (fme->state == IFPGA_FME_IMPLEMENTED)
> + return feature_init(fme_feature_drvs, &fme->feature_list);
>
> return 0;
> }
> diff --git a/drivers/raw/ifpga/base/ifpga_hw.h
> b/drivers/raw/ifpga/base/ifpga_hw.h
> index ed5edc6..4d56deb 100644
> --- a/drivers/raw/ifpga/base/ifpga_hw.h
> +++ b/drivers/raw/ifpga/base/ifpga_hw.h
> @@ -15,6 +15,7 @@
> enum ifpga_feature_state {
> IFPGA_FEATURE_UNUSED = 0,
> IFPGA_FEATURE_ATTACHED,
> + IFPGA_FEATURE_INITED
> };
>
> enum feature_type {
> @@ -134,6 +135,7 @@ struct ifpga_hw {
>
> struct ifpga_fme_hw fme;
> struct ifpga_port_hw port[MAX_FPGA_PORT_NUM];
> + int num_afus;
> };
>
> static inline bool is_ifpga_hw_pf(struct ifpga_hw *hw) diff --git
> a/drivers/raw/ifpga/base/opae_hw_api.c
> b/drivers/raw/ifpga/base/opae_hw_api.c
> index 11c9887..87256fc 100644
> --- a/drivers/raw/ifpga/base/opae_hw_api.c
> +++ b/drivers/raw/ifpga/base/opae_hw_api.c
> @@ -177,7 +177,7 @@ int opae_acc_get_region_info(struct opae_accelerator
> *acc, int opae_acc_set_irq(struct opae_accelerator *acc,
> u32 start, u32 count, s32 evtfds[]) {
> - if (!acc || !acc->data)
> + if (!acc)
> return -EINVAL;
>
> if (start + count <= start)
> diff --git a/drivers/raw/ifpga/base/opae_hw_api.h
> b/drivers/raw/ifpga/base/opae_hw_api.h
> index 7e04b56..fd40e09 100644
> --- a/drivers/raw/ifpga/base/opae_hw_api.h
> +++ b/drivers/raw/ifpga/base/opae_hw_api.h
> @@ -143,6 +143,7 @@ struct opae_accelerator {
> TAILQ_ENTRY(opae_accelerator) node;
> const char *name;
> int index;
> + struct opae_adapter *adapter;
> struct opae_bridge *br;
> struct opae_manager *mgr;
> struct opae_accelerator_ops *ops;
> @@ -240,6 +241,7 @@ struct opae_adapter_data {
>
> struct opae_reg_region {
> u64 phys_addr;
> +#define AFU_REGION_SIZE 0x8000
> u64 len;
> u8 *addr;
> };
> diff --git a/drivers/raw/ifpga/ifpga_rawdev.c
> b/drivers/raw/ifpga/ifpga_rawdev.c
> index 94df56c..ceb18ae 100644
> --- a/drivers/raw/ifpga/ifpga_rawdev.c
> +++ b/drivers/raw/ifpga/ifpga_rawdev.c
> @@ -47,11 +47,13 @@
> #define PCIE_DEVICE_ID_PF_INT_6_X 0xBCC0
> #define PCIE_DEVICE_ID_PF_DSC_1_X 0x09C4
> #define PCIE_DEVICE_ID_PAC_N3000 0x0B30
> +#define PCIE_DEVICE_ID_PAC_N6000 0xBCCE
> /* VF Device */
> #define PCIE_DEVICE_ID_VF_INT_5_X 0xBCBF
> #define PCIE_DEVICE_ID_VF_INT_6_X 0xBCC1
> #define PCIE_DEVICE_ID_VF_DSC_1_X 0x09C5
> #define PCIE_DEVICE_ID_VF_PAC_N3000 0x0B31
> +#define PCIE_DEVICE_ID_VF_PAC_N6000 0xBCCF
> #define RTE_MAX_RAW_DEVICE 10
>
> static const struct rte_pci_id pci_ifpga_map[] = { @@ -63,6 +65,8 @@
> { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_DSC_1_X) },
> { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PAC_N3000),},
> { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_PAC_N3000),},
> + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PAC_N6000),},
> + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_PAC_N6000),},
> { .vendor_id = 0, /* sentinel */ },
> };
>
> @@ -110,6 +114,7 @@ struct ifpga_rawdev *
>
> return IFPGA_RAWDEV_NUM;
> }
> +
> static struct ifpga_rawdev *
> ifpga_rawdev_allocate(struct rte_rawdev *rawdev) { @@ -365,7 +370,7 @@
> static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev)
> return -ENODEV;
>
> mgr = opae_adapter_get_mgr(adapter);
> - if (!mgr)
> + if (!mgr || !mgr->sensor_list)
> return -ENODEV;
>
> opae_mgr_for_each_sensor(mgr, sensor) { @@ -377,7 +382,7 @@
> static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev)
> goto fail;
>
> if (value == 0xdeadbeef) {
> - IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value
> %x\n",
> + IFPGA_RAWDEV_PMD_DEBUG("dev_id %d sensor %s
> value %x\n",
> raw_dev->dev_id, sensor->name,
> value);
> continue;
> }
> @@ -893,7 +898,7 @@ static int set_surprise_link_check_aer( {
> struct opae_adapter *adapter;
> struct opae_manager *mgr;
> - struct opae_board_info *info;
> + struct opae_board_info *info = NULL;
> struct rte_afu_pr_conf *afu_pr_conf;
> int ret;
> struct uuid uuid;
> @@ -921,17 +926,14 @@ static int set_surprise_link_check_aer(
> }
>
> mgr = opae_adapter_get_mgr(adapter);
> - if (!mgr) {
> - IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is
> NULL");
> - return -1;
> - }
> -
> - if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
> - IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info
> fail!");
> - return -1;
> + if (mgr) {
> + if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
> + IFPGA_RAWDEV_PMD_ERR("ifpga manager
> get_board_info fail!");
> + return -1;
> + }
> }
>
> - if (info->lightweight) {
> + if (info && info->lightweight) {
> /* set uuid to all 0, when fpga is lightweight image */
> memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
> memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
> @@ -953,7 +955,7 @@ static int set_surprise_link_check_aer(
> __func__,
> (unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
> (unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
> - }
> + }
> return 0;
> }
>
> @@ -1592,7 +1594,7 @@ static int fme_clean_fme_error(struct opae_manager
> *mgr)
> ret = opae_adapter_init(adapter, pci_dev->device.name, data);
> if (ret) {
> ret = -ENOMEM;
> - goto free_adapter_data;
> + goto cleanup;
> }
>
> rawdev->dev_ops = &ifpga_rawdev_ops;
> @@ -1602,29 +1604,23 @@ static int fme_clean_fme_error(struct
> opae_manager *mgr)
> /* must enumerate the adapter before use it */
> ret = opae_adapter_enumerate(adapter);
> if (ret)
> - goto free_adapter_data;
> + goto cleanup;
>
> /* get opae_manager to rawdev */
> mgr = opae_adapter_get_mgr(adapter);
> if (mgr) {
> - /* PF function */
> - IFPGA_RAWDEV_PMD_INFO("this is a PF function");
> + ret = ifpga_register_msix_irq(dev, 0, IFPGA_FME_IRQ, 0, 0,
> + fme_interrupt_handler, "fme_irq", mgr);
> + if (ret)
> + goto cleanup;
> }
>
> - ret = ifpga_register_msix_irq(dev, 0, IFPGA_FME_IRQ, 0, 0,
> - fme_interrupt_handler, "fme_irq", mgr);
> - if (ret)
> - goto free_adapter_data;
> -
> ret = ifpga_monitor_start_func(dev);
> if (ret)
> - goto free_adapter_data;
> + goto cleanup;
>
> return ret;
>
> -free_adapter_data:
> - if (data)
> - opae_adapter_data_free(data);
> cleanup:
> if (rawdev)
> rte_rawdev_pmd_release(rawdev);
> --
> 1.8.3.1
It looks good for me, you can add:
Acked-by: Tianfei Zhang <tianfei.zh...@intel.com>
