Add API to rebase memory registrations. The HCA driver already supports specifying an arbitrary starting address for an MR. This patch expose that functionality as a new entry point.
Signed-off-by: Fab Tillier <[email protected]> diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_mr.h .\core\al\al_mr.h --- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_mr.h Thu May 31 11:22:16 2012 +++ .\core\al\al_mr.h Wed May 23 18:26:47 2012 @@ -83,6 +83,7 @@ ib_api_status_t reg_mem( IN const ib_pd_handle_t h_pd, IN const ib_mr_create_t* const p_mr_create, + IN const uint64_t mapaddr, OUT net32_t* const p_lkey, OUT net32_t* const p_rkey, OUT ib_mr_handle_t* const ph_mr, diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_mr_shared.c .\core\al\al_mr_shared.c --- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_mr_shared.c Thu May 31 11:22:16 2012 +++ .\core\al\al_mr_shared.c Wed May 23 18:26:47 2012 @@ -172,7 +172,14 @@ ib_reg_mem( return IB_INVALID_PD_HANDLE; } - status = reg_mem( h_pd, p_mr_create, p_lkey, p_rkey, ph_mr, FALSE ); + if( !p_mr_create ) + { + AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("IB_INVALID_PARAMETER\n") ); + return IB_INVALID_PARAMETER; + } + + status = reg_mem( h_pd, p_mr_create, (ULONG_PTR)p_mr_create->vaddr, + p_lkey, p_rkey, ph_mr, FALSE ); /* Release the reference taken in alloc_mr for initialization. */ if( status == IB_SUCCESS ) @@ -188,6 +195,7 @@ ib_api_status_t reg_mem( IN const ib_pd_handle_t h_pd, IN const ib_mr_create_t* const p_mr_create, + IN const uint64_t mapaddr, OUT net32_t* const p_lkey, OUT net32_t* const p_rkey, OUT ib_mr_handle_t* const ph_mr, @@ -198,7 +206,7 @@ reg_mem( AL_ENTER( AL_DBG_MR ); - if( !p_mr_create || !p_lkey || !p_rkey || !ph_mr ) + if( !p_lkey || !p_rkey || !ph_mr ) { AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("IB_INVALID_PARAMETER\n") ); return IB_INVALID_PARAMETER; diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_verbs.h .\core\al\al_verbs.h --- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_verbs.h Thu Aug 02 12:53:26 2012 +++ .\core\al\al_verbs.h Wed May 23 18:26:47 2012 @@ -116,8 +116,8 @@ verbs_create_cq( IB_UNSUPPORTED ) #define verbs_register_mr(h_pd, p_mr_create, p_lkey, p_rkey, h_mr) \ - h_mr->obj.p_ci_ca->verbs.register_mr( h_pd->h_ci_pd,\ - p_mr_create, p_lkey, p_rkey, &h_mr->h_ci_mr, um_call ) + h_mr->obj.p_ci_ca->verbs.register_mr_remap( h_pd->h_ci_pd,\ + p_mr_create, mapaddr, p_lkey, p_rkey, &h_mr->h_ci_mr, um_call ) #define verbs_register_pmr(h_pd, p_phys_create, p_vaddr,\ p_lkey, p_rkey, h_mr) \ @@ -462,7 +462,8 @@ verbs_create_cq( #define verbs_register_mr(h_pd, p_mr_create, p_lkey, p_rkey, h_mr) \ ual_reg_mem(h_pd, p_mr_create, p_lkey, p_rkey, h_mr); \ - UNUSED_PARAM( um_call ) + UNUSED_PARAM( um_call ); \ + UNUSED_PARAM( mapaddr ) #define verbs_register_pmr(h_pd, p_phys_create, p_vaddr, p_lkey, p_rkey, h_mr) \ IB_UNSUPPORTED; \ diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\kernel\al_proxy_verbs.c .\core\al\kernel\al_proxy_verbs.c --- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\kernel\al_proxy_verbs.c Thu Aug 02 16:18:42 2012 +++ .\core\al\kernel\al_proxy_verbs.c Fri Aug 03 14:21:53 2012 @@ -2967,7 +2967,7 @@ proxy_register_mr( goto proxy_register_mr_err; } - status = reg_mem( h_pd, &p_ioctl->in.mem_create, &p_ioctl->out.lkey, + status = reg_mem( h_pd, &p_ioctl->in.mem_create, (ULONG_PTR)p_ioctl->in.mem_create.vaddr, &p_ioctl->out.lkey, &p_ioctl->out.rkey, &h_mr, TRUE ); if( status == IB_SUCCESS ) diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\mr.c .\hw\mlx4\kernel\hca\mr.c --- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\mr.c Thu Aug 02 13:58:26 2012 +++ .\hw\mlx4\kernel\hca\mr.c Thu May 31 12:35:11 2012 @@ -42,9 +42,10 @@ */ ib_api_status_t -mlnx_register_mr ( +mlnx_register_mr_remap ( IN const ib_pd_handle_t h_pd, IN const ib_mr_create_t *p_mr_create, + IN const uint64_t mapaddr, OUT net32_t* const p_lkey, OUT net32_t* const p_rkey, OUT ib_mr_handle_t *ph_mr, @@ -69,6 +71,14 @@ mlnx_register_mr ( status = IB_INVALID_PARAMETER; goto err_invalid_parm; } + if( BYTE_OFFSET( p_mr_create->vaddr ) != BYTE_OFFSET( mapaddr ) ) + { + HCA_PRINT(TRACE_LEVEL_WARNING, HCA_DBG_MEMORY, + ("mapaddr offset != vaddr offset\n")); + status = IB_INVALID_PARAMETER; + goto err_invalid_parm; + } + /* * Local write permission is required if remote write or * remote atomic permission is also requested. @@ -88,7 +98,7 @@ mlnx_register_mr ( // register mr p_ib_mr = ibv_reg_mr(p_ib_pd, (ULONG_PTR)p_mr_create->vaddr, - p_mr_create->length, (ULONG_PTR)p_mr_create->vaddr, + p_mr_create->length, mapaddr, access_ctrl); if (IS_ERR(p_ib_mr)) { err = PTR_ERR(p_ib_mr); @@ -116,6 +126,35 @@ err_unsupported: } ib_api_status_t +mlnx_register_mr ( + IN const ib_pd_handle_t h_pd, + IN const ib_mr_create_t *p_mr_create, + OUT net32_t* const p_lkey, + OUT net32_t* const p_rkey, + OUT ib_mr_handle_t *ph_mr, + IN boolean_t um_call ) +{ + ib_api_status_t status; + + HCA_ENTER(HCA_DBG_MEMORY); + + // sanity checks + if (!p_mr_create) { + HCA_PRINT(TRACE_LEVEL_WARNING ,HCA_DBG_MEMORY, + ("invalid attributes\n")); + status = IB_INVALID_PARAMETER; + goto err_invalid_parm; + } + + status = mlnx_register_mr_remap( + h_pd, p_mr_create, (ULONG_PTR)p_mr_create->vaddr, p_lkey, p_rkey, ph_mr, um_call ); + +err_invalid_parm: + HCA_EXIT(HCA_DBG_MEMORY); + return status; +} + +ib_api_status_t mlnx_register_pmr ( IN const ib_pd_handle_t h_pd, IN const ib_phys_create_t* const p_pmr_create, @@ -662,6 +701,8 @@ mlnx_mr_if( p_interface->alloc_fast_reg_mr = mlnx_alloc_fast_reg_mr; p_interface->alloc_fast_reg_page_list = mlnx_alloc_fast_reg_page_list; p_interface->free_fast_reg_page_list = mlnx_free_fast_reg_page_list; + + p_interface->register_mr_remap = mlnx_register_mr_remap; } void diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\inc\iba\ib_ci.h .\inc\iba\ib_ci.h --- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\inc\iba\ib_ci.h Thu Aug 02 16:18:42 2012 +++ .\inc\iba\ib_ci.h Fri Aug 03 14:21:53 2012 @@ -1622,6 +1622,78 @@ typedef ib_api_status_t ****** */ + +/****f* Verbs/ci_register_mr +* NAME +* ci_register_mr -- Register a memory region with the HCA. +* SYNOPSIS +*/ +typedef ib_api_status_t +(*ci_register_mr_remap) ( + IN const ib_pd_handle_t h_pd, + IN const ib_mr_create_t* const p_mr_create, + IN const uint64_t mapaddr, + OUT net32_t* const p_lkey, + OUT net32_t* const p_rkey, + OUT ib_mr_handle_t * const ph_mr, + IN boolean_t um_call ); +/* +* DESCRIPTION +* This routine registers a virtually contiguous region of memory with the +* HCA. All memory regions that need to be used by the HCA must be registered +* prior to use in data transfer operations. On successful completion +* the region handle, lkey are returned. If remote access rights are specified +* then the rkey is also returned. +* PARAMETERS +* h_pd +* [in] Handle to the PD on which memory is being registered +* p_mr_create +* [in] Holds attributes for the region being registered. Look at +* ib_mr_create_t for more details. +* mapaddr +* [in] Holds the requested address to use as the base for the MR. Must have the same +* page offset as the address provided in p_mr_create. +* p_lkey +* [out] Local Key Attributes of the registered memory region +* p_rkey +* [out] Remote key of the registered memory region. The verbs provider +* is required to give this in the expected ordering on the wire. When +* rkey's are exchanged between remote nodes, no swapping of this data +* will be performed. +* ph_mr +* [out] Handle to the registered memory region. This handle is used when +* submitting work requests to refer to this region of memory. +* um_call +* [in] Boolean indicating whether the registration originated in user-mode. +* RETURN VALUE +* IB_SUCCESS +* Registration with the adapter was successful. +* IB_INSUFFICIENT_RESOURCES +* Insufficient resources to satisfy request. +* IB_INVALID_PARAMETER +* One of the input pointers was NULL. +* IB_INVALID_PD_HANDLE +* Invalid mr_pdhandle +* IB_INVALID_PERMISSION +* Invalid access rights. +* NOTES +* In addition to registration, the routine also pins memory, so that the +* physical page associated with the virtual address does not get swapped +* out during the time the HCA is attempting to transfer data to this +* address. If the memory is not pinned, this could lead to data-corruption +* and unpredictable behavior by the operating environment. +* +* This routine allows mapping an arbitrary address to the MR, as long as the +* offset into the first page of the buffer is identical between the real +* virtual address and the requested mapping address. +* +* SEE ALSO +* ci_deregister_mr, ci_query_mr, ci_register_pmr, ci_modify_mr, +* ci_register_smr +****** +*/ + + /****f* Verbs/ci_register_pmr * NAME * ci_register_pmr -- Register a physical memory region with the HCA. @@ -3269,6 +3341,11 @@ typedef struct _ci_interface /* 2.5 verbs */ ci_create_qp_ex create_qp_ex; + /* + * Extended functionality to support kernel clients. + */ + ci_register_mr_remap register_mr_remap; + } ci_interface_t; /********/
ndv2.17.patch
Description: ndv2.17.patch
_______________________________________________ ofw mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
