WHAT: Adding communicator add/delete callbacks to MTL.
WHY: MTL will be able to separate messages on different contexts.
WHEN: On trunk (later on v1.5 as well), Tuesday telconf, 5 July 2011
TIMEOUT: Tuesday telconf, 12 July 2011
DETAILS:
Currently MTLs do no handle communicator contexts in any special way,
they only add the context id to the tag selection of the underlying
messaging meachinsm.
We would like to enable an MTL to maintain its own context data
per-communicator. This way an MTL will be able to queue incoming eager
messages and rendezvous requests per-communicator basis.
The MTL will be allowed to override comm->c_pml_comm member,
since it's unused in pml_cm anyway.
diff -r 0f757f8f3efb -r 8fcf276ee527 ompi/mca/mtl/mtl.h
--- a/ompi/mca/mtl/mtl.h Thu Jun 30 14:13:33 2011 +0300
+++ b/ompi/mca/mtl/mtl.h Fri Jul 01 13:54:20 2011 +0300
@@ -355,6 +355,34 @@
/**
+ * Downcall from PML layer when a new communicator is created.
+ *
+ * @param comm Communicator
+ * @return OMPI_SUCCESS or failure status.
+ *
+ * Provides the MTL the opportunity to initialize/cache a data structure
+ * on the communicator.
+ */
+typedef int (*mca_mtl_base_module_add_comm_fn_t)(
+ struct mca_mtl_base_module_t* mtl,
+ struct ompi_communicator_t* comm);
+
+
+/**
+ * Downcall from PML layer when a communicator is destroyed.
+ *
+ * @param comm Communicator
+ * @return OMPI_SUCCESS or failure status.
+ *
+ * Provides the MTL the opportunity to cleanup any datastructures
+ * associated with the communicator.
+ */
+typedef int (*mca_mtl_base_module_del_comm_fn_t)(
+ struct mca_mtl_base_module_t* mtl,
+ struct ompi_communicator_t* comm);
+
+
+/**
* MTL module interface functions and attributes.
*/
struct mca_mtl_base_module_t {
@@ -368,7 +396,6 @@
mca_mtl_base_module_add_procs_fn_t mtl_add_procs;
mca_mtl_base_module_del_procs_fn_t mtl_del_procs;
mca_mtl_base_module_finalize_fn_t mtl_finalize;
-
mca_mtl_base_module_send_fn_t mtl_send;
mca_mtl_base_module_isend_fn_t mtl_isend;
mca_mtl_base_module_irecv_fn_t mtl_irecv;
@@ -376,6 +403,8 @@
/* Optional MTL functions */
mca_mtl_base_module_cancel_fn_t mtl_cancel;
+ mca_mtl_base_module_add_comm_fn_t mtl_add_comm;
+ mca_mtl_base_module_del_comm_fn_t mtl_del_comm;
};
typedef struct mca_mtl_base_module_t mca_mtl_base_module_t;
diff -r 0f757f8f3efb -r 8fcf276ee527 ompi/mca/pml/cm/pml_cm.c
--- a/ompi/mca/pml/cm/pml_cm.c Thu Jun 30 14:13:33 2011 +0300
+++ b/ompi/mca/pml/cm/pml_cm.c Fri Jul 01 13:54:20 2011 +0300
@@ -82,14 +82,22 @@
int
mca_pml_cm_add_comm(ompi_communicator_t* comm)
{
+ int ret;
+
/* should never happen, but it was, so check */
if (comm->c_contextid > ompi_pml_cm.super.pml_max_contextid) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
- /* setup our per-communicator data */
+ /* initialize per-communicator data. MTLs may override this. */
comm->c_pml_comm = NULL;
+ /* notify the MTL about the added communicator */
+ if ((NULL != ompi_mtl->mtl_add_comm) &&
+ (OMPI_SUCCESS != (ret = OMPI_MTL_CALL(add_comm(ompi_mtl, comm)))))
{
+ return ret;
+ }
+
return OMPI_SUCCESS;
}
@@ -97,8 +105,13 @@
int
mca_pml_cm_del_comm(ompi_communicator_t* comm)
{
- /* clean up our per-communicator data */
- comm->c_pml_comm = NULL;
+ int ret;
+
+ /* notify the MTL about the deleted communicator */
+ if ((NULL != ompi_mtl->mtl_del_comm) &&
+ (OMPI_SUCCESS != (ret = OMPI_MTL_CALL(del_comm(ompi_mtl, comm)))))
{
+ return ret;
+ }
return OMPI_SUCCESS;
}
