? tmp.conf
? src/apps/admin/pvfs2-dspace-dump.c
Index: src/io/bmi/bmi-method-callback.h
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi-method-callback.h,v
retrieving revision 1.5
diff -u -a -p -r1.5 bmi-method-callback.h
--- src/io/bmi/bmi-method-callback.h	28 Jul 2004 14:32:39 -0000	1.5
+++ src/io/bmi/bmi-method-callback.h	8 Oct 2007 16:12:52 -0000
@@ -9,7 +9,8 @@
 
 #include "bmi-method-support.h"
 
-int bmi_method_addr_reg_callback(method_addr_p map);
+PVFS_BMI_addr_t bmi_method_addr_reg_callback(method_addr_p map);
+int bmi_method_addr_forget_callback(PVFS_BMI_addr_t addr);
 
 #endif /* __BMI_METHOD_CALLBACK_H */
 
Index: src/io/bmi/bmi.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi.c,v
retrieving revision 1.86
diff -u -a -p -r1.86 bmi.c
--- src/io/bmi/bmi.c	17 Aug 2007 19:31:19 -0000	1.86
+++ src/io/bmi/bmi.c	8 Oct 2007 16:12:52 -0000
@@ -1215,6 +1215,9 @@ int BMI_set_info(PVFS_BMI_addr_t addr,
     int i = 0;
     ref_st_p tmp_ref = NULL;
 
+    gossip_debug(GOSSIP_BMI_DEBUG_CONTROL,
+                 "[BMI CONTROL]: %s: set_info: %llu option: %d\n",
+                 __func__, llu(addr), option);
     /* if the addr is NULL, then the set_info should apply to all
      * available methods.
      */
@@ -1242,6 +1245,9 @@ int BMI_set_info(PVFS_BMI_addr_t addr,
 	return (0);
     }
 
+    gossip_debug(GOSSIP_BMI_DEBUG_CONTROL,
+                 "[BMI CONTROL]: %s: searching for ref %llu\n",
+                 __func__, llu(addr));
     /* find a reference that matches this address */
     gen_mutex_lock(&ref_mutex);
     tmp_ref = ref_list_search_addr(cur_ref_list, addr);
@@ -1255,12 +1261,18 @@ int BMI_set_info(PVFS_BMI_addr_t addr,
     if(option == BMI_INC_ADDR_REF)
     {
 	tmp_ref->ref_count++;
+        gossip_debug(GOSSIP_BMI_DEBUG_CONTROL,
+                     "[BMI CONTROL]: %s: incremented ref %llu to: %d\n",
+                     __func__, llu(addr), tmp_ref->ref_count);
 	gen_mutex_unlock(&ref_mutex);
 	return(0);
     }
     if(option == BMI_DEC_ADDR_REF)
     {
 	tmp_ref->ref_count--;
+        gossip_debug(GOSSIP_BMI_DEBUG_CONTROL,
+                     "[BMI CONTROL]: %s: decremented ref %llu to: %d\n",
+                     __func__, llu(addr), tmp_ref->ref_count);
 	assert(tmp_ref->ref_count >= 0);
 
 	if(tmp_ref->ref_count == 0)
@@ -1278,7 +1290,8 @@ int BMI_set_info(PVFS_BMI_addr_t addr,
 	    {
 		/* kill the address */
 		gossip_debug(GOSSIP_BMI_DEBUG_CONTROL,
-		    "bmi discarding address: %llu\n", llu(addr));
+		    "[BMI CONTROL]: %s: bmi discarding address: %llu\n",
+                    __func__, llu(addr));
 		ref_list_rem(cur_ref_list, addr);
 		/* NOTE: this triggers request to module to free underlying
 		 * resources if it wants to
@@ -1300,7 +1313,10 @@ int BMI_set_info(PVFS_BMI_addr_t addr,
              * out the entire address structure and anything linked to it so 
              * that the next addr_lookup starts from scratch
              */
-	    gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, "Closing bmi_tcp connection at caller's request.\n"); 
+	    gossip_debug(GOSSIP_BMI_DEBUG_CONTROL,
+                         "[BMI CONTROL]: %s: Closing bmi_tcp "
+                         "connection at caller's request.\n",
+                         __func__); 
             ref_list_rem(cur_ref_list, addr);
             dealloc_ref_st(tmp_ref);
         }
@@ -1562,7 +1578,7 @@ int BMI_addr_lookup(PVFS_BMI_addr_t * ne
     /* make sure one was successful */
     if (!meth_addr)
     {
-	return (bmi_errno_to_pvfs(-ENOPROTOOPT));
+        return bmi_errno_to_pvfs(-ENOPROTOOPT);
     }
 
     /* create a new reference for the addr */
@@ -1578,7 +1594,7 @@ int BMI_addr_lookup(PVFS_BMI_addr_t * ne
     new_ref->id_string = (char *) malloc(strlen(id_string) + 1);
     if (!new_ref->id_string)
     {
-	ret = -errno;
+	ret = bmi_errno_to_pvfs(errno);
 	goto bmi_addr_lookup_failure;
     }
     strcpy(new_ref->id_string, id_string);
@@ -1865,7 +1881,7 @@ int BMI_cancel(bmi_op_id_t id, 
  *
  * returns 0 on success, -errno on failure
  */
-int bmi_method_addr_reg_callback(method_addr_p map)
+PVFS_BMI_addr_t bmi_method_addr_reg_callback(method_addr_p map)
 {
     ref_st_p new_ref = NULL;
 
@@ -1877,7 +1893,7 @@ int bmi_method_addr_reg_callback(method_
     new_ref = alloc_ref_st();
     if (!new_ref)
     {
-	return (bmi_errno_to_pvfs(-ENOMEM));
+	return 0;
     }
 
     /*
@@ -1891,8 +1907,33 @@ int bmi_method_addr_reg_callback(method_
     new_ref->interface = active_method_table[map->method_type];
 
     /* add the reference structure to the list */
+    gen_mutex_lock(&ref_mutex);
     ref_list_add(cur_ref_list, new_ref);
+    gen_mutex_unlock(&ref_mutex);
 
+    return new_ref->bmi_addr;
+}
+
+int bmi_method_addr_forget_callback(PVFS_BMI_addr_t addr)
+{
+    ref_st_p ref;
+
+    gen_mutex_lock(&ref_mutex);
+    ref = ref_list_search_addr(cur_ref_list, addr);
+    if (!ref)
+    {
+	gen_mutex_unlock(&ref_mutex);
+	return (bmi_errno_to_pvfs(-EPROTO));
+    }
+    gen_mutex_unlock(&ref_mutex);
+
+    ref_list_rem(cur_ref_list, ref->bmi_addr);
+
+    /* have to set the method_addr to null before deallocating, since
+     * dealloc_ref_st tries to enter the method again to drop the addr
+     */
+    ref->method_addr = NULL;
+    dealloc_ref_st(ref);
     return (0);
 }
 
Index: src/io/bmi/bmi_gm/bmi-gm.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_gm/bmi-gm.c,v
retrieving revision 1.86
diff -u -a -p -r1.86 bmi-gm.c
--- src/io/bmi/bmi_gm/bmi-gm.c	22 Aug 2007 16:12:45 -0000	1.86
+++ src/io/bmi/bmi_gm/bmi-gm.c	8 Oct 2007 16:12:52 -0000
@@ -3147,11 +3147,11 @@ static int recv_event_handler(gm_recv_ev
 		gm_addr_data->node_id = gm_ntohs(poll_event->recv.sender_node_id);
 		gm_addr_data->port_id = gm_ntohc(poll_event->recv.sender_port_id);
 		/* let the bmi layer know about it */
-		ret = bmi_method_addr_reg_callback(map);
-		if (ret < 0)
+		gm_addr_data->bmi_addr = bmi_method_addr_reg_callback(map);
+		if (!gm_addr_data->bmi_addr)
 		{
 		    dealloc_gm_method_addr(map);
-		    return (ret);
+		    return (-BMI_ENOMEM);
 		}
 		/* keep up with it ourselves also */
 		gm_addr_add(&gm_addr_list, map);
Index: src/io/bmi/bmi_ib/ib.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/ib.c,v
retrieving revision 1.56
diff -u -a -p -r1.56 ib.c
--- src/io/bmi/bmi_ib/ib.c	7 Sep 2007 16:02:14 -0000	1.56
+++ src/io/bmi/bmi_ib/ib.c	8 Oct 2007 16:12:52 -0000
@@ -1822,9 +1822,9 @@ static int ib_tcp_server_check_new_conne
 
 	c->remote_map = ib_alloc_method_addr(c, hostname, port);
 	/* register this address with the method control layer */
-	ret = bmi_method_addr_reg_callback(c->remote_map);
-	if (ret < 0)
-	    error_xerrno(ret, "%s: bmi_method_addr_reg_callback", __func__);
+	c->bmi_addr = bmi_method_addr_reg_callback(c->remote_map);
+	if (c->bmi_addr == 0)
+	    error_xerrno(ENOMEM, "%s: bmi_method_addr_reg_callback", __func__);
 
 	debug(2, "%s: accepted new connection %s at server", __func__,
 	  c->peername);
@@ -1936,6 +1936,7 @@ static int BMI_ib_set_info(int option, v
 	ib_method_addr_t *ibmap = map->method_data;
 	free(ibmap->hostname);
 	free(map);
+	bmi_method_addr_forget_callback(ibmap->c->bmi_addr);
 	break;
     }
     case BMI_OPTIMISTIC_BUFFER_REG: {
Index: src/io/bmi/bmi_ib/ib.h
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/ib.h,v
retrieving revision 1.28
diff -u -a -p -r1.28 ib.h
--- src/io/bmi/bmi_ib/ib.h	8 May 2007 21:28:01 -0000	1.28
+++ src/io/bmi/bmi_ib/ib.h	8 Oct 2007 16:12:52 -0000
@@ -60,6 +60,7 @@ typedef struct {
 
     void *priv;
 
+    PVFS_BMI_addr_t bmi_addr;
 } ib_connection_t;
 
 /*
Index: src/io/bmi/bmi_tcp/bmi-tcp-addressing.h
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_tcp/bmi-tcp-addressing.h,v
retrieving revision 1.15
diff -u -a -p -r1.15 bmi-tcp-addressing.h
--- src/io/bmi/bmi_tcp/bmi-tcp-addressing.h	11 Sep 2006 20:22:04 -0000	1.15
+++ src/io/bmi/bmi_tcp/bmi-tcp-addressing.h	8 Oct 2007 16:12:52 -0000
@@ -46,6 +46,7 @@ struct tcp_allowed_connection_s {
  * connections are made */
 struct tcp_addr
 {
+    PVFS_BMI_addr_t bmi_addr;
     method_addr_p map;		/* points back to generic address */
     /* stores error code for addresses that are broken for some reason */
     int addr_error;		
Index: src/io/bmi/bmi_tcp/bmi-tcp.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_tcp/bmi-tcp.c,v
retrieving revision 1.117
diff -u -a -p -r1.117 bmi-tcp.c
--- src/io/bmi/bmi_tcp/bmi-tcp.c	22 Aug 2007 16:12:47 -0000	1.117
+++ src/io/bmi/bmi_tcp/bmi-tcp.c	8 Oct 2007 16:12:52 -0000
@@ -915,7 +915,7 @@ int BMI_tcp_get_info(int option,
 	 * an error and there is no way to reconnect
 	 */
 	if(tcp_addr_data->addr_error != 0 &&
-	    tcp_addr_data->dont_reconnect == 1)
+           tcp_addr_data->dont_reconnect == 1)
 	{
 	    query->response = 1;
 	}
@@ -1854,6 +1854,8 @@ void tcp_forget_addr(method_addr_p map,
 	BMI_socket_collection_testglobal(tcp_socket_collection_p,
 	    0, &tmp_outcount, &tmp_addr, &tmp_status, 0, &interface_mutex);
     }
+
+    bmi_method_addr_forget_callback(tcp_addr_data->bmi_addr);
     tcp_shutdown_addr(map);
     tcp_cleanse_addr(map, error_code);
     tcp_addr_data->addr_error = error_code;
@@ -2849,7 +2851,7 @@ static int handle_new_connection(method_
      */
     tcp_addr_data->dont_reconnect = 1;
     /* register this address with the method control layer */
-    ret = bmi_method_addr_reg_callback(new_addr);
+    tcp_addr_data->bmi_addr = bmi_method_addr_reg_callback(new_addr);
     if (ret < 0)
     {
 	tcp_shutdown_addr(new_addr);
@@ -2984,10 +2986,7 @@ static int tcp_do_work_recv(method_addr_
 
     *stall_flag = 0;
     gossip_ldebug(GOSSIP_BMI_DEBUG_TCP, "Reading header for new op.\n");
-    /* NOTE: we only allow a blocking call here because we peeked to see
-     * if this amount of data was ready above.  
-     */
-    ret = BMI_sockio_brecv(tcp_addr_data->socket,
+    ret = BMI_sockio_nbrecv(tcp_addr_data->socket,
                            new_header.enc_hdr, TCP_ENC_HDR_SIZE);
     if (ret < TCP_ENC_HDR_SIZE)
     {
Index: src/io/bmi/bmi_tcp/socket-collection-epoll.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_tcp/socket-collection-epoll.c,v
retrieving revision 1.4
diff -u -a -p -r1.4 socket-collection-epoll.c
--- src/io/bmi/bmi_tcp/socket-collection-epoll.c	12 Sep 2006 00:49:31 -0000	1.4
+++ src/io/bmi/bmi_tcp/socket-collection-epoll.c	8 Oct 2007 16:12:52 -0000
@@ -201,12 +201,14 @@ int BMI_socket_collection_testglobal(soc
 	tcp_addr_data = qlist_entry(iterator, struct tcp_addr, sc_link);
 	qlist_del(&tcp_addr_data->sc_link);
         
+
         /* take out of the epoll set */
         if(tcp_addr_data->sc_index > -1)
         {
             memset(&event, 0, sizeof(event));
             event.events = 0;
             event.data.ptr = tcp_addr_data->map;
+
             ret = epoll_ctl(scp->epfd, EPOLL_CTL_DEL, tcp_addr_data->socket,
                 &event);
 
@@ -228,6 +230,7 @@ int BMI_socket_collection_testglobal(soc
     {
 	tcp_addr_data = qlist_entry(iterator, struct tcp_addr, sc_link);
 	qlist_del(&tcp_addr_data->sc_link);
+
 	if(tcp_addr_data->sc_index > -1)
 	{
             memset(&event, 0, sizeof(event));
@@ -236,8 +239,9 @@ int BMI_socket_collection_testglobal(soc
             event.events = (EPOLLIN|EPOLLERR|EPOLLHUP);
 	    if(tcp_addr_data->write_ref_count > 0)
                 event.events |= EPOLLOUT;
+
             ret = epoll_ctl(scp->epfd, EPOLL_CTL_MOD, tcp_addr_data->socket,
-                &event);
+                            &event);
 
             if(ret < 0 && errno != ENOENT)
             {
@@ -257,6 +261,7 @@ int BMI_socket_collection_testglobal(soc
             event.events = (EPOLLIN|EPOLLERR|EPOLLHUP);
 	    if(tcp_addr_data->write_ref_count > 0)
                 event.events |= EPOLLOUT;
+
             ret = epoll_ctl(scp->epfd, EPOLL_CTL_ADD, tcp_addr_data->socket,
                 &event);
             if(ret < 0 && errno != EEXIST)