We were seeing pvfs2-client-core exhibiting large amounts of memory
usage, exceeding 1GB and continuing to rise. Investigating, it appears
there are cases where PVFS_hints get allocated but the state machine
don't always exit via terminate so the hints go un-freed. The culprit I
noticed the most is get-attr.
Attached are a set of patches to clean up the leaks I found.
1) hint_related_leaks.patch
Modifies PINT_client_state_machine_post to free hints that were
added at the top of the function if something causes it to return before
posting the job. Also, modify PINT_client_state_machine_release to free
hints before free the control block. Finally, adding a free of hints to
PINT_sys_release_smcb so state machines that don't go through terminate
still get their hints freed. These leaks cause continual increased
memory usage throughout the life of pvfs2-client-core.
2) pint_event_related_leaks.patch
Modifies PINT_event_finalize to call qhash_destroy_and_finalize
with a custom free method so that both the event and group and their
strdup'd name gets freed.
3) req_sched_related_leaks.patch
Modifies PINT_req_sched_finalize to both free the entries in the
timer_queue but also free user_ptr allocation in both timer_queue
elements and req_sched_table list elements.
4) perf_related_leaks.patch
Modifies main() to track the three smcbs it allocates for cache related
state machines. Previously each smcb was thrown away as the next was
allocated. Since these run the life of the client it isn't a large issue
this is just clean up on exit. Also, the globally-defined
PINT_perf_counters were not getting finalized. I stuck all the cleanup
into a single function to keep main from getting over-run with cleanup
code at each return point.
If all looks okay, can you apply?
Thanks,
Michael
Index: pvfs2/src/client/sysint/client-state-machine.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/client/sysint/client-state-machine.c,v
retrieving revision 1.106
diff -a -u -p -r1.106 client-state-machine.c
--- pvfs2/src/client/sysint/client-state-machine.c 26 Mar 2010 02:07:54
-0000 1.106
+++ pvfs2/src/client/sysint/client-state-machine.c 8 Apr 2010 16:07:38
-0000
@@ -423,6 +423,8 @@ PVFS_error PINT_client_state_machine_pos
if (!smcb)
{
+ /* give back the hint added above */
+ PVFS_hint_free( sm_p->hints );
return ret;
}
@@ -443,6 +445,9 @@ PVFS_error PINT_client_state_machine_pos
{
/* state machine code failed */
gen_mutex_unlock(&test_mutex);
+
+ /* give back the hint added above */
+ PVFS_hint_free( sm_p->hints );
return sm_ret;
}
@@ -491,6 +496,11 @@ PVFS_error PINT_client_state_machine_rel
{
PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
+ if( sm_p )
+ {
+ PVFS_hint_free( sm_p->hints );
+ }
+
PINT_smcb_set_complete(smcb);
PINT_id_gen_safe_unregister(sm_p->sys_op_id);
@@ -903,6 +913,8 @@ static void PINT_sys_release_smcb(PINT_s
else
{
cred_p = sm_p->cred_p;
+ /* free the hint if sm_p isn't null */
+ PVFS_hint_free( sm_p->hints );
}
if (PINT_smcb_op(smcb) && cred_p)
Index: pvfs2/src/apps/kernel/linux/pvfs2-client-core.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/apps/kernel/linux/pvfs2-client-core.c,v
retrieving revision 1.109
diff -a -u -p -r1.109 pvfs2-client-core.c
--- pvfs2/src/apps/kernel/linux/pvfs2-client-core.c 26 Mar 2010 02:07:54
-0000 1.109
+++ pvfs2/src/apps/kernel/linux/pvfs2-client-core.c 8 Apr 2010 16:06:46
-0000
@@ -239,6 +239,7 @@ static int set_acache_parameters(options
static void set_device_parameters(options_t *s_opts);
static void reset_ncache_timeout(void);
static int set_ncache_parameters(options_t* s_opts);
+static void finalize_perf_items(int n, ... );
inline static void fill_hints(PVFS_hint *hints, vfs_request_t *req);
static PVFS_object_ref perform_lookup_on_create_error(
@@ -3318,7 +3319,9 @@ int main(int argc, char **argv)
uint64_t debug_mask = GOSSIP_NO_DEBUG;
PINT_client_sm *acache_timer_sm_p = NULL;
PINT_client_sm *static_acache_timer_sm_p = NULL;
- PINT_smcb *smcb = NULL;
+ PINT_smcb *acache_smcb = NULL;
+ PINT_smcb *acache_static_smcb = NULL;
+ PINT_smcb *ncache_smcb = NULL;
PINT_client_sm *ncache_timer_sm_p = NULL;
#ifdef __PVFS2_SEGV_BACKTRACE__
@@ -3472,6 +3475,7 @@ int main(int argc, char **argv)
if(ret < 0)
{
gossip_err("Error: PINT_perf_set_info (history_size).\n");
+ finalize_perf_items( 0 );
return(ret);
}
@@ -3479,6 +3483,7 @@ int main(int argc, char **argv)
if(!static_acache_pc)
{
gossip_err("Error: PINT_perf_initialize failure.\n");
+ finalize_perf_items( 0 );
return(-PVFS_ENOMEM);
}
ret = PINT_perf_set_info(static_acache_pc, PINT_PERF_HISTORY_SIZE,
@@ -3486,6 +3491,7 @@ int main(int argc, char **argv)
if(ret < 0)
{
gossip_err("Error: PINT_perf_set_info (history_size).\n");
+ finalize_perf_items( 0 );
return(ret);
}
@@ -3496,6 +3502,7 @@ int main(int argc, char **argv)
if(!ncache_pc)
{
gossip_err("Error: PINT_perf_initialize failure.\n");
+ finalize_perf_items( 0 );
return(-PVFS_ENOMEM);
}
ret = PINT_perf_set_info(ncache_pc, PINT_PERF_HISTORY_SIZE,
@@ -3503,68 +3510,76 @@ int main(int argc, char **argv)
if(ret < 0)
{
gossip_err("Error: PINT_perf_set_info (history_size).\n");
+ finalize_perf_items( 0 );
return(ret);
}
PINT_ncache_enable_perf_counter(ncache_pc);
/* start a timer to roll over performance counters (acache) */
- PINT_smcb_alloc(&smcb, PVFS_CLIENT_PERF_COUNT_TIMER,
+ PINT_smcb_alloc(&acache_smcb, PVFS_CLIENT_PERF_COUNT_TIMER,
sizeof(struct PINT_client_sm),
client_op_state_get_machine,
client_state_machine_terminate,
s_client_dev_context);
- if (!smcb)
+ if (!acache_smcb)
{
+ finalize_perf_items( 0 );
return(-PVFS_ENOMEM);
}
- acache_timer_sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
+ acache_timer_sm_p = PINT_sm_frame(acache_smcb, PINT_FRAME_CURRENT);
acache_timer_sm_p->u.perf_count_timer.interval_secs =
&s_opts.perf_time_interval_secs;
acache_timer_sm_p->u.perf_count_timer.pc = acache_pc;
- ret = PINT_client_state_machine_post(smcb, NULL, NULL);
+ ret = PINT_client_state_machine_post(acache_smcb, NULL, NULL);
if (ret < 0)
{
gossip_lerr("Error posting acache timer.\n");
+ finalize_perf_items( 1, acache_smcb );
return(ret);
}
- PINT_smcb_alloc(&smcb, PVFS_CLIENT_PERF_COUNT_TIMER,
+ PINT_smcb_alloc(&acache_static_smcb, PVFS_CLIENT_PERF_COUNT_TIMER,
sizeof(struct PINT_client_sm),
client_op_state_get_machine,
client_state_machine_terminate,
s_client_dev_context);
- if (!smcb)
+ if (!acache_static_smcb)
{
+ finalize_perf_items( 1, acache_smcb );
return(-PVFS_ENOMEM);
}
- static_acache_timer_sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
+ static_acache_timer_sm_p = PINT_sm_frame(acache_static_smcb,
+ PINT_FRAME_CURRENT);
static_acache_timer_sm_p->u.perf_count_timer.interval_secs =
&s_opts.perf_time_interval_secs;
static_acache_timer_sm_p->u.perf_count_timer.pc = static_acache_pc;
- ret = PINT_client_state_machine_post(smcb, NULL, NULL);
+ ret = PINT_client_state_machine_post(acache_static_smcb, NULL, NULL);
if (ret < 0)
{
gossip_lerr("Error posting acache timer.\n");
+ finalize_perf_items( 2, acache_smcb, acache_static_smcb );
return(ret);
}
- PINT_smcb_alloc(&smcb, PVFS_CLIENT_PERF_COUNT_TIMER,
+ PINT_smcb_alloc(&ncache_smcb, PVFS_CLIENT_PERF_COUNT_TIMER,
sizeof(struct PINT_client_sm),
client_op_state_get_machine,
client_state_machine_terminate,
s_client_dev_context);
- if (!smcb)
+ if (!ncache_smcb)
{
+ finalize_perf_items( 2, acache_smcb, acache_static_smcb );
return(-PVFS_ENOMEM);
}
- ncache_timer_sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
+ ncache_timer_sm_p = PINT_sm_frame(ncache_smcb, PINT_FRAME_CURRENT);
ncache_timer_sm_p->u.perf_count_timer.interval_secs =
&s_opts.perf_time_interval_secs;
ncache_timer_sm_p->u.perf_count_timer.pc = ncache_pc;
- ret = PINT_client_state_machine_post(smcb, NULL, NULL);
+ ret = PINT_client_state_machine_post(ncache_smcb, NULL, NULL);
if (ret < 0)
{
gossip_lerr("Error posting ncache timer.\n");
+ finalize_perf_items( 3, acache_smcb, acache_static_smcb, ncache_smcb );
return(ret);
}
@@ -3572,6 +3587,7 @@ int main(int argc, char **argv)
if (ret)
{
PVFS_perror("initialize_ops_in_progress_table", ret);
+ finalize_perf_items( 3, acache_smcb, acache_static_smcb, ncache_smcb );
return ret;
}
@@ -3579,6 +3595,7 @@ int main(int argc, char **argv)
if (ret < 0)
{
PVFS_perror("PINT_dev_initialize", ret);
+ finalize_perf_items( 3, acache_smcb, acache_static_smcb, ncache_smcb );
return -PVFS_EDEVINIT;
}
@@ -3588,6 +3605,7 @@ int main(int argc, char **argv)
if (ret < 0)
{
PVFS_perror("PINT_dev_get_mapped_region", ret);
+ finalize_perf_items( 3, acache_smcb, acache_static_smcb, ncache_smcb );
return ret;
}
@@ -3595,6 +3613,7 @@ int main(int argc, char **argv)
if (ret < 0)
{
PVFS_perror("device job_open_context failed", ret);
+ finalize_perf_items( 3, acache_smcb, acache_static_smcb, ncache_smcb );
return ret;
}
@@ -3607,6 +3626,7 @@ int main(int argc, char **argv)
if (pthread_create(&remount_thread, NULL, exec_remount, NULL))
{
gossip_err("Cannot create remount thread!");
+ finalize_perf_items( 3, acache_smcb, acache_static_smcb, ncache_smcb );
return -1;
}
@@ -3645,6 +3665,8 @@ int main(int argc, char **argv)
PINT_dev_finalize();
PINT_dev_put_mapped_regions(NUM_MAP_DESC, s_io_desc);
+ finalize_perf_items( 3, acache_smcb, acache_static_smcb, ncache_smcb );
+
gossip_err("pvfs2-client-core shutting down.\n");
if (PVFS_sys_finalize())
{
@@ -4032,6 +4054,41 @@ static void reset_ncache_timeout(void)
gossip_debug(GOSSIP_CLIENTCORE_DEBUG, "All file systems "
"unmounted. Not resetting the ncache.\n");
}
+}
+
+static void finalize_perf_items(int n, ... )
+{
+
+ int i=0;
+ PINT_smcb *smcb;
+ va_list v_args;
+
+ va_start(v_args, n);
+ for( i=0; i < n; i++ )
+ {
+ smcb = va_arg(v_args, PINT_smcb *);
+ if( smcb )
+ {
+ PINT_client_state_machine_release( smcb );
+ }
+ }
+ va_end( v_args );
+
+ if( acache_pc != NULL )
+ {
+ PINT_perf_finalize( acache_pc );
+ }
+
+ if( static_acache_pc != NULL )
+ {
+ PINT_perf_finalize( static_acache_pc );
+ }
+
+ if( ncache_pc != NULL )
+ {
+ PINT_perf_finalize( ncache_pc );
+ }
+ return;
}
#ifndef GOSSIP_DISABLE_DEBUG
Index: pvfs2/src/common/misc/pint-event.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/common/misc/pint-event.c,v
retrieving revision 1.14
diff -a -u -p -r1.14 pint-event.c
--- pvfs2/src/common/misc/pint-event.c 20 Nov 2008 01:16:53 -0000 1.14
+++ pvfs2/src/common/misc/pint-event.c 8 Apr 2010 16:08:45 -0000
@@ -69,6 +69,31 @@ static void PINT_event_tau_thread_fini(v
#endif /* HAVE_TAU */
+void PINT_event_free( struct PINT_event *p )
+{
+ if( p != NULL )
+ {
+ if( p->name != NULL )
+ {
+ free( p->name );
+ }
+ free( p );
+ }
+ return;
+}
+
+void PINT_group_free( struct PINT_group *g )
+{
+ if( g != NULL )
+ {
+ if( g->name != NULL )
+ {
+ free( g->name );
+ }
+ free( g );
+ }
+ return;
+}
static int PINT_group_compare(void *key, struct qhash_head *link)
{
struct PINT_group *eg = qhash_entry(link, struct PINT_group, link);
@@ -137,8 +162,11 @@ void PINT_event_finalize(void)
PINT_event_tau_fini();
#endif
- qhash_finalize(groups_table);
- qhash_finalize(events_table);
+ /* need to free contents as well */
+ qhash_destroy_and_finalize( groups_table, struct PINT_group, link,
+ PINT_group_free );
+ qhash_destroy_and_finalize( events_table, struct PINT_event, link,
+ PINT_event_free);
return;
}
Index: pvfs2/src/server/request-scheduler/request-scheduler.c
===================================================================
RCS file:
/projects/cvsroot/pvfs2/src/server/request-scheduler/request-scheduler.c,v
retrieving revision 1.68
diff -a -u -p -r1.68 request-scheduler.c
--- pvfs2/src/server/request-scheduler/request-scheduler.c 4 Apr 2008
17:57:38 -0000 1.68
+++ pvfs2/src/server/request-scheduler/request-scheduler.c 8 Apr 2010
16:08:13 -0000
@@ -173,6 +173,11 @@ int PINT_req_sched_finalize(
{
tmp_element = qlist_entry(iterator2, struct req_sched_element,
list_link);
+ /* cleanup user_ptr */
+ if( tmp_element->user_ptr )
+ {
+ free(tmp_element->user_ptr);
+ }
free(tmp_element);
/* note: no need to delete from list; we are
* destroying it as we go
@@ -183,6 +188,19 @@ int PINT_req_sched_finalize(
* it as we go
*/
}
+ }
+
+ /* also cleanup the timer queue */
+ qlist_for_each_safe(iterator, scratch, &timer_queue)
+ {
+ tmp_element = qlist_entry(iterator, struct req_sched_element,
+ list_link);
+ /* clean up user_ptr */
+ if( tmp_element->user_ptr )
+ {
+ free(tmp_element->user_ptr);
+ }
+ free(tmp_element);
}
sched_count = 0;
_______________________________________________
Pvfs2-developers mailing list
[email protected]
http://www.beowulf-underground.org/mailman/listinfo/pvfs2-developers
