I noticed this while looking at Subversion issue #602. It is about Subversion consuming too much memory when importing large tree. http://subversion.tigris.org/issues/show_bug.cgi?id=602
I found that memory consumption doesn't go too high if I compiled APR with --enable-pool-debug so I glanced memory/unix/apr_pools.c. There I found that non-DEBUG build does not free any memory unless the pool holding the allocator is destroyed or cleared. I think it should free memory if it already has enough memory in free list. Here's a sample code to test that behavior. It is not as portable as APR itself (it uses fprintf and printf) and almost no error check is done, but it should compile on most UNIX platforms. It recursively stat directories and read the contents of files into memory. It creates subpool for each recursive entry to new directories and destroys it upon exit. Run it like ./test ~/ and see how the process continues to grow despite subpool destruction.
#include <apr_general.h> #include <apr_pools.h> #include <apr_file_info.h> #include <apr_file_io.h> #include <stdio.h> void read_files (const char *dir_path, apr_pool_t *pool) { apr_dir_t *dir; apr_finfo_t finfo; apr_dir_open (&dir, dir_path, pool); /* Throw away . and .. */ apr_dir_read (&finfo, APR_FINFO_SIZE|APR_FINFO_TYPE, dir); apr_dir_read (&finfo, APR_FINFO_SIZE|APR_FINFO_TYPE, dir); while (apr_dir_read (&finfo, APR_FINFO_NORM, dir) == APR_SUCCESS) { char *path; apr_filepath_merge (&path, dir_path, finfo.name, APR_FILEPATH_NATIVE, pool); if (finfo.filetype == APR_REG) { void *buf; apr_file_t *file; apr_size_t nbytes = finfo.size; buf = apr_palloc (pool, finfo.size); apr_file_open (&file, path, APR_READ, APR_OS_DEFAULT, pool); printf ("%s\n", path); apr_file_read (file, buf, &nbytes); apr_file_close (file); } else if (finfo.filetype == APR_DIR) { apr_pool_t *subpool; apr_pool_create (&subpool, pool); read_files (path, subpool); apr_pool_destroy (subpool); } } apr_dir_close (dir); } int main (int argc, char **argv) { apr_pool_t *pool; if (argc != 2) { fprintf (stderr, "USAGE: %s DIRNAME\n", argv[0]); exit (1); } apr_initialize (); apr_pool_create (&pool, NULL); read_files (argv[1], pool); apr_pool_destroy (pool); apr_terminate (); return 0; }
-- Yoshiki Hayashi