Oops, I forgot the hwloc_topology_destroy() and also hwloc_bitmap_free(cpuset);
Added them, I attach new code using hwloc_set_area_membind function directly and new Valgrind output. 2012/9/6 Brice Goglin <brice.gog...@inria.fr> > Le 06/09/2012 10:13, Gabriele Fatigati a écrit : > > Downsizing the array, up to 4GB, > > valgrind gives many warnings reported in the attached file. > > > Adding hwloc_topology_destroy() at the end of the file would likely remove > most of them. > > But that won't fix the problem since the leaks are small. > ==28082== LEAK SUMMARY: > ==28082== definitely lost: 4,080 bytes in 3 blocks > ==28082== indirectly lost: 51,708 bytes in 973 blocks > ==28082== possibly lost: 304 bytes in 1 blocks > ==28082== still reachable: 1,786 bytes in 4 blocks > ==28082== suppressed: 0 bytes in 0 blocks > > I don't know where to look, sorry. > > Brice > > > > > > > > > > > > > > 2012/9/6 Gabriele Fatigati <g.fatig...@cineca.it> > >> Sorry, >> >> I used a wrong hwloc installation. Using the hwloc with the printf >> controls: >> >> mbind hwloc_linux_set_area_membind() fails: >> >> Error from HWLOC mbind: Cannot allocate memory >> >> so this is the origin of bad allocation. >> >> I attach the right valgrind output >> >> valgrind --track-origins=yes --log-file=output_valgrind >> --leak-check=full --tool=memcheck --show-reachable=yes >> ./main_hybrid_bind_mem >> >> >> >> >> >> 2012/9/6 Gabriele Fatigati <g.fatig...@cineca.it> >> >>> Hi Brice, hi Jeff, >>> >>> >Can you add some printf inside hwloc_linux_set_area_membind() in >>> src/topology-linux.c to see if ENOMEM comes from the mbind >syscall or not? >>> >>> I added printf inside that function, but ENOMEM does not come from >>> there. >>> >>> >Have you run your application through valgrind or another >>> memory-checking debugger? >>> >>> I tried with valgrind : >>> >>> valgrind --track-origins=yes --log-file=output_valgrind >>> --leak-check=full --tool=memcheck --show-reachable=yes >>> ./main_hybrid_bind_mem >>> >>> ==25687== Warning: set address range perms: large range [0x39454040, >>> 0x2218d4040) (undefined) >>> ==25687== >>> ==25687== Valgrind's memory management: out of memory: >>> ==25687== newSuperblock's request for 4194304 bytes failed. >>> ==25687== 34253180928 bytes have already been allocated. >>> ==25687== Valgrind cannot continue. Sorry. >>> >>> >>> I attach the full output. >>> >>> >>> The code dies also using OpenMP pure code. Very misteriously. >>> >>> >>> >>> 2012/9/5 Jeff Squyres <jsquy...@cisco.com> >>> >>>> On Sep 5, 2012, at 2:36 PM, Gabriele Fatigati wrote: >>>> >>>> > I don't think is a simply out of memory since NUMA node has 48 GB, >>>> and I'm allocating just 8 GB. >>>> >>>> Mmm. Probably right. >>>> >>>> Have you run your application through valgrind or another >>>> memory-checking debugger? >>>> >>>> I've seen cases of heap corruption lead to malloc incorrectly failing >>>> with ENOMEM. >>>> >>>> -- >>>> Jeff Squyres >>>> jsquy...@cisco.com >>>> For corporate legal information go to: >>>> http://www.cisco.com/web/about/doing_business/legal/cri/ >>>> >>>> >>>> _______________________________________________ >>>> hwloc-users mailing list >>>> hwloc-us...@open-mpi.org >>>> http://www.open-mpi.org/mailman/listinfo.cgi/hwloc-users >>>> >>> >>> >>> >>> -- >>> Ing. Gabriele Fatigati >>> >>> HPC specialist >>> >>> SuperComputing Applications and Innovation Department >>> >>> Via Magnanelli 6/3, Casalecchio di Reno (BO) Italy >>> >>> www.cineca.it Tel: +39 051 >>> 6171722<%2B39%20051%206171722> >>> >>> g.fatigati [AT] cineca.it >>> >> >> >> >> -- >> Ing. Gabriele Fatigati >> >> HPC specialist >> >> SuperComputing Applications and Innovation Department >> >> Via Magnanelli 6/3, Casalecchio di Reno (BO) Italy >> >> www.cineca.it Tel: +39 051 >> 6171722<%2B39%20051%206171722> >> >> g.fatigati [AT] cineca.it >> > > > > -- > Ing. Gabriele Fatigati > > HPC specialist > > SuperComputing Applications and Innovation Department > > Via Magnanelli 6/3, Casalecchio di Reno (BO) Italy > > www.cineca.it Tel: +39 051 6171722 > > g.fatigati [AT] cineca.it > > > _______________________________________________ > hwloc-users mailing > listhwloc-users@open-mpi.orghttp://www.open-mpi.org/mailman/listinfo.cgi/hwloc-users > > > > _______________________________________________ > hwloc-users mailing list > hwloc-us...@open-mpi.org > http://www.open-mpi.org/mailman/listinfo.cgi/hwloc-users > -- Ing. Gabriele Fatigati HPC specialist SuperComputing Applications and Innovation Department Via Magnanelli 6/3, Casalecchio di Reno (BO) Italy www.cineca.it Tel: +39 051 6171722 g.fatigati [AT] cineca.it
output_valgrind
Description: Binary data
//#include <mpi.h>
#include <stdio.h>
#include <numa.h>
#include <hwloc.h>
#define PAGE_SIZE 4096
int main(int argc,char *argv[]){
/* Bind memory example: each thread bind a piece of allocated memory in local node
*/
//MPI_Init (&argc, &argv);
int rank;
int result;
// MPI_Comm_rank (MPI_COMM_WORLD, &rank);
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
size_t i;
// allocate 8 GB
size_t len=4096000000;
long free_mem = 0;
numa_node_size(0,&free_mem);
printf("free memory node 0: %li \n", free_mem);
numa_node_size(1,&free_mem);
printf("free memory node 1: %li \n", free_mem);
char* array;
array=(char*)malloc(len);
if(array==NULL) {
printf( " Error allocation memory \n");
return -1;
}
#pragma omp parallel num_threads(2)
{
size_t chunk = len/omp_get_num_threads();
int tid = omp_get_thread_num();
int my_pu_id, my_node_id;
int res;
size_t i;
// hwloc_obj_t obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NODE, tid);
hwloc_obj_t obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, tid);
hwloc_cpuset_t cpuset = hwloc_bitmap_dup(obj->cpuset);
hwloc_bitmap_singlify(cpuset);
hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_THREAD);
for( i = chunk*tid; i < len; i+=PAGE_SIZE) {
// res = hwloc_set_area_membind_nodeset(topology, &array[i], PAGE_SIZE, obj->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_THREAD);
res = hwloc_set_area_membind(topology, &array[i], PAGE_SIZE, cpuset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_THREAD);
if(res<0) {
printf( " ERRORE: %s \n", strerror(errno));
break;
}
}
hwloc_bitmap_free(cpuset);
}
numa_node_size(0,&free_mem);
printf("free memory node 0: %li \n", free_mem);
numa_node_size(1,&free_mem);
printf("free memory node 1: %li \n", free_mem);
free(array);
hwloc_topology_destroy(topology);
// MPI_Finalize();
return 0;
}
