Hi, I have a problem with MPI_Senrecv communication where I send columns on edges between processes. For debugging, I show you below a basic example where I initialize a 10x10 matrix ("x0" array) with x_domain=4 and y_domain=4. For the test, I simply initialize the 2D array values with x0[i][j] = i+j . After, in updateBound.c", I'm using the MPI_Sendrecv functions for the North-South and Est-West process.
Here's the main program "example.c" : ------------------------------------------------------------------------------------------- #include <stdio.h> #include <stdlib.h> #include <math.h> #include "mpi.h" int main(int argc, char *argv[]) { /* size of the discretization */ double** x; double** x0; int i,j,k,l; int nproc; int ndims; int S=0, E=1, N=2, W=3; int NeighBor[4]; int xcell, ycell, size_tot_x, size_tot_y; int *xs,*ys,*xe,*ye; int size_x = 4; int size_y = 4; int me; int x_domains=2; int y_domains=2; MPI_Comm comm, comm2d; int dims[2]; int periods[2]; int reorganisation = 0; int row; MPI_Datatype column_type; size_tot_x=size_x+2*x_domains+2; size_tot_y=size_y+2*y_domains+2; xcell=(size_x/x_domains); ycell=(size_y/y_domains); MPI_Init(&argc, &argv); comm = MPI_COMM_WORLD; MPI_Comm_size(comm,&nproc); MPI_Comm_rank(comm,&me); x = malloc(size_tot_y*sizeof(double*)); x0 = malloc(size_tot_y*sizeof(double*)); for(j=0;j<=size_tot_y-1;j++) { x[j] = malloc(size_tot_x*sizeof(double)); x0[j] = malloc(size_tot_x*sizeof(double)); } xs = malloc(nproc*sizeof(int)); xe = malloc(nproc*sizeof(int)); ys = malloc(nproc*sizeof(int)); ye = malloc(nproc*sizeof(int)); /* Create 2D cartesian grid */ periods[0] = 0; periods[1] = 0; ndims = 2; dims[0]=x_domains; dims[1]=y_domains; MPI_Cart_create(comm, ndims, dims, periods, reorganisation, &comm2d); /* Identify neighbors */ NeighBor[0] = MPI_PROC_NULL; NeighBor[1] = MPI_PROC_NULL; NeighBor[2] = MPI_PROC_NULL; NeighBor[3] = MPI_PROC_NULL; /* Left/West and right/Est neigbors */ MPI_Cart_shift(comm2d,0,1,&NeighBor[W],&NeighBor[E]); /* Bottom/South and Upper/North neigbors */ MPI_Cart_shift(comm2d,1,1,&NeighBor[S],&NeighBor[N]); /* coordinates of current cell with me rank */ xcell=(size_x/x_domains); ycell=(size_y/y_domains); ys[me]=(y_domains-me%(y_domains)-1)*(ycell+2)+2; ye[me]=ys[me]+ycell-1; for(i=0;i<=y_domains-1;i++) {xs[i]=2;} for(i=0;i<=y_domains-1;i++) {xe[i]=xs[i]+xcell-1;} for(i=1;i<=(x_domains-1);i++) { for(j=0;j<=(y_domains-1);j++) { xs[i*y_domains+j]=xs[(i-1)*y_domains+j]+xcell+2; xe[i*y_domains+j]=xs[i*y_domains+j]+xcell-1; } } for(i=0;i<=size_tot_y-1;i++) { for(j=0;j<=size_tot_x-1;j++) { x0[i][j]= i+j; // printf("%f\n",x0[i][j]); } } /* Create column data type to communicate with South and North neighbors */ MPI_Type_vector( ycell, 1, size_tot_x, MPI_DOUBLE, &column_type); MPI_Type_commit(&column_type); updateBound(x0, NeighBor, comm2d, column_type, me, xs, ys, xe, ye, xcell); for(i=0;i<=size_tot_y-1;i++) { free(x[i]); free(x0[i]); } free(x); free(x0); free(xs); free(xe); free(ys); free(ye); MPI_Finalize(); return 0; } ------------------------------------------------------------------------------------------- and the second file "updateBound.c" which sends the columns and rows ------------------------------------------------------------------------------------------- #include "mpi.h" #include <stdio.h> /*******************************************************************/ /* Update Bounds of subdomain with me process */ /*******************************************************************/ void updateBound(double** x,int NeighBor[], MPI_Comm comm2d, MPI_Datatype column_type , int me, int* xs, int* ys, int* xe, int* ye, int xcell) { int S=0, E=1, N=2, W=3; int flag; MPI_Status status; int i,j; if(me==0) {printf("verif_update_before\n"); for(i=ys[me]-1;i<=ye[me]+1;i++) { for(j=xs[me]-1;j<=xe[me]+1;j++) { printf("%f ",x[i][j]); } printf("\n"); } printf("\n"); } /********* North/South communication **********************************/ flag = 1; /*Send my boundary to North and receive from South*/ MPI_Sendrecv(&x[ys[me]][xs[me]], xcell, MPI_DOUBLE, NeighBor[N], flag, &x[ye[me]+1][xs[me]], xcell, MPI_DOUBLE, NeighBor[S], flag, comm2d, &status); /*Send my boundary to South and receive from North*/ MPI_Sendrecv(&x[ye[me]][xs[me]], xcell, MPI_DOUBLE, NeighBor[S], flag, &x[ys[me]-1][xs[me]], xcell, MPI_DOUBLE, NeighBor[N], flag, comm2d, &status); /********* Est/West communication ************************************/ flag = 2; /*Send my boundary to Est and receive from West*/ MPI_Sendrecv(&x[ys[me]][xe[me]], 1, column_type, NeighBor[E], flag, &x[ys[me]][xs[me]-1], 1, column_type, NeighBor[W], flag, comm2d, &status); /*Send my boundary to West and receive from Est*/ MPI_Sendrecv(&x[ys[me]][xs[me]], 1, column_type, NeighBor[W], flag, &x[ys[me]][xe[me]+1], 1, column_type, NeighBor[E], flag, comm2d, &status); if(me==0) {printf("verif_update_after\n"); for(i=ys[me]-1;i<=ye[me]+1;i++) { for(j=xs[me]-1;j<=xe[me]+1;j++) { printf("%f ",x[i][j]); } printf("\n"); } printf("\n"); } } ------------------------------------------------------------------------------ Running with nproc=4, I print the values of the subarray with rank=0 (so at left bottom of the grid) and I get before and after the bounds udpate : verif_update_before 6.000000 7.000000 8.000000 9.000000 7.000000 8.000000 9.000000 10.000000 8.000000 9.000000 10.000000 11.000000 9.000000 10.000000 11.000000 12.000000 verif_update_after 6.000000 5.000000 6.000000 9.000000 7.000000 8.000000 9.000000 12.000000 8.000000 9.000000 10.000000 *11.000000 * 9.000000 10.000000 11.000000 12.000000 As you can see, after the udpate, I don't have the correct value ( in underligned bold : 11.0 ) at the second element of the column coming from the Est. I expected 13.0 instead of 11.0. So there's a problem with the column datatype which only send the first element of this column. In "example.c", I define the column as following : MPI_Type_vector( ycell, 1, size_tot_x, MPI_DOUBLE, &column_type); MPI_Type_commit(&column_type); However, It seems ok and the computation of begin and end coordinates as a function of rank "me" is also good. I make you notice there's no problem between the exchange of rows between the North and the South, only between columns. If you could help me, I don't know what to do. Regards