Hi Barry
Thanks for pointing out the error. I think the problem coming from the zero
fieldsplit in proc 0. In this modified example, I parameterized the matrix
size and block size, so when you're executing
mpirun -np 2 ./ex -mat_size 40 -block_size 2 -method 1
everything was fine. With method = 1, fieldsplit size of B is nonzero and
is divided by the block size.
With method=2, i.e mpirun -np 2 ./ex -mat_size 40 -block_size 2 -method 2,
the fieldsplit B is zero on proc 0, and the error is thrown
[1]PETSC ERROR: --------------------- Error Message
--------------------------------------------------------------
[1]PETSC ERROR: Arguments are incompatible
[1]PETSC ERROR: Local size 11 not compatible with block size 2
This is somehow not logical, because 0 is divided by block_size.
Furthermore, if you execute "mpirun -np 2 ./ex -mat_size 20 -block_size 2
-method 2", the code hangs at ISSetBlockSize, which is pretty similar to my
original problem. Probably the original one also hangs at ISSetBlockSize,
which I may not realize at that time.
Giang
On Wed, Jun 14, 2017 at 5:29 PM, Barry Smith <[email protected]> wrote:
>
> You can't do this
>
> ierr = MatSetSizes(A,PETSC_DECIDE,N,N,N);CHKERRQ(ierr);
>
> use PETSC_DECIDE for the third argument
>
> Also this is wrong
>
> for (i = Istart; i < Iend; ++i)
> {
> ierr = MatSetValue(A,i,i,2,INSERT_VALUES);CHKERRQ(ierr);
> ierr = MatSetValue(A,i+1,i,-1,INSERT_VALUES);CHKERRQ(ierr);
> ierr = MatSetValue(A,i,i+1,-1,INSERT_VALUES);CHKERRQ(ierr);
> }
>
> you will get
>
> $ petscmpiexec -n 2 ./ex1
> 0: Istart = 0, Iend = 60
> 1: Istart = 60, Iend = 120
> [1]PETSC ERROR: --------------------- Error Message
> --------------------------------------------------------------
> [1]PETSC ERROR: Argument out of range
> [1]PETSC ERROR: Row too large: row 120 max 119
> [1]PETSC ERROR: See http://www.mcs.anl.gov/petsc/documentation/faq.html
> for trouble shooting.
> [1]PETSC ERROR: Petsc Development GIT revision: v3.7.6-4103-g93161b8192
> GIT Date: 2017-06-11 14:49:39 -0500
> [1]PETSC ERROR: ./ex1 on a arch-basic named Barrys-MacBook-Pro.local by
> barrysmith Wed Jun 14 18:26:52 2017
> [1]PETSC ERROR: Configure options PETSC_ARCH=arch-basic
> [1]PETSC ERROR: #1 MatSetValues_MPIAIJ() line 550 in
> /Users/barrysmith/Src/petsc/src/mat/impls/aij/mpi/mpiaij.c
> [1]PETSC ERROR: #2 MatSetValues() line 1270 in
> /Users/barrysmith/Src/petsc/src/mat/interface/matrix.c
> [1]PETSC ERROR: #3 main() line 30 in /Users/barrysmith/Src/petsc/te
> st-dir/ex1.c
> [1]PETSC ERROR: PETSc Option Table entries:
> [1]PETSC ERROR: -malloc_test
>
> You need to get the example working so it ends with the error you reported
> previously not these other bugs.
>
>
> > On Jun 12, 2017, at 10:19 AM, Hoang Giang Bui <[email protected]>
> wrote:
> >
> > Dear Barry
> >
> > I made a small example with 2 process with one empty split in proc 0.
> But it gives another strange error
> >
> > [1]PETSC ERROR: --------------------- Error Message
> --------------------------------------------------------------
> > [1]PETSC ERROR: Arguments are incompatible
> > [1]PETSC ERROR: Local size 31 not compatible with block size 2
> >
> > The local size is always 60, so this is confusing.
> >
> > Giang
> >
> > On Sun, Jun 11, 2017 at 8:11 PM, Barry Smith <[email protected]> wrote:
> > Could be, send us a simple example that demonstrates the problem and
> we'll track it down.
> >
> >
> > > On Jun 11, 2017, at 12:34 PM, Hoang Giang Bui <[email protected]>
> wrote:
> > >
> > > Hello
> > >
> > > I noticed that my code stopped very long, possibly hang, at
> PCFieldSplitSetIS. There are two splits and one split is empty in one
> process. May that be the possible reason that PCFieldSplitSetIS hang ?
> > >
> > > Giang
> >
> >
> > <ex.c>
>
>
#include <petsc.h>
int main(int argc, char *argv[])
{
Mat A;
Vec u,b;
PetscErrorCode ierr;
KSP ksp;
PC pc;
unsigned int i;
PetscInt Istart,Iend,local_m,local_n;
int rank;
PetscInt method;
PetscInt mat_size;
PetscInt block_size;
PetscInitialize(&argc,&argv,(char*)0,"void");
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
PetscOptionsGetInt(PETSC_NULL,PETSC_NULL,"-mat_size",&mat_size,PETSC_NULL);
PetscOptionsGetInt(PETSC_NULL,PETSC_NULL,"-method",&method,PETSC_NULL);
PetscOptionsGetInt(PETSC_NULL,PETSC_NULL,"-block_size",&block_size,PETSC_NULL);
if(rank == 0)
printf(": matrix size = %d, block size = %d\n",mat_size,block_size);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,mat_size,mat_size,mat_size);CHKERRQ(ierr);
ierr = MatSetType(A,MATMPIAIJ);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr);
printf("%d: Istart = %d, Iend = %d\n",rank,Istart,Iend);
for (i = Istart; i < Iend; ++i)
{
ierr = MatSetValue(A,i,i,2,INSERT_VALUES);CHKERRQ(ierr);
if(i < mat_size-1)
ierr = MatSetValue(A,i,i+1,-1,INSERT_VALUES);CHKERRQ(ierr);
if(i > 0)
ierr = MatSetValue(A,i,i-1,-1,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&local_m,&local_n);CHKERRQ(ierr);
printf("%d: local_m = %d, local_n = %d\n",rank,local_m,local_n);
// Create Index Sets
PetscInt* A_indices;
PetscInt* B_indices;
PetscInt A_size,B_size;
if(rank == 0)
{
if(method > 1)
{
/* with method = 1, the fieldsplit B is set to zero */
A_size = Iend-Istart;
B_size = 0;
}
else
{
/* with method = 2, the fieldsplit A and B is equal. It is noticed that A_size, or N/4, must be divided by block_size */
A_size = (Iend-Istart)/2;
B_size = (Iend-Istart)/2;
}
A_indices = (PetscInt*) calloc(A_size,sizeof(PetscInt));
B_indices = (PetscInt*) calloc(B_size,sizeof(PetscInt));
for(i = 0; i < A_size; ++i)
A_indices[i] = Istart + i;
}
else if(rank == 1)
{
A_size = (Iend-Istart)/2;
B_size = (Iend-Istart)/2;
A_indices = (PetscInt*) calloc(A_size,sizeof(PetscInt));
B_indices = (PetscInt*) calloc(B_size,sizeof(PetscInt));
for(i = 0; i < A_size; ++i)
A_indices[i] = Istart + i;
for(i = 0; i < B_size; ++i)
B_indices[i] = Istart + i + A_size;
}
IS A_IS, B_IS;
ierr = ISCreateGeneral(PETSC_COMM_WORLD,A_size,A_indices,PETSC_COPY_VALUES,&A_IS);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,B_size,B_indices,PETSC_COPY_VALUES,&B_IS);CHKERRQ(ierr);
printf("%d: A_size = %d, B_size = %d\n",rank,A_size,B_size);
printf("%d: ISCreateGeneral completed\n",rank);
// Solve the system
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPGMRES);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
printf("%d: KSPSetOperators completed\n",rank);
// Define the fieldsplit for the global matrix
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCFIELDSPLIT);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc,"a",A_IS);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc,"b",B_IS);CHKERRQ(ierr);
printf("%d: PCFieldSplitSetIS completed\n",rank);
ierr = ISSetBlockSize(A_IS,block_size);CHKERRQ(ierr);
ierr = ISSetBlockSize(B_IS,block_size);CHKERRQ(ierr);
printf("%d: ISSetBlockSize completed\n",rank);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
free(A_indices);
free(B_indices);
ierr = ISDestroy(&A_IS);CHKERRQ(ierr);
ierr = ISDestroy(&B_IS);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
