Hello,
I have run into a problem using KSP. Attached is a hopefully short,
simple example program (which has nothing to do with my actual
application) demonstrating the problem. Also attached is an output
log from several runs of the program. The program runs fine if I
choose local ownership according to PetscSplitOwnership(). If,
however, I slightly modify that local ownership scheme, the program
will not run in parallel. In my view, KSP appears to make its own
coefficient (or preconditioner, or whatever) matrix that has different
local ownership than the one I give to KSPSetOperators(), and so
cannot do matrix/vector multiplies.
I would like to be able to choose my own local ownership. Any help
would be appreciated. Hopefully, I'm not doing something stupid.
Thanks.
Bill
--
Bill Perkins
Research Engineer
Hydrology Group
Pacific Northwest National Laboratory
902 Battelle Boulevard
P.O. Box 999, MSIN K9-36
Richland, WA 99352 USA
Tel: 509-372-6131
Fax: 509-372-6089
[email protected]
www.pnnl.gov
/* -------------------------------------------------------------
file: versteeg.cpp
This is a simple test of PETSC Matrix operations and Linear
equation solvers. This problem comes from Example 7.2 in
Versteeg, H.K. and W. Malalasekera, 1995. An introduction to
computational fluid dynamics, the finite volume method. Prentice
Hall. 257 pp.
------------------------------------------------------------- */
/* -------------------------------------------------------------
Created October 10, 2002 by William A. Perkins
Last Change: 2013-09-26 14:32:33 d3g096
------------------------------------------------------------- */
static const char* SCCS_ID = "$Id: versteeg.c,v 1.3 2008/05/24 23:04:39 d3g096
Exp $ Battelle PNNL";
static char help[] =
"Solves an example heat transfer problem from Versteeg and Malalasekera with
SLES.\n\
The following options are available:\n\
-view_solution yes/no: write the initial and final solution to stdout\n\
-problem yes/no: cause problem by choosing local ownership\n\n";
#include <stdio.h>
#include <petscksp.h>
/* -------------------------------------------------------------
assemble
------------------------------------------------------------- */
int
assemble(const int imax, const int jmax, const double scale, Mat A, Vec b)
{
static const float k = 1000; /* conductivity, W/m/K */
static const float t = 0.01; /* plate thickness, m */
static const float W = 0.3; /* plate width, m */
static const float H = 0.4; /* plate height, m */
PetscErrorCode ierr;
int i, j;
float ap, aw, ae, as, an, bp;
float dx = W/(float)imax;
float dy = H/(float)jmax;
int iP, iN, iS, iE, iW;
PetscScalar v;
PetscInt lo, hi;
ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRQ(ierr);
for (i = 0; i < imax; i++) {
for (j = 0; j < jmax; j++) {
iP = i*jmax + j;
if (! (lo <= iP && iP < hi) ) continue;
iE = (i+1)*jmax + j;
iW = (i-1)*jmax + j;
iN = i*jmax + (j+1);
iS = i*jmax + (j-1);
bp = 0.0;
ap = 0.0;
if (j == 0) { /* insulated south boundary */
as = 0.0;
bp += 0.0;
ap -= 0.0;
} else {
as = (k/dx)*(dx*t);
}
if (j == jmax - 1) { /* constant tempurature (100C) north boundary */
an = 0.0;
bp += 2*k/dy*(dy*t)*100.0;
ap -= -2*k/dy*(dy*t);
} else {
an = (k/dx)*(dx*t);
}
if (i == 0) { /* constant flux (500kw/m2) west boundary */
aw = 0.0;
bp += 500000.0*dy*t;
ap -= 0.0;
} else {
aw = (k/dx)*(dx*t);
}
if (i == imax - 1) { /* insulated east boundary */
ae = 0.0;
bp += 0.0;
ap -= 0.0;
} else {
ae = (k/dx)*(dx*t);
}
ap += as + an + aw + ae;
v = ap*scale;
ierr = MatSetValues(A,1,&iP,1,&iP,&v,INSERT_VALUES); CHKERRQ(ierr);
if (an != 0.0) {
v = -an*scale;
ierr = MatSetValues(A,1,&iP,1,&iN,&v,INSERT_VALUES); CHKERRQ(ierr);
}
if (as != 0.0) {
v = -as*scale;
ierr = MatSetValues(A,1,&iP,1,&iS,&v,INSERT_VALUES); CHKERRQ(ierr);
}
if (ae != 0.0) {
v = -ae*scale;
ierr = MatSetValues(A,1,&iP,1,&iE,&v,INSERT_VALUES); CHKERRQ(ierr);
}
if (aw != 0.0) {
v = -aw*scale;
ierr = MatSetValues(A,1,&iP,1,&iW,&v,INSERT_VALUES); CHKERRQ(ierr);
}
v = bp*scale;
ierr = VecSetValues(b, 1, &iP, &v, INSERT_VALUES); CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecAssemblyBegin(b); CHKERRQ(ierr);
ierr = VecAssemblyEnd(b); CHKERRQ(ierr);
return(0);
}
/* -------------------------------------------------------------
Main Program
------------------------------------------------------------- */
int
main(int argc, char **args)
{
static const int imax = 3;
static const int jmax = 4;
PetscBool print_solution(PETSC_FALSE);
PetscBool set_local_size(PETSC_FALSE);
int its;
PetscErrorCode ierr;
Vec x, b;
Mat A;
PetscScalar v;
KSP ksp;
VecScatter ctx;
int rank;
int nproc;
PetscInt local_size(PETSC_DECIDE);
PetscInt global_size(imax*jmax);
PetscInitialize(&argc,&args,(char *)0,help);
PetscOptionsGetBool(NULL, "-print_solution", &print_solution, NULL);
PetscOptionsGetBool(NULL, "-problem", &set_local_size, NULL);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&nproc);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscSplitOwnership(PETSC_COMM_WORLD, &local_size, &global_size);
CHKERRQ(ierr);
if (set_local_size) {
// If there is more than one processor, alter the default local
// ownership so that process 0 has one less row and the last
// process has more row.
if (nproc > 1) {
if (rank == 0) {
local_size -= 1;
} else if (rank == nproc - 1) {
local_size += 1;
}
}
}
ierr = MatCreate(PETSC_COMM_WORLD, &A); CHKERRQ(ierr);
ierr = MatSetSizes(A,
local_size, PETSC_DECIDE,
imax*jmax, imax*jmax); CHKERRXX(ierr);
ierr = MatSetFromOptions(A); CHKERRQ(ierr);
ierr = MatSetUp(A); CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,local_size,imax*jmax);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = assemble(imax, jmax, 1.0, A, b); CHKERRQ(ierr);
v = 0.0;
ierr = VecSet(x,v);CHKERRQ(ierr);
if (print_solution) {
ierr = VecView(x, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
}
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp,PETSC_TRUE); CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp, 1e-06, 1e-12, PETSC_DEFAULT, PETSC_DEFAULT);
CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPSetUp(ksp); CHKERRQ(ierr);
ierr = KSPSolve(ksp, b, x);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"iterations %d\n",its);CHKERRQ(ierr);
if (print_solution) {
ierr = VecView(x, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return(ierr);
}
>>> mpirun -np 1 versteeg -print_solution
Vector Object: 1 MPI processes
type: seq
0
0
0
0
0
0
0
0
0
0
0
0
iterations 8
Vector Object: 1 MPI processes
type: seq
260.037
242.275
205.592
146.322
227.799
211.195
178.178
129.696
212.164
196.53
166.23
123.982
>>> mpirun -np 2 versteeg -print_solution
Vector Object: 2 MPI processes
type: mpi
Process [0]
0
0
0
0
0
0
Process [1]
0
0
0
0
0
0
iterations 9
Vector Object: 2 MPI processes
type: mpi
Process [0]
260.037
242.275
205.592
146.322
227.799
211.195
Process [1]
178.178
129.696
212.164
196.53
166.23
123.982
>>> mpirun -np 4 versteeg -print_solution
Vector Object: 4 MPI processes
type: mpi
Process [0]
0
0
0
Process [1]
0
0
0
Process [2]
0
0
0
Process [3]
0
0
0
iterations 12
Vector Object: 4 MPI processes
type: mpi
Process [0]
260.037
242.275
205.592
Process [1]
146.322
227.799
211.195
Process [2]
178.178
129.696
212.164
Process [3]
196.53
166.23
123.982
>>> mpirun -np 2 versteeg -print_solution -problem
Vector Object: 2 MPI processes
type: mpi
Process [0]
0
0
0
0
0
Process [1]
0
0
0
0
0
0
0
[0]PETSC ERROR: [1]PETSC ERROR: --------------------- Error Message
------------------------------------
[1]PETSC ERROR: Invalid argument!
[1]PETSC ERROR: Must be square matrix, rows 7 columns 6!
[1]PETSC ERROR:
------------------------------------------------------------------------
[1]PETSC ERROR: Petsc Release Version 3.4.2, Jul, 02, 2013
[1]PETSC ERROR: See docs/changes/index.html for recent updates.
[1]PETSC ERROR: See docs/faq.html for hints about trouble shooting.
[1]PETSC ERROR: See docs/index.html for manual pages.
[1]PETSC ERROR:
------------------------------------------------------------------------
[1]PETSC ERROR: versteeg on a arch-complex named flophouse by d3g096 Thu Sep 26
14:30:33 2013
[1]PETSC ERROR: Libraries linked from
/net/flophouse/files0/perksoft/petsc-3.4.2/arch-complex/lib
[1]PETSC ERROR: Configure run at Wed Sep 11 06:43:12 2013
[1]PETSC ERROR: Configure options --with-scalar-type=complex
--with-clanguage=C++ PETSC_ARCH=arch-complex --with-fortran-kernels=generic
--download-superlu_dist --download-mumps --download-scalapack
--download-parmetis --download-metis --download-elemental --with-mpi=1
--download-blacs
[1]PETSC ERROR:
------------------------------------------------------------------------
[1]PETSC ERROR: MatGetOrdering() line 257 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/order/sorder.c
[1]PETSC ERROR: PCSetUp_ILU() line 194 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/factor/ilu/ilu.c
[1]PETSC ERROR: PCSetUp() line 890 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[1]PETSC ERROR: KSPSetUp() line 278 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[1]PETSC ERROR: PCSetUpOnBlocks_BJacobi_Singleblock() line 646 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/bjacobi/bjacobi.c
[1]PETSC ERROR: PCSetUpOnBlocks() line 924 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[1]PETSC ERROR: KSPSetUpOnBlocks() line 165 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[1]PETSC ERROR: KSPSolve() line 400 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[1]PETSC ERROR: main() line 216 in
/home/d3g096/Projects/PerformanceLDRD/src/sandbox/petsc/versteeg.cpp
--------------------- Error Message ------------------------------------
[0]PETSC ERROR: Invalid argument!
[0]PETSC ERROR: Must be square matrix, rows 5 columns 6!
[0]PETSC ERROR:
------------------------------------------------------------------------
[0]PETSC ERROR: Petsc Release Version 3.4.2, Jul, 02, 2013
[0]PETSC ERROR: See docs/changes/index.html for recent updates.
[0]PETSC ERROR: See docs/faq.html for hints about trouble shooting.
[0]PETSC ERROR: See docs/index.html for manual pages.
[0]PETSC ERROR:
------------------------------------------------------------------------
[0]PETSC ERROR: versteeg on a arch-complex named flophouse by d3g096 Thu Sep 26
14:30:33 2013
[0]PETSC ERROR: Libraries linked from
/net/flophouse/files0/perksoft/petsc-3.4.2/arch-complex/lib
[0]PETSC ERROR: Configure run at Wed Sep 11 06:43:12 2013
[0]PETSC ERROR: Configure options --with-scalar-type=complex
--with-clanguage=C++ PETSC_ARCH=arch-complex --with-fortran-kernels=generic
--download-superlu_dist --download-mumps --download-scalapack
--download-parmetis --download-metis --download-elemental --with-mpi=1
--download-blacs
[0]PETSC ERROR:
------------------------------------------------------------------------
[0]PETSC ERROR: MatGetOrdering() line 257 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/order/sorder.c
[0]PETSC ERROR: PCSetUp_ILU() line 194 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/factor/ilu/ilu.c
[0]PETSC ERROR: PCSetUp() line 890 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[0]PETSC ERROR: KSPSetUp() line 278 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[0]PETSC ERROR: PCSetUpOnBlocks_BJacobi_Singleblock() line 646 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/bjacobi/bjacobi.c
[0]PETSC ERROR: PCSetUpOnBlocks() line 924 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[0]PETSC ERROR: KSPSetUpOnBlocks() line 165 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[0]PETSC ERROR: KSPSolve() line 400 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[0]PETSC ERROR: main() line 216 in
/home/d3g096/Projects/PerformanceLDRD/src/sandbox/petsc/versteeg.cpp
--------------------------------------------------------------------------
MPI_ABORT was invoked on rank 1 in communicator MPI_COMM_WORLD
with errorcode 62.
NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.
You may or may not see output from other processes, depending on
exactly when Open MPI kills them.
--------------------------------------------------------------------------
--------------------------------------------------------------------------
mpirun has exited due to process rank 1 with PID 12459 on
node flophouse exiting without calling "finalize". This may
have caused other processes in the application to be
terminated by signals sent by mpirun (as reported here).
--------------------------------------------------------------------------
[flophouse:12457] 1 more process has sent help message help-mpi-api.txt /
mpi-abort
[flophouse:12457] Set MCA parameter "orte_base_help_aggregate" to 0 to see all
help / error messages
>>> mpirun -np 4 versteeg -print_solution -problem
Vector Object: 4 MPI processes
type: mpi
Process [0]
0
0
Process [1]
0
0
0
Process [2]
0
0
0
Process [3]
0
0
0
0
[0]PETSC ERROR: --------------------- Error Message
------------------------------------
[0]PETSC ERROR: Invalid argument!
[0]PETSC ERROR: [3]PETSC ERROR: --------------------- Error Message
------------------------------------
[3]PETSC ERROR: Invalid argument!
[3]PETSC ERROR: Must be square matrix, rows 4 columns 3!
[3]PETSC ERROR:
------------------------------------------------------------------------
[3]PETSC ERROR: Petsc Release Version 3.4.2, Jul, 02, 2013
[3]PETSC ERROR: See docs/changes/index.html for recent updates.
[3]PETSC ERROR: See docs/faq.html for hints about trouble shooting.
[3]PETSC ERROR: See docs/index.html for manual pages.
[3]PETSC ERROR:
------------------------------------------------------------------------
[3]PETSC ERROR: versteeg on a arch-complex named flophouse by d3g096 Thu Sep 26
14:30:51 2013
[3]PETSC ERROR: Libraries linked from
/net/flophouse/files0/perksoft/petsc-3.4.2/arch-complex/lib
[3]PETSC ERROR: Configure run at Wed Sep 11 06:43:12 2013
[3]PETSC ERROR: Configure options --with-scalar-type=complex
--with-clanguage=C++ PETSC_ARCH=arch-complex --with-fortran-kernels=generic
--download-superlu_dist --download-mumps --download-scalapack
--download-parmetis --download-metis --download-elemental --with-mpi=1
--download-blacs
[3]PETSC ERROR:
------------------------------------------------------------------------
[3]PETSC ERROR: MatGetOrdering() line 257 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/order/sorder.c
[3]PETSC ERROR: PCSetUp_ILU() line 194 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/factor/ilu/ilu.c
[3]PETSC ERROR: PCSetUp() line 890 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[3]PETSC ERROR: KSPSetUp() line 278 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[3]PETSC ERROR: PCSetUpOnBlocks_BJacobi_Singleblock() line 646 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/bjacobi/bjacobi.c
[3]PETSC ERROR: PCSetUpOnBlocks() line 924 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[3]PETSC ERROR: KSPSetUpOnBlocks() line 165 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[3]PETSC ERROR: KSPSolve() line 400 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[3]PETSC ERROR: main() line 216 in
/home/d3g096/Projects/PerformanceLDRD/src/sandbox/petsc/versteeg.cpp
Must be square matrix, rows 2 columns 3!
[0]PETSC ERROR:
------------------------------------------------------------------------
[0]PETSC ERROR: Petsc Release Version 3.4.2, Jul, 02, 2013
[0]PETSC ERROR: See docs/changes/index.html for recent updates.
[0]PETSC ERROR: See docs/faq.html for hints about trouble shooting.
[0]PETSC ERROR: See docs/index.html for manual pages.
[0]PETSC ERROR:
------------------------------------------------------------------------
[0]PETSC ERROR: versteeg on a arch-complex named flophouse by d3g096 Thu Sep 26
14:30:51 2013
[0]PETSC ERROR: Libraries linked from
/net/flophouse/files0/perksoft/petsc-3.4.2/arch-complex/lib
[0]PETSC ERROR: Configure run at Wed Sep 11 06:43:12 2013
[0]PETSC ERROR: Configure options --with-scalar-type=complex
--with-clanguage=C++ PETSC_ARCH=arch-complex --with-fortran-kernels=generic
--download-superlu_dist --download-mumps --download-scalapack
--download-parmetis --download-metis --download-elemental --with-mpi=1
--download-blacs
[0]PETSC ERROR:
------------------------------------------------------------------------
[0]PETSC ERROR: MatGetOrdering() line 257 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/order/sorder.c
[0]PETSC ERROR: PCSetUp_ILU() line 194 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/factor/ilu/ilu.c
[0]PETSC ERROR: PCSetUp() line 890 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[0]PETSC ERROR: KSPSetUp() line 278 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[0]PETSC ERROR: PCSetUpOnBlocks_BJacobi_Singleblock() line 646 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/bjacobi/bjacobi.c
[0]PETSC ERROR: PCSetUpOnBlocks() line 924 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[0]PETSC ERROR: KSPSetUpOnBlocks() line 165 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[0]PETSC ERROR: KSPSolve() line 400 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[0]PETSC ERROR: main() line 216 in
/home/d3g096/Projects/PerformanceLDRD/src/sandbox/petsc/versteeg.cpp
[2]PETSC ERROR: --------------------- Error Message
------------------------------------
[2]PETSC ERROR: Object is in wrong state!
[2]PETSC ERROR: Matrix is missing diagonal entry 2!
[2]PETSC ERROR:
------------------------------------------------------------------------
[2]PETSC ERROR: Petsc Release Version 3.4.2, Jul, 02, 2013
[2]PETSC ERROR:
--------------------------------------------------------------------------
MPI_ABORT was invoked on rank 3 in communicator MPI_COMM_WORLD
with errorcode 62.
NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.
You may or may not see output from other processes, depending on
exactly when Open MPI kills them.
--------------------------------------------------------------------------
See docs/changes/index.html for recent updates.
[2]PETSC ERROR: See docs/faq.html for hints about trouble shooting.
[2]PETSC ERROR: See docs/index.html for manual pages.
[2]PETSC ERROR:
------------------------------------------------------------------------
[2]PETSC ERROR: versteeg on a arch-complex named flophouse by d3g096 Thu Sep 26
14:30:51 2013
[2]PETSC ERROR: Libraries linked from
/net/flophouse/files0/perksoft/petsc-3.4.2/arch-complex/lib
[2]PETSC ERROR: Configure run at Wed Sep 11 06:43:12 2013
[2]PETSC ERROR: Configure options --with-scalar-type=complex
--with-clanguage=C++ PETSC_ARCH=arch-complex --with-fortran-kernels=generic
--download-superlu_dist --download-mumps --download-scalapack
--download-parmetis --download-metis --download-elemental --with-mpi=1
--download-blacs
[2]PETSC ERROR:
------------------------------------------------------------------------
[2]PETSC ERROR: MatILUFactorSymbolic_SeqAIJ_ilu0() line 1653 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/impls/aij/seq/aijfact.c
[2]PETSC ERROR: MatILUFactorSymbolic_SeqAIJ() line 1756 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/impls/aij/seq/aijfact.c
[2]PETSC ERROR: MatILUFactorSymbolic() line 6238 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/interface/matrix.c
[2]PETSC ERROR: PCSetUp_ILU() line 204 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/factor/ilu/ilu.c
[2]PETSC ERROR: PCSetUp() line 890 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[2]PETSC ERROR: KSPSetUp() line 278 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[2]PETSC ERROR: PCSetUpOnBlocks_BJacobi_Singleblock() line 646 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/bjacobi/bjacobi.c
[2]PETSC ERROR: PCSetUpOnBlocks() line 924 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[2]PETSC ERROR: KSPSetUpOnBlocks() line 165 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[2]PETSC ERROR: KSPSolve() line 400 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[2]PETSC ERROR: main() line 216 in
/home/d3g096/Projects/PerformanceLDRD/src/sandbox/petsc/versteeg.cpp
[1]PETSC ERROR: --------------------- Error Message
------------------------------------
[1]PETSC ERROR: Argument out of range!
[1]PETSC ERROR: Cannot log negative flops!
[1]PETSC ERROR:
------------------------------------------------------------------------
[1]PETSC ERROR: Petsc Release Version 3.4.2, Jul, 02, 2013
[1]PETSC ERROR: See docs/changes/index.html for recent updates.
[1]PETSC ERROR: See docs/faq.html for hints about trouble shooting.
[1]PETSC ERROR: See docs/index.html for manual pages.
[1]PETSC ERROR:
------------------------------------------------------------------------
[1]PETSC ERROR: versteeg on a arch-complex named flophouse by d3g096 Thu Sep 26
14:30:51 2013
[1]PETSC ERROR: Libraries linked from
/net/flophouse/files0/perksoft/petsc-3.4.2/arch-complex/lib
[1]PETSC ERROR: Configure run at Wed Sep 11 06:43:12 2013
[1]PETSC ERROR: Configure options --with-scalar-type=complex
--with-clanguage=C++ PETSC_ARCH=arch-complex --with-fortran-kernels=generic
--download-superlu_dist --download-mumps --download-scalapack
--download-parmetis --download-metis --download-elemental --with-mpi=1
--download-blacs
[1]PETSC ERROR:
------------------------------------------------------------------------
[1]PETSC ERROR: PetscLogFlops() line 204 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/include/petsclog.h
[1]PETSC ERROR: MatLUFactorNumeric_SeqAIJ_Inode() line 1312 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/impls/aij/seq/inode.c
[1]PETSC ERROR: MatLUFactorNumeric() line 2889 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/mat/interface/matrix.c
[1]PETSC ERROR: PCSetUp_ILU() line 232 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/factor/ilu/ilu.c
[1]PETSC ERROR: PCSetUp() line 890 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[1]PETSC ERROR: KSPSetUp() line 278 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[1]PETSC ERROR: PCSetUpOnBlocks_BJacobi_Singleblock() line 646 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/impls/bjacobi/bjacobi.c
[1]PETSC ERROR: PCSetUpOnBlocks() line 924 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/pc/interface/precon.c
[1]PETSC ERROR: KSPSetUpOnBlocks() line 165 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[1]PETSC ERROR: KSPSolve() line 400 in
/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/net/flophouse/files0/perksoft/petsc-3.4.2/src/ksp/ksp/interface/itfunc.c
[1]PETSC ERROR: main() line 216 in
/home/d3g096/Projects/PerformanceLDRD/src/sandbox/petsc/versteeg.cpp
--------------------------------------------------------------------------
mpirun has exited due to process rank 3 with PID 12483 on
node flophouse exiting without calling "finalize". This may
have caused other processes in the application to be
terminated by signals sent by mpirun (as reported here).
--------------------------------------------------------------------------
[flophouse:12479] 3 more processes have sent help message help-mpi-api.txt /
mpi-abort
[flophouse:12479] Set MCA parameter "orte_base_help_aggregate" to 0 to see all
help / error messages
