Stefano, I've realized you are right, and that, unfortunately the initial
problem is not solved...
In the target code I work on, MatIS is created this way:
MatCreateIS(..., rmap, cmap, ...)
MatISSetLocalMat()
MatAssemblyBegin/End()
Note that I do not use MatSetValues directly on the MatIS (I do it on local
matrices only). Also, I do not call MatSetLocalToGlobalMapping (as you said
Stefano, it's done in MatCreateIS).
On my target code, when trying to MatView a MatIS in binary format, I got this
error (note that ASCII and matlab are OK):
[0]PETSC ERROR: No support for this operation for this object type
[0]PETSC ERROR: Cannot get subcomm viewer for binary files or sockets unless
SubViewer contains the rank 0 process
Googling this lead me to
https://lists.mcs.anl.gov/mailman/htdig/petsc-users/2016-September/030438.html,
from where I restarted hoping to get a reduced toy/minimal problem (this
example uses MatCreate+MatSetType but not MatCreateIS as I use it in the target
code: I realized this slight difference after I sent the mail).
When Pierre replied, I was in a rush... And tested the wrong command line (!) :
I believed the problem was solved... It's unfortunately not !
In the toy example I added MatSetLocalToGlobalMapping and MatISSetLocalMat, as
you can see:
>> more matExport.cpp
/*
* mpirun -n 2 ./matExport.exe
* mpirun -n 2 ./matExport.exe is
*/
#include <petscmat.h>
#include <petscsys.h>
#include <string.h>
int main(int argc, char *argv[])
{
PetscInitialize(&argc, &argv, "", NULL);
Mat matrix;
MatCreate(PETSC_COMM_WORLD, &matrix);
const char* matType = (argc >= 2) ? argv[1] : MATMPIAIJ;
PetscPrintf(PETSC_COMM_WORLD, "mat type is %s \n", matType);
MatSetType(matrix, matType);
MatSetSizes(matrix, PETSC_DECIDE, PETSC_DECIDE, 10, 10);
if (strcmp(matType, "is") == 0) {
ISLocalToGlobalMapping is;
PetscInt idx[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD, 1, 10, idx, PETSC_COPY_VALUES,
&is);
MatSetLocalToGlobalMapping(matrix, is, is);
ISLocalToGlobalMappingDestroy(&is);
PetscPrintf(PETSC_COMM_WORLD, "MatSetLocalToGlobalMapping done\n");
Mat local;
MatCreateSeqDense(PETSC_COMM_SELF, 10, 10, NULL, &local);
for (int i = 0; i < 10; i++) MatSetValue(local, i, i, (double)(i+1.),
ADD_VALUES);
MatAssemblyBegin(local, MAT_FINAL_ASSEMBLY); MatAssemblyBegin(local,
MAT_FINAL_ASSEMBLY);
MatISSetLocalMat(matrix, local);
MatDestroy(&local);
}
else {
MatSetUp(matrix);
for (int i = 0; i < 10; i++) MatSetValue(matrix, i, i, (double)(i+1.),
ADD_VALUES);
}
MatAssemblyBegin(matrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(matrix, MAT_FINAL_ASSEMBLY);
PetscViewer viewer;
PetscViewerBinaryOpen(PETSC_COMM_WORLD, "matExport.bin", FILE_MODE_WRITE,
&viewer);
MatView(matrix, viewer);
PetscViewerDestroy(&viewer);
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "matExport.log", &viewer);
MatView(matrix, viewer);
PetscViewerDestroy(&viewer);
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "matExport.mat", &viewer);
PetscViewerPushFormat(viewer, PETSC_VIEWER_ASCII_MATLAB);
MatView(matrix, viewer);
PetscViewerDestroy(&viewer);
MatDestroy(&matrix);
PetscFinalize();
}
MPIAIJ is still OK:
>> mpirun -n 2 ./matExport.exe
mat type is mpiaij
MATIS is still KO :
>> mpirun -n 2 ./matExport.exe is
mat type is is
MatSetLocalToGlobalMapping done
[0]PETSC ERROR: --------------------- Error Message
--------------------------------------------------------------
[0]PETSC ERROR: No support for this operation for this object type
[0]PETSC ERROR: Cannot get subcomm viewer for binary files or sockets unless
SubViewer contains the rank 0 process
[0]PETSC ERROR: See http://www.mcs.anl.gov/petsc/documentation/faq.html for
trouble shooting.
>From gdb I got this stack (still need to recompile in debug tomorrow - I won't
>have time tonight)
#0 0x00007ff47bdb466d in nanosleep () at ../sysdeps/unix/syscall-template.S:84
#1 0x00007ff47bdb45aa in __sleep (seconds=0) at ../sysdeps/posix/sleep.c:55
#2 0x00007ff47cd59d75 in PetscSleep (s=<optimized out>) at
/home/fghoussen/Documents/INRIA/petsc/src/sys/utils/psleep.c:50
#3 0x00007ff47cd15c11 in PetscAttachDebugger () at
/home/fghoussen/Documents/INRIA/petsc/src/sys/error/adebug.c:388
#4 0x00007ff47cd16434 in PetscAttachDebuggerErrorHandler (comm=<optimized out>,
line=<optimized out>, fun=<optimized out>, file=<optimized out>, num=<optimized
out>, p=<optimized out>,
mess=0x7ffe1e7f6fe0 "Cannot get subcomm viewer for binary files or sockets
unless SubViewer contains the rank 0 process", ctx=0x0) at
/home/fghoussen/Documents/INRIA/petsc/src/sys/error/adebug.c:450
#5 0x00007ff47cd16a57 in PetscError (comm=0x556a39840420, line=line@entry=45,
func=func@entry=0x7ff47d9581f0 <__func__.15647>
"PetscViewerGetSubViewer_Binary",
file=file@entry=0x7ff47d9578f8
"/home/fghoussen/Documents/INRIA/petsc/src/sys/classes/viewer/impls/binary/binv.c",
n=n@entry=56, p=p@entry=PETSC_ERROR_INITIAL,
mess=0x7ff47d957a08 "Cannot get subcomm viewer for binary files or sockets
unless SubViewer contains the rank 0 process") at
/home/fghoussen/Documents/INRIA/petsc/src/sys/error/err.c:367
#6 0x00007ff47cda148b in PetscViewerGetSubViewer_Binary (viewer=0x556a3984e260,
comm=<optimized out>, outviewer=0x7ffe1e7f7958)
at
/home/fghoussen/Documents/INRIA/petsc/src/sys/classes/viewer/impls/binary/binv.c:45
#7 0x00007ff47cdbbfb9 in PetscViewerGetSubViewer
(viewer=viewer@entry=0x556a3984e260, comm=comm@entry=0x556a377e40e0
<ompi_mpi_comm_self>, outviewer=outviewer@entry=0x7ffe1e7f7958)
at
/home/fghoussen/Documents/INRIA/petsc/src/sys/classes/viewer/interface/dupl.c:41
#8 0x00007ff47cefa6b4 in MatView_IS (A=<optimized out>, viewer=0x556a3984e260)
at /home/fghoussen/Documents/INRIA/petsc/src/mat/impls/is/matis.c:1569
#9 0x00007ff47ceda6a3 in MatView (mat=0x556a3983f930, viewer=<optimized out>)
at /home/fghoussen/Documents/INRIA/petsc/src/mat/interface/matrix.c:1015
#10 0x0000556a375e300a in main ()
The problem is only for binary export. When I comment it, I get no error :
>> mpirun -n 2 ./matExport.exe
mat type is mpiaij
>> mpirun -n 2 ./matExport.exe is
mat type is is
MatSetLocalToGlobalMapping done
----- Mail original -----
> De: "Stefano Zampini" <[email protected]>
> À: "Franck Houssen" <[email protected]>
> Cc: "Pierre Jolivet" <[email protected]>, "petsc-dev"
> <[email protected]>
> Envoyé: Mercredi 13 Décembre 2017 18:29:13
> Objet: Re: [petsc-dev] MatIS export
> MatSetType + MatSetLocalToGlobalMapping
> is equivalent to MatCreateIS. Look at the code for MatCreateIS.
> Il 13 Dic 2017 7:32 PM, "Franck Houssen" < [email protected] > ha
> scritto:
> > Thanks. Working now. I missed that. I believed "MatSetType +
> > MatSetLocalToGlobalMapping" <=> "MatCreateIS(..., rmap, cmap, ...)" : I got
> > that wrong.
>
> > Franck
>
> > > De: "Pierre Jolivet" < [email protected] >
> >
>
> > > À: "Franck Houssen" < [email protected] >
> >
>
> > > Cc: [email protected]
> >
>
> > > Envoyé: Mardi 12 Décembre 2017 20:43:50
> >
>
> > > Objet: Re: [petsc-dev] MatIS export
> >
>
> > > Hello Franck,
> >
>
> > > From the documentation: "You must call MatSetLocalToGlobalMapping()
> > > before
> > > using this matrix type.” (
> > > http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/Mat/MATIS.html
> > > )
> >
>
> > > You are missing this call when your Mat is of type MATIS, thus the
> > > runtime
> > > error.
> >
>
> > > Thanks,
> >
>
> > > Pierre
> >
>
> > > > On 11 Dec 2017, at 5:42 PM, Franck Houssen < [email protected] >
> > > > wrote:
> > >
> >
>
> > > > Seems there is a problem with the attachment. Here is the file :
> > >
> >
>
> > > > >> more matExport.cpp
> > >
> >
>
> > > > #include <petscmat.h>
> > >
> >
>
> > > > #include <petscsys.h>
> > >
> >
>
> > > > int main(int argc, char *argv[])
> > >
> >
>
> > > > {
> > >
> >
>
> > > > PetscInitialize(&argc, &argv, "", NULL);
> > >
> >
>
> > > > Mat matrix;
> > >
> >
>
> > > > MatCreate(PETSC_COMM_WORLD, &matrix);
> > >
> >
>
> > > > const char* matType = (argc >= 2) ? argv[1] : MATMPIAIJ;
> > >
> >
>
> > > > PetscPrintf(PETSC_COMM_WORLD, "mat type is %s \n", matType);
> > >
> >
>
> > > > MatSetType(matrix, matType);
> > >
> >
>
> > > > MatSetSizes(matrix, PETSC_DECIDE, PETSC_DECIDE, 10, 10);
> > >
> >
>
> > > > MatSetUp(matrix);
> > >
> >
>
> > > > for (int i = 0; i < 10; i++) MatSetValue(matrix, i, i, (double)(i+1.),
> > > > ADD_VALUES);
> > >
> >
>
> > > > MatAssemblyBegin(matrix, MAT_FINAL_ASSEMBLY);
> > >
> >
>
> > > > MatAssemblyEnd(matrix, MAT_FINAL_ASSEMBLY);
> > >
> >
>
> > > > PetscViewer viewer;
> > >
> >
>
> > > > PetscViewerBinaryOpen(PETSC_COMM_WORLD, "matExport.bin",
> > > > FILE_MODE_WRITE,
> > > > &viewer);
> > >
> >
>
> > > > MatView(matrix, viewer);
> > >
> >
>
> > > > PetscViewerDestroy(&viewer);
> > >
> >
>
> > > > PetscViewerASCIIOpen(PETSC_COMM_WORLD, "matExport.log", &viewer);
> > >
> >
>
> > > > MatView(matrix, viewer);
> > >
> >
>
> > > > PetscViewerDestroy(&viewer);
> > >
> >
>
> > > > PetscViewerASCIIOpen(PETSC_COMM_WORLD, "matExport.mat", &viewer);
> > >
> >
>
> > > > PetscViewerPushFormat(viewer, PETSC_VIEWER_ASCII_MATLAB);
> > >
> >
>
> > > > MatView(matrix, viewer);
> > >
> >
>
> > > > PetscViewerDestroy(&viewer);
> > >
> >
>
> > > > MatDestroy(&matrix);
> > >
> >
>
> > > > PetscFinalize();
> > >
> >
>
> > > > }
> > >
> >
>
> > > > > De: "Franck Houssen" < [email protected] >
> > > >
> > >
> >
>
> > > > > À: [email protected]
> > > >
> > >
> >
>
> > > > > Envoyé: Lundi 11 Décembre 2017 17:39:40
> > > >
> > >
> >
>
> > > > > Objet: [petsc-dev] MatIS export
> > > >
> > >
> >
>
> > > > > Reporting a small bug related to MatIS export (or is this not meant
> > > > > to
> > > > > be
> > > > > supported ?).
> > > >
> > >
> >
>
> > > > > Franck
> > > >
> > >
> >
>
> > > > > >> rm -f matExport.log matExport.mat matExport.bin; mpirun -n 2
> > > > > >> ./matExport.exe is ; more matExport.log matExport.mat
> > > > > >> matExport.bin
> > > >
> > >
> >
>
> > > > > mat type is is
> > > >
> > >
> >
>
> > > > > [0]PETSC ERROR:
> > > > > ------------------------------------------------------------------------
> > > >
> > >
> >
>
> > > > > [0]PETSC ERROR: Caught signal number 11 SEGV: Segmentation Violation,
> > > > > probably memory access out of range
> > > >
> > >
> >
>
> > > > > [0]PETSC ERROR: Try option -start_in_debugger or
> > > > > -on_error_attach_debugger
> > > >
> > >
> >
>
> > > > > [0]PETSC ERROR: or see
> > > > > http://www.mcs.anl.gov/petsc/documentation/faq.html#valgrind
> > > >
> > >
> >
>
> > > > > [0]PETSC ERROR: or try http://valgrind.org on GNU/linux and Apple Mac
> > > > > OS
> > > > > X
> > > > > to
> > > > > find memory corruption errors
> > > >
> > >
> >
>
> > > > > [0]PETSC ERROR: configure using --with-debugging=yes, recompile,
> > > > > link,
> > > > > and
> > > > > run
> > > >
> > >
> >
>
> > > > > [0]PETSC ERROR: to get more information on the crash.
> > > >
> > >
> >
>
> > > > > [1]PETSC ERROR:
> > > > > ------------------------------------------------------------------------
> > > >
> > >
> >
>
> > > > > [1]PETSC ERROR: Caught signal number 11 SEGV: Segmentation Violation,
> > > > > probably memory access out of range
> > > >
> > >
> >
>
> > > > > [1]PETSC ERROR: Try option -start_in_debugger or
> > > > > -on_error_attach_debugger
> > > >
> > >
> >
>
> > > > > [1]PETSC ERROR: or see
> > > > > http://www.mcs.anl.gov/petsc/documentation/faq.html#valgrind
> > > >
> > >
> >
>
> > > > > [1]PETSC ERROR: or try http://valgrind.org on GNU/linux and Apple Mac
> > > > > OS
> > > > > X
> > > > > to
> > > > > find memory corruption errors
> > > >
> > >
> >
>
> > > > > [1]PETSC ERROR: configure using --with-debugging=yes, recompile,
> > > > > link,
> > > > > and
> > > > > run
> > > >
> > >
> >
>
> > > > > [1]PETSC ERROR: to get more information on the crash.
> > > >
> > >
> >
>
> > > > > [0]PETSC ERROR: --------------------- Error Message
> > > > > --------------------------------------------------------------
> > > >
> > >
> >
>
> > > > > >> rm -f matExport.log matExport.mat matExport.bin; mpirun -n 2
> > > > > >> ./matExport.exe mpiaij ; more matExport.log matExport.mat
> > > > > >> matExport.bin
> > > >
> > >
> >
>
> > > > > mat type is mpiaij
> > > >
> > >
> >
>
> > > > > ::::::::::::::
> > > >
> > >
> >
>
> > > > > matExport.log
> > > >
> > >
> >
>
> > > > > ::::::::::::::
> > > >
> > >
> >
>
> > > > > Mat Object: 2 MPI processes
> > > >
> > >
> >
>
> > > > > type: mpiaij
> > > >
> > >
> >
>
> > > > > row 0: (0, 2.)
> > > >
> > >
> >
>
> > > > > row 1: (1, 4.)
> > > >
> > >
> >
>
> > > > > row 2: (2, 6.)
> > > >
> > >
> >
>
> > > > > row 3: (3, 8.)
> > > >
> > >
> >
>
> > > > > row 4: (4, 10.)
> > > >
> > >
> >
>
> > > > > row 5: (5, 12.)
> > > >
> > >
> >
>
> > > > > row 6: (6, 14.)
> > > >
> > >
> >
>
> > > > > row 7: (7, 16.)
> > > >
> > >
> >
>
> > > > > row 8: (8, 18.)
> > > >
> > >
> >
>
> > > > > row 9: (9, 20.)
> > > >
> > >
> >
>
> > > > > ::::::::::::::
> > > >
> > >
> >
>
> > > > > matExport.mat
> > > >
> > >
> >
>
> > > > > ::::::::::::::
> > > >
> > >
> >
>
> > > > > %Mat Object: 2 MPI processes
> > > >
> > >
> >
>
> > > > > % type: mpiaij
> > > >
> > >
> >
>
> > > > > % Size = 10 10
> > > >
> > >
> >
>
> > > > > % Nonzeros = 10
> > > >
> > >
> >
>
> > > > > zzz = zeros(10,3);
> > > >
> > >
> >
>
> > > > > zzz = [
> > > >
> > >
> >
>
> > > > > 1 1 2.0000000000000000e+00
> > > >
> > >
> >
>
> > > > > 2 2 4.0000000000000000e+00
> > > >
> > >
> >
>
> > > > > 3 3 6.0000000000000000e+00
> > > >
> > >
> >
>
> > > > > 4 4 8.0000000000000000e+00
> > > >
> > >
> >
>
> > > > > 5 5 1.0000000000000000e+01
> > > >
> > >
> >
>
> > > > > 6 6 1.2000000000000000e+01
> > > >
> > >
> >
>
> > > > > 7 7 1.4000000000000000e+01
> > > >
> > >
> >
>
> > > > > 8 8 1.6000000000000000e+01
> > > >
> > >
> >
>
> > > > > 9 9 1.8000000000000000e+01
> > > >
> > >
> >
>
> > > > > 10 10 2.0000000000000000e+01
> > > >
> > >
> >
>
> > > > > ];
> > > >
> > >
> >
>
> > > > > Mat_0x557ffe276ba0_0 = spconvert(zzz);
> > > >
> > >
> >
>
> > > > > ::::::::::::::
> > > >
> > >
> >
>
> > > > > matExport.bin
> > > >
> > >
> >
>
> > > > > ::::::::::::::
> > > >
> > >
> >
>
/*
* mpirun -n 2 ./matExport.exe
* mpirun -n 2 ./matExport.exe is
*/
#include <petscmat.h>
#include <petscsys.h>
#include <string.h>
int main(int argc, char *argv[])
{
PetscInitialize(&argc, &argv, "", NULL);
Mat matrix;
MatCreate(PETSC_COMM_WORLD, &matrix);
const char* matType = (argc >= 2) ? argv[1] : MATMPIAIJ;
PetscPrintf(PETSC_COMM_WORLD, "mat type is %s \n", matType);
MatSetType(matrix, matType);
MatSetSizes(matrix, PETSC_DECIDE, PETSC_DECIDE, 10, 10);
if (strcmp(matType, "is") == 0) {
ISLocalToGlobalMapping is;
PetscInt idx[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD, 1, 10, idx, PETSC_COPY_VALUES, &is);
MatSetLocalToGlobalMapping(matrix, is, is);
ISLocalToGlobalMappingDestroy(&is);
PetscPrintf(PETSC_COMM_WORLD, "MatSetLocalToGlobalMapping done\n");
Mat local;
MatCreateSeqDense(PETSC_COMM_SELF, 10, 10, NULL, &local);
for (int i = 0; i < 10; i++) MatSetValue(local, i, i, (double)(i+1.), ADD_VALUES);
MatAssemblyBegin(local, MAT_FINAL_ASSEMBLY); MatAssemblyBegin(local, MAT_FINAL_ASSEMBLY);
MatISSetLocalMat(matrix, local);
MatDestroy(&local);
}
else {
MatSetUp(matrix);
for (int i = 0; i < 10; i++) MatSetValue(matrix, i, i, (double)(i+1.), ADD_VALUES);
}
MatAssemblyBegin(matrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(matrix, MAT_FINAL_ASSEMBLY);
PetscViewer viewer;
PetscViewerBinaryOpen(PETSC_COMM_WORLD, "matExport.bin", FILE_MODE_WRITE, &viewer);
MatView(matrix, viewer);
PetscViewerDestroy(&viewer);
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "matExport.log", &viewer);
MatView(matrix, viewer);
PetscViewerDestroy(&viewer);
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "matExport.mat", &viewer);
PetscViewerPushFormat(viewer, PETSC_VIEWER_ASCII_MATLAB);
MatView(matrix, viewer);
PetscViewerDestroy(&viewer);
MatDestroy(&matrix);
PetscFinalize();
}
