Re: [petsc-users] Cannot do make on ex55k
Satish is right. ex55k is not standalone.
There are many examples in petsc tests that can run with kokkos. Just
search "requires: kokkos_kernels" and you will find them.
ex55k.kokkos.cxx is an example showing how to use petsc/kokkos functions
like DMDAVecRestoreKokkosOffsetView, MatSetValuesCOO etc.
--Junchao Zhang
On Wed, Feb 14, 2024 at 11:23 AM Satish Balay via petsc-users <
petsc-users@mcs.anl.gov> wrote:
> Looks like ex55 is the one to use - that links in with ex55k
>
> But it needs a fix for a build from 'make'
>
> >>
> balay@petsc-gpu-01:/scratch/balay/petsc/src/snes/tutorials$ git diff
> diff --git a/src/snes/tutorials/makefile b/src/snes/tutorials/makefile
> index 672a62aa5a0..eed127f7eae 100644
> --- a/src/snes/tutorials/makefile
> +++ b/src/snes/tutorials/makefile
> @@ -8,6 +8,8 @@ CLEANFILES = ex5f90t
> include ${PETSC_DIR}/lib/petsc/conf/variables
> include ${PETSC_DIR}/lib/petsc/conf/rules
>
> +ex55: ex55.o ex55k.o
> +
> #-
>
> # these tests are used by the makefile in PETSC_DIR for basic tests of
> the install and should not be removed
> balay@petsc-gpu-01:/scratch/balay/petsc/src/snes/tutorials$ make ex55
> mpicc -o ex55.o -c -fPIC -Wall -Wwrite-strings -Wno-unknown-pragmas
> -Wno-lto-type-mismatch -Wno-stringop-overflow -fstack-protector
> -fvisibility=hidden -g3 -O0 -I/scratch/balay/petsc/include
> -I/scratch/balay/petsc/arch-linux-c-debug/include
> -I/usr/local/cuda/include`pwd`/ex55.c
> PATH=`dirname
> /usr/local/cuda/bin/nvcc`:/nfs/gce/projects/petsc/soft/u22.04/mpich-4.0.2/bin:/usr/lib/ccache:/home/balay/.local/bin:/home/balay/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin::/nfs/gce/projects/gce/bin
> NVCC_WRAPPER_DEFAULT_COMPILER="mpicxx"
> /scratch/balay/petsc/arch-linux-c-debug/bin/nvcc_wrapper
> --expt-extended-lambda -c -ccbin mpicxx -std=c++17 -Xcompiler -fPIC -g
> -lineinfo -gencode arch=compute_86,code=sm_86
> -I/scratch/balay/petsc/include
> -I/scratch/balay/petsc/arch-linux-c-debug/include -I/usr/local/cuda/include
> -Wall -Wwrite-strings -Wno-strict-aliasing -Wno-unknown-pragmas
> -Wno-lto-type-mismatch -Wno-psabi -fstack-protector -g
> -I/nfs/gce/projects/petsc/soft/u22.04/mpich-4.0.2/include -o ex55k.o
> `pwd`/ex55k.kokkos.cxx
> mpicc -fPIC -Wall -Wwrite-strings -Wno-unknown-pragmas
> -Wno-lto-type-mismatch -Wno-stringop-overflow -fstack-protector
> -fvisibility=hidden -g3 -O0 -Wl,-export-dynamic ex55.o ex55k.o
> -Wl,-rpath,/scratch/balay/petsc/arch-linux-c-debug/lib
> -L/scratch/balay/petsc/arch-linux-c-debug/lib
> -Wl,-rpath,/scratch/balay/petsc/arch-linux-c-debug/lib
> -L/scratch/balay/petsc/arch-linux-c-debug/lib
> -Wl,-rpath,/usr/local/cuda/lib64 -L/usr/local/cuda/lib64
> -L/usr/local/cuda/lib64/stubs
> -Wl,-rpath,/nfs/gce/projects/petsc/soft/u22.04/mpich-4.0.2/lib
> -L/nfs/gce/projects/petsc/soft/u22.04/mpich-4.0.2/lib
> -Wl,-rpath,/usr/lib/gcc/x86_64-linux-gnu/11
> -L/usr/lib/gcc/x86_64-linux-gnu/11 -lpetsc -lkokkoskernels
> -lkokkoscontainers -lkokkoscore -lkokkossimd -llapack -lblas -lm -lcudart
> -lnvToolsExt -lcufft -lcublas -lcusparse -lcusolver -lcurand -lcuda -lX11
> -lmpifort -lmpi -lgfortran -lm -lgfortran -lm -lgcc_s -lquadmath -lstdc++
> -lquadmath -o ex55
> balay@petsc-gpu-01:/scratch/balay/petsc/src/snes/tutorials$
> <<<
>
> Satish
>
>
> On Wed, 14 Feb 2024, Uralovich, Ibragimov Iskander wrote:
>
> > Hello!
> >
> > I want to start porting part of our PETSc-based code to GPU through
> kokkos and I want to start with excercises given in tutorials folder.
> >
> > For that I found two examples of using PETSC with kokkos in folder
> petsc/src/snes/tutorials
> >
> > I managed to succesfully compile and run ex3k.kokkos.cxx, but I cannot
> compile ex55k.kokkos.cxx.
> > when I do make i got a following error:
> >
> >
> > iskander@apollo:~/software/petsc3_20/src/snes/tutorials$ make ex55k
> > PATH=`dirname
> nvcc`:/local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin:/usr/local/cuda-12.2/bin:/local/home/iskander/.vscode-server/bin/05047486b6df5eb8d44b2ecd70ea3bdf775fd937/bin/remote-cli:/local/home/iskander/.local/bin:/local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin:/usr/local/cuda-12.2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/opt/paraview/v5.10.0/bin
> NVCC_WRAPPER_DEFAULT_COMPILER="/local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin"/mpicxx
> /local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin/nvcc_wrapper
> --expt-extended-lambda -ccbin
> /local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin/mpicxx
> -std=c++17 -Xcompiler -fPIC -Xcompiler -fvisibility=hidden -g -lineinfo
> -gencode arch=compute_80,code=sm_80
> -I/local/home/iskander/software/petsc3_20/petsc-3.20-deb/include
> -I/local/home/iskander/software/petsc3_20/include
> -I/local/home/iskander/software/petsc3_20/arch-linux-c-d
> ebug/inc
>
Re: [petsc-users] Cannot do make on ex55k
Looks like ex55 is the one to use - that links in with ex55k
But it needs a fix for a build from 'make'
>>
balay@petsc-gpu-01:/scratch/balay/petsc/src/snes/tutorials$ git diff
diff --git a/src/snes/tutorials/makefile b/src/snes/tutorials/makefile
index 672a62aa5a0..eed127f7eae 100644
--- a/src/snes/tutorials/makefile
+++ b/src/snes/tutorials/makefile
@@ -8,6 +8,8 @@ CLEANFILES = ex5f90t
include ${PETSC_DIR}/lib/petsc/conf/variables
include ${PETSC_DIR}/lib/petsc/conf/rules
+ex55: ex55.o ex55k.o
+
#-
# these tests are used by the makefile in PETSC_DIR for basic tests of the
install and should not be removed
balay@petsc-gpu-01:/scratch/balay/petsc/src/snes/tutorials$ make ex55
mpicc -o ex55.o -c -fPIC -Wall -Wwrite-strings -Wno-unknown-pragmas
-Wno-lto-type-mismatch -Wno-stringop-overflow -fstack-protector
-fvisibility=hidden -g3 -O0 -I/scratch/balay/petsc/include
-I/scratch/balay/petsc/arch-linux-c-debug/include -I/usr/local/cuda/include
`pwd`/ex55.c
PATH=`dirname
/usr/local/cuda/bin/nvcc`:/nfs/gce/projects/petsc/soft/u22.04/mpich-4.0.2/bin:/usr/lib/ccache:/home/balay/.local/bin:/home/balay/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin::/nfs/gce/projects/gce/bin
NVCC_WRAPPER_DEFAULT_COMPILER="mpicxx"
/scratch/balay/petsc/arch-linux-c-debug/bin/nvcc_wrapper --expt-extended-lambda
-c -ccbin mpicxx -std=c++17 -Xcompiler -fPIC -g -lineinfo -gencode
arch=compute_86,code=sm_86 -I/scratch/balay/petsc/include
-I/scratch/balay/petsc/arch-linux-c-debug/include -I/usr/local/cuda/include
-Wall -Wwrite-strings -Wno-strict-aliasing -Wno-unknown-pragmas
-Wno-lto-type-mismatch -Wno-psabi -fstack-protector -g
-I/nfs/gce/projects/petsc/soft/u22.04/mpich-4.0.2/include -o ex55k.o
`pwd`/ex55k.kokkos.cxx
mpicc -fPIC -Wall -Wwrite-strings -Wno-unknown-pragmas -Wno-lto-type-mismatch
-Wno-stringop-overflow -fstack-protector -fvisibility=hidden -g3 -O0
-Wl,-export-dynamic ex55.o ex55k.o
-Wl,-rpath,/scratch/balay/petsc/arch-linux-c-debug/lib
-L/scratch/balay/petsc/arch-linux-c-debug/lib
-Wl,-rpath,/scratch/balay/petsc/arch-linux-c-debug/lib
-L/scratch/balay/petsc/arch-linux-c-debug/lib -Wl,-rpath,/usr/local/cuda/lib64
-L/usr/local/cuda/lib64 -L/usr/local/cuda/lib64/stubs
-Wl,-rpath,/nfs/gce/projects/petsc/soft/u22.04/mpich-4.0.2/lib
-L/nfs/gce/projects/petsc/soft/u22.04/mpich-4.0.2/lib
-Wl,-rpath,/usr/lib/gcc/x86_64-linux-gnu/11 -L/usr/lib/gcc/x86_64-linux-gnu/11
-lpetsc -lkokkoskernels -lkokkoscontainers -lkokkoscore -lkokkossimd -llapack
-lblas -lm -lcudart -lnvToolsExt -lcufft -lcublas -lcusparse -lcusolver
-lcurand -lcuda -lX11 -lmpifort -lmpi -lgfortran -lm -lgfortran -lm -lgcc_s
-lquadmath -lstdc++ -lquadmath -o ex55
balay@petsc-gpu-01:/scratch/balay/petsc/src/snes/tutorials$
<<<
Satish
On Wed, 14 Feb 2024, Uralovich, Ibragimov Iskander wrote:
> Hello!
>
> I want to start porting part of our PETSc-based code to GPU through kokkos
> and I want to start with excercises given in tutorials folder.
>
> For that I found two examples of using PETSC with kokkos in folder
> petsc/src/snes/tutorials
>
> I managed to succesfully compile and run ex3k.kokkos.cxx, but I cannot
> compile ex55k.kokkos.cxx.
> when I do make i got a following error:
>
>
> iskander@apollo:~/software/petsc3_20/src/snes/tutorials$ make ex55k
> PATH=`dirname
> nvcc`:/local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin:/usr/local/cuda-12.2/bin:/local/home/iskander/.vscode-server/bin/05047486b6df5eb8d44b2ecd70ea3bdf775fd937/bin/remote-cli:/local/home/iskander/.local/bin:/local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin:/usr/local/cuda-12.2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/opt/paraview/v5.10.0/bin
>
> NVCC_WRAPPER_DEFAULT_COMPILER="/local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin"/mpicxx
> /local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin/nvcc_wrapper
> --expt-extended-lambda -ccbin
> /local/home/iskander/software/petsc3_20/petsc-3.20-deb/bin/mpicxx -std=c++17
> -Xcompiler -fPIC -Xcompiler -fvisibility=hidden -g -lineinfo -gencode
> arch=compute_80,code=sm_80
> -I/local/home/iskander/software/petsc3_20/petsc-3.20-deb/include
> -I/local/home/iskander/software/petsc3_20/include
> -I/local/home/iskander/software/petsc3_20/arch-linux-c-d
ebug/inc
lude -I/local/home/iskander/software/petsc3_20/petsc-3.20-deb/include
-I/usr/local/cuda-12.2/include-Wall -Wwrite-strings -Wno-strict-aliasing
-Wno-unknown-pragmas -Wno-lto-type-mismatch -Wno-psabi -fstack-protector
-fvisibility=hidden -march=znver2
-L/local/home/iskander/software/petsc3_20/petsc-3.20-deb/lib -Wl,-rpath
-Wl,/local/home/iskander/software/petsc3_20/petsc-3.20-deb/lib
-Wl,--enable-new-dtags -lmpi ex55k.kokkos.cxx
