I was doing 'make ex5k.kokkos'
It would be nice if that failed, but it building now.
Thanks,
On Thu, Jun 3, 2021 at 10:57 AM Junchao Zhang <junchao.zh...@gmail.com>
wrote:
> (Moved to petsc-dev)
>
> Mark,
> Is it because your branch is out of date? petsc should use nvcc_wapper
> to compile ex5k.kokkos.cxx. See mine
> $cd ~/petsc/src/mat/tutorials
> $ make ex5k
> PATH=/home/jczhang/spack/opt/spack/linux-ubuntu18.04-skylake_avx512/gcc-8.3.0/gdb-9.1-bzjzolog57gom5anscfcb3oe6uqr6s6m/bin:/home/jczhang/spack/opt/spack/linux-ubuntu18.04-skylake_avx512/gcc-8.3.0/bear-2.2.0-xhfk3moyh7jled6o62f3om557iia6oun/bin:/home/jczhang/spack/opt/spack/linux-ubuntu18.04-skylake_avx512/gcc-8.3.0/perl-5.30.3-2boymujkmxezayez4emfw5sw5wuqgar6/bin:/home/jczhang/spack/opt/spack/linux-ubuntu18.04-skylake_avx512/gcc-8.3.0/openmpi-4.0.2-7vmyqiyk4iyvdeoqpux7fyoce6mjt7iw/bin:/home/jczhang/spack/opt/spack/linux-ubuntu18.04-skylake_avx512/gcc-8.3.0/cuda-10.2.89-uy4hlcnd7svrcahaguwx4bzl7ujoqx2v/bin:/nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.3.0/cmake-3.20.0-vov726r/bin:/nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.3.0/gcc-8.3.0-fjpc5ys/bin:/home/jczhang/soft/bin:/home/jczhang/spack/bin:/home/jczhang/arm/forge/20.2/bin:/home/jczhang/.vscode-server/data/User/globalStorage/llvm-vs-code-extensions.vscode-clangd/install/10.0.0/clangd_10.0.0/bin:/home/jczhang/.vscode-server/bin/054a9295330880ed74ceaedda236253b4f39a335/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:`dirname
> nvcc` NVCC_WRAPPER_DEFAULT_COMPILER=gcc
> /home/jczhang/petsc/linux-kokkos-dbg/bin/nvcc_wrapper --expt-extended-lambda
> -Xcompiler -fPIC -g -gencode arch=compute_70,code=sm_70
>
> -I/home/jczhang/spack/opt/spack/linux-ubuntu18.04-skylake_avx512/gcc-8.3.0/openmpi-4.0.2-7vmyqiyk4iyvdeoqpux7fyoce6mjt7iw/include
> -Wno-deprecated-gpu-targets -I/home/jczhang/petsc/include
> -I/home/jczhang/petsc/linux-kokkos-dbg/include
>
> -I/home/jczhang/spack/opt/spack/linux-ubuntu18.04-skylake_avx512/gcc-8.3.0/openmpi-4.0.2-7vmyqiyk4iyvdeoqpux7fyoce6mjt7iw/include
> -Wall -Wwrite-strings -Wno-strict-aliasing -Wno-unknown-pragmas
> -fstack-protector -fvisibility=hidden -g -O0 ex5k.kokkos.cxx
> -Wl,-rpath,/home/jczhang/petsc/linux-kokkos-dbg/lib
> -L/home/jczhang/petsc/linux-kokkos-dbg/lib
> -Wl,-rpath,/home/jczhang/petsc/linux-kokkos-dbg/lib
> -L/home/jczhang/petsc/linux-kokkos-dbg/lib -lpetsc -lkokkoskernels
> -lkokkoscontainers -lkokkoscore -llapack -lblas -lm -lcudart -lcufft
> -lcublas -lcusparse -lcusolver -lcurand -lX11 -lquadmath -lstdc++ -ldl -o
> ex5k
>
> --Junchao Zhang
>
>
> On Thu, Jun 3, 2021 at 8:32 AM Mark Adams <mfad...@lbl.gov> wrote:
>
>> I am getting this error:
>>
>> 09:22 adams/landau-mass-opt=
>> /gpfs/alpine/csc314/scratch/adams/petsc/src/mat/tutorials$ make
>> PETSC_DIR=/gpfs/alpine/csc314/scratch/adams/petsc
>> PETSC_ARCH=arch-summit-opt-gnu-kokkos-notpl-cuda10 ex5k.kokkos
>> mpicxx -fPIC -g -DLANDAU_DIM=2 -DLANDAU_MAX_SPECIES=10
>> -DPETSC_HAVE_CUDA_ATOMIC -DLANDAU_MAX_Q=4 -O0 -fPIC -g -DLANDAU_DIM=2
>> -DLANDAU_MAX_SPECIES=10 -DPETSC_HAVE_CUDA_ATOMIC -DLANDAU_MAX_Q=4 -O0
>> -fPIC -I/gpfs/alpine/csc314/scratch/adams/petsc/include
>> -I/gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include
>> -I/sw/summit/cuda/10.1.243/include ex5k.kokkos.cxx
>>
>> -Wl,-rpath,/gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/lib
>> -L/gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/lib
>> -Wl,-rpath,/gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/lib
>> -L/gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/lib
>> -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/netlib-lapack-3.8.0-wcabdyqhdi5rooxbkqa6x5d7hxyxwdkm/lib64
>> -Wl,-rpath,/sw/summit/cuda/10.1.243/lib64 -L/sw/summit/cuda/10.1.243/lib64
>> -lpetsc -lkokkoskernels -lkokkoscontainers -lkokkoscore -lp4est -lsc -lblas
>> -llapack -ltriangle -lm -lz -lcudart -lcufft -lcublas -lcusparse -lcusolver
>> -lcurand -lstdc++ -ldl -o ex5k.kokkos
>> In file included from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/KokkosCore_Config_SetupBackend.hpp:47:0,
>> from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/Kokkos_Macros.hpp:109,
>> from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/Kokkos_Core_fwd.hpp:52,
>> from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/Kokkos_Core.hpp:51,
>> from ex5k.kokkos.cxx:10:
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/setup/Kokkos_Setup_Cuda.hpp:54:2:
>> error: #error "KOKKOS_ENABLE_CUDA defined but the compiler is not defining
>> the __CUDACC__ macro as expected"
>> #error \
>> ^~~~~
>> In file included from
>> /sw/summit/cuda/10.1.243/include/crt/common_functions.h:267:0,
>> from /sw/summit/cuda/10.1.243/include/cuda_runtime.h:115,
>> from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/setup/Kokkos_Setup_Cuda.hpp:68,
>> from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/KokkosCore_Config_SetupBackend.hpp:47,
>> from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/Kokkos_Macros.hpp:109,
>> from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/Kokkos_Core_fwd.hpp:52,
>> from
>> /gpfs/alpine/csc314/scratch/adams/petsc/arch-summit-opt-gnu-kokkos-notpl-cuda10/include/Kokkos_Core.hpp:51,
>> from ex5k.kokkos.cxx:10:
>> /sw/summit/cuda/10.1.243/include/crt/math_functions.h:8944:72: error:
>> 'int std::isnan(double)' conflicts with a previous declaration
>> __DEVICE_FUNCTIONS_DECL__ __cudart_builtin__ int isnan(double x) throw();
>> ^
>> Here is my config file:
>>
>> #!/usr/bin/env python
>> # -lnvToolsExt
>>
>> if __name__ == '__main__':
>> import sys
>> import os
>> sys.path.insert(0, os.path.abspath('config'))
>> import configure
>> configure_options = [
>> '--with-fc=0',
>> '--CFLAGS=-fPIC -g -DLANDAU_DIM=2 -DLANDAU_MAX_SPECIES=10
>> -DPETSC_HAVE_CUDA_ATOMIC -DLANDAU_MAX_Q=4',
>> '--CXXFLAGS=-fPIC -g -DLANDAU_DIM=2 -DLANDAU_MAX_SPECIES=10
>> -DPETSC_HAVE_CUDA_ATOMIC -DLANDAU_MAX_Q=4',
>> '--COPTFLAGS=-O0',
>> '--CXXOPTFLAGS=-O0',
>> '--CUDAOPTFLAGS=-O0',
>> # 'LIBS= -L'+os.environ['CUDAPATH']+'/lib64 -lnvToolsExt',
>> '--CUDAFLAGS=-Xcompiler -rdynamic -lineinfo -DLANDAU_DIM=2
>> -DLANDAU_MAX_SPECIES=10 -DPETSC_HAVE_CUDA_ATOMIC -DLANDAU_MAX_Q=4',
>> '--with-ssl=0',
>> '--with-batch=0',
>> '--with-cxx=mpicxx',
>> '--with-mpiexec=jsrun -g1 ',
>> '--with-openmp=0',
>> # '--with-threadsafety=1',
>> # '--with-log=0',
>> '--with-cuda=1',
>> '--with-cuda-gencodearch=70',
>> '--with-cudac=nvcc',
>> '--download-p4est=1',
>> '--download-zlib',
>> # '--download-hdf5=1',
>> # '--download-metis',
>> # '--download-superlu_dist',
>> # '--download-superlu_dist-commit=HEAD',
>> # '--download-hypre-configure-arguments=HYPRE_CUDA_SM=70',
>> #'--with-hwloc=0',
>> # '--download-parmetis',
>> #'--download-hypre',
>> '--download-triangle',
>> #'--download-amgx',
>> #'--download-fblaslapack',
>> '--with-blaslapack-lib=-L' + os.environ['OLCF_NETLIB_LAPACK_ROOT'] +
>> '/lib64 -lblas -llapack',
>> #'--download-openblas',
>> '--with-cc=mpicc',
>> #'--with-fc=mpif90',
>> '--with-shared-libraries=1',
>> # '--known-mpi-shared-libraries=1',
>> '--with-x=0',
>> '--with-64-bit-indices=0',
>> '--with-debugging=1',
>> '--download-kokkos',
>> '--with-ctable=0',
>> '--with-make-np=8',
>> '--download-kokkos-kernels',
>> # '--with-kokkos-cuda-arch=VOLTA70',
>> '--with-kokkos-kernels-tpl=0',
>>
>> '--download-kokkos-cmake-arguments=-DKokkos_ENABLE_AGGRESSIVE_VECTORIZATION=ON
>> -DKokkos_ARCH_POWER9=ON',
>> # '--download-kokkos-commit=origin/develop',
>> # '--download-kokkos-kernels-commit=origin/develop',
>> 'PETSC_ARCH=arch-summit-opt-gnu-kokkos-notpl-cuda10',
>> ]
>> configure.petsc_configure(configure_options)
>>
>> ~
>> ~
>> ~
>> ~
>> ~
>> ~
>> ~
>>
>
