17 GB for a 1D DMDA, wow. :-)
Could you try applying this diff to make it work for DMDA (it's currently
handled by DMPlex)?
diff --git i/src/dm/impls/da/fdda.c w/src/dm/impls/da/fdda.c
index cad4d926504..bd2a3bda635 100644
--- i/src/dm/impls/da/fdda.c
+++ w/src/dm/impls/da/fdda.c
@@ -675,19 +675,21 @@ PetscErrorCode DMCreateMatrix_DA(DM da, Mat *J)
specialized setting routines depend only on the particular preallocation
details of the matrix, not the type itself.
*/
- PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatMPIAIJSetPreallocation_C", &aij));
- if (!aij) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatSeqAIJSetPreallocation_C", &aij));
- if (!aij) {
- PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatMPIBAIJSetPreallocation_C", &baij));
- if (!baij) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatSeqBAIJSetPreallocation_C", &baij));
- if (!baij) {
- PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatMPISBAIJSetPreallocation_C", &sbaij));
- if (!sbaij) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatSeqSBAIJSetPreallocation_C", &sbaij));
- if (!sbaij) {
- PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatMPISELLSetPreallocation_C", &sell));
- if (!sell) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatSeqSELLSetPreallocation_C", &sell));
+ if (!dm->prealloc_skip) { // Flag is likely set when user intends to use
MatSetPreallocationCOO()
+ PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatMPIAIJSetPreallocation_C", &aij));
+ if (!aij) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatSeqAIJSetPreallocation_C", &aij));
+ if (!aij) {
+ PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatMPIBAIJSetPreallocation_C", &baij));
+ if (!baij) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatSeqBAIJSetPreallocation_C", &baij));
+ if (!baij) {
+ PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatMPISBAIJSetPreallocation_C", &sbaij));
+ if (!sbaij) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatSeqSBAIJSetPreallocation_C", &sbaij));
+ if (!sbaij) {
+ PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatMPISELLSetPreallocation_C", &sell));
+ if (!sell) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatSeqSELLSetPreallocation_C", &sell));
+ }
+ if (!sell) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatISSetPreallocation_C", &is));
}
- if (!sell) PetscCall(PetscObjectQueryFunction((PetscObject)A,
"MatISSetPreallocation_C", &is));
}
}
if (aij) {
"Fackler, Philip via petsc-users" <petsc-users@mcs.anl.gov> writes:
> I'm using the following sequence of functions related to the Jacobian matrix:
>
> DMDACreate1d(..., &da);
> DMSetFromOptions(da);
> DMSetUp(da);
> DMSetMatType(da, MATAIJKOKKOS);
> DMSetMatrixPreallocateSkip(da, PETSC_TRUE);
> Mat J;
> DMCreateMatrix(da, &J);
> MatSetPreallocationCOO(J, ...);
>
> I recently added the call to DMSetMatrixPreallocateSkip, hoping the
> allocation would be delayed to MatSetPreallocationCOO, and that it would
> require less memory. The
> documentation<https://petsc.org/release/manualpages/DM/DMSetMatrixPreallocateSkip/>
> says that the data structures will not be preallocated. The following data
> from heaptrack shows that the allocation is still happening in the call to
> DMCreateMatrix.
>
> [cid:bda9ef12-a46f-47b2-9b9b-a4b2808b6b13]
>
> Can someone help me understand this?
>
> Thanks,
>
> Philip Fackler
> Research Software Engineer, Application Engineering Group
> Advanced Computing Systems Research Section
> Computer Science and Mathematics Division
> Oak Ridge National Laboratory
