On Wed, Jul 13, 2016 at 3:57 AM, Morten Nobel-Jørgensen <[email protected]> wrote:
> I’m having problems distributing a simple FEM model using DMPlex. For test
> case I use 1x1x2 hex box elements (/cells) with 12 vertices. Each vertex
> has one DOF.
> When I distribute the system to two processors, each get a single element
> and the local vector has the size 8 (one DOF for each vertex of a hex box)
> as expected.
>
> My problem is that when I manually assemble the global stiffness matrix (a
> 12x12 matrix) it seems like my ghost values are ignored. I’m sure that I’m
> missing something obvious but cannot see what it is.
>
> In the attached example, I’m assembling the global stiffness matrix using
> a simple local stiffness matrix of ones. This makes it very easy to see if
> the matrix is assembled correctly. If I run it on one process, then global
> stiffness matrix consists of 0’s, 1’s and 2’s and its trace is 16.0. But if
> I run it distributed on on two, then it consists only of 0's and 1’s and
> its trace is 12.0.
>
> I hope that somebody can spot my mistake and help me in the right
> direction :)
>
This is my fault, and Stefano Zampini had already tried to tell me this was
broken. I normally use DMPlexMatSetClosure(), which handles global indices
correctly.
I have fixed this in the branch
knepley/fix-plex-l2g
which is also merged to 'next'. I am attaching a version of your sample
where all objects are freed correctly. Let me know if that works for you.
Thanks,
Matt
> Kind regards,
> Morten
>
--
What most experimenters take for granted before they begin their
experiments is infinitely more interesting than any results to which their
experiments lead.
-- Norbert Wiener
static char help[] = "";
#include <petscdmplex.h>
const int dof = 1;
#undef __FUNCT__
#define __FUNCT__ "SetupDOFs"
/* Assign dofs (3 for each node) */
PetscErrorCode SetupDOFs(DM dm) {
PetscSection s;
PetscInt pStart, pEnd, cStart, cEnd, c, fStart, fEnd, f, vStart, vEnd, v, eStart, eEnd, e;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &s); CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(s, 1); CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd); CHKERRQ(ierr);
PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd); CHKERRQ(ierr);
ierr = PetscSectionSetChart(s, pStart, pEnd); CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
ierr = PetscSectionSetDof(s, v, dof); CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(s, v, 0, dof); CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(s); CHKERRQ(ierr);
ierr = DMSetDefaultSection(dm, s); CHKERRQ(ierr);
ierr = PetscSectionDestroy(&s); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#undef __FUNCT__
#define __FUNCT__ "CreateGlobalStiffnessMatrix"
PetscInt CreateGlobalStiffnessMatrix(DM dm, Mat *K) {
PetscSection localSection;
PetscInt cStart, cEnd, nStart, nEnd, e;
PetscInt edof[dof * 8]; // edof vector
PetscScalar ke[(dof * 8) * (dof * 8)]; // element matrix
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd); CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &nStart, &nEnd); CHKERRQ(ierr);
ierr = DMCreateMatrix(dm, K); CHKERRQ(ierr);
ierr = MatViewFromOptions(*K, NULL, "-mat_view"); CHKERRQ(ierr);
ierr = DMGetDefaultSection(dm, &localSection); CHKERRQ(ierr);
for (e = 0; e < (8 * dof) * (8 * dof); e++) {
ke[e] = 1.0; // placeholder value - replace with local stiffness matrix
}
for (e = cStart; e < cEnd; e++) {
PetscBool useCone = PETSC_TRUE;
PetscInt clojureCount;
PetscInt *clojureIndex = NULL;
PetscInt edofIndex = 0, v;
ierr = DMPlexGetTransitiveClosure(dm, e, useCone, &clojureCount, &clojureIndex); CHKERRQ(ierr);
for (v = 0; v < clojureCount; ++v) {
PetscInt cPoint = clojureIndex[v * 2];
PetscInt cOrientation = clojureIndex[v * 2 + 1];
PetscBool isVertex = cPoint >= nStart && cPoint < nEnd ? PETSC_TRUE : PETSC_FALSE; // is vertex if it exist in the vertex range
if (isVertex) {
PetscInt offset, dofSize, i;
ierr = PetscSectionGetOffset(localSection, cPoint, &offset); CHKERRQ(ierr);
ierr = PetscSectionGetDof(localSection, cPoint, &dofSize); CHKERRQ(ierr);
for (i = 0; i < dofSize; ++i) {
edof[edofIndex] = offset + i;
edofIndex++;
}
}
}
ierr = DMPlexRestoreTransitiveClosure(dm, e, useCone, &clojureCount, &clojureIndex); CHKERRQ(ierr);
ierr = MatSetValuesLocal(*K, 8 * dof, edof, 8 * dof, edof, ke, ADD_VALUES); CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(*K, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(*K, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#undef __FUNCT__
#define __FUNCT__ "main"
int main(int argc, char **argv) {
DM dm, dist;
Mat mat;
PetscInt dim = 3;
PetscInt cells[] = {1, 1, 2};
PetscInt overlap = 0;
PetscErrorCode ierr;
PetscViewer view;
ierr = PetscInitialize(&argc, &argv, NULL, help); CHKERRQ(ierr);
ierr = DMPlexCreateHexBoxMesh(PETSC_COMM_WORLD, dim, cells, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,
&dm); CHKERRQ(ierr);
//ierr = DMPlexSetAdjacencyUseCone(dm, PETSC_FALSE); CHKERRQ(ierr);
//ierr = DMPlexSetAdjacencyUseClosure(dm, PETSC_TRUE); CHKERRQ(ierr);
ierr = DMPlexDistribute(dm, overlap, NULL, &dist); CHKERRQ(ierr);
if (dist) {
ierr = DMDestroy(&dm); CHKERRQ(ierr);
dm = dist;
}
ierr = SetupDOFs(dm); CHKERRQ(ierr);
ierr = CreateGlobalStiffnessMatrix(dm, &mat); CHKERRQ(ierr);
Vec v;
ierr = DMCreateLocalVector(dm, &v); CHKERRQ(ierr);
PetscInt vsize;
ierr = VecGetSize(v, &vsize); CHKERRQ(ierr);
PetscSynchronizedPrintf(PETSC_COMM_WORLD, "Loc size: %i\n", vsize);
PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT);
PetscScalar trace;
ierr = MatGetTrace(mat, &trace); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD, "Trace of matrix: %f\n", trace);
ierr = MatDestroy(&mat); CHKERRQ(ierr);
ierr = VecDestroy(&v); CHKERRQ(ierr);
ierr = DMDestroy(&dm); CHKERRQ(ierr);
PetscFinalize();
return 0;
}
