Dear all,
I want to build the scalar product between rows of a
TrilinosWrappers::SparseMatrix and a TrilinosWrappers::MPI::Vector.
I implemented this by looping over the elements of the rows via iterators
(it = systemMatrix.begin(row) , itend = systemMatrix.end(row)
-> while (it != itend) { ... it++; } )
Inside while, I calculate it->value (to get the matrix element) times
vec(it->column())
(to get the corresponding vector element) and sum it up.
I calculate the scalar product for a set of rows and store the result in a
FullMatrix
which I finally write to a file from the master process,
after gathering the local full matrices from all processors.
Attached is a small example which demonstrates this in practice.
In a parallel program, vec(it->column()) only works if it->column() is a
locally owned element of the vector. So I wrapped this read access in
if( vec.in_local_range(it->column() ) .
Another problem is the following:
Say row 5 lives on processor 0 and has non-zero entries at columns
10,15,20,25,30.
Columns 10,15,20 live on processor 0 as well but columns 25,30 live on
processor 1.
Then, I will never add the contributions to the scalar product from columns
25 and 30 because on processor 1,
systemMatrix.begin(5) == systemMatrix.end(5)
which is also stated in the docu.
How can I deal with this?
Best,
Simon
--
The deal.II project is located at http://www.dealii.org/
For mailing list/forum options, see
https://groups.google.com/d/forum/dealii?hl=en
---
You received this message because you are subscribed to the Google Groups
"deal.II User Group" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/dealii/03085575-2931-4755-abcd-7b7a31df102dn%40googlegroups.com.
#include <deal.II/lac/trilinos_vector.h>
#include <deal.II/lac/affine_constraints.h>
#include <deal.II/lac/trilinos_sparse_matrix.h>
#include <deal.II/lac/trilinos_sparsity_pattern.h>
#include <deal.II/lac/full_matrix.h>
#include <deal.II/dofs/dof_renumbering.h>
#include <deal.II/dofs/dof_tools.h>
#include <deal.II/base/conditional_ostream.h>
#include <deal.II/base/index_set.h>
#include <deal.II/base/utilities.h>
#include <deal.II/dofs/dof_handler.h>
#include <deal.II/distributed/grid_refinement.h>
#include <deal.II/distributed/tria.h>
#include <deal.II/grid/grid_generator.h>
#include <deal.II/fe/fe_q.h>
#include <iostream>
#include <stdlib.h>
#include <fstream>
#include <numeric>
namespace dealiiLA
{
using namespace dealii::TrilinosWrappers;
} // namespace dealiiLA
int main(int argc, char *argv[])
{
const unsigned int masterProcessRank = 0;
dealii::Utilities::MPI::MPI_InitFinalize mpiInitialization(argc, argv, 1);
MPI_Comm const &mpiCommunicator(MPI_COMM_WORLD);
unsigned int processorRank =
dealii::Utilities::MPI::this_mpi_process(mpiCommunicator);
dealii::ConditionalOStream pCout(std::cout, processorRank ==
masterProcessRank);
// make grid
dealii::parallel::distributed::Triangulation<3> tria(mpiCommunicator);
dealii::GridGenerator::hyper_cube(tria);
tria.refine_global(2);
// distribute dofs
dealii::DoFHandler<3> dofHandler(tria);
dofHandler.distribute_dofs(dealii::FE_Q<3>(1));
dealii::IndexSet locallyOwnedDofs = dofHandler.locally_owned_dofs();
dealii::IndexSet locallyRelevantDofs;
dealii::DoFTools::extract_locally_relevant_dofs(dofHandler,
locallyRelevantDofs);
dealii::AffineConstraints<double> constraintMatrix;
constraintMatrix.clear();
constraintMatrix.reinit(locallyRelevantDofs);
dealii::DoFTools::make_hanging_node_constraints(dofHandler,
constraintMatrix);
constraintMatrix.close();
// sparse matrix and sparsity pattern
dealiiLA::SparsityPattern sparsityPattern;
sparsityPattern.reinit(locallyOwnedDofs, mpiCommunicator);
dealii::DoFTools::make_sparsity_pattern(dofHandler,
sparsityPattern,
constraintMatrix,
false,
dealii::Utilities::MPI::this_mpi_process(
mpiCommunicator));
sparsityPattern.compress();
dealiiLA::SparseMatrix systemMatrix;
systemMatrix.reinit(sparsityPattern);
// create dummy vector
dealii::TrilinosWrappers::MPI::Vector vec;
vec.reinit(locallyOwnedDofs, mpiCommunicator);
for(unsigned int i=0; i<vec.size(); ++i)
vec(i) = i;
vec.compress(dealii::VectorOperation::insert);
std::vector<unsigned int> rows{10, 40, 70, 100, 120}; //nDoFs = 125
dealii::FullMatrix<double> matrix(rows.size(), 1);
// populate fullmatrix
for(unsigned int i = 0; i < rows.size(); ++i)
{
unsigned int idx = rows[i];
auto it = systemMatrix.begin(idx);
auto itend = systemMatrix.end(idx);
// loop over row idx (first contribution)
while(it != itend)
{
matrix(i, 0) += vec.in_local_range((*it).column()) ? ((*it).value() +
1) * vec((*it).column()) : 0.0;
it++;
}
// second contribution
if(vec.in_local_range(idx))
matrix(i, 0) += vec(idx);
}
// send local matrices to processor 0
std::vector<dealii::FullMatrix<double>> gathered =
dealii::Utilities::MPI::gather(mpiCommunicator, matrix, 0);
// write the matrix from processor 0
if(processorRank == 0)
{
dealii::FullMatrix<double> sum(gathered[0].m(), gathered[0].n());
for(const auto & matrix : gathered)
{
sum.add(1.0, matrix);
}
std::ofstream myMatrix("./myMatrix.txt");
sum.print_formatted(myMatrix, 1, false);
}
return 0;
}
