globalindices.hh

Go to the documentation of this file.

// -*- mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
// vi: set et ts=4 sw=4 sts=4:
/*
  This file is part of the Open Porous Media project (OPM).
 
  OPM is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 2 of the License, or
  (at your option) any later version.
 
  OPM is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
 
  You should have received a copy of the GNU General Public License
  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
 
  Consult the COPYING file in the top-level source directory of this
  module for the precise wording of the license and the list of
  copyright holders.
*/
#ifndef EWOMS_GLOBAL_INDICES_HH
#define EWOMS_GLOBAL_INDICES_HH
 
#include <dune/grid/common/datahandleif.hh>
#include <dune/istl/bcrsmatrix.hh>
#include <dune/istl/scalarproducts.hh>
#include <dune/istl/operators.hh>
 
#include <algorithm>
#include <set>
#include <map>
#include <iostream>
#include <tuple>
 
#if HAVE_MPI
#include <mpi.h>
#endif
 
#include "overlaptypes.hh"
 
namespace Opm {
namespace Linear {
template <class ForeignOverlap>
class GlobalIndices
{
    GlobalIndices(const GlobalIndices& ) = delete;
 
    using GlobalToDomesticMap = std::map<Index, Index>;
    using DomesticToGlobalMap = std::map<Index, Index>;
 
public:
    GlobalIndices(const ForeignOverlap& foreignOverlap)
        : foreignOverlap_(foreignOverlap)
    {
        myRank_ = 0;
        mpiSize_ = 1;
 
#if HAVE_MPI
        {
            int tmp;
            MPI_Comm_rank(MPI_COMM_WORLD, &tmp);
            myRank_ = static_cast<ProcessRank>(tmp);
            MPI_Comm_size(MPI_COMM_WORLD, &tmp);
            mpiSize_ = static_cast<size_t>(tmp);
        }
#endif
 
        // calculate the domestic overlap (i.e. all overlap indices in
        // foreign processes which the current process overlaps.)
        // This requires communication via MPI.
        buildGlobalIndices_();
    }
 
    Index domesticToGlobal(Index domesticIdx) const
    {
        assert(domesticToGlobal_.find(domesticIdx) != domesticToGlobal_.end());
 
        return domesticToGlobal_.find(domesticIdx)->second;
    }
 
    Index globalToDomestic(Index globalIdx) const
    {
        const auto& tmp = globalToDomestic_.find(globalIdx);
 
        if (tmp == globalToDomestic_.end())
            return -1;
 
        return tmp->second;
    }
 
    size_t numLocal() const
    { return foreignOverlap_.numLocal(); }
 
    size_t numDomestic() const
    { return numDomestic_; }
 
    void addIndex(Index domesticIdx, Index globalIdx)
    {
        domesticToGlobal_[domesticIdx] = globalIdx;
        globalToDomestic_[globalIdx] = domesticIdx;
        numDomestic_ = domesticToGlobal_.size();
 
        assert(domesticToGlobal_.size() == globalToDomestic_.size());
    }
 
    void sendBorderIndex([[maybe_unused]] ProcessRank peerRank,
                         [[maybe_unused]] Index domesticIdx,
                         [[maybe_unused]] Index peerLocalIdx)
    {
#if HAVE_MPI
        PeerIndexGlobalIndex sendBuf;
        sendBuf.peerIdx = peerLocalIdx;
        sendBuf.globalIdx = domesticToGlobal(domesticIdx);
        MPI_Send(&sendBuf,                     // buff
                 sizeof(PeerIndexGlobalIndex), // count
                 MPI_BYTE,                     // data type
                 static_cast<int>(peerRank),   // peer process
                 0,                            // tag
                 MPI_COMM_WORLD);              // communicator
#endif
    }
 
    void receiveBorderIndex([[maybe_unused]] ProcessRank peerRank)
    {
#if HAVE_MPI
        PeerIndexGlobalIndex recvBuf;
        MPI_Recv(&recvBuf,                     // buff
                 sizeof(PeerIndexGlobalIndex), // count
                 MPI_BYTE,                     // data type
                 static_cast<int>(peerRank),   // peer process
                 0,                            // tag
                 MPI_COMM_WORLD,               // communicator
                 MPI_STATUS_IGNORE);           // status
 
        Index domesticIdx = foreignOverlap_.nativeToLocal(recvBuf.peerIdx);
        if (domesticIdx >= 0) {
            Index globalIdx = recvBuf.globalIdx;
            addIndex(domesticIdx, globalIdx);
        }
#endif // HAVE_MPI
    }
 
    bool hasGlobalIndex(Index globalIdx) const
    { return globalToDomestic_.find(globalIdx) != globalToDomestic_.end(); }
 
    void print() const
    {
        std::cout << "(domestic index, global index, domestic->global->domestic)"
                  << " list for rank " << myRank_ << "\n";
 
        for (size_t domIdx = 0; domIdx < domesticToGlobal_.size(); ++domIdx)
            std::cout << "(" << domIdx << ", " << domesticToGlobal(domIdx)
                      << ", " << globalToDomestic(domesticToGlobal(domIdx)) << ") ";
        std::cout << "\n" << std::flush;
    }
 
protected:
    // retrieve the offset for the indices where we are master in the
    // global index list
    void buildGlobalIndices_()
    {
#if HAVE_MPI
        numDomestic_ = 0;
#else
        numDomestic_ = foreignOverlap_.numLocal();
#endif
 
#if HAVE_MPI
        if (myRank_ == 0) {
            // the first rank starts at index zero
            domesticOffset_ = 0;
        }
        else {
            // all other ranks retrieve their offset from the next
            // lower rank
            MPI_Recv(&domesticOffset_, // buffer
                     1,                // count
                     MPI_INT,          // data type
                     static_cast<int>(myRank_ - 1), // peer rank
                     0,                // tag
                     MPI_COMM_WORLD,   // communicator
                     MPI_STATUS_IGNORE);
        }
 
        // create maps for all indices for which the current process
        // is the master
        int numMaster = 0;
        for (unsigned i = 0; i < foreignOverlap_.numLocal(); ++i) {
            if (!foreignOverlap_.iAmMasterOf(static_cast<Index>(i)))
                continue;
 
            addIndex(static_cast<Index>(i),
                     static_cast<Index>(domesticOffset_ + numMaster));
            ++numMaster;
        }
 
        if (myRank_ < mpiSize_ - 1) {
            // send the domestic offset plus the number of master
            // indices to the process which is one rank higher
            int tmp = domesticOffset_ + numMaster;
            MPI_Send(&tmp,            // buff
                     1,               // count
                     MPI_INT,         // data type
                     static_cast<int>(myRank_ + 1), // peer rank
                     0,               // tag
                     MPI_COMM_WORLD); // communicator
        }
 
        typename PeerSet::const_iterator peerIt;
        typename PeerSet::const_iterator peerEndIt = peerSet_().end();
        // receive the border indices from the lower ranks
        peerIt = peerSet_().begin();
        for (; peerIt != peerEndIt; ++peerIt) {
            if (*peerIt < myRank_)
                receiveBorderFrom_(*peerIt);
        }
 
        // send the border indices to the higher ranks
        peerIt = peerSet_().begin();
        for (; peerIt != peerEndIt; ++peerIt) {
            if (*peerIt > myRank_)
                sendBorderTo_(*peerIt);
        }
 
        // receive the border indices from the higher ranks
        peerIt = peerSet_().begin();
        for (; peerIt != peerEndIt; ++peerIt) {
            if (*peerIt > myRank_)
                receiveBorderFrom_(*peerIt);
        }
 
        // send the border indices to the lower ranks
        peerIt = peerSet_().begin();
        for (; peerIt != peerEndIt; ++peerIt) {
            if (*peerIt < myRank_)
                sendBorderTo_(*peerIt);
        }
#endif // HAVE_MPI
    }
 
    void sendBorderTo_([[maybe_unused]] ProcessRank peerRank)
    {
#if HAVE_MPI
        // send (local index on myRank, global index) pairs to the
        // peers
        BorderList::const_iterator borderIt = borderList_().begin();
        BorderList::const_iterator borderEndIt = borderList_().end();
        for (; borderIt != borderEndIt; ++borderIt) {
            ProcessRank borderPeer = borderIt->peerRank;
            BorderDistance borderDistance = borderIt->borderDistance;
            if (borderPeer != peerRank || borderDistance != 0)
                continue;
 
            Index localIdx = foreignOverlap_.nativeToLocal(borderIt->localIdx);
            Index peerIdx = borderIt->peerIdx;
            assert(localIdx >= 0);
            if (foreignOverlap_.iAmMasterOf(localIdx)) {
                sendBorderIndex(borderPeer, localIdx, peerIdx);
            }
        }
#endif // HAVE_MPI
    }
 
    void receiveBorderFrom_([[maybe_unused]] ProcessRank peerRank)
    {
#if HAVE_MPI
        // retrieve the global indices for which we are not master
        // from the processes with lower rank
        BorderList::const_iterator borderIt = borderList_().begin();
        BorderList::const_iterator borderEndIt = borderList_().end();
        for (; borderIt != borderEndIt; ++borderIt) {
            ProcessRank borderPeer = borderIt->peerRank;
            BorderDistance borderDistance = borderIt->borderDistance;
            if (borderPeer != peerRank || borderDistance != 0)
                continue;
 
            Index nativeIdx = borderIt->localIdx;
            Index localIdx = foreignOverlap_.nativeToLocal(nativeIdx);
            if (localIdx >= 0 && foreignOverlap_.masterRank(localIdx) == borderPeer)
                receiveBorderIndex(borderPeer);
        }
#endif // HAVE_MPI
    }
 
    const PeerSet& peerSet_() const
    { return foreignOverlap_.peerSet(); }
 
    const BorderList& borderList_() const
    { return foreignOverlap_.borderList(); }
 
    ProcessRank myRank_;
    size_t mpiSize_;
 
    int domesticOffset_;
    size_t numDomestic_;
    const ForeignOverlap& foreignOverlap_;
 
    GlobalToDomesticMap globalToDomestic_;
    DomesticToGlobalMap domesticToGlobal_;
};
 
} // namespace Linear
} // namespace Opm
 
#endif