ParallelWellInfo.hpp

/*
  Copyright 2020 OPM-OP AS

  This file is part of the Open Porous Media project (OPM).

  OPM is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  OPM is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPM_PARALLELWELLINFO_HEADER_INCLUDED
#define OPM_PARALLELWELLINFO_HEADER_INCLUDED

#include <dune/common/parallel/communicator.hh>
#include <dune/common/parallel/interface.hh>
#include <dune/common/parallel/mpihelper.hh>
#include <dune/common/parallel/plocalindex.hh>
#include <dune/common/parallel/remoteindices.hh>

#include <opm/simulators/utils/ParallelCommunication.hpp>

#include <memory>
#include <unordered_map>

namespace Opm {

class Well;

template<class Scalar>
class CommunicateAboveBelow
{
public:
    enum Attribute {
      owner=1,
      overlap=2,
      // there is a bug in older versions of DUNE that will skip
      // entries with matching attributes in RemoteIndices that are local
      // therefore we add one more version for above.
      ownerAbove = 3,
      overlapAbove = 4
    };
    using LocalIndex = Dune::ParallelLocalIndex<Attribute>;
    using IndexSet = Dune::ParallelIndexSet<int,LocalIndex,50>;
#if HAVE_MPI
    using RI = Dune::RemoteIndices<IndexSet>;
#endif

    explicit CommunicateAboveBelow(const Parallel::Communication& comm);
    void pushBackEclIndex(int above, int current, bool owner = true);

    void clear();

    void beginReset();

    int endReset();

    std::vector<Scalar> communicateAbove(Scalar first_value,
                                         const Scalar* current,
                                         std::size_t size);

    std::vector<Scalar> communicateBelow(Scalar first_value,
                                         const Scalar* current,
                                         std::size_t size);

    template<class RAIterator>
    void partialSumPerfValues(RAIterator begin, RAIterator end) const;

    const IndexSet& getIndexSet() const;

    int numLocalPerfs() const;

private:
    Parallel::Communication comm_;
    IndexSet current_indices_;
#if HAVE_MPI
    IndexSet above_indices_;
    RI remote_indices_;
    Dune::Interface interface_;
    Dune::BufferedCommunicator communicator_;
#endif
    std::size_t num_local_perfs_{};
};

template<class Scalar>
class GlobalPerfContainerFactory
{
public:
    using IndexSet = typename CommunicateAboveBelow<Scalar>::IndexSet;
    using Attribute = typename CommunicateAboveBelow<Scalar>::Attribute;
    using GlobalIndex = typename IndexSet::IndexPair::GlobalIndex;

    GlobalPerfContainerFactory(const IndexSet& local_indices,
                               const Parallel::Communication comm,
                               int num_local_perfs);

    std::vector<Scalar> createGlobal(const std::vector<Scalar>& local_perf_container,
                                     std::size_t num_quantities) const;

    void copyGlobalToLocal(const std::vector<Scalar>& global, std::vector<Scalar>& local,
                           std::size_t num_quantities) const;

    int numGlobalPerfs() const;
    int globalToLocal(const int globalIndex) const;
    int localToGlobal(std::size_t localIndex) const;

private:
    void buildLocalToGlobalMap() const;
    void buildGlobalToLocalMap() const;
    mutable std::unordered_map<std::size_t, int> local_to_global_map_; // Cache for L2G mapping
    mutable std::unordered_map<int, std::size_t> global_to_local_map_; // Cache for G2L mapping
    mutable bool l2g_map_built_ = false;
    mutable bool g2l_map_built_ = false;
    const IndexSet& local_indices_;
    Parallel::Communication comm_;
    int num_global_perfs_;
    std::vector<int> sizes_;
    std::vector<int> displ_;
    std::vector<int> map_received_;
    std::vector<int> perf_ecl_index_;
};

template<class Scalar>
class ParallelWellInfo
{
public:
    static constexpr int INVALID_ECL_INDEX = -1;

    explicit ParallelWellInfo(const std::string& name = {""},
                              bool hasLocalCells = true);

    ParallelWellInfo(const std::pair<std::string,bool>& well_info,
                     Parallel::Communication allComm);

    const Parallel::Communication& communication() const
    {
        return *comm_;
    }

    void communicateFirstPerforation(bool hasFirst);

    // \brief Set the activePerfToLocalPerf-Map for multisegment wells, to be called from WellState::initWellStateMSWell
    void setActivePerfToLocalPerfMap(const std::unordered_map<int,int> active_to_local_map) const;
    // \brief Convert a global active perforation index to a local active perforation index
    int activePerfToLocalPerf(const int activeIndex) const;
    // \brief Convert a local active perforation index to a global active perforation index
    int localPerfToActivePerf(std::size_t localIndex) const;
    // \brief Convert a global perforation index to a local perforation index
    int globalPerfToLocalPerf(const int globalIndex) const;
    // \brief Convert a local perforation index to a global perforation index
    int localPerfToGlobalPerf(std::size_t localIndex) const;

    template<class T>
    T broadcastFirstPerforationValue(const T& t) const;

    std::vector<Scalar> communicateAboveValues(Scalar first_value,
                                               const Scalar* current,
                                               std::size_t size) const;

    std::vector<Scalar> communicateAboveValues(Scalar first_value,
                                               const std::vector<Scalar>& current) const;

    std::vector<Scalar> communicateBelowValues(Scalar last_value,
                                               const Scalar* current,
                                               std::size_t size) const;

    std::vector<Scalar> communicateBelowValues(Scalar last_value,
                                               const std::vector<Scalar>& current) const;

    void pushBackEclIndex(int above, int current);

    const std::string& name() const
    {
        return name_;
    }

    bool hasLocalCells() const
    {
        return hasLocalCells_;
    }
    bool isOwner() const
    {
        return isOwner_;
    }

    void beginReset();

    void endReset();

    template<typename It>
    typename It::value_type sumPerfValues(It begin, It end) const;

    template<class RAIterator>
    void partialSumPerfValues(RAIterator begin, RAIterator end) const
    {
        commAboveBelow_->partialSumPerfValues(begin, end);
    }

    void clear();

    const GlobalPerfContainerFactory<Scalar>& getGlobalPerfContainerFactory() const;

private:

    struct DestroyComm
    {
        void operator()(Parallel::Communication* comm);
    };


    std::string name_;
    bool hasLocalCells_;
    bool isOwner_;
    int rankWithFirstPerf_;
    std::unique_ptr<Parallel::Communication, DestroyComm> comm_;

    std::unique_ptr<CommunicateAboveBelow<Scalar>> commAboveBelow_;

    std::unique_ptr<GlobalPerfContainerFactory<Scalar>> globalPerfCont_;

    mutable std::unordered_map<int,int> active_to_local_map_; // Cache for active perforation index to local perforation index mapping
    mutable std::unordered_map<int,int> local_to_active_map_; // Cache for local perforation index to active perforation index mapping
};

template<class Scalar>
class CheckDistributedWellConnections
{
public:
    CheckDistributedWellConnections(const Well& well,
                                    const ParallelWellInfo<Scalar>& info);

    void connectionFound(std::size_t index);

    bool checkAllConnectionsFound();

private:
    std::vector<std::size_t> foundConnections_;
    const Well& well_;
    const ParallelWellInfo<Scalar>& pwinfo_;
};

template<class Scalar>
bool operator<(const ParallelWellInfo<Scalar>& well1, const ParallelWellInfo<Scalar>& well2);

template<class Scalar>
bool operator==(const ParallelWellInfo<Scalar>& well1, const ParallelWellInfo<Scalar>& well2);

template<class Scalar>
bool operator!=(const ParallelWellInfo<Scalar>& well1, const ParallelWellInfo<Scalar>& well2);

template<class Scalar>
bool operator<(const std::pair<std::string, bool>& pair, const ParallelWellInfo<Scalar>& well);

template<class Scalar>
bool operator<( const ParallelWellInfo<Scalar>& well, const std::pair<std::string, bool>& pair);

template<class Scalar>
bool operator==(const std::pair<std::string, bool>& pair, const ParallelWellInfo<Scalar>& well);

template<class Scalar>
bool operator==(const ParallelWellInfo<Scalar>& well, const std::pair<std::string, bool>& pair);

template<class Scalar>
bool operator!=(const std::pair<std::string, bool>& pair, const ParallelWellInfo<Scalar>& well);

template<class Scalar>
bool operator!=(const ParallelWellInfo<Scalar>& well, const std::pair<std::string, bool>& pair);

} // end namespace Opm

#endif //  OPM_PARALLELWELLINFO_HEADER_INCLUDED