MPISerializer.hpp

/*
  This file is part of the Open Porous Media project (OPM).

  OPM is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 2 of the License, or
  (at your option) any later version.

  OPM is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with OPM.  If not, see <http://www.gnu.org/licenses/>.

  Consult the COPYING file in the top-level source directory of this
  module for the precise wording of the license and the list of
  copyright holders.
*/
#ifndef MPI_SERIALIZER_HPP
#define MPI_SERIALIZER_HPP

#include <opm/common/utility/Serializer.hpp>
#include <opm/simulators/utils/MPIPacker.hpp>
#include <opm/simulators/utils/ParallelCommunication.hpp>

namespace Opm::Parallel {

struct RootRank
{
    int value;
};

class MpiSerializer : public Serializer<Mpi::Packer> {
public:
    explicit MpiSerializer(Parallel::Communication comm)
        : Serializer<Mpi::Packer>(m_packer)
        , m_packer(comm)
        , m_comm(comm)
    {}

    template<typename... Args>
    void broadcast(RootRank rootrank, Args&&... args)
    {
        if (m_comm.size() == 1)
            return;

        const int root = rootrank.value;
        if (m_comm.rank() == root) {
            try {
                this->pack(std::forward<Args>(args)...);
                m_comm.broadcast(&m_packSize, 1, root);
                broadcast_chunked(root);
            } catch (...) {
                m_packSize = std::numeric_limits<size_t>::max();
                m_comm.broadcast(&m_packSize, 1, root);
                throw;
            }
        } else {
            m_comm.broadcast(&m_packSize, 1, root);
            if (m_packSize == std::numeric_limits<size_t>::max()) {
                throw std::runtime_error("Error detected in parallel serialization");
            }
            m_buffer.resize(m_packSize);
            broadcast_chunked(root);
            this->unpack(std::forward<Args>(args)...);
        }
    }


    template<class T>
    void append(T& data, int root = 0)
    {
        if (m_comm.size() == 1)
            return;

        T tmp;
        T& bcast = m_comm.rank() == root ? data : tmp;
        broadcast(RootRank{root}, bcast);

        if (m_comm.rank() != root)
            data.append(tmp);
    }

private:
    void broadcast_chunked(int root) {
        const int maxChunkSize = std::numeric_limits<int>::max();
        std::size_t remainingSize = m_packSize;
        std::size_t pos = 0;
        while (remainingSize > maxChunkSize) {
            m_comm.broadcast(m_buffer.data()+pos, maxChunkSize, root);
            pos += maxChunkSize;
            remainingSize -= maxChunkSize;
        }
        m_comm.broadcast(m_buffer.data()+pos, static_cast<int>(remainingSize), root);
    }

    const Mpi::Packer m_packer; 
    Parallel::Communication m_comm; 
};

}

#endif