ReservoirCouplingMaster.hpp

Go to the documentation of this file.

/*
  Copyright 2024 Equinor ASA
 
  This file is part of the Open Porous Media project (OPM).
 
  OPM is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.
 
  OPM is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
 
  You should have received a copy of the GNU General Public License
  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#ifndef OPM_RESERVOIR_COUPLING_MASTER_HPP
#define OPM_RESERVOIR_COUPLING_MASTER_HPP
 
#include <opm/simulators/utils/ParallelCommunication.hpp>
#include <opm/simulators/flow/rescoup/ReservoirCoupling.hpp>
#include <opm/simulators/flow/rescoup/ReservoirCouplingMasterReportStep.hpp>
#include <opm/simulators/flow/rescoup/ReservoirCouplingTimeStepper.hpp>
#include <opm/input/eclipse/EclipseState/Phase.hpp>
#include <opm/input/eclipse/Schedule/Schedule.hpp>
#include <opm/common/OpmLog/OpmLog.hpp>
 
#include <mpi.h>
 
#include <filesystem>
#include <map>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
 
namespace Opm {
 
template <class Scalar>
class ReservoirCouplingMaster {
public:
    using MessageTag = ReservoirCoupling::MessageTag;
    using Seconds = ReservoirCoupling::Seconds;
    using Potentials = ReservoirCoupling::Potentials<Scalar>;
    using SlaveGroupProductionData = ReservoirCoupling::SlaveGroupProductionData<Scalar>;
    using SlaveGroupInjectionData = ReservoirCoupling::SlaveGroupInjectionData<Scalar>;
    using InjectionGroupTarget = ReservoirCoupling::InjectionGroupTarget<Scalar>;
    using MasterGroupNodePressure = ReservoirCoupling::MasterGroupNodePressure<Scalar>;
    using ProductionGroupConstraints = ReservoirCoupling::ProductionGroupConstraints<Scalar>;
 
    ReservoirCouplingMaster(
        const Parallel::Communication &comm,
        const Schedule &schedule,
        int argc, char **argv
    );
 
    bool activated() { return this->activated_; }
    void addSlaveCommunicator(MPI_Comm comm) {
         this->master_slave_comm_.push_back(comm);
    }
    void addSlaveName(const std::string &name) { this->slave_names_.push_back(name); }
    void addSlaveActivationDate(double date) { this->slave_activation_dates_.push_back(date); }
    void addSlaveStartDate(std::time_t date) { this->slave_start_dates_.push_back(date); }
    void clearDeferredLogger() { logger_.clearDeferredLogger(); }
 
    int effectiveGCW(const std::string& group_name) const {
        const auto it = this->effective_gcw_.find(group_name);
        return (it == this->effective_gcw_.end()) ? 1 : it->second;
    }
    void setEffectiveGCW(const std::string& group_name, int value) {
        this->effective_gcw_[group_name] = value;
    }
 
    int effectiveInjectionGCW(const std::string& group_name, Phase phase) const {
        const auto it = this->effective_injection_gcw_.find({group_name, phase});
        return (it == this->effective_injection_gcw_.end()) ? 1 : it->second;
    }
    void setEffectiveInjectionGCW(const std::string& group_name, Phase phase, int value) {
        this->effective_injection_gcw_[{group_name, phase}] = value;
    }
 
    void resetEffectiveGCW() {
        this->effective_gcw_.clear();
        this->effective_injection_gcw_.clear();
    }
 
    double getActivationDate() const { return this->activation_date_; }
    int getArgc() const { return this->argc_; }
    char *getArgv(int index) const { return this->argv_[index]; }
    char **getArgv() const { return this->argv_; }
    const Parallel::Communication &getComm() const { return this->comm_; }
 
    const std::vector<std::string>& getMasterGroupNamesForSlave(std::size_t slave_idx) const;
 
    std::size_t getMasterGroupCanonicalIdx(
        const std::string &slave_name, const std::string &master_group_name) const;
 
    Scalar getMasterGroupRate(
        const std::string &group_name, ReservoirCoupling::Phase phase, ReservoirCoupling::RateKind kind) const;
    std::map<std::string, std::string>& getMasterGroupToSlaveNameMap() {
         return this->master_group_slave_names_;
    }
    double getSimulationStartDate() const
    { return TimeService::to_time_t(this->schedule_.getStartTime()); }
    double getSlaveActivationDate(int index) const { return this->slave_activation_dates_[index]; }
    const double *getSlaveActivationDates() const { return this->slave_activation_dates_.data(); }
    MPI_Comm getSlaveComm(int index) const { return this->master_slave_comm_[index]; }
    std::map<std::string, std::vector<std::string>> &getSlaveNameToMasterGroupsMap() {
        return this->slave_name_to_master_groups_map_;
    }
    int getSlaveGroupNumGroupControlledInjectors(const std::string &master_group_name,
                                                 ReservoirCoupling::Phase phase) const;
    int getSlaveGroupNumGroupControlledProducers(const std::string &master_group_name) const;
    const Potentials& getSlaveGroupPotentials(const std::string &master_group_name);
    int getSlaveIdx(const std::string &slave_name) const;
    const std::string &getSlaveName(int index) const { return this->slave_names_[index]; }
    double getSlaveStartDate(int index) const { return this->slave_start_dates_[index]; }
    const double *getSlaveStartDates() { return this->slave_start_dates_.data(); }
    void initStartOfReportStep(int report_step_idx);
    void initTimeStepping();
    bool isFirstSubstepOfSyncTimestep() const;
    bool isMasterGroup(const std::string &group_name) const;
 
    bool needsSlaveDataReceive() const;
 
    void setNeedsSlaveDataReceive(bool value);
 
    ReservoirCoupling::Logger& logger() { return this->logger_; }
    ReservoirCoupling::Logger& logger() const { return this->logger_; }
    void maybeActivate(int report_step);
    void maybeReceiveActivationHandshakeFromSlaves(double current_time);
    double maybeChopSubStep(double suggested_timestep, double current_time) const;
    void maybeSpawnSlaveProcesses(int report_step);
    std::size_t numSlaveGroups(unsigned int index);
    std::size_t numSlaves() const { return this->numSlavesStarted(); }
    std::size_t numSlavesStarted() const;
 
    std::size_t numCoupledSlaves() const;
    void rebuildSlaveIdxToMasterGroupsVector();
    void receiveNextReportDateFromSlaves();
    void receiveProductionDataFromSlaves();
    void receiveInjectionDataFromSlaves();
    void resizeNextReportDates(int size);
    void resizeSlaveActivationDates(int size) { this->slave_activation_dates_.resize(size); }
    void resizeSlaveStartDates(int size) { this->slave_start_dates_.resize(size); }
    const Schedule& schedule() const { return this->schedule_; }
    void sendNextTimeStepToSlaves(double next_time_step) {
        this->report_step_data_->setSyncStepLength(next_time_step);
        this->time_stepper_->sendNextTimeStepToSlaves(next_time_step);
    }
 
    void saveGroupFlowsAtStartOfSyncStep() { this->report_step_data_->saveGroupFlowsAtStartOfSyncStep(); }
 
    void updateFlowChangeTimeStepLimit() { this->report_step_data_->updateFlowChangeTimeStepLimit(); }
 
    double flowChangeTimeStepLimit() const { return this->report_step_data_->flowChangeTimeStepLimit(); }
 
    std::tuple<std::string, double, double> flowChangeLimitingGroup() const {
        return this->report_step_data_->flowChangeLimitingGroup();
    }
 
    void sendCoupledNetworkActiveStatusToSlave(
        std::size_t slave_idx,
        bool active
    ) const;
    void sendInjectionTargetsToSlave(
        std::size_t slave_idx,
        const std::vector<InjectionGroupTarget>& injection_targets
    ) const;
 
    void sendMasterGroupNodePressuresToSlave(
        std::size_t slave_idx,
        const std::vector<MasterGroupNodePressure>& pressures
    ) const;
    void sendNumGroupConstraintsToSlave(
        std::size_t slave_idx,
        std::size_t num_injection_targets,
        std::size_t num_production_constraints
    ) const;
 
    void sendNumMasterGroupNodePressuresToSlave(
        std::size_t slave_idx,
        std::size_t num_pressures,
        bool is_final
    ) const;
    void sendProductionConstraintsToSlave(
        std::size_t slave_idx,
        const std::vector<ProductionGroupConstraints>& production_constraints
    ) const;
    void setDeferredLogger(DeferredLogger *deferred_logger) {
         this->logger_.setDeferredLogger(deferred_logger);
    }
    void setFirstSubstepOfSyncTimestep(bool value);
 
    data::ReservoirCouplingGroupRates collectGroupRatesForSummary() const;
    // These are currently only used for unit testing
    void setSlaveActivated(int index)
    {
        if (this->slave_activation_status_.size() < this->numSlavesStarted()) {
            this->slave_activation_status_.resize(this->numSlavesStarted(), 0);
        }
        this->slave_activation_status_[index] = 1;
    }
    void setSlaveActivationDate(int index, double date) { this->slave_activation_dates_[index] = date; }
    void setSlaveGroupInjectionData(const std::string& slave_name,
                                    std::vector<SlaveGroupInjectionData> injection_data)
    {
        this->report_step_data_->setSlaveGroupInjectionData(slave_name, std::move(injection_data));
    }
    void setSlaveGroupProductionData(const std::string& slave_name,
                                     std::vector<SlaveGroupProductionData> production_data)
    {
        this->report_step_data_->setSlaveGroupProductionData(slave_name, std::move(production_data));
    }
    void setSlaveNextReportTimeOffset(int index, double offset);
    void setSlaveStartDate(int index, std::time_t date) { this->slave_start_dates_[index] = date; }
    bool slaveIsActivated(int index) const { return this->slave_activation_status_[index] != 0; }
 
    bool slaveHasEnded(int index) const
    {
        return (static_cast<std::size_t>(index) < this->slave_ended_.size())
            && (this->slave_ended_[index] != 0);
    }
 
    bool slaveIsCoupled(int index) const
    {
        return this->slaveIsActivated(index) && !this->slaveHasEnded(index);
    }
 
    void markSlaveEndedAndDisconnect(int index);
 
    bool syncAtReportSteps() const { return this->sync_at_report_steps_; }
    void updateMasterGroupNameOrderMap(
        const std::string& slave_name, const std::map<std::string, std::size_t>& master_group_map);
 
    void sendDontTerminateSignalToSlaves();
 
    void sendTerminateAndDisconnect();
 
private:
    double getMasterActivationDate_() const;
 
    const Parallel::Communication &comm_;
    const Schedule& schedule_;
    int argc_;
    char **argv_;
    // Whether the master process has activated the reservoir coupling
    bool activated_{false};
 
    // NOTE: MPI_Comm is just an integer handle, so we can just copy it into the vector
    std::vector<MPI_Comm> master_slave_comm_; // MPI communicators for the slave processes
    std::vector<std::string> slave_names_;
 
    // The start dates are in whole seconds since the epoch. We use a double to store the value
    // since both schedule_.getStartTime(), converted to seconds since the epoch, and
    // schedule_.stepLength(report_step) represent whole seconds.
    // However, note that schedule_[report_step].start_time() returns a time_point
    // which can include milliseconds. The double values are also convenient when we need to
    // to add fractions of seconds for sub steps to the start date.
    std::vector<double> slave_start_dates_;
 
    // The activation dates are in whole seconds since the epoch.
    std::vector<double> slave_activation_dates_;
    double activation_date_{0.0};  // The date when SLAVES is encountered in the schedule
 
    // A mapping from a slave name to the master group name order used when slaves send
    //  potentials to the master process.
    std::map<std::string, std::map<std::string, std::size_t>> master_group_name_order_;
 
    // Effective group-controlled-wells count per master group, used by guide-rate
    // distribution independently of the production control mode (see effectiveGCW()).
    // Reset and repopulated on each master-group constraint calculation.
    std::map<std::string, int> effective_gcw_;
 
    // Effective group-controlled-wells count per (master group, injection phase),
    // the injection counterpart of effective_gcw_ (see effectiveInjectionGCW()).
    std::map<std::pair<std::string, Phase>, int> effective_injection_gcw_;
 
    mutable ReservoirCoupling::Logger logger_;
 
    // Whether the slave has activated. Unfortunatley, we cannot use std::vector<bool> since
    // it is not supported to get a pointer to the underlying array of bools needed
    // with MPI broadcast().
    std::vector<std::uint8_t> slave_activation_status_;
 
    // Whether the slave has reached the end of its own schedule and disconnected. Same
    // std::uint8_t-instead-of-bool reasoning as slave_activation_status_ above: the value is
    // derived on every rank from the broadcast next-report-date vector, so it must be a plain
    // array of bytes.
    std::vector<std::uint8_t> slave_ended_;
 
    // A mapping from master group names to slave names
    std::map<std::string, std::string> master_group_slave_names_;
 
    // A mapping from slave names to master group names
    // NOTE: The order of the master groups in the vector is important,
    //   as the slaves will communicate the indices of the master groups in
    //   the vector instead of the group names themselves.
    // NOTE: This map is created by ReservoirCouplingSpawnSlaves.cpp
    std::map<std::string, std::vector<std::string>> slave_name_to_master_groups_map_;
 
    // Direct index-based lookup for performance optimization (O(1) instead of O(log n))
    // This vector is populated in parallel with slave_name_to_master_groups_map_
    // and maintains the same ordering as slave_names_ vector for consistent indexing
    std::vector<std::vector<std::string>> slave_idx_to_master_groups_;
 
    // Stores data that changes for a single report step or for timesteps within a report step.
    std::unique_ptr<ReservoirCouplingMasterReportStep<Scalar>> report_step_data_{nullptr};
 
    // Handles time stepping for the master and slaves
    std::unique_ptr<ReservoirCouplingTimeStepper<Scalar>> time_stepper_{nullptr};
 
    // CLI flag --rescoup-sync-at-report-steps.  When true, the master syncs with
    // its slaves at report-step boundaries (RSYNC).  When false
    // (default), it syncs at every master time step (TSYNC).
    bool sync_at_report_steps_{false};
};
 
} // namespace Opm
 
#endif // OPM_RESERVOIR_COUPLING_MASTER_HPP