simulator.hh

Go to the documentation of this file.

// -*- mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
// vi: set et ts=4 sw=4 sts=4:
/*
  This file is part of the Open Porous Media project (OPM).
 
  OPM is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 2 of the License, or
  (at your option) any later version.
 
  OPM is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
 
  You should have received a copy of the GNU General Public License
  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
 
  Consult the COPYING file in the top-level source directory of this
  module for the precise wording of the license and the list of
  copyright holders.
*/
#ifndef EWOMS_SIMULATOR_HH
#define EWOMS_SIMULATOR_HH
 
#include <opm/models/io/restart.hh>
#include <opm/models/utils/parametersystem.hh>
 
#include <opm/models/utils/basicproperties.hh>
#include <opm/models/utils/propertysystem.hh>
#include <opm/models/utils/timer.hh>
#include <opm/models/utils/timerguard.hh>
#include <opm/models/parallel/mpiutil.hh>
#include <opm/models/discretization/common/fvbaseproperties.hh>
 
#include <dune/common/parallel/mpihelper.hh>
 
#include <iostream>
#include <fstream>
#include <iomanip>
#include <vector>
#include <string>
#include <memory>
 
namespace Opm
{
namespace detail
{
inline auto getMPIHelperCommunication()
{
    return Dune::MPIHelper::getCommunication();
}
} // end namespace detail
} // end namespace Opm
 
#define EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(code)                     \
    {                                                                   \
        const auto& comm = ::Opm::detail::getMPIHelperCommunication();  \
        bool exceptionThrown = false;                                   \
        try { code; }                                                   \
        catch (const Dune::Exception& e) {                              \
            exceptionThrown = true;                                     \
            std::cerr << "Process " << comm.rank() << " threw a fatal exception: " \
                      << e.what() << ". Abort!" << std::endl;           \
        }                                                               \
        catch (const std::exception& e) {                               \
            exceptionThrown = true;                                     \
            std::cerr << "Process " << comm.rank() << " threw a fatal exception: " \
                      << e.what() << ". Abort!" << std::endl;           \
        }                                                               \
        catch (...) {                                                   \
            exceptionThrown = true;                                     \
            std::cerr << "Process " << comm.rank() << " threw a fatal exception. " \
                      <<" Abort!" << std::endl;                         \
        }                                                               \
                                                                        \
        if (comm.max(exceptionThrown))                                  \
            std::abort();                                               \
    }
 
namespace Opm {
 
template <class TypeTag>
class Simulator
{
    using Scalar = GetPropType<TypeTag, Properties::Scalar>;
    using Vanguard = GetPropType<TypeTag, Properties::Vanguard>;
    using GridView = GetPropType<TypeTag, Properties::GridView>;
    using Model = GetPropType<TypeTag, Properties::Model>;
    using Problem = GetPropType<TypeTag, Properties::Problem>;
 
    using MPIComm = typename Dune::MPIHelper::MPICommunicator;
    using Communication = Dune::Communication<MPIComm>;
 
public:
    // do not allow to copy simulators around
    Simulator(const Simulator& ) = delete;
 
    Simulator(bool verbose = true)
        :Simulator(Communication(), verbose)   
    {  
    }
 
    Simulator(Communication comm, bool verbose = true)
    {
        TimerGuard setupTimerGuard(setupTimer_);
 
        setupTimer_.start();
 
        verbose_ = verbose && comm.rank() == 0;
 
        timeStepIdx_ = 0;
        startTime_ = 0.0;
        time_ = 0.0;
        endTime_ = Parameters::get<TypeTag, Properties::EndTime>();
        timeStepSize_ = Parameters::get<TypeTag, Properties::InitialTimeStepSize>();
        assert(timeStepSize_ > 0);
        const std::string& predetTimeStepFile =
            Parameters::get<TypeTag, Properties::PredeterminedTimeStepsFile>();
        if (!predetTimeStepFile.empty()) {
            std::ifstream is(predetTimeStepFile);
            while (!is.eof()) {
                Scalar dt;
                is >> dt;
                forcedTimeSteps_.push_back(dt);
            }
        }
 
        episodeIdx_ = 0;
        episodeStartTime_ = 0;
        episodeLength_ = std::numeric_limits<Scalar>::max();
 
        finished_ = false;
 
        if (verbose_)
            std::cout << "Allocating the simulation vanguard\n" << std::flush;
 
        int exceptionThrown = 0;
        std::string what;
 
        auto catchAction =
            [&exceptionThrown, &what, comm](const std::exception& e,
                                            bool doPrint) {
            exceptionThrown = 1;
            what = e.what();
            if (comm.size() > 1) {
                what += " (on rank " + std::to_string(comm.rank()) + ")";
            }
            if (doPrint)
                std::cerr << "Rank " << comm.rank() << " threw an exception: " << e.what() << std::endl;
        };
 
        auto checkParallelException =
            [comm](const std::string& prefix,
                   int exceptionThrown_,
                   const std::string& what_)
        {
            if (comm.max(exceptionThrown_)) {
                auto all_what = gatherStrings(what_);
                assert(!all_what.empty());
                throw std::runtime_error(prefix + all_what.front());
            }
        };
 
        try
        { vanguard_.reset(new Vanguard(*this)); }
        catch (const std::exception& e) {
            catchAction(e, verbose_);
        }
        checkParallelException("Allocating the simulation vanguard failed: ",
                               exceptionThrown, what);
 
        if (verbose_)
            std::cout << "Distributing the vanguard's data\n" << std::flush;
 
        try
        { vanguard_->loadBalance(); }
        catch (const std::exception& e) {
            catchAction(e, verbose_);
        }
        checkParallelException("Could not distribute the vanguard data: ",
                               exceptionThrown, what);
 
        if (verbose_)
            std::cout << "Allocating the model\n" << std::flush;
        try {
            model_.reset(new Model(*this));
        }
        catch (const std::exception& e) {
            catchAction(e, verbose_);
        }
        checkParallelException("Could not allocate model: ",
                               exceptionThrown, what);
 
        if (verbose_)
            std::cout << "Allocating the problem\n" << std::flush;
 
        try {
            problem_.reset(new Problem(*this));
        }
        catch (const std::exception& e) {
            catchAction(e, verbose_);
        }
        checkParallelException("Could not allocate the problem: ",
                               exceptionThrown, what);
 
        if (verbose_)
            std::cout << "Initializing the model\n" << std::flush;
 
        try
        { model_->finishInit(); }
        catch (const std::exception& e) {
            catchAction(e, verbose_);
        }
        checkParallelException("Could not initialize the  model: ",
                               exceptionThrown, what);
 
        if (verbose_)
            std::cout << "Initializing the problem\n" << std::flush;
 
        try
        { problem_->finishInit(); }
        catch (const std::exception& e) {
            catchAction(e, verbose_);
        }
        checkParallelException("Could not initialize the problem: ",
                               exceptionThrown, what);
 
        setupTimer_.stop();
 
        if (verbose_)
            std::cout << "Simulator successfully set up\n" << std::flush;
    }
 
    static void registerParameters()
    {
        Parameters::registerParam<TypeTag, Properties::EndTime>
            ("The simulation time at which the simulation is finished [s]");
        Parameters::registerParam<TypeTag, Properties::InitialTimeStepSize>
            ("The size of the initial time step [s]");
        Parameters::registerParam<TypeTag, Properties::RestartTime>
            ("The simulation time at which a restart should be attempted [s]");
        Parameters::registerParam<TypeTag, Properties::PredeterminedTimeStepsFile>
            ("A file with a list of predetermined time step sizes (one "
             "time step per line)");
 
        Vanguard::registerParameters();
        Model::registerParameters();
        Problem::registerParameters();
    }
 
    Vanguard& vanguard()
    { return *vanguard_; }
 
    const Vanguard& vanguard() const
    { return *vanguard_; }
 
    const GridView& gridView() const
    { return vanguard_->gridView(); }
 
    Model& model()
    { return *model_; }
 
    const Model& model() const
    { return *model_; }
 
    Problem& problem()
    { return *problem_; }
 
    const Problem& problem() const
    { return *problem_; }
 
    void setStartTime(Scalar t)
    { startTime_ = t; }
 
    Scalar startTime() const
    { return startTime_; }
 
    void setTime(Scalar t)
    { time_ = t; }
 
    void setTime(Scalar t, unsigned stepIdx)
    {
        time_ = t;
        timeStepIdx_ = stepIdx;
    }
 
    Scalar time() const
    { return time_; }
 
    void setEndTime(Scalar t)
    { endTime_ = t; }
 
    Scalar endTime() const
    { return endTime_; }
 
    const Timer& setupTimer() const
    { return setupTimer_; }
 
    const Timer& executionTimer() const
    { return executionTimer_; }
    Timer& executionTimer()
    { return executionTimer_; }
 
    const Timer& prePostProcessTimer() const
    { return prePostProcessTimer_; }
 
    const Timer& linearizeTimer() const
    { return linearizeTimer_; }
 
    const Timer& solveTimer() const
    { return solveTimer_; }
 
    const Timer& updateTimer() const
    { return updateTimer_; }
 
    const Timer& writeTimer() const
    { return writeTimer_; }
 
    void setTimeStepSize(Scalar value)
    {
         timeStepSize_ = value;
    }
 
    void setTimeStepIndex(unsigned value)
    { timeStepIdx_ = value; }
 
    Scalar timeStepSize() const
    { return timeStepSize_; }
 
    int timeStepIndex() const
    { return timeStepIdx_; }
 
    void setFinished(bool yesno = true)
    { finished_ = yesno; }
 
    bool finished() const
    {
        assert(timeStepSize_ >= 0.0);
        Scalar eps =
            std::max(Scalar(std::abs(this->time())), timeStepSize())
            *std::numeric_limits<Scalar>::epsilon()*1e3;
        return finished_ || (this->time()*(1.0 + eps) >= endTime());
    }
 
    bool willBeFinished() const
    {
        static const Scalar eps = std::numeric_limits<Scalar>::epsilon()*1e3;
 
        return finished_ || (this->time() + timeStepSize_)*(1.0 + eps) >= endTime();
    }
 
    Scalar maxTimeStepSize() const
    {
        if (finished())
            return 0.0;
 
        return std::min(episodeMaxTimeStepSize(),
                        std::max<Scalar>(0.0, endTime() - this->time()));
    }
 
    void startNextEpisode(Scalar episodeStartTime, Scalar episodeLength)
    {
        ++episodeIdx_;
        episodeStartTime_ = episodeStartTime;
        episodeLength_ = episodeLength;
    }
 
    void startNextEpisode(Scalar len = std::numeric_limits<Scalar>::max())
    {
        ++episodeIdx_;
        episodeStartTime_ = startTime_ + time_;
        episodeLength_ = len;
    }
 
    void setEpisodeIndex(int episodeIdx)
    { episodeIdx_ = episodeIdx; }
 
    int episodeIndex() const
    { return episodeIdx_; }
 
    Scalar episodeStartTime() const
    { return episodeStartTime_; }
 
    void setEpisodeLength(Scalar dt)
    { episodeLength_ = dt; }
 
    Scalar episodeLength() const
    { return episodeLength_; }
 
    bool episodeStarts() const
    {
        static const Scalar eps = std::numeric_limits<Scalar>::epsilon()*1e3;
 
        return this->time() <= (episodeStartTime_ - startTime())*(1 + eps);
    }
 
    bool episodeIsOver() const
    {
        static const Scalar eps = std::numeric_limits<Scalar>::epsilon()*1e3;
 
        return this->time() >= (episodeStartTime_ - startTime() + episodeLength())*(1 - eps);
    }
 
    bool episodeWillBeOver() const
    {
        static const Scalar eps = std::numeric_limits<Scalar>::epsilon()*1e3;
 
        return this->time() + timeStepSize()
            >=  (episodeStartTime_ - startTime() + episodeLength())*(1 - eps);
    }
 
    Scalar episodeMaxTimeStepSize() const
    {
        // if the current episode is over and the simulation
        // wants to give it some extra time, we will return
        // the time step size it suggested instead of trying
        // to align it to the end of the episode.
        if (episodeIsOver())
            return 0.0;
 
        // make sure that we don't exceed the end of the
        // current episode.
        return std::max<Scalar>(0.0,
                                (episodeStartTime() + episodeLength())
                                - (this->time() + this->startTime()));
    }
 
    /*
     * \}
     */
 
    void run()
    {
        // create TimerGuard objects to hedge for exceptions
        TimerGuard setupTimerGuard(setupTimer_);
        TimerGuard executionTimerGuard(executionTimer_);
        TimerGuard prePostProcessTimerGuard(prePostProcessTimer_);
        TimerGuard writeTimerGuard(writeTimer_);
 
        setupTimer_.start();
        Scalar restartTime = Parameters::get<TypeTag, Properties::RestartTime>();
        if (restartTime > -1e30) {
            // try to restart a previous simulation
            time_ = restartTime;
 
            Restart res;
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(res.deserializeBegin(*this, time_));
            if (verbose_)
                std::cout << "Deserialize from file '" << res.fileName() << "'\n" << std::flush;
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(this->deserialize(res));
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->deserialize(res));
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(model_->deserialize(res));
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(res.deserializeEnd());
            if (verbose_)
                std::cout << "Deserialization done."
                          << " Simulator time: " << time() << humanReadableTime(time())
                          << " Time step index: " << timeStepIndex()
                          << " Episode index: " << episodeIndex()
                          << "\n" << std::flush;
        }
        else {
            // if no restart is done, apply the initial solution
            if (verbose_)
                std::cout << "Applying the initial solution of the \"" << problem_->name()
                          << "\" problem\n" << std::flush;
 
            Scalar oldTimeStepSize = timeStepSize_;
            int oldTimeStepIdx = timeStepIdx_;
            timeStepSize_ = 0.0;
            timeStepIdx_ = -1;
 
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(model_->applyInitialSolution());
 
            // write initial condition
            if (problem_->shouldWriteOutput())
                EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->writeOutput());
 
            timeStepSize_ = oldTimeStepSize;
            timeStepIdx_ = oldTimeStepIdx;
        }
        setupTimer_.stop();
 
        executionTimer_.start();
        bool episodeBegins = episodeIsOver() || (timeStepIdx_ == 0);
        // do the time steps
        while (!finished()) {
            prePostProcessTimer_.start();
            if (episodeBegins) {
                // notify the problem that a new episode has just been
                // started.
                EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->beginEpisode());
 
                if (finished()) {
                    // the problem can chose to terminate the simulation in
                    // beginEpisode(), so we have handle this case.
                    EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->endEpisode());
                    prePostProcessTimer_.stop();
 
                    break;
                }
            }
            episodeBegins = false;
 
            if (verbose_) {
                std::cout << "Begin time step " << timeStepIndex() + 1 << ". "
                          << "Start time: " << this->time() << " seconds" << humanReadableTime(this->time())
                          << ", step size: " << timeStepSize() << " seconds" << humanReadableTime(timeStepSize())
                          << "\n";
            }
 
            // pre-process the current solution
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->beginTimeStep());
 
            if (finished()) {
                // the problem can chose to terminate the simulation in
                // beginTimeStep(), so we have handle this case.
                EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->endTimeStep());
                EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->endEpisode());
                prePostProcessTimer_.stop();
 
                break;
            }
            prePostProcessTimer_.stop();
 
            try {
                // execute the time integration scheme
                problem_->timeIntegration();
            }
            catch (...) {
                // exceptions in the time integration might be recoverable. clean up in
                // case they are
                const auto& model = problem_->model();
                prePostProcessTimer_ += model.prePostProcessTimer();
                linearizeTimer_ += model.linearizeTimer();
                solveTimer_ += model.solveTimer();
                updateTimer_ += model.updateTimer();
 
                throw;
            }
 
            const auto& model = problem_->model();
            prePostProcessTimer_ += model.prePostProcessTimer();
            linearizeTimer_ += model.linearizeTimer();
            solveTimer_ += model.solveTimer();
            updateTimer_ += model.updateTimer();
 
            // post-process the current solution
            prePostProcessTimer_.start();
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->endTimeStep());
            prePostProcessTimer_.stop();
 
            // write the result to disk
            writeTimer_.start();
            if (problem_->shouldWriteOutput())
                EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->writeOutput());
            writeTimer_.stop();
 
            // do the next time integration
            Scalar oldDt = timeStepSize();
            EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->advanceTimeLevel());
 
            if (verbose_) {
                std::cout << "Time step " << timeStepIndex() + 1 << " done. "
                          << "CPU time: " << executionTimer_.realTimeElapsed() << " seconds" << humanReadableTime(executionTimer_.realTimeElapsed())
                          << ", end time: " << this->time() + oldDt << " seconds" << humanReadableTime(this->time() + oldDt)
                          << ", step size: " << oldDt << " seconds" << humanReadableTime(oldDt)
                          << "\n" << std::flush;
            }
 
            // advance the simulated time by the current time step size
            time_ += oldDt;
            ++timeStepIdx_;
 
            prePostProcessTimer_.start();
            // notify the problem if an episode is finished
            if (episodeIsOver()) {
                // Notify the problem about the end of the current episode...
                EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->endEpisode());
                episodeBegins = true;
            }
            else {
                Scalar dt;
                if (timeStepIdx_ < static_cast<int>(forcedTimeSteps_.size()))
                    // use the next time step size from the input file
                    dt = forcedTimeSteps_[timeStepIdx_];
                else
                    // ask the problem to provide the next time step size
                    dt = std::min(maxTimeStepSize(), problem_->nextTimeStepSize());
                assert(finished() || dt > 0);
                setTimeStepSize(dt);
            }
            prePostProcessTimer_.stop();
 
            // write restart file if mandated by the problem
            writeTimer_.start();
            if (problem_->shouldWriteRestartFile())
                EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(serialize());
            writeTimer_.stop();
        }
        executionTimer_.stop();
 
        EWOMS_CATCH_PARALLEL_EXCEPTIONS_FATAL(problem_->finalize());
    }
 
    static std::string humanReadableTime(Scalar timeInSeconds, bool isAmendment=true)
    {
        std::ostringstream oss;
        oss << std::setprecision(4);
        if (isAmendment)
            oss << " (";
        if (timeInSeconds >= 365.25*24*60*60) {
            int years = static_cast<int>(timeInSeconds/(365.25*24*60*60));
            int days = static_cast<int>((timeInSeconds - years*(365.25*24*60*60))/(24*60*60));
 
            double accuracy = 1e-2;
            double hours =
                std::round(1.0/accuracy*
                           (timeInSeconds
                            - years*(365.25*24*60*60)
                            - days*(24*60*60))/(60*60))
                *accuracy;
 
            oss << years << " years, " << days << " days, "  << hours << " hours";
        }
        else if (timeInSeconds >= 24.0*60*60) {
            int days = static_cast<int>(timeInSeconds/(24*60*60));
            int hours = static_cast<int>((timeInSeconds - days*(24*60*60))/(60*60));
 
            double accuracy = 1e-2;
            double minutes =
                std::round(1.0/accuracy*
                           (timeInSeconds
                            - days*(24*60*60)
                            - hours*(60*60))/60)
                *accuracy;
 
            oss << days << " days, " << hours << " hours, " << minutes << " minutes";
        }
        else if (timeInSeconds >= 60.0*60) {
            int hours = static_cast<int>(timeInSeconds/(60*60));
            int minutes = static_cast<int>((timeInSeconds - hours*(60*60))/60);
 
            double accuracy = 1e-2;
            double seconds =
                std::round(1.0/accuracy*
                           (timeInSeconds
                            - hours*(60*60)
                            - minutes*60))
                *accuracy;
 
            oss << hours << " hours, " << minutes << " minutes, "  << seconds << " seconds";
        }
        else if (timeInSeconds >= 60.0) {
            int minutes = static_cast<int>(timeInSeconds/60);
 
            double accuracy = 1e-3;
            double seconds =
                std::round(1.0/accuracy*
                           (timeInSeconds
                            - minutes*60))
                *accuracy;
 
            oss << minutes << " minutes, "  << seconds << " seconds";
        }
        else if (!isAmendment)
            oss << timeInSeconds << " seconds";
        else
            return "";
        if (isAmendment)
            oss << ")";
 
        return oss.str();
    }
 
    void serialize()
    {
        using Restarter = Restart;
        Restarter res;
        res.serializeBegin(*this);
        if (gridView().comm().rank() == 0)
            std::cout << "Serialize to file '" << res.fileName() << "'"
                      << ", next time step size: " << timeStepSize()
                      << "\n" << std::flush;
 
        this->serialize(res);
        problem_->serialize(res);
        model_->serialize(res);
        res.serializeEnd();
    }
 
    template <class Restarter>
    void serialize(Restarter& restarter)
    {
        restarter.serializeSectionBegin("Simulator");
        restarter.serializeStream()
            << episodeIdx_ << " "
            << episodeStartTime_ << " "
            << episodeLength_ << " "
            << startTime_ << " "
            << time_ << " "
            << timeStepIdx_ << " ";
        restarter.serializeSectionEnd();
    }
 
    template <class Restarter>
    void deserialize(Restarter& restarter)
    {
        restarter.deserializeSectionBegin("Simulator");
        restarter.deserializeStream()
            >> episodeIdx_
            >> episodeStartTime_
            >> episodeLength_
            >> startTime_
            >> time_
            >> timeStepIdx_;
        restarter.deserializeSectionEnd();
    }
 
    template<class Serializer>
    void serializeOp(Serializer& serializer)
    {
        serializer(*vanguard_);
        serializer(*model_);
        serializer(*problem_);
        serializer(episodeIdx_);
        serializer(episodeStartTime_);
        serializer(episodeLength_);
        serializer(startTime_);
        serializer(time_);
        serializer(timeStepIdx_);
    }
 
private:
    std::unique_ptr<Vanguard> vanguard_;
    std::unique_ptr<Model> model_;
    std::unique_ptr<Problem> problem_;
 
    int episodeIdx_;
    Scalar episodeStartTime_;
    Scalar episodeLength_;
 
    Timer setupTimer_;
    Timer executionTimer_;
    Timer prePostProcessTimer_;
    Timer linearizeTimer_;
    Timer solveTimer_;
    Timer updateTimer_;
    Timer writeTimer_;
 
    std::vector<Scalar> forcedTimeSteps_;
    Scalar startTime_;
    Scalar time_;
    Scalar endTime_;
 
    Scalar timeStepSize_;
    int timeStepIdx_;
 
    bool finished_;
    bool verbose_;
};
 
namespace Properties {
template<class TypeTag>
struct Simulator<TypeTag, TTag::NumericModel> { using type = ::Opm::Simulator<TypeTag>; };
}
 
} // namespace Opm
 
#endif