22#ifndef OPM_FLOW_MAIN_HEADER_INCLUDED
23#define OPM_FLOW_MAIN_HEADER_INCLUDED
25#include <opm/input/eclipse/EclipseState/EclipseState.hpp>
26#include <opm/input/eclipse/EclipseState/IOConfig/IOConfig.hpp>
27#include <opm/input/eclipse/EclipseState/InitConfig/InitConfig.hpp>
37#define RESERVOIR_COUPLING_ENABLED
40#include <dune/fem/misc/mpimanager.hh>
42#include <dune/common/parallel/mpihelper.hh>
56struct EnableLoggingFalloutWarning {
static constexpr bool value =
false; };
57struct OutputInterval {
static constexpr int value = 1; };
59struct DebugVerbosityLevel {
static constexpr int value = 1; };
67 template <
class TypeTag>
71 using MaterialLawManager =
typename GetProp<TypeTag, Properties::MaterialLaw>::EclMaterialLawManager;
72 using ModelSimulator = GetPropType<TypeTag, Properties::Simulator>;
73 using Grid = GetPropType<TypeTag, Properties::Grid>;
74 using GridView = GetPropType<TypeTag, Properties::GridView>;
75 using Problem = GetPropType<TypeTag, Properties::Problem>;
76 using Scalar = GetPropType<TypeTag, Properties::Scalar>;
77 using FluidSystem = GetPropType<TypeTag, Properties::FluidSystem>;
79 using Simulator = SimulatorFullyImplicitBlackoil<TypeTag>;
81 FlowMain(
int argc,
char **argv,
bool output_cout,
bool output_files )
82 : argc_{argc}, argv_{argv},
83 output_cout_{output_cout}, output_files_{output_files}
102 Parameters::Register<Parameters::OutputInterval>
103 (
"Specify the number of report steps between two consecutive writes of restart data");
104 Parameters::Register<Parameters::EnableLoggingFalloutWarning>
105 (
"Developer option to see whether logging was on non-root processors. "
106 "In that case it will be appended to the *.DBG or *.PRT files");
107 Parameters::Register<Parameters::DebugVerbosityLevel>
108 (
"Set debug verbosity level globally. Default is 1, increasing values give additional output and 0 disables most messages to the .DBG file");
111 registerAllParameters_<TypeTag>(
false);
115 detail::hideUnusedParameters<Scalar>();
119 int mpiRank = comm.rank();
122 int status = ::Opm::setupParameters_<TypeTag>(argc,
123 const_cast<const char**
>(argv),
132 int unknownKeyWords = 0;
136 int globalUnknownKeyWords = comm.sum(unknownKeyWords);
137 unknownKeyWords = globalUnknownKeyWords;
138 if ( unknownKeyWords )
142 std::string msg =
"Aborting simulation due to unknown "
143 "parameters. Please query \"flow --help\" for "
144 "supported command line parameters.";
145 if (OpmLog::hasBackend(
"STREAMLOG"))
150 std::cerr << msg << std::endl;
157 if (Parameters::Get<Parameters::PrintParameters>() == 1) {
176 return execute_(&FlowMain::runSimulator,
true);
179 int executeInitStep()
181 return execute_(&FlowMain::runSimulatorInit,
false);
188 return simulator_->runStep(*simtimer_);
193 int executeStepsCleanup()
195 SimulatorReport report = simulator_->finalize();
196 runSimulatorAfterSim_(report);
197 return report.success.exit_status;
200 ModelSimulator* getSimulatorPtr()
202 return modelSimulator_.get();
205 SimulatorTimer* getSimTimer()
207 return simtimer_.get();
211 double getPreviousReportStepSize()
213 return simtimer_->stepLengthTaken();
218 int execute_(
int (FlowMain::* runOrInitFunc)(),
bool cleanup)
220 auto logger = [
this](
const std::exception& e,
const std::string& message_start) {
221 std::ostringstream message;
222 message << message_start << e.what();
224 if (this->output_cout_) {
227 if (OpmLog::hasBackend(
"STREAMLOG")) {
228 OpmLog::error(message.str());
231 std::cout << message.str() <<
"\n";
241 Dune::Timer setupTimerAfterReadingDeck;
242 setupTimerAfterReadingDeck.start();
250 setupModelSimulator();
253 this->deck_read_time_ = modelSimulator_->vanguard().setupTime();
254 this->total_setup_time_ = setupTimerAfterReadingDeck.elapsed() + this->deck_read_time_;
257 int exitCode = (this->*runOrInitFunc)();
263 catch (
const TimeSteppingBreakdown& e) {
264 auto exitCode = logger(e,
"Simulation aborted: ");
268 catch (
const std::exception& e) {
269 auto exitCode = logger(e,
"Simulation aborted as program threw an unexpected exception: ");
275 void executeCleanup_() {
281 void setupParallelism()
287 mpi_rank_ = comm.rank();
288 mpi_size_ = comm.size();
293 static void setMaxThreads()
300 constexpr int default_threads = 2;
301 const bool isSet = Parameters::IsSet<Parameters::ThreadsPerProcess>();
302 const int requested_threads = Parameters::Get<Parameters::ThreadsPerProcess>();
303 int threads = requested_threads > 0 ? requested_threads : default_threads;
305 const char* env_var = getenv(
"OMP_NUM_THREADS");
307 int omp_num_threads = -1;
308 auto result = std::from_chars(env_var, env_var + std::strlen(env_var), omp_num_threads);
309 if (result.ec == std::errc() && omp_num_threads > 0) {
311 threads = omp_num_threads;
313 OpmLog::warning(
"Environment variable OMP_NUM_THREADS takes precedence over the --threads-per-process cmdline argument.");
316 OpmLog::warning(
"Invalid value for OMP_NUM_THREADS environment variable.");
322 if (env_var || !(isSet && requested_threads == -1)) {
323 omp_set_num_threads(threads);
327 using TM = GetPropType<TypeTag, Properties::ThreadManager>;
334 OpmLog::removeAllBackends();
336 if (mpi_rank_ != 0 || mpi_size_ < 2 || !this->output_files_ || !modelSimulator_) {
341 Parameters::Get<Parameters::EclDeckFileName>(),
342 Parameters::Get<Parameters::EnableLoggingFalloutWarning>());
345 void setupModelSimulator()
348 modelSimulator_->executionTimer().start();
349 modelSimulator_->model().applyInitialSolution();
352 const EclipseState& eclState()
const
353 {
return modelSimulator_->vanguard().eclState(); }
355 EclipseState& eclState()
356 {
return modelSimulator_->vanguard().eclState(); }
358 const Schedule& schedule()
const
359 {
return modelSimulator_->vanguard().schedule(); }
364 return runSimulatorInitOrRun_(&FlowMain::runSimulatorRunCallback_);
367 int runSimulatorInit()
369 return runSimulatorInitOrRun_(&FlowMain::runSimulatorInitCallback_);
374 int runSimulatorRunCallback_()
376#ifdef RESERVOIR_COUPLING_ENABLED
377 SimulatorReport report = simulator_->run(*simtimer_, this->argc_, this->argv_);
379 SimulatorReport report = simulator_->run(*simtimer_);
381 runSimulatorAfterSim_(report);
382 return report.success.exit_status;
386 int runSimulatorInitCallback_()
388#ifdef RESERVOIR_COUPLING_ENABLED
389 simulator_->init(*simtimer_, this->argc_, this->argv_);
391 simulator_->init(*simtimer_);
397 void runSimulatorAfterSim_(SimulatorReport &report)
399 if (simulator_->model().hasNlddSolver()) {
400 const auto& odir = eclState().getIOConfig().getOutputDir();
402 simulator_->model().writeNonlinearIterationsPerCell(odir);
405 simulator_->model().localAccumulatedReports(),
410 if (! this->output_cout_) {
415#if !defined(_OPENMP) || !_OPENMP
418 = omp_get_max_threads();
421 printFlowTrailer(mpi_size_, threads, total_setup_time_, deck_read_time_, report);
424 Parameters::Get<Parameters::OutputExtraConvergenceInfo>(),
425 R
"(OutputExtraConvergenceInfo (--output-extra-convergence-info))",
426 eclState().getIOConfig().getOutputDir(),
427 eclState().getIOConfig().getBaseName());
431 int runSimulatorInitOrRun_(
int (FlowMain::* initOrRunFunc)())
434 const auto& schedule = this->schedule();
435 auto& ioConfig = eclState().getIOConfig();
436 simtimer_ = std::make_unique<SimulatorTimer>();
439 const auto& initConfig = eclState().getInitConfig();
440 simtimer_->init(schedule,
static_cast<std::size_t
>(initConfig.getRestartStep()));
442 if (this->output_cout_) {
443 std::ostringstream oss;
448 std::cout <<
"----------------- Unrecognized parameters: -----------------\n";
449 std::cout << oss.str();
450 std::cout <<
"----------------------------------------------------------------" << std::endl;
454 if (!ioConfig.initOnly()) {
455 if (this->output_cout_) {
457 msg =
"\n\n================ Starting main simulation loop ===============\n";
461 return (this->*initOrRunFunc)();
464 if (this->output_cout_) {
465 std::cout <<
"\n\n================ Simulation turned off ===============\n" << std::flush;
477 void createSimulator()
480 simulator_ = std::make_unique<Simulator>(*modelSimulator_);
484 {
return modelSimulator_->vanguard().grid(); }
487 std::unique_ptr<ModelSimulator> modelSimulator_;
490 std::any parallel_information_;
491 std::unique_ptr<Simulator> simulator_;
492 std::unique_ptr<SimulatorTimer> simtimer_;
497 double total_setup_time_ = 0.0;
498 double deck_read_time_ = 0.0;
static Parallel::Communication & comm()
Obtain global communicator.
Definition: FlowGenericVanguard.hpp:332
static void registerParameters()
Registers all runtime parameters used by the simulation.
Definition: simulator.hh:214
Dune::Communication< MPIComm > Communication
Definition: ParallelCommunication.hpp:30
Definition: blackoilnewtonmethodparams.hpp:31
void printValues(std::ostream &os)
Print values of the run-time parameters.
bool IsRegistrationOpen()
Query whether parameter registration is open or not.
void endRegistration()
Indicate that all parameters are registered for a given type tag.
bool printUnused(std::ostream &os)
Print the list of unused run-time parameters.
void handleExtraConvergenceOutput(const SimulatorReport &report, std::string_view option, std::string_view optionName, std::string_view output_dir, std::string_view base_name)
void mergeParallelLogFiles(std::string_view output_dir, std::string_view deck_filename, bool enableLoggingFalloutWarning)
void checkAllMPIProcesses()
Definition: blackoilboundaryratevector.hh:39
void printFlowTrailer(int nprocs, int nthreads, const double total_setup_time, const double deck_read_time, const SimulatorReport &report)
void reportNlddStatistics(const std::vector< SimulatorReport > &domain_reports, const SimulatorReport &local_report, const bool output_cout, const Parallel::Communication &comm)
Provides convenience routines to bring up the simulation at runtime.