FlowMain.hpp
Go to the documentation of this file.
1/*
2 Copyright 2013, 2014, 2015 SINTEF ICT, Applied Mathematics.
3 Copyright 2014 Dr. Blatt - HPC-Simulation-Software & Services
4 Copyright 2015 IRIS AS
5 Copyright 2014 STATOIL ASA.
6
7 This file is part of the Open Porous Media project (OPM).
8
9 OPM is free software: you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation, either version 3 of the License, or
12 (at your option) any later version.
13
14 OPM is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with OPM. If not, see <http://www.gnu.org/licenses/>.
21*/
22#ifndef OPM_FLOW_MAIN_HEADER_INCLUDED
23#define OPM_FLOW_MAIN_HEADER_INCLUDED
24
25#include <opm/input/eclipse/EclipseState/EclipseState.hpp>
26#include <opm/input/eclipse/EclipseState/IOConfig/IOConfig.hpp>
27#include <opm/input/eclipse/EclipseState/InitConfig/InitConfig.hpp>
28
30
35
36#if HAVE_MPI
37#define RESERVOIR_COUPLING_ENABLED
38#endif
39#if HAVE_DUNE_FEM
40#include <dune/fem/misc/mpimanager.hh>
41#else
42#include <dune/common/parallel/mpihelper.hh>
43#endif
44
45#ifdef _OPENMP
46#include <omp.h>
47#endif
48
49#include <charconv>
50#include <cstddef>
51#include <memory>
52
53namespace Opm::Parameters {
54
55// Do not merge parallel output files or warn about them
56struct EnableLoggingFalloutWarning { static constexpr bool value = false; };
57struct OutputInterval { static constexpr int value = 1; };
58// Set global debug verbosity level
59struct DebugVerbosityLevel { static constexpr int value = 1; };
60} // namespace Opm::Parameters
61
62namespace Opm {
63
64 class Deck;
65
66 // The FlowMain class is the black-oil simulator.
67 template <class TypeTag>
68 class FlowMain
69 {
70 public:
71 using MaterialLawManager = typename GetProp<TypeTag, Properties::MaterialLaw>::EclMaterialLawManager;
72 using ModelSimulator = GetPropType<TypeTag, Properties::Simulator>;
73 using Grid = GetPropType<TypeTag, Properties::Grid>;
74 using GridView = GetPropType<TypeTag, Properties::GridView>;
75 using Problem = GetPropType<TypeTag, Properties::Problem>;
76 using Scalar = GetPropType<TypeTag, Properties::Scalar>;
77 using FluidSystem = GetPropType<TypeTag, Properties::FluidSystem>;
78
79 using Simulator = SimulatorFullyImplicitBlackoil<TypeTag>;
80
81 FlowMain(int argc, char **argv, bool output_cout, bool output_files )
82 : argc_{argc}, argv_{argv},
83 output_cout_{output_cout}, output_files_{output_files}
84 {
85
86 }
87
88 // Read the command line parameters. Throws an exception if something goes wrong.
89 static int setupParameters_(int argc, char** argv, Parallel::Communication comm)
90 {
92 // We have already successfully run setupParameters_().
93 // For the dynamically chosen runs (as from the main flow
94 // executable) we must run this function again with the
95 // real typetag to be used, as the first time was with the
96 // "FlowEarlyBird" typetag. However, for the static ones (such
97 // as 'flow_onephase_energy') it has already been run with the
98 // correct typetag.
99 return EXIT_SUCCESS;
100 }
101 // register the flow specific parameters
102 Parameters::Register<Parameters::OutputInterval>
103 ("Specify the number of report steps between two consecutive writes of restart data");
104 Parameters::Register<Parameters::EnableLoggingFalloutWarning>
105 ("Developer option to see whether logging was on non-root processors. "
106 "In that case it will be appended to the *.DBG or *.PRT files");
107 Parameters::Register<Parameters::DebugVerbosityLevel>
108 ("Set debug verbosity level globally. Default is 1, increasing values give additional output and 0 disables most messages to the .DBG file");
109
110 // register the base parameters
111 registerAllParameters_<TypeTag>(/*finalizeRegistration=*/false);
112
114
115 detail::hideUnusedParameters<Scalar>();
116
118
119 int mpiRank = comm.rank();
120
121 // read in the command line parameters
122 int status = ::Opm::setupParameters_<TypeTag>(argc,
123 const_cast<const char**>(argv),
124 /*doRegistration=*/false,
125 /*allowUnused=*/true,
126 /*handleHelp=*/(mpiRank==0),
127 mpiRank);
128 if (status == 0) {
129
130 // deal with unknown parameters.
131
132 int unknownKeyWords = 0;
133 if (mpiRank == 0) {
134 unknownKeyWords = Parameters::printUnused(std::cerr);
135 }
136 int globalUnknownKeyWords = comm.sum(unknownKeyWords);
137 unknownKeyWords = globalUnknownKeyWords;
138 if ( unknownKeyWords )
139 {
140 if ( mpiRank == 0 )
141 {
142 std::string msg = "Aborting simulation due to unknown "
143 "parameters. Please query \"flow --help\" for "
144 "supported command line parameters.";
145 if (OpmLog::hasBackend("STREAMLOG"))
146 {
147 OpmLog::error(msg);
148 }
149 else {
150 std::cerr << msg << std::endl;
151 }
152 }
153 return EXIT_FAILURE;
154 }
155
156 // deal with --print-parameters and unknown parameters.
157 if (Parameters::Get<Parameters::PrintParameters>() == 1) {
158 if (mpiRank == 0) {
159 Parameters::printValues(std::cout);
160 }
161 return -1;
162 }
163 }
164
165 // set the maximum limit on OMP threads
166 setMaxThreads();
167
168 return status;
169 }
170
174 int execute()
175 {
176 return execute_(&FlowMain::runSimulator, /*cleanup=*/true);
177 }
178
179 int executeInitStep()
180 {
181 return execute_(&FlowMain::runSimulatorInit, /*cleanup=*/false);
182 }
183
184 // Returns true unless "EXIT" was encountered in the schedule
185 // section of the input datafile.
186 int executeStep()
187 {
188 return simulator_->runStep(*simtimer_);
189 }
190
191 // Called from Python to cleanup after having executed the last
192 // executeStep()
193 int executeStepsCleanup()
194 {
195 SimulatorReport report = simulator_->finalize();
196 runSimulatorAfterSim_(report);
197 return report.success.exit_status;
198 }
199
200 ModelSimulator* getSimulatorPtr()
201 {
202 return modelSimulator_.get();
203 }
204
205 SimulatorTimer* getSimTimer()
206 {
207 return simtimer_.get();
208 }
209
211 double getPreviousReportStepSize()
212 {
213 return simtimer_->stepLengthTaken();
214 }
215
216 private:
217 // called by execute() or executeInitStep()
218 int execute_(int (FlowMain::* runOrInitFunc)(), bool cleanup)
219 {
220 auto logger = [this](const std::exception& e, const std::string& message_start) {
221 std::ostringstream message;
222 message << message_start << e.what();
223
224 if (this->output_cout_) {
225 // in some cases exceptions are thrown before the logging system is set
226 // up.
227 if (OpmLog::hasBackend("STREAMLOG")) {
228 OpmLog::error(message.str());
229 }
230 else {
231 std::cout << message.str() << "\n";
232 }
233 }
235 return EXIT_FAILURE;
236 };
237
238 try {
239 // deal with some administrative boilerplate
240
241 Dune::Timer setupTimerAfterReadingDeck;
242 setupTimerAfterReadingDeck.start();
243
244 int status = setupParameters_(this->argc_, this->argv_, FlowGenericVanguard::comm());
245 if (status) {
246 return status;
247 }
248
249 setupParallelism();
250 setupModelSimulator();
251 createSimulator();
252
253 this->deck_read_time_ = modelSimulator_->vanguard().setupTime();
254 this->total_setup_time_ = setupTimerAfterReadingDeck.elapsed() + this->deck_read_time_;
255
256 // if run, do the actual work, else just initialize
257 int exitCode = (this->*runOrInitFunc)();
258 if (cleanup) {
259 executeCleanup_();
260 }
261 return exitCode;
262 }
263 catch (const TimeSteppingBreakdown& e) {
264 auto exitCode = logger(e, "Simulation aborted: ");
265 executeCleanup_();
266 return exitCode;
267 }
268 catch (const std::exception& e) {
269 auto exitCode = logger(e, "Simulation aborted as program threw an unexpected exception: ");
270 executeCleanup_();
271 return exitCode;
272 }
273 }
274
275 void executeCleanup_() {
276 // clean up
278 }
279
280 protected:
281 void setupParallelism()
282 {
283 // determine the rank of the current process and the number of processes
284 // involved in the simulation. MPI must have already been initialized
285 // here. (yes, the name of this method is misleading.)
286 auto comm = FlowGenericVanguard::comm();
287 mpi_rank_ = comm.rank();
288 mpi_size_ = comm.size();
289
290 setMaxThreads();
291 }
292
293 static void setMaxThreads()
294 {
295#if _OPENMP
296 // If openMP is available, default to 2 threads per process unless
297 // OMP_NUM_THREADS is set or command line --threads-per-process used.
298 // Issue a warning if both OMP_NUM_THREADS and --threads-per-process are set,
299 // but let the environment variable take precedence.
300 constexpr int default_threads = 2;
301 const bool isSet = Parameters::IsSet<Parameters::ThreadsPerProcess>();
302 const int requested_threads = Parameters::Get<Parameters::ThreadsPerProcess>();
303 int threads = requested_threads > 0 ? requested_threads : default_threads;
304
305 const char* env_var = getenv("OMP_NUM_THREADS");
306 if (env_var) {
307 int omp_num_threads = -1;
308 auto result = std::from_chars(env_var, env_var + std::strlen(env_var), omp_num_threads);
309 if (result.ec == std::errc() && omp_num_threads > 0) {
310 // Set threads to omp_num_threads if it was successfully parsed and is positive
311 threads = omp_num_threads;
312 if (isSet) {
313 OpmLog::warning("Environment variable OMP_NUM_THREADS takes precedence over the --threads-per-process cmdline argument.");
314 }
315 } else {
316 OpmLog::warning("Invalid value for OMP_NUM_THREADS environment variable.");
317 }
318 }
319
320 // Requesting -1 thread will let OMP automatically deduce the number
321 // but setting OMP_NUM_THREADS takes precedence.
322 if (env_var || !(isSet && requested_threads == -1)) {
323 omp_set_num_threads(threads);
324 }
325#endif
326
327 using TM = GetPropType<TypeTag, Properties::ThreadManager>;
328 TM::init(false);
329 }
330
332 {
333 // force closing of all log files.
334 OpmLog::removeAllBackends();
335
336 if (mpi_rank_ != 0 || mpi_size_ < 2 || !this->output_files_ || !modelSimulator_) {
337 return;
338 }
339
340 detail::mergeParallelLogFiles(eclState().getIOConfig().getOutputDir(),
341 Parameters::Get<Parameters::EclDeckFileName>(),
342 Parameters::Get<Parameters::EnableLoggingFalloutWarning>());
343 }
344
345 void setupModelSimulator()
346 {
347 modelSimulator_ = std::make_unique<ModelSimulator>(FlowGenericVanguard::comm(), /*verbose=*/false);
348 modelSimulator_->executionTimer().start();
349 modelSimulator_->model().applyInitialSolution();
350 }
351
352 const EclipseState& eclState() const
353 { return modelSimulator_->vanguard().eclState(); }
354
355 EclipseState& eclState()
356 { return modelSimulator_->vanguard().eclState(); }
357
358 const Schedule& schedule() const
359 { return modelSimulator_->vanguard().schedule(); }
360
361 // Run the simulator.
362 int runSimulator()
363 {
364 return runSimulatorInitOrRun_(&FlowMain::runSimulatorRunCallback_);
365 }
366
367 int runSimulatorInit()
368 {
369 return runSimulatorInitOrRun_(&FlowMain::runSimulatorInitCallback_);
370 }
371
372 private:
373 // Callback that will be called from runSimulatorInitOrRun_().
374 int runSimulatorRunCallback_()
375 {
376#ifdef RESERVOIR_COUPLING_ENABLED
377 SimulatorReport report = simulator_->run(*simtimer_, this->argc_, this->argv_);
378#else
379 SimulatorReport report = simulator_->run(*simtimer_);
380#endif
381 runSimulatorAfterSim_(report);
382 return report.success.exit_status;
383 }
384
385 // Callback that will be called from runSimulatorInitOrRun_().
386 int runSimulatorInitCallback_()
387 {
388#ifdef RESERVOIR_COUPLING_ENABLED
389 simulator_->init(*simtimer_, this->argc_, this->argv_);
390#else
391 simulator_->init(*simtimer_);
392#endif
393 return EXIT_SUCCESS;
394 }
395
396 // Output summary after simulation has completed
397 void runSimulatorAfterSim_(SimulatorReport &report)
398 {
399 if (simulator_->model().hasNlddSolver()) {
400 const auto& odir = eclState().getIOConfig().getOutputDir();
401 // Write the number of nonlinear iterations per cell to a file in ResInsight compatible format
402 simulator_->model().writeNonlinearIterationsPerCell(odir);
403 // Write the NLDD statistics to the DBG file
404 reportNlddStatistics(simulator_->model().domainAccumulatedReports(),
405 simulator_->model().localAccumulatedReports(),
406 this->output_cout_,
408 }
409
410 if (! this->output_cout_) {
411 return;
412 }
413
414 const int threads
415#if !defined(_OPENMP) || !_OPENMP
416 = 1;
417#else
418 = omp_get_max_threads();
419#endif
420
421 printFlowTrailer(mpi_size_, threads, total_setup_time_, deck_read_time_, report);
422
424 Parameters::Get<Parameters::OutputExtraConvergenceInfo>(),
425 R"(OutputExtraConvergenceInfo (--output-extra-convergence-info))",
426 eclState().getIOConfig().getOutputDir(),
427 eclState().getIOConfig().getBaseName());
428 }
429
430 // Run the simulator.
431 int runSimulatorInitOrRun_(int (FlowMain::* initOrRunFunc)())
432 {
433
434 const auto& schedule = this->schedule();
435 auto& ioConfig = eclState().getIOConfig();
436 simtimer_ = std::make_unique<SimulatorTimer>();
437
438 // initialize variables
439 const auto& initConfig = eclState().getInitConfig();
440 simtimer_->init(schedule, static_cast<std::size_t>(initConfig.getRestartStep()));
441
442 if (this->output_cout_) {
443 std::ostringstream oss;
444
445 // This allows a user to catch typos and misunderstandings in the
446 // use of simulator parameters.
447 if (Parameters::printUnused(oss)) {
448 std::cout << "----------------- Unrecognized parameters: -----------------\n";
449 std::cout << oss.str();
450 std::cout << "----------------------------------------------------------------" << std::endl;
451 }
452 }
453
454 if (!ioConfig.initOnly()) {
455 if (this->output_cout_) {
456 std::string msg;
457 msg = "\n\n================ Starting main simulation loop ===============\n";
458 OpmLog::info(msg);
459 }
460
461 return (this->*initOrRunFunc)();
462 }
463 else {
464 if (this->output_cout_) {
465 std::cout << "\n\n================ Simulation turned off ===============\n" << std::flush;
466 }
467 return EXIT_SUCCESS;
468 }
469 }
470
471 protected:
472
474 // Create simulator instance.
475 // Writes to:
476 // simulator_
477 void createSimulator()
478 {
479 // Create the simulator instance.
480 simulator_ = std::make_unique<Simulator>(*modelSimulator_);
481 }
482
483 Grid& grid()
484 { return modelSimulator_->vanguard().grid(); }
485
486 private:
487 std::unique_ptr<ModelSimulator> modelSimulator_;
488 int mpi_rank_ = 0;
489 int mpi_size_ = 1;
490 std::any parallel_information_;
491 std::unique_ptr<Simulator> simulator_;
492 std::unique_ptr<SimulatorTimer> simtimer_;
493 int argc_;
494 char **argv_;
495 bool output_cout_;
496 bool output_files_;
497 double total_setup_time_ = 0.0;
498 double deck_read_time_ = 0.0;
499 };
500
501} // namespace Opm
502
503#endif // OPM_FLOW_MAIN_HEADER_INCLUDED
static Parallel::Communication & comm()
Obtain global communicator.
Definition: FlowGenericVanguard.hpp:332
static void registerParameters()
Registers all runtime parameters used by the simulation.
Definition: simulator.hh:214
Dune::Communication< MPIComm > Communication
Definition: ParallelCommunication.hpp:30
Definition: blackoilnewtonmethodparams.hpp:31
void printValues(std::ostream &os)
Print values of the run-time parameters.
bool IsRegistrationOpen()
Query whether parameter registration is open or not.
void endRegistration()
Indicate that all parameters are registered for a given type tag.
bool printUnused(std::ostream &os)
Print the list of unused run-time parameters.
void handleExtraConvergenceOutput(const SimulatorReport &report, std::string_view option, std::string_view optionName, std::string_view output_dir, std::string_view base_name)
void mergeParallelLogFiles(std::string_view output_dir, std::string_view deck_filename, bool enableLoggingFalloutWarning)
void checkAllMPIProcesses()
Definition: blackoilboundaryratevector.hh:39
void printFlowTrailer(int nprocs, int nthreads, const double total_setup_time, const double deck_read_time, const SimulatorReport &report)
void reportNlddStatistics(const std::vector< SimulatorReport > &domain_reports, const SimulatorReport &local_report, const bool output_cout, const Parallel::Communication &comm)
Provides convenience routines to bring up the simulation at runtime.