22 #ifndef OPM_FLOW_MAIN_EBOS_HEADER_INCLUDED
23 #define OPM_FLOW_MAIN_EBOS_HEADER_INCLUDED
26 #include <sys/utsname.h>
28 #include <opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp>
29 #include <opm/simulators/utils/ParallelFileMerger.hpp>
30 #include <opm/simulators/utils/moduleVersion.hpp>
31 #include <opm/simulators/utils/ParallelEclipseState.hpp>
33 #include <opm/parser/eclipse/EclipseState/EclipseState.hpp>
34 #include <opm/parser/eclipse/EclipseState/IOConfig/IOConfig.hpp>
35 #include <opm/parser/eclipse/EclipseState/InitConfig/InitConfig.hpp>
36 #include <opm/parser/eclipse/EclipseState/Schedule/UDQ/UDQState.hpp>
37 #include <opm/common/utility/String.hpp>
39 #include <fmt/format.h>
42 #include <dune/fem/misc/mpimanager.hh>
44 #include <dune/common/parallel/mpihelper.hh>
47 namespace Opm::Properties {
49 template<
class TypeTag,
class MyTypeTag>
51 using type = UndefinedProperty;
53 template<
class TypeTag,
class MyTypeTag>
55 using type = UndefinedProperty;
57 template<
class TypeTag,
class MyTypeTag>
59 using type = UndefinedProperty;
63 template<
class TypeTag>
65 static constexpr
auto value =
"auto";
68 template<
class TypeTag>
70 static constexpr
bool value =
false;
72 template<
class TypeTag>
74 static constexpr
int value = 1;
85 template <
class TypeTag>
89 using MaterialLawManager =
typename GetProp<TypeTag, Properties::MaterialLaw>::EclMaterialLawManager;
90 using EbosSimulator = GetPropType<TypeTag, Properties::Simulator>;
91 using Grid = GetPropType<TypeTag, Properties::Grid>;
92 using GridView = GetPropType<TypeTag, Properties::GridView>;
93 using Problem = GetPropType<TypeTag, Properties::Problem>;
94 using Scalar = GetPropType<TypeTag, Properties::Scalar>;
95 using FluidSystem = GetPropType<TypeTag, Properties::FluidSystem>;
99 FlowMainEbos(
int argc,
char **argv,
bool output_cout,
bool output_files )
100 : argc_{argc}, argv_{argv},
101 output_cout_{output_cout}, output_files_{output_files}
107 static int setupParameters_(
int argc,
char** argv, Parallel::Communication comm)
109 using ParamsMeta = GetProp<TypeTag, Properties::ParameterMetaData>;
110 if (!ParamsMeta::registrationOpen()) {
121 EWOMS_REGISTER_PARAM(TypeTag, std::string, EnableDryRun,
122 "Specify if the simulation ought to be actually run, or just pretended to be");
123 EWOMS_REGISTER_PARAM(TypeTag,
int, OutputInterval,
124 "Specify the number of report steps between two consecutive writes of restart data");
125 EWOMS_REGISTER_PARAM(TypeTag,
bool, EnableLoggingFalloutWarning,
126 "Developer option to see whether logging was on non-root processors. In that case it will be appended to the *.DBG or *.PRT files");
128 Simulator::registerParameters();
131 registerAllParameters_<TypeTag>(
false);
134 EWOMS_HIDE_PARAM(TypeTag, EnableGravity);
135 EWOMS_HIDE_PARAM(TypeTag, EnableGridAdaptation);
140 EWOMS_HIDE_PARAM(TypeTag, EnableIntensiveQuantityCache);
144 EWOMS_HIDE_PARAM(TypeTag, EnableThermodynamicHints);
147 EWOMS_HIDE_PARAM(TypeTag, EndTime);
150 EWOMS_HIDE_PARAM(TypeTag, InitialTimeStepSize);
151 EWOMS_HIDE_PARAM(TypeTag, MaxTimeStepDivisions);
152 EWOMS_HIDE_PARAM(TypeTag, MaxTimeStepSize);
153 EWOMS_HIDE_PARAM(TypeTag, MinTimeStepSize);
154 EWOMS_HIDE_PARAM(TypeTag, PredeterminedTimeStepsFile);
156 EWOMS_HIDE_PARAM(TypeTag, EclMaxTimeStepSizeAfterWellEvent);
157 EWOMS_HIDE_PARAM(TypeTag, EclRestartShrinkFactor);
158 EWOMS_HIDE_PARAM(TypeTag, EclEnableTuning);
161 EWOMS_HIDE_PARAM(TypeTag, NewtonMaxError);
162 EWOMS_HIDE_PARAM(TypeTag, NewtonMaxIterations);
163 EWOMS_HIDE_PARAM(TypeTag, NewtonTolerance);
164 EWOMS_HIDE_PARAM(TypeTag, NewtonTargetIterations);
165 EWOMS_HIDE_PARAM(TypeTag, NewtonVerbose);
166 EWOMS_HIDE_PARAM(TypeTag, NewtonWriteConvergence);
167 EWOMS_HIDE_PARAM(TypeTag, EclNewtonSumTolerance);
168 EWOMS_HIDE_PARAM(TypeTag, EclNewtonSumToleranceExponent);
169 EWOMS_HIDE_PARAM(TypeTag, EclNewtonStrictIterations);
170 EWOMS_HIDE_PARAM(TypeTag, EclNewtonRelaxedVolumeFraction);
171 EWOMS_HIDE_PARAM(TypeTag, EclNewtonRelaxedTolerance);
174 EWOMS_HIDE_PARAM(TypeTag, RestartTime);
175 EWOMS_HIDE_PARAM(TypeTag, RestartWritingInterval);
178 EWOMS_HIDE_PARAM(TypeTag, VtkWriteOilFormationVolumeFactor);
179 EWOMS_HIDE_PARAM(TypeTag, VtkWriteOilSaturationPressure);
180 EWOMS_HIDE_PARAM(TypeTag, VtkWriteOilVaporizationFactor);
181 EWOMS_HIDE_PARAM(TypeTag, VtkWritePorosity);
182 EWOMS_HIDE_PARAM(TypeTag, VtkWritePotentialGradients);
183 EWOMS_HIDE_PARAM(TypeTag, VtkWritePressures);
184 EWOMS_HIDE_PARAM(TypeTag, VtkWritePrimaryVars);
185 EWOMS_HIDE_PARAM(TypeTag, VtkWritePrimaryVarsMeaning);
186 EWOMS_HIDE_PARAM(TypeTag, VtkWriteProcessRank);
187 EWOMS_HIDE_PARAM(TypeTag, VtkWriteRelativePermeabilities);
188 EWOMS_HIDE_PARAM(TypeTag, VtkWriteSaturatedGasOilVaporizationFactor);
189 EWOMS_HIDE_PARAM(TypeTag, VtkWriteSaturatedOilGasDissolutionFactor);
190 EWOMS_HIDE_PARAM(TypeTag, VtkWriteSaturationRatios);
191 EWOMS_HIDE_PARAM(TypeTag, VtkWriteSaturations);
192 EWOMS_HIDE_PARAM(TypeTag, VtkWriteTemperature);
193 EWOMS_HIDE_PARAM(TypeTag, VtkWriteViscosities);
194 EWOMS_HIDE_PARAM(TypeTag, VtkWriteWaterFormationVolumeFactor);
195 EWOMS_HIDE_PARAM(TypeTag, VtkWriteGasDissolutionFactor);
196 EWOMS_HIDE_PARAM(TypeTag, VtkWriteGasFormationVolumeFactor);
197 EWOMS_HIDE_PARAM(TypeTag, VtkWriteGasSaturationPressure);
198 EWOMS_HIDE_PARAM(TypeTag, VtkWriteIntrinsicPermeabilities);
199 EWOMS_HIDE_PARAM(TypeTag, VtkWriteEclTracerConcentration);
200 EWOMS_HIDE_PARAM(TypeTag, VtkWriteExtrusionFactor);
201 EWOMS_HIDE_PARAM(TypeTag, VtkWriteFilterVelocities);
202 EWOMS_HIDE_PARAM(TypeTag, VtkWriteDensities);
203 EWOMS_HIDE_PARAM(TypeTag, VtkWriteDofIndex);
204 EWOMS_HIDE_PARAM(TypeTag, VtkWriteMobilities);
206 EWOMS_HIDE_PARAM(TypeTag, VtkWriteAverageMolarMasses);
207 EWOMS_HIDE_PARAM(TypeTag, VtkWriteFugacities);
208 EWOMS_HIDE_PARAM(TypeTag, VtkWriteFugacityCoeffs);
209 EWOMS_HIDE_PARAM(TypeTag, VtkWriteMassFractions);
210 EWOMS_HIDE_PARAM(TypeTag, VtkWriteMolarities);
211 EWOMS_HIDE_PARAM(TypeTag, VtkWriteMoleFractions);
212 EWOMS_HIDE_PARAM(TypeTag, VtkWriteTotalMassFractions);
213 EWOMS_HIDE_PARAM(TypeTag, VtkWriteTotalMoleFractions);
215 EWOMS_END_PARAM_REGISTRATION(TypeTag);
217 int mpiRank = comm.rank();
220 int status = ::Opm::setupParameters_<TypeTag>(argc,
const_cast<const char**
>(argv),
false,
true, (mpiRank==0));
225 int unknownKeyWords = 0;
227 unknownKeyWords = Parameters::printUnused<TypeTag>(std::cerr);
229 int globalUnknownKeyWords = comm.sum(unknownKeyWords);
230 unknownKeyWords = globalUnknownKeyWords;
231 if ( unknownKeyWords )
235 std::string msg =
"Aborting simulation due to unknown "
236 "parameters. Please query \"flow --help\" for "
237 "supported command line parameters.";
238 if (OpmLog::hasBackend(
"STREAMLOG"))
243 std::cerr << msg << std::endl;
253 if (EWOMS_GET_PARAM(TypeTag,
int, PrintProperties) == 1) {
256 Properties::printValues<TypeTag>();
259 if (EWOMS_GET_PARAM(TypeTag,
int, PrintParameters) == 1) {
262 Parameters::printValues<TypeTag>();
272 static void printBanner(Parallel::Communication comm)
274 const int lineLen = 70;
276 const std::string banner =
"This is flow "+version;
277 const int bannerPreLen = (lineLen - 2 - banner.size())/2;
278 const int bannerPostLen = bannerPreLen + (lineLen - 2 - banner.size())%2;
279 std::cout <<
"**********************************************************************\n";
280 std::cout <<
"* *\n";
281 std::cout <<
"*" << std::string(bannerPreLen,
' ') << banner << std::string(bannerPostLen,
' ') <<
"*\n";
282 std::cout <<
"* *\n";
283 std::cout <<
"* Flow is a simulator for fully implicit three-phase black-oil flow, *\n";
284 std::cout <<
"* including solvent and polymer capabilities. *\n";
285 std::cout <<
"* For more information, see https://opm-project.org *\n";
286 std::cout <<
"* *\n";
287 std::cout <<
"**********************************************************************\n\n";
295 if (getenv(
"OMP_NUM_THREADS"))
296 threads = omp_get_max_threads();
298 threads = std::min(2, omp_get_max_threads());
300 const int input_threads = EWOMS_GET_PARAM(TypeTag,
int, ThreadsPerProcess);
302 if (input_threads > 0)
303 threads = std::min(input_threads, omp_get_max_threads());
306 int mpiSize = comm.size();
308 std::cout <<
"Using "<< mpiSize <<
" MPI processes with "<< threads <<
" OMP threads on each \n\n";
316 return execute_(&FlowMainEbos::runSimulator,
true);
319 int executeInitStep()
321 return execute_(&FlowMainEbos::runSimulatorInit,
false);
328 return simulator_->runStep(*simtimer_);
333 int executeStepsCleanup()
335 SimulatorReport report = simulator_->finalize();
336 runSimulatorAfterSim_(report);
337 return report.success.exit_status;
342 static void printPRTHeader(
bool output_cout)
346 const double megabyte = 1024 * 1024;
347 unsigned num_cpu = std::thread::hardware_concurrency();
349 const char* user = getlogin();
350 time_t now = std::time(0);
353 tstruct = *localtime(&now);
354 strftime(tmstr,
sizeof(tmstr),
"%d-%m-%Y at %X", &tstruct);
355 const double mem_size = getTotalSystemMemory() / megabyte;
356 std::ostringstream ss;
358 ss <<
" ######## # ###### # #\n";
359 ss <<
" # # # # # # \n";
360 ss <<
" ##### # # # # # # \n";
361 ss <<
" # # # # # # # # \n";
362 ss <<
" # ####### ###### # # \n\n";
363 ss <<
"Flow is a simulator for fully implicit three-phase black-oil flow,";
364 ss <<
" and is part of OPM.\nFor more information visit: https://opm-project.org \n\n";
365 ss <<
"Flow Version = " + version +
"\n";
366 if (uname(&arch) == 0) {
367 ss <<
"Machine name = " << arch.nodename <<
" (Number of logical cores: " << num_cpu;
368 ss <<
", Memory size: " << std::fixed << std::setprecision (2) << mem_size <<
" MB) \n";
369 ss <<
"Operating system = " << arch.sysname <<
" " << arch.machine <<
" (Kernel: " << arch.release;
370 ss <<
", " << arch.version <<
" )\n";
374 ss <<
"User = " << user << std::endl;
376 ss <<
"Simulation started on " << tmstr <<
" hrs\n";
378 ss <<
"Parameters used by Flow:\n";
379 Parameters::printValues<TypeTag>(ss);
381 OpmLog::note(ss.str());
385 EbosSimulator *getSimulatorPtr() {
386 return ebosSimulator_.get();
391 int execute_(
int (FlowMainEbos::* runOrInitFunc)(),
bool cleanup)
396 int status = setupParameters_(this->argc_, this->argv_, EclGenericVanguard::comm());
401 setupEbosSimulator();
405 int exitCode = (this->*runOrInitFunc)();
411 catch (
const std::exception& e) {
412 std::ostringstream message;
413 message <<
"Program threw an exception: " << e.what();
415 if (this->output_cout_) {
418 if (OpmLog::hasBackend(
"STREAMLOG")) {
419 OpmLog::error(message.str());
422 std::cout << message.str() <<
"\n";
426 if (this->mpi_size_ > 1)
427 MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
433 void executeCleanup_() {
435 mergeParallelLogFiles();
439 void setupParallelism()
444 auto comm = EclGenericVanguard::comm();
445 mpi_rank_ = comm.rank();
446 mpi_size_ = comm.size();
450 if (!getenv(
"OMP_NUM_THREADS"))
451 omp_set_num_threads(std::min(2, omp_get_num_procs()));
454 using ThreadManager = GetPropType<TypeTag, Properties::ThreadManager>;
455 ThreadManager::init();
460 void mergeParallelLogFiles()
463 OpmLog::removeAllBackends();
465 if (mpi_rank_ != 0 || mpi_size_ < 2 || !this->output_files_) {
469 namespace fs = ::Opm::filesystem;
470 const std::string& output_dir = eclState().getIOConfig().getOutputDir();
471 fs::path output_path(output_dir);
472 fs::path deck_filename(EWOMS_GET_PARAM(TypeTag, std::string, EclDeckFileName));
473 std::string basename;
475 std::string extension = uppercase(deck_filename.extension().string());
476 if ( extension ==
".DATA" || extension ==
"." )
478 basename = uppercase(deck_filename.stem().string());
482 basename = uppercase(deck_filename.filename().string());
484 std::for_each(fs::directory_iterator(output_path),
485 fs::directory_iterator(),
486 detail::ParallelFileMerger(output_path, basename,
487 EWOMS_GET_PARAM(TypeTag,
bool, EnableLoggingFalloutWarning)));
490 void setupEbosSimulator()
492 ebosSimulator_.reset(
new EbosSimulator(EclGenericVanguard::comm(),
false));
493 ebosSimulator_->executionTimer().start();
494 ebosSimulator_->model().applyInitialSolution();
498 const std::string& dryRunString = EWOMS_GET_PARAM(TypeTag, std::string, EnableDryRun);
499 if (dryRunString !=
"" && dryRunString !=
"auto") {
501 if (dryRunString ==
"true"
502 || dryRunString ==
"t"
503 || dryRunString ==
"1")
505 else if (dryRunString ==
"false"
506 || dryRunString ==
"f"
507 || dryRunString ==
"0")
510 throw std::invalid_argument(
"Invalid value for parameter EnableDryRun: '"
512 auto& ioConfig = eclState().getIOConfig();
513 ioConfig.overrideNOSIM(yesno);
516 catch (
const std::invalid_argument& e) {
517 std::cerr <<
"Failed to create valid EclipseState object" << std::endl;
518 std::cerr <<
"Exception caught: " << e.what() << std::endl;
523 const Deck& deck()
const
524 {
return ebosSimulator_->vanguard().deck(); }
527 {
return ebosSimulator_->vanguard().deck(); }
529 const EclipseState& eclState()
const
530 {
return ebosSimulator_->vanguard().eclState(); }
532 EclipseState& eclState()
533 {
return ebosSimulator_->vanguard().eclState(); }
535 const Schedule& schedule()
const
536 {
return ebosSimulator_->vanguard().schedule(); }
541 return runSimulatorInitOrRun_(&FlowMainEbos::runSimulatorRunCallback_);
544 int runSimulatorInit()
546 return runSimulatorInitOrRun_(&FlowMainEbos::runSimulatorInitCallback_);
551 int runSimulatorRunCallback_()
553 SimulatorReport report = simulator_->run(*simtimer_);
554 runSimulatorAfterSim_(report);
555 return report.success.exit_status;
559 int runSimulatorInitCallback_()
561 simulator_->init(*simtimer_);
566 void runSimulatorAfterSim_(SimulatorReport &report)
568 if (this->output_cout_) {
569 std::ostringstream ss;
570 ss <<
"\n\n================ End of simulation ===============\n\n";
571 ss << fmt::format(
"Number of MPI processes: {:9}\n", mpi_size_ );
573 int threads = omp_get_max_threads();
577 ss << fmt::format(
"Threads per MPI process: {:9}\n", threads);
578 report.reportFullyImplicit(ss);
579 OpmLog::info(ss.str());
580 const std::string dir = eclState().getIOConfig().getOutputDir();
581 namespace fs = ::Opm::filesystem;
582 fs::path output_dir(dir);
584 std::string filename = eclState().getIOConfig().getBaseName() +
".INFOSTEP";
585 fs::path fullpath = output_dir / filename;
586 std::ofstream os(fullpath.string());
587 report.fullReports(os);
593 int runSimulatorInitOrRun_(
int (FlowMainEbos::* initOrRunFunc)())
596 const auto& schedule = this->schedule();
597 auto& ioConfig = eclState().getIOConfig();
598 simtimer_ = std::make_unique<SimulatorTimer>();
601 const auto& initConfig = eclState().getInitConfig();
602 simtimer_->init(schedule, (
size_t)initConfig.getRestartStep());
604 if (this->output_cout_) {
605 std::ostringstream oss;
609 if (Parameters::printUnused<TypeTag>(oss)) {
610 std::cout <<
"----------------- Unrecognized parameters: -----------------\n";
611 std::cout << oss.str();
612 std::cout <<
"----------------------------------------------------------------" << std::endl;
616 if (!ioConfig.initOnly()) {
617 if (this->output_cout_) {
619 msg =
"\n\n================ Starting main simulation loop ===============\n";
623 return (this->*initOrRunFunc)();
626 if (this->output_cout_) {
627 std::cout <<
"\n\n================ Simulation turned off ===============\n" << std::flush;
642 simulator_.reset(
new Simulator(*ebosSimulator_));
645 static unsigned long long getTotalSystemMemory()
647 long pages = sysconf(_SC_PHYS_PAGES);
648 long page_size = sysconf(_SC_PAGE_SIZE);
649 return pages * page_size;
654 {
return ebosSimulator_->vanguard().grid(); }
657 std::unique_ptr<EbosSimulator> ebosSimulator_;
660 std::any parallel_information_;
661 std::unique_ptr<Simulator> simulator_;
662 std::unique_ptr<SimulatorTimer> simtimer_;
Definition: FlowMainEbos.hpp:87
int execute()
This is the main function of Flow.
Definition: FlowMainEbos.hpp:314
void createSimulator()
This is the main function of Flow.
Definition: FlowMainEbos.hpp:639
a simulator for the blackoil model
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:72
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition: BlackoilPhases.hpp:26
std::string moduleVersionName()
Return the version name of the module, for example "2015.10" (for a release branch) or "2016....
Definition: moduleVersion.cpp:29
std::string compileTimestamp()
Return a string "dd-mm-yyyy at HH::MM::SS hrs" which is the time the binary was compiled.
Definition: moduleVersion.cpp:52
std::string moduleVersion()
Return a string containing both the name and hash, if N is the name and H is the hash it will be "N (...
Definition: moduleVersion.cpp:45
Definition: FlowMainEbos.hpp:50
Definition: FlowMainEbos.hpp:58
Definition: FlowMainEbos.hpp:54