www.mooseframework.org
MultiApp.h
Go to the documentation of this file.
1 //* This file is part of the MOOSE framework
2 //* https://www.mooseframework.org
3 //*
4 //* All rights reserved, see COPYRIGHT for full restrictions
5 //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 //*
7 //* Licensed under LGPL 2.1, please see LICENSE for details
8 //* https://www.gnu.org/licenses/lgpl-2.1.html
9 
10 #pragma once
11 
12 #include "MooseObject.h"
13 #include "SetupInterface.h"
14 #include "Restartable.h"
15 #include "PerfGraphInterface.h"
16 #include "Backup.h"
17 
18 #include "libmesh/communicator.h"
19 #include "libmesh/point.h"
20 
21 #include "nlohmann/json.h"
22 
23 class UserObject;
24 class FEProblemBase;
25 class FEProblem;
26 class Executioner;
27 class MooseApp;
28 class MultiAppTransfer;
30 class Positions;
31 
32 // libMesh forward declarations
33 namespace libMesh
34 {
35 class BoundingBox;
36 namespace MeshTools
37 {
38 class BoundingBox;
39 }
40 template <typename T>
41 class NumericVector;
42 } // namespace libMesh
43 
47 {
75 };
76 
89  processor_id_type nprocs,
90  dof_id_type napps,
91  processor_id_type min_app_procs,
92  processor_id_type max_app_procs,
93  bool batch_mode = false);
94 
100 class SubAppBackups : public std::vector<std::unique_ptr<Backup>>
101 {
102 };
103 
112 class MultiApp : public MooseObject,
113  public SetupInterface,
114  public Restartable,
115  public PerfGraphInterface
116 {
117 public:
118  static InputParameters validParams();
119 
121 
122  virtual void preExecute() {}
123 
127  virtual void finalize();
128 
132  virtual void postExecute();
133 
138  void setupPositions();
139 
144  virtual void createLocalApp(const unsigned int i);
145 
149  virtual void initialSetup() override;
150 
155  virtual void preTransfer(Real dt, Real target_time);
156 
170  virtual bool solveStep(Real dt, Real target_time, bool auto_advance = true) = 0;
171 
178  virtual void incrementTStep(Real /*target_time*/) {}
179 
188  virtual void finishStep(bool /*recurse_through_multiapp_levels*/ = false) {}
189 
195  virtual void backup();
196 
204  virtual void restore(bool force = true);
205 
210  virtual bool needsRestoration() { return true; }
211 
216  virtual Executioner * getExecutioner(unsigned int app);
217 
229  virtual BoundingBox getBoundingBox(unsigned int app,
230  bool displaced_mesh,
231  const MultiAppCoordTransform * coord_transform = nullptr);
232 
237 
242  FEProblemBase & appProblemBase(unsigned int app);
243 
248  FEProblem & appProblem(unsigned int app);
249 
255  const UserObject & appUserObjectBase(unsigned int app, const std::string & name);
256 
262  Real appPostprocessorValue(unsigned int app, const std::string & name);
263 
271  virtual NumericVector<Number> & appTransferVector(unsigned int app, std::string var_name);
272 
276  unsigned int numGlobalApps() const { return _total_num_apps; }
277 
281  unsigned int numLocalApps() { return _apps.size(); }
282 
286  unsigned int firstLocalApp() { return _first_local_app; }
287 
291  bool isFirstLocalRank() const;
292 
296  bool hasApp() { return _has_an_app; }
297 
303  bool hasLocalApp(unsigned int global_app) const;
304 
309  MooseApp * localApp(unsigned int local_app);
310 
316  const Point & position(unsigned int app) const;
317 
330  virtual void resetApp(unsigned int global_app, Real time = 0.0);
331 
338  virtual void moveApp(unsigned int global_app, Point p);
339 
344  virtual void parentOutputPositionChanged();
345 
350  MPI_Comm & comm() { return _my_comm; }
351 
356  bool isRootProcessor() { return _my_rank == 0; }
357 
361  bool usingPositions() const { return _use_positions; }
362 
367  bool runningInPosition() const { return _run_in_position; }
368 
372  void addAssociatedTransfer(MultiAppTransfer & transfer);
373 
378  static void transformBoundingBox(BoundingBox & box, const MultiAppCoordTransform & transform);
379 
383  void setAppOutputFileBase();
384 
385 protected:
387  virtual std::vector<std::string> cliArgs() const
388  {
389  return std::vector<std::string>(_cli_args.begin(), _cli_args.end());
390  }
391 
395  virtual void fillPositions();
396 
401 
408  void createApp(unsigned int i, Real start_time);
409 
415  void buildComm();
416 
424  unsigned int globalAppToLocal(unsigned int global_app);
425 
427  virtual void preRunInputFile();
428 
434  virtual std::string getCommandLineArgsParamHelper(unsigned int local_app);
435 
439  void init(unsigned int num_apps, bool batch_mode = false);
440 
444  void init(unsigned int num_apps, const LocalRankConfig & config);
445 
451  void createApps();
452 
457  void keepSolutionDuringRestore(bool keep_solution_during_restore);
458 
465  void setAppOutputFileBase(unsigned int index);
466 
475  static std::string
476  getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total);
477 
480 
482  std::string _app_type;
483 
485  std::vector<Point> _positions;
487  std::vector<const Positions *> _positions_objs;
489  std::vector<unsigned int> _positions_index_offsets;
490 
493  const bool _use_positions;
494 
496  std::vector<FileName> _input_files;
497 
500 
502  std::vector<unsigned int> _npositions_inputfile;
503 
505  std::string _output_base;
506 
508  unsigned int _total_num_apps;
509 
511  unsigned int _my_num_apps;
512 
514  unsigned int _first_local_app;
515 
517  const MPI_Comm & _orig_comm;
518 
521 
523  MPI_Comm & _my_comm;
524 
527 
530 
532  std::string _node_name;
533 
535  int _my_rank;
536 
538  std::vector<std::shared_ptr<MooseApp>> _apps;
539 
541  std::vector<bool> _has_bounding_box;
542 
544  std::vector<BoundingBox> _bounding_box;
545 
548 
551 
554 
557 
560 
563 
565  std::vector<Real> _reset_times;
566 
568  std::vector<unsigned int> _reset_apps;
569 
571  std::vector<bool> _reset_happened;
572 
575 
577  std::vector<unsigned int> _move_apps;
578 
580  std::vector<Point> _move_positions;
581 
584 
587 
589  const std::vector<CLIArgString> & _cli_args;
590 
592  std::vector<std::string> _cli_args_from_file;
593 
596 
598  std::vector<std::unique_ptr<NumericVector<Real>>> _end_solutions;
599 
602 
604  std::vector<MultiAppTransfer *> _associated_transfers;
605 
607  const bool _run_in_position;
608 
611 
618 };
619 
620 void dataStore(std::ostream & stream, SubAppBackups & backups, void * context);
621 void dataLoad(std::istream & stream, SubAppBackups & backups, void * context);
void keepSolutionDuringRestore(bool keep_solution_during_restore)
Reserve the solution from the previous simulation, and it is used as an initial guess for the next ru...
Definition: MultiApp.C:812
bool hasLocalApp(unsigned int global_app) const
Whether or not the given global app number is on this processor.
Definition: MultiApp.C:990
virtual std::vector< std::string > cliArgs() const
function that provides cli_args to subapps
Definition: MultiApp.h:387
std::vector< unsigned int > _reset_apps
The apps to be reset.
Definition: MultiApp.h:568
A class for creating restricted objects.
Definition: Restartable.h:28
MultiApp(const InputParameters &parameters)
Definition: MultiApp.C:249
std::vector< const Positions * > _positions_objs
The positions of all of the apps, using the Positions system.
Definition: MultiApp.h:487
virtual void restore(bool force=true)
Restore the state of every Sub App.
Definition: MultiApp.C:737
virtual void backup()
Save off the state of every Sub App.
Definition: MultiApp.C:722
const bool _use_positions
Toggle use of "positions".
Definition: MultiApp.h:493
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
Definition: FEProblem.h:20
const std::vector< CLIArgString > & _cli_args
CommandLine arguments.
Definition: MultiApp.h:589
std::vector< unsigned int > _npositions_inputfile
Number of positions for each input file.
Definition: MultiApp.h:502
void setupPositions()
Called just after construction to allow derived classes to set _positions and create sub-apps accordi...
Definition: MultiApp.C:351
std::vector< unsigned int > _positions_index_offsets
The offsets, in case multiple Positions objects are specified.
Definition: MultiApp.h:489
Real _move_time
The time at which to move apps.
Definition: MultiApp.h:574
LocalRankConfig rankConfig(processor_id_type rank, processor_id_type nprocs, dof_id_type napps, processor_id_type min_app_procs, processor_id_type max_app_procs, bool batch_mode=false)
Returns app partitioning information relevant to the given rank for a multiapp scenario with the give...
Definition: MultiApp.C:1210
unsigned int numLocalApps()
Definition: MultiApp.h:281
Real appPostprocessorValue(unsigned int app, const std::string &name)
Get a Postprocessor value for a specified global app.
Definition: MultiApp.C:969
int _orig_rank
The mpi "rank" of this processor in the original communicator.
Definition: MultiApp.h:529
LocalRankConfig _rank_config
The app configuration resulting from calling init.
Definition: MultiApp.h:601
virtual std::string getCommandLineArgsParamHelper(unsigned int local_app)
Method to aid in getting the "cli_args" parameters.
Definition: MultiApp.C:1191
const PerfID _solve_step_timer
Timers.
Definition: MultiApp.h:613
std::vector< std::shared_ptr< MooseApp > > _apps
Pointers to each of the Apps.
Definition: MultiApp.h:538
Positions objects are under the hood Reporters.
Definition: Positions.h:19
dof_id_type first_local_app_index
The (global) index of the first local app for this rank.
Definition: MultiApp.h:64
virtual void parentOutputPositionChanged()
For apps outputting in position we need to change their output positions if their parent app moves...
Definition: MultiApp.C:1047
std::vector< Point > _move_positions
The new positions for the apps to be moved.
Definition: MultiApp.h:580
Base class for MOOSE-based applications.
Definition: MooseApp.h:73
The main MOOSE class responsible for handling user-defined parameters in almost every MOOSE system...
const PerfID _restore_timer
Definition: MultiApp.h:616
The following methods are specializations for using the libMesh::Parallel::packed_range_* routines fo...
Real _inflation
Relative bounding box inflation.
Definition: MultiApp.h:547
FEProblemBase & _fe_problem
The FEProblemBase this MultiApp is part of.
Definition: MultiApp.h:479
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
processor_id_type _min_procs_per_app
Minimum number of processors to give to each app.
Definition: MultiApp.h:556
virtual const std::string & name() const
Get the name of the class.
Definition: MooseBase.h:57
int _orig_num_procs
The number of processors in the original comm.
Definition: MultiApp.h:526
const PerfID _backup_timer
Definition: MultiApp.h:615
unsigned int PerfID
Definition: MooseTypes.h:201
virtual void finishStep(bool=false)
Calls multi-apps executioners&#39; endStep and postStep methods which creates output and advances time (n...
Definition: MultiApp.h:188
std::vector< Real > _reset_times
The times at which to reset apps.
Definition: MultiApp.h:565
virtual void preTransfer(Real dt, Real target_time)
Gets called just before transfers are done to the MultiApp (Which is just before the MultiApp is solv...
Definition: MultiApp.C:632
void createApps()
Create the provided number of apps.
Definition: MultiApp.C:362
virtual BoundingBox getBoundingBox(unsigned int app, bool displaced_mesh, const MultiAppCoordTransform *coord_transform=nullptr)
Get the BoundingBox for the mesh associated with app The bounding box will be shifted to be in the co...
Definition: MultiApp.C:862
uint8_t processor_id_type
const bool & _wait_for_first_app_init
Whether to create the first app on rank 0 while all other MPI ranks are idle.
Definition: MultiApp.h:499
virtual NumericVector< Number > & appTransferVector(unsigned int app, std::string var_name)
Get the vector to transfer to for this MultiApp.
Definition: MultiApp.C:978
unsigned int numGlobalApps() const
Definition: MultiApp.h:276
bool _keep_solution_during_restore
Flag indicates if or not restart from the latest solution.
Definition: MultiApp.h:595
Every object that can be built by the factory should be derived from this class.
Definition: MooseObject.h:33
processor_id_type _max_procs_per_app
Maximum number of processors to give to each app.
Definition: MultiApp.h:553
virtual void fillPositions()
must fill in _positions with the positions of the sub-aps
Definition: MultiApp.C:519
bool _move_happened
Whether or not the move has happened.
Definition: MultiApp.h:583
std::vector< MultiAppTransfer * > _associated_transfers
Transfers associated with this multiapp.
Definition: MultiApp.h:604
Helper class for holding Sub-app backups.
Definition: MultiApp.h:100
bool hasApp()
Whether or not this MultiApp has an app on this processor.
Definition: MultiApp.h:296
const PerfID _reset_timer
Definition: MultiApp.h:617
bool runningInPosition() const
Whether or not this MultiApp is being run in position, eg with the coordinate transform already appli...
Definition: MultiApp.h:367
std::string _output_base
The output file basename for each multiapp.
Definition: MultiApp.h:505
std::string _app_type
The type of application to build.
Definition: MultiApp.h:482
bool isRootProcessor()
Whether or not this processor is the "root" processor for the sub communicator.
Definition: MultiApp.h:356
FEProblemBase & appProblemBase(unsigned int app)
Get the FEProblemBase for the global app desired.
Definition: MultiApp.C:936
std::vector< unsigned int > _move_apps
The apps to be moved.
Definition: MultiApp.h:577
Executioners are objects that do the actual work of solving your problem.
Definition: Executioner.h:30
void buildComm()
Create an MPI communicator suitable for each app.
Definition: MultiApp.C:1288
virtual void resetApp(unsigned int global_app, Real time=0.0)
"Reset" the App corresponding to the global App number passed in.
Definition: MultiApp.C:1007
bool isFirstLocalRank() const
Definition: MultiApp.C:984
Interface for objects interacting with the PerfGraph.
unsigned int _total_num_apps
The total number of apps to simulate.
Definition: MultiApp.h:508
dof_id_type num_local_sims
The number of simulations that should/will be run locally on this rank.
Definition: MultiApp.h:49
std::vector< BoundingBox > _bounding_box
This multi-app&#39;s bounding box.
Definition: MultiApp.h:544
virtual bool solveStep(Real dt, Real target_time, bool auto_advance=true)=0
Re-solve all of the Apps.
unsigned int _my_num_apps
The number of apps this object is involved in simulating.
Definition: MultiApp.h:511
static std::string getMultiAppName(const std::string &base_name, dof_id_type index, dof_id_type total)
Helper for constructing the name of the multiapp.
Definition: MultiApp.C:1375
std::string _node_name
Node Name.
Definition: MultiApp.h:532
bool _has_an_app
Whether or not this processor as an App at all
Definition: MultiApp.h:586
libMesh::Parallel::Communicator _my_communicator
The communicator object that holds the MPI_Comm that we&#39;re going to use.
Definition: MultiApp.h:520
void createApp(unsigned int i, Real start_time)
Helper function for creating an App instance.
Definition: MultiApp.C:1055
static void transformBoundingBox(BoundingBox &box, const MultiAppCoordTransform &transform)
Transform a bounding box according to the transformations in the provided coordinate transformation o...
Definition: MultiApp.C:822
bool usingPositions() const
Whether or not this MultiApp is using positions or its own way for constructing sub-apps.
Definition: MultiApp.h:361
const PerfID _init_timer
Definition: MultiApp.h:614
unsigned int _first_local_app
The number of the first app on this processor.
Definition: MultiApp.h:514
void addAssociatedTransfer(MultiAppTransfer &transfer)
Add a transfer that is associated with this multiapp.
Definition: MultiApp.C:1354
void dataStore(std::ostream &stream, SubAppBackups &backups, void *context)
Definition: MultiApp.C:1395
virtual void preExecute()
Definition: MultiApp.h:122
unsigned int firstLocalApp()
Definition: MultiApp.h:286
DIE A HORRIBLE DEATH HERE typedef LIBMESH_DEFAULT_SCALAR_TYPE Real
MooseApp * localApp(unsigned int local_app)
Get the local MooseApp object.
Definition: MultiApp.C:1000
virtual void initialSetup() override
Method to be called in main-app initial setup for create sub-apps if using positions is false...
Definition: MultiApp.C:413
virtual bool needsRestoration()
Whether or not this MultiApp should be restored at the beginning of each Picard iteration.
Definition: MultiApp.h:210
Base class for all MultiAppTransfer objects.
MPI_Comm & comm()
Get the MPI communicator this MultiApp is operating on.
Definition: MultiApp.h:350
bool is_first_local_rank
This is true if this rank is the primary/zero rank for a (sub)app slot.
Definition: MultiApp.h:71
FEProblem & appProblem(unsigned int app)
Get the FEProblem for the global app is part of.
Definition: MultiApp.C:947
SubAppBackups & _sub_app_backups
The cached subapp backups (passed from the parent app)
Definition: MultiApp.h:610
virtual void postExecute()
Method called at the end of the simulation (after finalize).
Definition: MultiApp.C:710
bool _output_in_position
Whether or not to move the output of the MultiApp into position.
Definition: MultiApp.h:559
void dataLoad(std::istream &stream, SubAppBackups &backups, void *context)
Definition: MultiApp.C:1406
std::vector< Point > _positions
The positions of all of the apps, using input constant vectors (to be deprecated) ...
Definition: MultiApp.h:485
const InputParameters & parameters() const
Get the parameters of the object.
virtual void finalize()
Method called towards the end of the simulation to execute on final.
Definition: MultiApp.C:697
void readCommandLineArguments()
Fill command line arguments for sub apps.
Definition: MultiApp.C:422
int _my_rank
The mpi "rank" of this processor in the sub communicator.
Definition: MultiApp.h:535
dof_id_type num_local_apps
The number of (sub)apps that should/will be run locally on this rank.
Definition: MultiApp.h:58
virtual void createLocalApp(const unsigned int i)
Create the i-th local app.
Definition: MultiApp.C:406
std::vector< FileName > _input_files
The input file for each app&#39;s simulation.
Definition: MultiApp.h:496
Holds app partitioning information relevant to the a particular rank for a multiapp scenario...
Definition: MultiApp.h:46
dof_id_type first_local_sim_index
The (global) index of the first local simulation for this rank.
Definition: MultiApp.h:53
virtual void preRunInputFile()
call back executed right before app->runInputFile()
Definition: MultiApp.C:1349
Point _bounding_box_padding
Additional padding added to the bounding box, useful for 1D meshes.
Definition: MultiApp.h:550
const Real _global_time_offset
The offset time so the MultiApp local time relative to the global time.
Definition: MultiApp.h:562
void init(unsigned int num_apps, bool batch_mode=false)
Build communicators and reserve backups.
Definition: MultiApp.C:314
const MPI_Comm & _orig_comm
The original comm handle.
Definition: MultiApp.h:517
virtual void incrementTStep(Real)
Advances the multi-apps time step which is important for dt selection.
Definition: MultiApp.h:178
static InputParameters validParams()
Definition: MultiApp.C:50
std::vector< std::unique_ptr< NumericVector< Real > > > _end_solutions
The solution from the end of the previous solve, this is cloned from the Nonlinear solution during re...
Definition: MultiApp.h:598
A MultiApp represents one or more MOOSE applications that are running simultaneously.
Definition: MultiApp.h:112
void setAppOutputFileBase()
Sets all the app&#39;s output file bases.
Definition: MultiApp.C:1360
processor_id_type my_first_rank
For every rank working on a subapp, we store the first rank on each process to make the communication...
Definition: MultiApp.h:74
FEProblemBase & problemBase()
Get the FEProblemBase this MultiApp is part of.
Definition: MultiApp.h:236
MPI_Comm & _my_comm
The MPI communicator this object is going to use.
Definition: MultiApp.h:523
const bool _run_in_position
Whether to run the child apps with their meshes transformed with the coordinate transforms.
Definition: MultiApp.h:607
virtual void moveApp(unsigned int global_app, Point p)
Move the global_app to Point p.
Definition: MultiApp.C:1028
virtual Executioner * getExecutioner(unsigned int app)
Definition: MultiApp.C:688
Base class for user-specific data.
Definition: UserObject.h:39
const Point & position(unsigned int app) const
The physical position of a global App number.
Definition: MultiApp.C:1384
std::vector< std::string > _cli_args_from_file
CommandLine arguments from files.
Definition: MultiApp.h:592
const UserObject & appUserObjectBase(unsigned int app, const std::string &name)
Get a UserObject base for a specific global app.
Definition: MultiApp.C:960
uint8_t dof_id_type
unsigned int globalAppToLocal(unsigned int global_app)
Map a global App number to the local number.
Definition: MultiApp.C:1330
This class contains transformation information that only exists in a context in which there are multi...
std::vector< bool > _reset_happened
Whether or not apps have been reset at each time.
Definition: MultiApp.h:571
std::vector< bool > _has_bounding_box
Flag if this multi-app computed its bounding box (valid only for non-displaced meshes) ...
Definition: MultiApp.h:541