www.mooseframework.org
MultiApp.C
Go to the documentation of this file.
1 //* This file is part of the MOOSE framework
2 //* https://www.mooseframework.org
3 //*
4 //* All rights reserved, see COPYRIGHT for full restrictions
5 //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 //*
7 //* Licensed under LGPL 2.1, please see LICENSE for details
8 //* https://www.gnu.org/licenses/lgpl-2.1.html
9 
10 // MOOSE includes
11 #include "MultiApp.h"
12 
13 #include "AppFactory.h"
14 #include "AuxiliarySystem.h"
15 #include "DisplacedProblem.h"
16 #include "Console.h"
17 #include "Executioner.h"
18 #include "FEProblem.h"
19 #include "MooseMesh.h"
20 #include "MooseUtils.h"
21 #include "OutputWarehouse.h"
22 #include "SetupInterface.h"
23 #include "UserObject.h"
24 #include "CommandLine.h"
25 #include "Conversion.h"
26 #include "NonlinearSystemBase.h"
27 #include "DelimitedFileReader.h"
28 #include "MooseAppCoordTransform.h"
29 #include "MultiAppTransfer.h"
30 #include "Positions.h"
31 #include "Transient.h"
32 #include "Backup.h"
33 #include "Parser.h"
34 
35 #include "libmesh/mesh_tools.h"
36 #include "libmesh/numeric_vector.h"
37 
38 // C++ includes
39 #include <fstream>
40 #include <iomanip>
41 #include <iterator>
42 #include <algorithm>
43 
44 // Call to "uname"
45 #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
46 #include <sys/utsname.h>
47 #endif
48 
51 {
53  params += SetupInterface::validParams();
54 
55  params.addParam<bool>("use_displaced_mesh",
56  false,
57  "Whether or not this object should use the "
58  "displaced mesh for computation. Note that "
59  "in the case this is true but no "
60  "displacements are provided in the Mesh block "
61  "the undisplaced mesh will still be used.");
62 
63  std::ostringstream app_types_strings;
64  for (const auto & name_bi_pair : AppFactory::instance().registeredObjects())
65  app_types_strings << name_bi_pair.first << " ";
66  MooseEnum app_types_options(app_types_strings.str(), "", true);
67 
68  // Dynamic loading
69  params.addParam<MooseEnum>("app_type",
70  app_types_options,
71  "The type of application to build (applications not "
72  "registered can be loaded with dynamic libraries. Parent "
73  "application type will be used if not provided.");
74  params.addParam<std::string>("library_path",
75  "",
76  "Path to search for dynamic libraries (please "
77  "avoid committing absolute paths in addition to "
78  "MOOSE_LIBRARY_PATH)");
79  params.addParam<std::string>(
80  "library_name",
81  "",
82  "The file name of the library (*.la file) that will be dynamically loaded.");
83  params.addParam<bool>("library_load_dependencies",
84  false,
85  "Tells MOOSE to manually load library dependencies. This should not be "
86  "necessary and is here for debugging/troubleshooting.");
87 
88  // Subapp positions
89  params.addParam<std::vector<Point>>(
90  "positions",
91  "The positions of the App locations. Each set of 3 values will represent a "
92  "Point. This and 'positions_file' cannot be both supplied. If this and "
93  "'positions_file'/'_objects' are not supplied, a single position (0,0,0) will be used");
94  params.addParam<std::vector<FileName>>("positions_file",
95  "Filename(s) that should be looked in for positions. Each"
96  " set of 3 values in that file will represent a Point. "
97  "This and 'positions(_objects)' cannot be both supplied");
98  params.addParam<std::vector<PositionsName>>("positions_objects",
99  "The name of a Positions object that will contain "
100  "the locations of the sub-apps created. This and "
101  "'positions(_file)' cannot be both supplied");
102  params.addParam<bool>(
103  "output_in_position",
104  false,
105  "If true this will cause the output from the MultiApp to be 'moved' by its position vector");
106  params.addParam<bool>(
107  "run_in_position",
108  false,
109  "If true this will cause the mesh from the MultiApp to be 'moved' by its position vector");
110 
111  params.addRequiredParam<std::vector<FileName>>(
112  "input_files",
113  "The input file for each App. If this parameter only contains one input file "
114  "it will be used for all of the Apps. When using 'positions_from_file' it is "
115  "also admissable to provide one input_file per file.");
116  params.addParam<Real>("bounding_box_inflation",
117  0.01,
118  "Relative amount to 'inflate' the bounding box of this MultiApp.");
119  params.addParam<Point>("bounding_box_padding",
120  RealVectorValue(),
121  "Additional padding added to the dimensions of the bounding box. The "
122  "values are added to the x, y and z dimension respectively.");
123 
124  params.addPrivateParam<MPI_Comm>("_mpi_comm");
125 
126  // Set the default execution time
127  params.set<ExecFlagEnum>("execute_on", true) = EXEC_TIMESTEP_BEGIN;
128 
129  params.addParam<processor_id_type>("max_procs_per_app",
131  "Maximum number of processors to give to each App in this "
132  "MultiApp. Useful for restricting small solves to just a few "
133  "procs so they don't get spread out");
134  params.addParam<processor_id_type>("min_procs_per_app",
135  1,
136  "Minimum number of processors to give to each App in this "
137  "MultiApp. Useful for larger, distributed mesh solves.");
138  params.addParam<bool>(
139  "wait_for_first_app_init",
140  false,
141  "Create the first sub-application on rank 0, then MPI_Barrier before "
142  "creating the next N-1 apps (on all ranks). "
143  "This is only needed if your sub-application needs to perform some setup "
144  "actions in quiet, without other sub-applications working at the same time.");
145 
146  params.addParam<Real>("global_time_offset",
147  0,
148  "The time offset relative to the parent application for the purpose of "
149  "starting a subapp at a different time from the parent application. The "
150  "global time will be ahead by the offset specified here.");
151 
152  // Resetting subapps
153  params.addParam<std::vector<Real>>(
154  "reset_time",
155  {},
156  "The time(s) at which to reset Apps given by the 'reset_apps' parameter. "
157  "Resetting an App means that it is destroyed and recreated, possibly "
158  "modeling the insertion of 'new' material for that app.");
159  params.addParam<std::vector<unsigned int>>(
160  "reset_apps",
161  {},
162  "The Apps that will be reset when 'reset_time' is hit. These are the App "
163  "'numbers' starting with 0 corresponding to the order of the App positions. "
164  "Resetting an App means that it is destroyed and recreated, possibly modeling "
165  "the insertion of 'new' material for that app.");
166 
167  // Moving subapps
168  params.addParam<Real>(
169  "move_time",
171  "The time at which Apps designated by move_apps are moved to move_positions.");
172 
173  params.addParam<std::vector<unsigned int>>(
174  "move_apps",
175  {},
176  "Apps, designated by their 'numbers' starting with 0 corresponding to the order "
177  "of the App positions, to be moved at move_time to move_positions");
178  params.addParam<std::vector<Point>>(
179  "move_positions", {}, "The positions corresponding to each move_app.");
180 
181  params.addParam<std::vector<CLIArgString>>(
182  "cli_args",
183  {},
184  "Additional command line arguments to pass to the sub apps. If one set is provided the "
185  "arguments are applied to all, otherwise there must be a set for each sub app.");
186 
187  params.addParam<std::vector<FileName>>(
188  "cli_args_files",
189  "File names that should be looked in for additional command line arguments "
190  "to pass to the sub apps. Each line of a file is set to each sub app. If only "
191  "one line is provided, it will be applied to all sub apps.");
192 
193  // Fixed point iterations
194  params.addRangeCheckedParam<Real>("relaxation_factor",
195  1.0,
196  "relaxation_factor>0 & relaxation_factor<2",
197  "Fraction of newly computed value to keep."
198  "Set between 0 and 2.");
199  params.addDeprecatedParam<std::vector<std::string>>(
200  "relaxed_variables",
201  {},
202  "Use transformed_variables.",
203  "List of subapp variables to relax during Multiapp coupling iterations");
204  params.addParam<std::vector<std::string>>(
205  "transformed_variables",
206  {},
207  "List of subapp variables to use coupling algorithm on during Multiapp coupling iterations");
208  params.addParam<std::vector<PostprocessorName>>(
209  "transformed_postprocessors",
210  {},
211  "List of subapp postprocessors to use coupling "
212  "algorithm on during Multiapp coupling iterations");
213  params.addParam<bool>("keep_solution_during_restore",
214  false,
215  "This is useful when doing MultiApp coupling iterations. It takes the "
216  "final solution from the previous coupling iteration"
217  "and re-uses it as the initial guess "
218  "for the next coupling iteration");
219 
220  params.addDeprecatedParam<bool>("clone_master_mesh",
221  false,
222  "True to clone parent app mesh and use it for this MultiApp.",
223  "clone_master_mesh is deprecated, use clone_parent_mesh instead");
224  params.addParam<bool>(
225  "clone_parent_mesh", false, "True to clone parent app mesh and use it for this MultiApp.");
226 
227  params.addPrivateParam<std::shared_ptr<CommandLine>>("_command_line");
228  params.addPrivateParam<bool>("use_positions", true);
229  params.declareControllable("enable");
230  params.declareControllable("cli_args", {EXEC_PRE_MULTIAPP_SETUP});
231  params.registerBase("MultiApp");
232 
233  params.addParamNamesToGroup("use_displaced_mesh wait_for_first_app_init", "Advanced");
234  params.addParamNamesToGroup("positions positions_file positions_objects run_in_position "
235  "output_in_position",
236  "Positions / transformations of the MultiApp frame of reference");
237  params.addParamNamesToGroup("min_procs_per_app max_procs_per_app", "Parallelism");
238  params.addParamNamesToGroup("reset_time reset_apps", "Reset MultiApp");
239  params.addParamNamesToGroup("move_time move_apps move_positions", "Timed move of MultiApps");
240  params.addParamNamesToGroup("relaxation_factor transformed_variables transformed_postprocessors "
241  "keep_solution_during_restore",
242  "Fixed point acceleration of MultiApp quantities");
243  params.addParamNamesToGroup("library_name library_path library_load_dependencies",
244  "Dynamic loading");
245  params.addParamNamesToGroup("cli_args cli_args_files", "Passing command line argument");
246  return params;
247 }
248 
250  : MooseObject(parameters),
251  SetupInterface(this),
252  Restartable(this, "MultiApps"),
253  PerfGraphInterface(this, std::string("MultiApp::") + _name),
254  _fe_problem(*getCheckedPointerParam<FEProblemBase *>("_fe_problem_base")),
255  _app_type(isParamValid("app_type") ? std::string(getParam<MooseEnum>("app_type"))
256  : _fe_problem.getMooseApp().type()),
257  _use_positions(getParam<bool>("use_positions")),
258  _input_files(getParam<std::vector<FileName>>("input_files")),
259  _wait_for_first_app_init(getParam<bool>("wait_for_first_app_init")),
260  _total_num_apps(0),
261  _my_num_apps(0),
262  _first_local_app(0),
263  _orig_comm(_communicator.get()),
264  _my_communicator(),
265  _my_comm(_my_communicator.get()),
266  _my_rank(0),
267  _inflation(getParam<Real>("bounding_box_inflation")),
268  _bounding_box_padding(getParam<Point>("bounding_box_padding")),
269  _max_procs_per_app(getParam<processor_id_type>("max_procs_per_app")),
270  _min_procs_per_app(getParam<processor_id_type>("min_procs_per_app")),
271  _output_in_position(getParam<bool>("output_in_position")),
272  _global_time_offset(getParam<Real>("global_time_offset")),
273  _reset_times(getParam<std::vector<Real>>("reset_time")),
274  _reset_apps(getParam<std::vector<unsigned int>>("reset_apps")),
275  _reset_happened(false),
276  _move_time(getParam<Real>("move_time")),
277  _move_apps(getParam<std::vector<unsigned int>>("move_apps")),
278  _move_positions(getParam<std::vector<Point>>("move_positions")),
279  _move_happened(false),
280  _has_an_app(true),
281  _cli_args(getParam<std::vector<CLIArgString>>("cli_args")),
282  _keep_solution_during_restore(getParam<bool>("keep_solution_during_restore")),
283  _run_in_position(getParam<bool>("run_in_position")),
284  _sub_app_backups(declareRestartableDataWithContext<SubAppBackups>("sub_app_backups", this)),
285  _solve_step_timer(registerTimedSection("solveStep", 3, "Executing MultiApps", false)),
286  _init_timer(registerTimedSection("init", 3, "Initializing MultiApp")),
287  _backup_timer(registerTimedSection("backup", 3, "Backing Up MultiApp")),
288  _restore_timer(registerTimedSection("restore", 3, "Restoring MultiApp")),
289  _reset_timer(registerTimedSection("resetApp", 3, "Resetting MultiApp"))
290 {
291  if (parameters.isParamSetByUser("cli_args") && parameters.isParamValid("cli_args") &&
292  parameters.isParamValid("cli_args_files"))
293  paramError("cli_args",
294  "'cli_args' and 'cli_args_files' cannot be specified simultaneously in MultiApp ");
295 
296  if (!_use_positions && (isParamValid("positions") || isParamValid("positions_file") ||
297  isParamValid("positions_objects")))
298  paramError("use_positions",
299  "This MultiApps has been set to not use positions, "
300  "but a 'positions' parameter has been set.");
301 
302  if ((_reset_apps.size() > 0 && _reset_times.size() == 0) ||
303  (_reset_apps.size() == 0 && _reset_times.size() > 0))
304  mooseError("reset_time and reset_apps may only be specified together");
305 
306  // Check that the reset times are sorted by the user
307  auto sorted_times = _reset_times;
308  std::sort(sorted_times.begin(), sorted_times.end());
309  if (_reset_times.size() && _reset_times != sorted_times)
310  paramError("reset_time", "List of reset times must be sorted in increasing order");
311 }
312 
313 void
314 MultiApp::init(unsigned int num_apps, bool batch_mode)
315 {
316  auto config = rankConfig(
317  processor_id(), n_processors(), num_apps, _min_procs_per_app, _max_procs_per_app, batch_mode);
318  init(num_apps, config);
319 }
320 
321 void
322 MultiApp::init(unsigned int num_apps, const LocalRankConfig & config)
323 {
324  TIME_SECTION(_init_timer);
325 
326  _total_num_apps = num_apps;
327  _rank_config = config;
328  buildComm();
330 
331  _has_bounding_box.resize(_my_num_apps, false);
332  _reset_happened.resize(_reset_times.size(), false);
333  _bounding_box.resize(_my_num_apps);
334 
335  if ((_cli_args.size() > 1) && (_total_num_apps != _cli_args.size()))
336  paramError("cli_args",
337  "The number of items supplied must be 1 or equal to the number of sub apps.");
338 
339  // if cliArgs() != _cli_args, then cliArgs() was overridden and we need to check it
340  auto cla = cliArgs();
341  if (cla != std::vector<std::string>(_cli_args.begin(), _cli_args.end()))
342  {
343  if ((cla.size() > 1) && (_total_num_apps != cla.size()))
344  mooseError("The number of items supplied as command line argument to subapps must be 1 or "
345  "equal to the number of sub apps. Note: you use a multiapp that provides its own "
346  "command line parameters so the error is not in cli_args");
347  }
348 }
349 
350 void
352 {
353  if (_use_positions)
354  {
355  fillPositions();
356  init(_positions.size());
357  createApps();
358  }
359 }
360 
361 void
363 {
364  if (!_has_an_app)
365  return;
366 
367  TIME_SECTION("createApps", 2, "Instantiating Sub-Apps", false);
368 
369  // Read commandLine arguments that will be used when creating apps
371 
373 
374  _apps.resize(_my_num_apps);
375 
376  // If the user provided an unregistered app type, see if we can load it dynamically
377  if (!AppFactory::instance().isRegistered(_app_type))
379  getParam<std::string>("library_path"),
380  getParam<std::string>("library_name"),
381  getParam<bool>("library_load_dependencies"));
382 
383  bool rank_did_quiet_init = false;
384  unsigned int local_app = libMesh::invalid_uint;
386  {
387  if (hasLocalApp(0))
388  {
389  rank_did_quiet_init = true;
390  local_app = globalAppToLocal(0);
391  createLocalApp(local_app);
392  }
393 
394  MPI_Barrier(_orig_comm);
395  }
396 
397  for (unsigned int i = 0; i < _my_num_apps; i++)
398  {
399  if (rank_did_quiet_init && i == local_app)
400  continue;
401  createLocalApp(i);
402  }
403 }
404 
405 void
406 MultiApp::createLocalApp(const unsigned int i)
407 {
409  _app.builder().hitCLIFilter(_apps[i]->name(), _app.commandLine()->getArguments());
410 }
411 
412 void
414 {
415  if (!_use_positions)
416  // if not using positions, we create the sub-apps in initialSetup instead of right after
417  // construction of MultiApp
418  createApps();
419 }
420 
421 void
423 {
424  if (isParamValid("cli_args_files"))
425  {
426  _cli_args_from_file.clear();
427 
428  std::vector<FileName> cli_args_files = getParam<std::vector<FileName>>("cli_args_files");
429  std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
430 
431  // If we use parameter "cli_args_files", at least one file should be provided
432  if (!cli_args_files.size())
433  paramError("cli_args_files", "You need to provide at least one commandLine argument file ");
434 
435  // If we multiple input files, then we need to check if the number of input files
436  // match with the number of argument files
437  if (cli_args_files.size() != 1 && cli_args_files.size() != input_files.size())
438  paramError("cli_args_files",
439  "The number of commandLine argument files ",
440  cli_args_files.size(),
441  " for MultiApp ",
442  name(),
443  " must either be only one or match the number of input files ",
444  input_files.size());
445 
446  // Go through all argument files
447  std::vector<std::string> cli_args;
448  for (unsigned int p_file_it = 0; p_file_it < cli_args_files.size(); p_file_it++)
449  {
450  std::string cli_args_file = cli_args_files[p_file_it];
451  // Clear up
452  cli_args.clear();
453  // Read the file on the root processor then broadcast it
454  if (processor_id() == 0)
455  {
456  MooseUtils::checkFileReadable(cli_args_file);
457 
458  std::ifstream is(cli_args_file.c_str());
459  std::copy(std::istream_iterator<std::string>(is),
460  std::istream_iterator<std::string>(),
461  std::back_inserter(cli_args));
462 
463  // We do not allow empty files
464  if (!cli_args.size())
465  paramError("cli_args_files",
466  "There is no commandLine argument in the commandLine argument file ",
467  cli_args_file);
468 
469  // If we have position files, we need to
470  // make sure the number of commandLine argument strings
471  // match with the number of positions
472  if (_npositions_inputfile.size())
473  {
474  auto num_positions = _npositions_inputfile[p_file_it];
475  // Check if the number of commandLine argument strings equal to
476  // the number of positions
477  if (cli_args.size() == 1)
478  for (MooseIndex(num_positions) num = 0; num < num_positions; num++)
479  _cli_args_from_file.push_back(cli_args.front());
480  else if (cli_args.size() == num_positions)
481  for (auto && cli_arg : cli_args)
482  _cli_args_from_file.push_back(cli_arg);
483  else if (cli_args.size() != num_positions)
484  paramError("cli_args_files",
485  "The number of commandLine argument strings ",
486  cli_args.size(),
487  " in the file ",
488  cli_args_file,
489  " must either be only one or match the number of positions ",
490  num_positions);
491  }
492  else
493  {
494  // If we do not have position files, we will check if the number of
495  // commandLine argument strings match with the total number of subapps
496  for (auto && cli_arg : cli_args)
497  _cli_args_from_file.push_back(cli_arg);
498  }
499  }
500  }
501 
502  // Broad cast all arguments to everyone
504  }
505 
506  if (_cli_args_from_file.size() && _cli_args_from_file.size() != 1 &&
508  mooseError(" The number of commandLine argument strings ",
509  _cli_args_from_file.size(),
510  " must either be only one or match the total "
511  "number of sub apps ",
513 
514  if (_cli_args_from_file.size() && cliArgs().size())
515  mooseError("Cannot set commandLine arguments from both input_file and external files");
516 }
517 
518 void
520 {
521  if (_move_apps.size() != _move_positions.size())
522  mooseError("The number of apps to move and the positions to move them to must be the same for "
523  "MultiApp ",
524  _name);
525 
526  if (isParamValid("positions") + isParamValid("positions_file") +
527  isParamValid("positions_objects") >
528  1)
529  mooseError("Only one 'positions' parameter may be specified");
530 
531  if (isParamValid("positions"))
532  {
533  _positions = getParam<std::vector<Point>>("positions");
534 
535  if (_positions.size() < _input_files.size())
536  mooseError("Not enough positions for the number of input files provided in MultiApp ",
537  name());
538  }
539  else if (isParamValid("positions_file"))
540  {
541  std::vector<FileName> positions_files = getParam<std::vector<FileName>>("positions_file");
542  std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
543 
544  if (input_files.size() != 1 && positions_files.size() != input_files.size())
545  mooseError("Number of input_files for MultiApp ",
546  name(),
547  " must either be only one or match the number of positions_file files");
548 
549  // Clear out the _input_files because we're going to rebuild it
550  if (input_files.size() != 1)
551  _input_files.clear();
552 
553  for (unsigned int p_file_it = 0; p_file_it < positions_files.size(); p_file_it++)
554  {
555  std::string positions_file = positions_files[p_file_it];
556  MooseUtils::DelimitedFileReader file(positions_file, &_communicator);
558  file.read();
559 
560  const std::vector<Point> & data = file.getDataAsPoints();
561  for (const auto & d : data)
562  _positions.push_back(d);
563 
564  // Save the number of positions for this input file
565  _npositions_inputfile.push_back(data.size());
566 
567  for (unsigned int i = 0; i < data.size(); ++i)
568  if (input_files.size() != 1)
569  _input_files.push_back(input_files[p_file_it]);
570  }
571  }
572  else if (isParamValid("positions_objects"))
573  {
574  const auto & positions_param_objs = getParam<std::vector<PositionsName>>("positions_objects");
575  const auto & input_files = getParam<std::vector<FileName>>("input_files");
576 
577  if (input_files.size() != 1 && positions_param_objs.size() != input_files.size())
578  mooseError("Number of input_files for MultiApp ",
579  name(),
580  " must either be only one or match the number of positions_objects specified");
581 
582  // Clear out the _input_files because we're going to rebuild it
583  if (input_files.size() != 1)
584  _input_files.clear();
585 
586  // Keeps track of where each positions object start in terms of subapp numbers
587  unsigned int offset = 0;
588 
589  for (const auto p_obj_it : index_range(positions_param_objs))
590  {
591  const std::string & positions_name = positions_param_objs[p_obj_it];
592  auto positions_obj = &_fe_problem.getPositionsObject(positions_name);
593 
594  const auto & data = positions_obj->getPositions(true);
595 
596  // Append all positions from this object
597  for (const auto & d : data)
598  _positions.push_back(d);
599 
600  // Save the number of positions for this input file
601  _npositions_inputfile.push_back(data.size());
602 
603  if (!positions_obj)
604  paramError("positions_objects",
605  "'" + positions_name + "' is not of the expected type. Should be a Positions");
606 
607  // Keep track of which positions is tied to what subapp
608  for (unsigned int i = 0; i < data.size(); ++i)
609  {
610  if (input_files.size() != 1)
611  _input_files.push_back(input_files[p_obj_it]);
612  _positions_objs.push_back(positions_obj);
613  _positions_index_offsets.push_back(offset);
614  }
615  offset += data.size();
616  }
617  }
618  else
619  {
620  _positions = {Point()};
621 
622  if (_positions.size() < _input_files.size())
623  mooseError("Not enough positions for the number of input files provided in MultiApp ",
624  name());
625  }
626 
627  mooseAssert(_input_files.size() == 1 || _positions.size() == _input_files.size(),
628  "Number of positions and input files are not the same!");
629 }
630 
631 void
632 MultiApp::preTransfer(Real /*dt*/, Real target_time)
633 {
634  // Get a transient executioner to get a user-set tolerance
635  Real timestep_tol = 1e-13;
636  if (dynamic_cast<Transient *>(_fe_problem.getMooseApp().getExecutioner()))
637  timestep_tol =
638  dynamic_cast<Transient *>(_fe_problem.getMooseApp().getExecutioner())->timestepTol();
639 
640  // First, see if any Apps need to be reset
641  for (unsigned int i = 0; i < _reset_times.size(); i++)
642  {
643  if (!_reset_happened[i] && (target_time + timestep_tol >= _reset_times[i]))
644  {
645  _reset_happened[i] = true;
646  if (_reset_apps.size() > 0)
647  for (auto & app : _reset_apps)
648  resetApp(app);
649 
650  // If we reset an application, then we delete the old objects, including the coordinate
651  // transformation classes. Consequently we need to reset the coordinate transformation classes
652  // in the associated transfer classes
653  for (auto * const transfer : _associated_transfers)
654  transfer->getAppInfo();
655 
656  // Similarly we need to transform the mesh again
657  if (_run_in_position)
658  for (const auto i : make_range(_my_num_apps))
659  {
660  auto app_ptr = _apps[i];
661  if (usingPositions())
662  app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
663  app_ptr->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
664  else
665  app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
666  app_ptr->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
667  }
668 
669  // If the time step covers multiple reset times, set them all as having 'happened'
670  for (unsigned int j = i; j < _reset_times.size(); j++)
671  if (target_time + timestep_tol >= _reset_times[j])
672  _reset_happened[j] = true;
673 
674  break;
675  }
676  }
677 
678  // Now move any apps that should be moved
679  if (_use_positions && !_move_happened && target_time + timestep_tol >= _move_time)
680  {
681  _move_happened = true;
682  for (unsigned int i = 0; i < _move_apps.size(); i++)
684  }
685 }
686 
687 Executioner *
688 MultiApp::getExecutioner(unsigned int app)
689 {
690  if (!_has_an_app)
691  mooseError("No app for ", name(), " on processor ", _orig_rank);
692 
693  return _apps[globalAppToLocal(app)]->getExecutioner();
694 }
695 
696 void
698 {
699  for (const auto & app_ptr : _apps)
700  {
701  auto * executioner = app_ptr->getExecutioner();
702  mooseAssert(executioner, "Executioner is nullptr");
703 
704  executioner->feProblem().execute(EXEC_FINAL);
705  executioner->feProblem().outputStep(EXEC_FINAL);
706  }
707 }
708 
709 void
711 {
712  for (const auto & app_ptr : _apps)
713  {
714  auto * executioner = app_ptr->getExecutioner();
715  mooseAssert(executioner, "Executioner is nullptr");
716 
717  executioner->postExecute();
718  }
719 }
720 
721 void
723 {
724  TIME_SECTION(_backup_timer);
725 
727  _console << "Backed up MultiApp ... ";
728 
729  for (unsigned int i = 0; i < _my_num_apps; i++)
730  _sub_app_backups[i] = _apps[i]->backup();
731 
733  _console << name() << std::endl;
734 }
735 
736 void
737 MultiApp::restore(bool force)
738 {
739  TIME_SECTION(_restore_timer);
740 
741  if (force || needsRestoration())
742  {
743  // Must be restarting / recovering from main app so hold off on restoring
744  // Instead - the restore will happen in sub-apps' initialSetup()
745  // Note that _backups was already populated by dataLoad() in the main app
747  return;
748 
749  // We temporarily copy and store solutions for all subapps
751  {
753 
754  for (unsigned int i = 0; i < _my_num_apps; i++)
755  {
756  _end_solutions[i] = _apps[i]
757  ->getExecutioner()
758  ->feProblem()
759  .getNonlinearSystemBase(/*nl_sys=*/0)
760  .solution()
761  .clone();
762  auto & sub_multiapps =
763  _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects();
764 
765  // multiapps of each subapp should do the same things
766  // It is implemented recursively
767  for (auto & multi_app : sub_multiapps)
768  multi_app->keepSolutionDuringRestore(_keep_solution_during_restore);
769  }
770  }
771 
773  _console << "Restoring MultiApp ... ";
774 
775  for (unsigned int i = 0; i < _my_num_apps; i++)
776  {
777  _apps[i]->restore(std::move(_sub_app_backups[i]), false);
778  _sub_app_backups[i] = _apps[i]->finalizeRestore();
779  mooseAssert(_sub_app_backups[i], "Should have a backup");
780  }
781 
783  _console << name() << std::endl;
784 
785  // Now copy the latest solutions back for each subapp
787  {
788  for (unsigned int i = 0; i < _my_num_apps; i++)
789  {
790  _apps[i]->getExecutioner()->feProblem().getNonlinearSystemBase(/*nl_sys=*/0).solution() =
791  *_end_solutions[i];
792 
793  // We need to synchronize solution so that local_solution has the right values
794  _apps[i]->getExecutioner()->feProblem().getNonlinearSystemBase(/*nl_sys=*/0).update();
795  }
796 
797  _end_solutions.clear();
798  }
799  }
800  else
801  {
802  for (unsigned int i = 0; i < _my_num_apps; i++)
803  {
804  for (auto & sub_app :
805  _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects())
806  sub_app->restore(false);
807  }
808  }
809 }
810 
811 void
812 MultiApp::keepSolutionDuringRestore(bool keep_solution_during_restore)
813 {
814  if (_pars.isParamSetByUser("keep_solution_during_restore"))
815  paramError("keep_solution_during_restore",
816  "This parameter should only be provided in parent app");
817 
818  _keep_solution_during_restore = keep_solution_during_restore;
819 }
820 
821 void
822 MultiApp::transformBoundingBox(BoundingBox & box, const MultiAppCoordTransform & transform)
823 {
824  const Real min_x = box.first(0);
825  const Real max_x = box.second(0);
826  const Real min_y = box.first(1);
827  const Real max_y = box.second(1);
828  const Real min_z = box.first(2);
829  const Real max_z = box.second(2);
830 
831  std::array<Point, 8> box_corners = {{Point(min_x, min_y, min_z),
832  Point(max_x, min_y, min_z),
833  Point(min_x, max_y, min_z),
834  Point(max_x, max_y, min_z),
835  Point(min_x, min_y, max_z),
836  Point(max_x, min_y, max_z),
837  Point(min_x, max_y, max_z),
838  Point(max_x, max_y, max_z)}};
839 
840  // transform each corner
841  for (auto & corner : box_corners)
842  corner = transform(corner);
843 
844  // Create new bounding box
845  Point new_box_min = box_corners[0];
846  Point new_box_max = new_box_min;
847  for (const auto p : make_range(1, 8))
848  for (const auto d : make_range(Moose::dim))
849  {
850  const Point & pt = box_corners[p];
851  if (new_box_min(d) > pt(d))
852  new_box_min(d) = pt(d);
853 
854  if (new_box_max(d) < pt(d))
855  new_box_max(d) = pt(d);
856  }
857  box.first = new_box_min;
858  box.second = new_box_max;
859 }
860 
861 BoundingBox
862 MultiApp::getBoundingBox(unsigned int app,
863  bool displaced_mesh,
864  const MultiAppCoordTransform * const coord_transform)
865 {
866  if (!_has_an_app)
867  mooseError("No app for ", name(), " on processor ", _orig_rank);
868 
869  unsigned int local_app = globalAppToLocal(app);
870  FEProblemBase & fe_problem_base = _apps[local_app]->getExecutioner()->feProblem();
871  MooseMesh & mesh = (displaced_mesh && fe_problem_base.getDisplacedProblem().get() != NULL)
872  ? fe_problem_base.getDisplacedProblem()->mesh()
873  : fe_problem_base.mesh();
874 
875  {
877  if (displaced_mesh)
878  _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
879  else
880  {
881  if (!_has_bounding_box[local_app])
882  {
883  _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
884  _has_bounding_box[local_app] = true;
885  }
886  }
887  }
888  BoundingBox bbox = _bounding_box[local_app];
889 
890  Point min = bbox.min();
892  Point max = bbox.max();
894 
895  Point inflation_amount = (max - min) * _inflation;
896 
897  Point inflated_min = min - inflation_amount;
898  Point inflated_max = max + inflation_amount;
899 
900  Point shifted_min = inflated_min;
901  Point shifted_max = inflated_max;
902 
903  if ((!coord_transform || coord_transform->skipCoordinateCollapsing()) &&
904  fe_problem_base.getCoordSystem(*(mesh.meshSubdomains().begin())) == Moose::COORD_RZ)
905  {
906  // If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
907  // FIXME: Assuming all subdomains are the same coordinate system type!
908  shifted_min(0) = -inflated_max(0);
909  shifted_min(1) = inflated_min(1);
910  shifted_min(2) = -inflated_max(0);
911 
912  shifted_max(0) = inflated_max(0);
913  shifted_max(1) = inflated_max(1);
914  shifted_max(2) = inflated_max(0);
915  }
916 
917  if (coord_transform)
918  {
919  BoundingBox transformed_bbox(shifted_min, shifted_max);
920  transformBoundingBox(transformed_bbox, *coord_transform);
921  return transformed_bbox;
922  }
923  else
924  {
925  // This is where the app is located. We need to shift by this amount.
926  Point p = position(app);
927 
928  // Shift them to the position they're supposed to be
929  shifted_min += p;
930  shifted_max += p;
931  return BoundingBox(shifted_min, shifted_max);
932  }
933 }
934 
936 MultiApp::appProblemBase(unsigned int app)
937 {
938  if (!_has_an_app)
939  mooseError("No app for ", name(), " on processor ", _orig_rank);
940 
941  unsigned int local_app = globalAppToLocal(app);
942 
943  return _apps[local_app]->getExecutioner()->feProblem();
944 }
945 
946 FEProblem &
947 MultiApp::appProblem(unsigned int app)
948 {
950  "MultiApp::appProblem() is deprecated, call MultiApp::appProblemBase() instead.\n");
951  if (!_has_an_app)
952  mooseError("No app for ", name(), " on processor ", _orig_rank);
953 
954  unsigned int local_app = globalAppToLocal(app);
955 
956  return dynamic_cast<FEProblem &>(_apps[local_app]->getExecutioner()->feProblem());
957 }
958 
959 const UserObject &
960 MultiApp::appUserObjectBase(unsigned int app, const std::string & name)
961 {
962  if (!_has_an_app)
963  mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
964 
966 }
967 
968 Real
969 MultiApp::appPostprocessorValue(unsigned int app, const std::string & name)
970 {
971  if (!_has_an_app)
972  mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
973 
975 }
976 
978 MultiApp::appTransferVector(unsigned int app, std::string var_name)
979 {
980  return *(appProblemBase(app).getSystem(var_name).solution);
981 }
982 
983 bool
985 {
987 }
988 
989 bool
990 MultiApp::hasLocalApp(unsigned int global_app) const
991 {
992  if (_has_an_app && global_app >= _first_local_app &&
993  global_app <= _first_local_app + (_my_num_apps - 1))
994  return true;
995 
996  return false;
997 }
998 
999 MooseApp *
1000 MultiApp::localApp(unsigned int local_app)
1001 {
1002  mooseAssert(local_app < _apps.size(), "Index out of range: " + Moose::stringify(local_app));
1003  return _apps[local_app].get();
1004 }
1005 
1006 void
1007 MultiApp::resetApp(unsigned int global_app, Real time)
1008 {
1009  TIME_SECTION(_reset_timer);
1010 
1012 
1013  if (hasLocalApp(global_app))
1014  {
1015  unsigned int local_app = globalAppToLocal(global_app);
1016 
1017  // Extract the file numbers from the output, so that the numbering is maintained after reset
1018  std::map<std::string, unsigned int> m = _apps[local_app]->getOutputWarehouse().getFileNumbers();
1019 
1020  createApp(local_app, time);
1021 
1022  // Reset the file numbers of the newly reset apps
1023  _apps[local_app]->getOutputWarehouse().setFileNumbers(m);
1024  }
1025 }
1026 
1027 void
1028 MultiApp::moveApp(unsigned int global_app, Point p)
1029 {
1030  if (_use_positions)
1031  {
1032  _positions[global_app] = p;
1033 
1034  if (hasLocalApp(global_app))
1035  {
1036  unsigned int local_app = globalAppToLocal(global_app);
1037 
1038  if (_output_in_position)
1039  _apps[local_app]->setOutputPosition(p);
1040  if (_run_in_position)
1041  paramError("run_in_position", "Moving apps and running apps in position is not supported");
1042  }
1043  }
1044 }
1045 
1046 void
1048 {
1050  for (unsigned int i = 0; i < _apps.size(); i++)
1051  _apps[i]->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1052 }
1053 
1054 void
1055 MultiApp::createApp(unsigned int i, Real start_time)
1056 {
1057  // Define the app name
1058  const std::string multiapp_name = getMultiAppName(name(), _first_local_app + i, _total_num_apps);
1059  std::string full_name;
1060 
1061  // Only add parent name if the parent is not the main app
1062  if (_app.multiAppLevel() > 0)
1063  full_name = _app.name() + "_" + multiapp_name;
1064  else
1065  full_name = multiapp_name;
1066 
1068  app_params.set<FEProblemBase *>("_parent_fep") = &_fe_problem;
1069  app_params.set<std::unique_ptr<Backup> *>("_initial_backup") = &_sub_app_backups[i];
1070 
1071  // Set the command line parameters with a copy of the main application command line parameters,
1072  // the copy is required so that the addArgument command below doesn't accumulate more and more
1073  // of the same cli_args, which is important when running in batch mode.
1074  std::shared_ptr<CommandLine> app_cli = std::make_shared<CommandLine>(*_app.commandLine());
1075 
1076  if (cliArgs().size() > 0 || _cli_args_from_file.size() > 0)
1077  {
1078  for (const std::string & str : MooseUtils::split(getCommandLineArgsParamHelper(i), ";"))
1079  {
1080  std::ostringstream oss;
1081  oss << full_name << ":" << str;
1082  app_cli->addArgument(oss.str());
1083  }
1084  }
1085  app_cli->initForMultiApp(full_name);
1086  app_params.set<std::shared_ptr<CommandLine>>("_command_line") = app_cli;
1087 
1089  _console << COLOR_CYAN << "Creating MultiApp " << name() << " of type " << _app_type
1090  << " of level " << _app.multiAppLevel() + 1 << " and number " << _first_local_app + i
1091  << " on processor " << processor_id() << " with full name " << full_name
1092  << COLOR_DEFAULT << std::endl;
1093  app_params.set<unsigned int>("_multiapp_level") = _app.multiAppLevel() + 1;
1094  app_params.set<unsigned int>("_multiapp_number") = _first_local_app + i;
1095  if (getParam<bool>("clone_master_mesh") || getParam<bool>("clone_parent_mesh"))
1096  {
1098  _console << COLOR_CYAN << "Cloned parent app mesh will be used for MultiApp " << name()
1099  << COLOR_DEFAULT << std::endl;
1100  app_params.set<const MooseMesh *>("_master_mesh") = &_fe_problem.mesh();
1101  auto displaced_problem = _fe_problem.getDisplacedProblem();
1102  if (displaced_problem)
1103  app_params.set<const MooseMesh *>("_master_displaced_mesh") = &displaced_problem->mesh();
1104  }
1105 
1106  // If only one input file was provided, use it for all the solves
1107  const auto input_index = _input_files.size() == 1 ? 0 : _first_local_app + i;
1108  const auto & input_file = _input_files[input_index];
1109 
1110  // create new parser tree for the application and parse
1111  auto parser = std::make_unique<Parser>(input_file);
1112 
1113  if (input_file.size())
1114  {
1115  parser->parse();
1116  const auto & app_type = parser->getAppType();
1117  if (app_type.empty() && _app_type.empty())
1118  mooseWarning("The application type is not specified for ",
1119  full_name,
1120  ". Please use [Application] block to specify the application type.");
1121  if (!app_type.empty() && app_type != _app_type &&
1122  !AppFactory::instance().isRegistered(app_type))
1123  mooseError("In the ",
1124  full_name,
1125  ", '",
1126  app_type,
1127  "' is not a registered application. The registered application is named: '",
1128  _app_type,
1129  "'. Please double check the [Application] block to make sure the correct "
1130  "application is provided. \n");
1131  }
1132 
1133  if (parser->getAppType().empty())
1134  parser->setAppType(_app_type);
1135 
1136  app_params.set<std::shared_ptr<Parser>>("_parser") = std::move(parser);
1137  _apps[i] = AppFactory::instance().createShared(_app_type, full_name, app_params, _my_comm);
1138  auto & app = _apps[i];
1139 
1140  app->setGlobalTimeOffset(start_time);
1141  app->setOutputFileNumbers(_app.getOutputWarehouse().getFileNumbers());
1142  app->setRestart(_app.isRestarting());
1143  app->setRecover(_app.isRecovering());
1144 
1145  if (_use_positions && getParam<bool>("output_in_position"))
1146  app->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1148  paramError("run_in_position",
1149  "Sub-apps are already displaced, so they are already output in position");
1150 
1151  // Update the MultiApp level for the app that was just created
1152  app->setupOptions();
1153  // if multiapp does not have file base in Outputs input block, output file base will
1154  // be empty here since setupOptions() does not set the default file base with the multiapp
1155  // input file name. Parent app will create the default file base for multiapp by taking the
1156  // output base of the parent app problem and appending the name of the multiapp plus a number to
1157  // it
1158  if (app->getOutputFileBase().empty())
1160  preRunInputFile();
1161 
1162  // Transfer coupling relaxation information to the subapps
1163  _apps[i]->fixedPointConfig().sub_relaxation_factor = getParam<Real>("relaxation_factor");
1164  _apps[i]->fixedPointConfig().sub_transformed_vars =
1165  getParam<std::vector<std::string>>("transformed_variables");
1166  // Handle deprecated parameter
1167  if (!parameters().isParamSetByAddParam("relaxed_variables"))
1168  _apps[i]->fixedPointConfig().sub_transformed_vars =
1169  getParam<std::vector<std::string>>("relaxed_variables");
1170  _apps[i]->fixedPointConfig().sub_transformed_pps =
1171  getParam<std::vector<PostprocessorName>>("transformed_postprocessors");
1172 
1173  app->runInputFile();
1174  auto fixed_point_solve = &(_apps[i]->getExecutioner()->fixedPointSolve());
1175  if (fixed_point_solve)
1176  fixed_point_solve->allocateStorage(false);
1177 
1178  // Transform the app mesh if requested
1179  if (_run_in_position)
1180  {
1181  if (usingPositions())
1182  app->getExecutioner()->feProblem().coordTransform().transformMesh(
1183  app->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
1184  else
1185  app->getExecutioner()->feProblem().coordTransform().transformMesh(
1186  app->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
1187  }
1188 }
1189 
1190 std::string
1192 {
1193  auto cla = cliArgs();
1194 
1195  mooseAssert(cla.size() || _cli_args_from_file.size(), "There is no commandLine argument \n");
1196 
1197  // Single set of "cli_args" to be applied to all sub apps
1198  if (cla.size() == 1)
1199  return cla[0];
1200  else if (_cli_args_from_file.size() == 1)
1201  return _cli_args_from_file[0];
1202  else if (cla.size())
1203  // Unique set of "cli_args" to be applied to each sub apps
1204  return cla[local_app + _first_local_app];
1205  else
1206  return _cli_args_from_file[local_app + _first_local_app];
1207 }
1208 
1211  processor_id_type nprocs,
1212  dof_id_type napps,
1213  processor_id_type min_app_procs,
1214  processor_id_type max_app_procs,
1215  bool batch_mode)
1216 {
1217  if (min_app_procs > nprocs)
1218  mooseError("minimum number of procs per app is higher than the available number of procs");
1219  else if (min_app_procs > max_app_procs)
1220  mooseError("minimum number of procs per app must be lower than the max procs per app");
1221 
1222  mooseAssert(rank < nprocs, "rank must be smaller than the number of procs");
1223 
1224  // A "slot" is a group of procs/ranks that are grouped together to run a
1225  // single (sub)app/sim in parallel.
1226 
1227  const processor_id_type slot_size =
1228  std::max(std::min(cast_int<processor_id_type>(nprocs / napps), max_app_procs), min_app_procs);
1229  const processor_id_type nslots = std::min(
1230  nprocs / slot_size,
1231  cast_int<processor_id_type>(std::min(
1232  static_cast<dof_id_type>(std::numeric_limits<processor_id_type>::max()), napps)));
1233  mooseAssert(nprocs >= (nslots * slot_size),
1234  "Ensure that leftover procs is represented by an unsigned type");
1235  const processor_id_type leftover_procs = nprocs - nslots * slot_size;
1236  const dof_id_type apps_per_slot = napps / nslots;
1237  const dof_id_type leftover_apps = napps % nslots;
1238 
1239  std::vector<int> slot_for_rank(nprocs);
1240  processor_id_type slot = 0;
1241  processor_id_type procs_in_slot = 0;
1242  for (processor_id_type rankiter = 0; rankiter <= rank; rankiter++)
1243  {
1244  if (slot < nslots)
1245  slot_for_rank[rankiter] = cast_int<int>(slot);
1246  else
1247  slot_for_rank[rankiter] = -1;
1248  procs_in_slot++;
1249  // this slot keeps growing until we reach slot size plus possibly an extra
1250  // proc if there were any leftover from the slotization of nprocs - this
1251  // must also make sure we don't go over max app procs.
1252  if (procs_in_slot == slot_size + 1 * (slot < leftover_procs && slot_size < max_app_procs))
1253  {
1254  procs_in_slot = 0;
1255  slot++;
1256  }
1257  }
1258 
1259  if (slot_for_rank[rank] < 0)
1260  // ranks assigned a negative slot don't have any apps running on them.
1261  return {0, 0, 0, 0, false, 0};
1262  const processor_id_type slot_num = cast_int<processor_id_type>(slot_for_rank[rank]);
1263 
1264  const bool is_first_local_rank = rank == 0 || (slot_for_rank[rank - 1] != slot_for_rank[rank]);
1265  const dof_id_type n_local_apps = apps_per_slot + 1 * (slot_num < leftover_apps);
1266 
1267  processor_id_type my_first_rank = 0;
1268  for (processor_id_type rankiter = rank; rankiter > 0; rankiter--)
1269  if (slot_for_rank[rank] != slot_for_rank[rankiter])
1270  {
1271  my_first_rank = cast_int<processor_id_type>(slot_for_rank[rankiter + 1]);
1272  break;
1273  }
1274 
1275  dof_id_type app_index = 0;
1276  for (processor_id_type slot = 0; slot < slot_num; slot++)
1277  {
1278  const dof_id_type num_slot_apps = apps_per_slot + 1 * (slot < leftover_apps);
1279  app_index += num_slot_apps;
1280  }
1281 
1282  if (batch_mode)
1283  return {n_local_apps, app_index, 1, slot_num, is_first_local_rank, my_first_rank};
1284  return {n_local_apps, app_index, n_local_apps, app_index, is_first_local_rank, my_first_rank};
1285 }
1286 
1287 void
1289 {
1290  int ierr;
1291 
1292  ierr = MPI_Comm_size(_communicator.get(), &_orig_num_procs);
1293  mooseCheckMPIErr(ierr);
1294  ierr = MPI_Comm_rank(_communicator.get(), &_orig_rank);
1295  mooseCheckMPIErr(ierr);
1296 
1297 #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
1298  struct utsname sysInfo;
1299  uname(&sysInfo);
1300  _node_name = sysInfo.nodename;
1301 #else
1302  _node_name = "Unknown";
1303 #endif
1304 
1305  int rank;
1306  ierr = MPI_Comm_rank(_communicator.get(), &rank);
1307  mooseCheckMPIErr(ierr);
1308 
1311 
1314  mooseError("Internal error, a processor has an undefined app.");
1315 
1316  if (_has_an_app)
1317  {
1319  ierr = MPI_Comm_rank(_my_comm, &_my_rank);
1320  mooseCheckMPIErr(ierr);
1321  }
1322  else
1323  {
1324  _communicator.split(MPI_UNDEFINED, rank, _my_communicator);
1325  _my_rank = 0;
1326  }
1327 }
1328 
1329 unsigned int
1330 MultiApp::globalAppToLocal(unsigned int global_app)
1331 {
1332  if (global_app >= _first_local_app && global_app <= _first_local_app + (_my_num_apps - 1))
1333  return global_app - _first_local_app;
1334 
1335  std::stringstream ss;
1336  ss << "Requesting app " << global_app << ", but processor " << processor_id() << " ";
1337  if (_my_num_apps == 0)
1338  ss << "does not own any apps";
1339  else if (_my_num_apps == 1)
1340  ss << "owns app " << _first_local_app;
1341  else
1342  ss << "owns apps " << _first_local_app << "-" << _first_local_app + (_my_num_apps - 1);
1343  ss << ".";
1344  mooseError("Invalid global_app!\n", ss.str());
1345  return 0;
1346 }
1347 
1348 void
1350 {
1351 }
1352 
1353 void
1355 {
1356  _associated_transfers.push_back(&transfer);
1357 }
1358 
1359 void
1361 {
1362  for (unsigned int i = 0; i < _my_num_apps; ++i)
1364 }
1365 
1366 void
1368 {
1369  const std::string multiapp_name =
1371  _apps[index]->setOutputFileBase(_app.getOutputFileBase() + "_" + multiapp_name);
1372 }
1373 
1374 std::string
1375 MultiApp::getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total)
1376 {
1377  std::ostringstream multiapp_name;
1378  multiapp_name << base_name << std::setw(std::ceil(std::log10(total))) << std::setprecision(0)
1379  << std::setfill('0') << std::right << index;
1380  return multiapp_name.str();
1381 }
1382 
1383 const Point &
1384 MultiApp::position(unsigned int app) const
1385 {
1386  // If we're not using positions, it won't have changed
1387  if (_positions_objs.empty())
1388  return _positions[app];
1389  else
1390  // Find which Positions object is specifying it, and query a potentially updated value
1391  return _positions_objs[app]->getPosition(app - _positions_index_offsets[app], false);
1392 }
1393 
1394 void
1395 dataStore(std::ostream & stream, SubAppBackups & backups, void * context)
1396 {
1397  MultiApp * multi_app = static_cast<MultiApp *>(context);
1398  mooseAssert(multi_app, "Not set");
1399 
1400  multi_app->backup();
1401 
1402  dataStore(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1403 }
1404 
1405 void
1406 dataLoad(std::istream & stream, SubAppBackups & backups, void * context)
1407 {
1408  MultiApp * multi_app = static_cast<MultiApp *>(context);
1409  mooseAssert(multi_app, "Not set");
1410 
1411  dataLoad(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1412 
1413  multi_app->restore();
1414 }
bool isRegistered(const std::string &app_name) const
Returns a Boolean indicating whether an application type has been registered.
Definition: AppFactory.h:119
void keepSolutionDuringRestore(bool keep_solution_during_restore)
Reserve the solution from the previous simulation, and it is used as an initial guess for the next ru...
Definition: MultiApp.C:812
const std::string & name() const
Get the name of the object.
Definition: MooseApp.h:106
bool hasLocalApp(unsigned int global_app) const
Whether or not the given global app number is on this processor.
Definition: MultiApp.C:990
virtual std::vector< std::string > cliArgs() const
function that provides cli_args to subapps
Definition: MultiApp.h:387
void read()
Perform the actual data reading.
std::vector< unsigned int > _reset_apps
The apps to be reset.
Definition: MultiApp.h:568
Transient executioners usually loop through a number of timesteps...
Definition: Transient.h:26
virtual System & getSystem(const std::string &var_name) override
Returns the equation system containing the variable provided.
A MultiMooseEnum object to hold "execute_on" flags.
Definition: ExecFlagEnum.h:21
A class for creating restricted objects.
Definition: Restartable.h:28
MultiApp(const InputParameters &parameters)
Definition: MultiApp.C:249
void addDeprecatedParam(const std::string &name, const T &value, const std::string &doc_string, const std::string &deprecation_message)
std::vector< const Positions * > _positions_objs
The positions of all of the apps, using the Positions system.
Definition: MultiApp.h:487
const unsigned int invalid_uint
const std::string & _name
The name of this class, reference to value stored in InputParameters.
Definition: MooseBase.h:75
void mooseDeprecated(Args &&... args) const
virtual void restore(bool force=true)
Restore the state of every Sub App.
Definition: MultiApp.C:737
virtual void backup()
Save off the state of every Sub App.
Definition: MultiApp.C:722
bool verboseMultiApps() const
Whether or not to use verbose printing for MultiApps.
const bool _use_positions
Toggle use of "positions".
Definition: MultiApp.h:493
void addPrivateParam(const std::string &name, const T &value)
These method add a parameter to the InputParameters object which can be retrieved like any other para...
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
Definition: FEProblem.h:20
const std::vector< CLIArgString > & _cli_args
CommandLine arguments.
Definition: MultiApp.h:589
std::vector< unsigned int > _npositions_inputfile
Number of positions for each input file.
Definition: MultiApp.h:502
void dataLoad(std::istream &stream, SubAppBackups &backups, void *context)
Definition: MultiApp.C:1406
void mooseError(Args &&... args)
Emit an error message with the given stringified, concatenated args and terminate the application...
Definition: MooseError.h:284
void setupPositions()
Called just after construction to allow derived classes to set _positions and create sub-apps accordi...
Definition: MultiApp.C:351
std::shared_ptr< CommandLine > commandLine() const
Get the command line.
Definition: MooseApp.h:425
std::vector< unsigned int > _positions_index_offsets
The offsets, in case multiple Positions objects are specified.
Definition: MultiApp.h:489
Utility class for reading delimited data (e.g., CSV data).
Real _move_time
The time at which to move apps.
Definition: MultiApp.h:574
MooseAppPtr createShared(const std::string &app_type, const std::string &name, InputParameters parameters, MPI_Comm COMM_WORLD_IN)
Build an application object (must be registered)
Definition: AppFactory.C:124
const ExecFlagType & getCurrentExecuteOnFlag() const
Return/set the current execution flag.
Real appPostprocessorValue(unsigned int app, const std::string &name)
Get a Postprocessor value for a specified global app.
Definition: MultiApp.C:969
int _orig_rank
The mpi "rank" of this processor in the original communicator.
Definition: MultiApp.h:529
LocalRankConfig _rank_config
The app configuration resulting from calling init.
Definition: MultiApp.h:601
virtual std::string getCommandLineArgsParamHelper(unsigned int local_app)
Method to aid in getting the "cli_args" parameters.
Definition: MultiApp.C:1191
std::string getOutputFileBase(bool for_non_moose_build_output=false) const
Get the output file base name.
Definition: MooseApp.C:1072
T * get(const std::unique_ptr< T > &u)
The MooseUtils::get() specializations are used to support making forwards-compatible code changes fro...
Definition: MooseUtils.h:1147
std::vector< std::shared_ptr< MooseApp > > _apps
Pointers to each of the Apps.
Definition: MultiApp.h:538
T & set(const std::string &name, bool quiet_mode=false)
Returns a writable reference to the named parameters.
dof_id_type first_local_app_index
The (global) index of the first local app for this rank.
Definition: MultiApp.h:64
unsigned int multiAppLevel() const
The MultiApp Level.
Definition: MooseApp.h:812
virtual void parentOutputPositionChanged()
For apps outputting in position we need to change their output positions if their parent app moves...
Definition: MultiApp.C:1047
MeshBase & mesh
std::vector< Point > _move_positions
The new positions for the apps to be moved.
Definition: MultiApp.h:580
Base class for MOOSE-based applications.
Definition: MooseApp.h:69
The main MOOSE class responsible for handling user-defined parameters in almost every MOOSE system...
void dataStore(std::ostream &stream, SubAppBackups &backups, void *context)
Definition: MultiApp.C:1395
static constexpr std::size_t dim
This is the dimension of all vector and tensor datastructures used in MOOSE.
Definition: Moose.h:148
const PerfID _restore_timer
Definition: MultiApp.h:616
const Parallel::Communicator & _communicator
Real _inflation
Relative bounding box inflation.
Definition: MultiApp.h:547
FEProblemBase & _fe_problem
The FEProblemBase this MultiApp is part of.
Definition: MultiApp.h:479
bool isRestarting() const
Whether or not this is a "restart" calculation.
Definition: MooseApp.C:1173
const Positions & getPositionsObject(const std::string &name) const
Get the Positions object by its name.
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
processor_id_type _min_procs_per_app
Minimum number of processors to give to each app.
Definition: MultiApp.h:556
virtual const std::string & name() const
Get the name of the class.
Definition: MooseBase.h:56
ierr
int _orig_num_procs
The number of processors in the original comm.
Definition: MultiApp.h:526
void mooseWarning(Args &&... args) const
Emits a warning prefixed with object name and type.
MooseApp & getMooseApp() const
Get the MooseApp this class is associated with.
Definition: MooseBase.h:44
void addRequiredParam(const std::string &name, const std::string &doc_string)
This method adds a parameter and documentation string to the InputParameters object that will be extr...
const PerfID _backup_timer
Definition: MultiApp.h:615
auto max(const L &left, const R &right)
std::vector< std::string > split(const std::string &str, const std::string &delimiter, std::size_t max_count=std::numeric_limits< std::size_t >::max())
Python like split functions for strings.
Definition: MooseUtils.C:1112
std::vector< Real > _reset_times
The times at which to reset apps.
Definition: MultiApp.h:565
const std::vector< Point > getDataAsPoints() const
Get the data in Point format.
bool isParamValid(const std::string &name) const
Test if the supplied parameter is valid.
virtual void preTransfer(Real dt, Real target_time)
Gets called just before transfers are done to the MultiApp (Which is just before the MultiApp is solv...
Definition: MultiApp.C:632
void registerBase(const std::string &value)
This method must be called from every base "Moose System" to create linkage with the Action System...
void createApps()
Create the provided number of apps.
Definition: MultiApp.C:362
virtual BoundingBox getBoundingBox(unsigned int app, bool displaced_mesh, const MultiAppCoordTransform *coord_transform=nullptr)
Get the BoundingBox for the mesh associated with app The bounding box will be shifted to be in the co...
Definition: MultiApp.C:862
uint8_t processor_id_type
bool isParamSetByAddParam(const std::string &name) const
Returns whether or not the parameter was set due to addParam.
processor_id_type n_processors() const
const bool & _wait_for_first_app_init
Whether to create the first app on rank 0 while all other MPI ranks are idle.
Definition: MultiApp.h:499
const auto & registeredObjects() const
Returns a reference to the map from names to AppFactoryBuildInfo pointers.
Definition: AppFactory.h:114
virtual NumericVector< Number > & appTransferVector(unsigned int app, std::string var_name)
Get the vector to transfer to for this MultiApp.
Definition: MultiApp.C:978
bool _keep_solution_during_restore
Flag indicates if or not restart from the latest solution.
Definition: MultiApp.h:595
Moose::Builder & builder()
Returns a writable reference to the builder.
Definition: MooseApp.h:215
Every object that can be built by the factory should be derived from this class.
Definition: MooseObject.h:33
std::map< std::string, unsigned int > getFileNumbers()
Extracts the file numbers from the output objects.
processor_id_type _max_procs_per_app
Maximum number of processors to give to each app.
Definition: MultiApp.h:553
const ExecFlagType EXEC_TIMESTEP_BEGIN
Definition: Moose.C:33
virtual void fillPositions()
must fill in _positions with the positions of the sub-aps
Definition: MultiApp.C:519
const ExecFlagType EXEC_PRE_MULTIAPP_SETUP
Definition: Moose.C:46
bool _move_happened
Whether or not the move has happened.
Definition: MultiApp.h:583
std::vector< MultiAppTransfer * > _associated_transfers
Transfers associated with this multiapp.
Definition: MultiApp.h:604
Helper class for holding Sub-app backups.
Definition: MultiApp.h:100
const PerfID _reset_timer
Definition: MultiApp.h:617
bool checkFileReadable(const std::string &filename, bool check_line_endings=false, bool throw_on_unreadable=true, bool check_for_git_lfs_pointer=true)
Checks to see if a file is readable (exists and permissions)
Definition: MooseUtils.C:253
void setFormatFlag(FormatFlag value)
std::string _app_type
The type of application to build.
Definition: MultiApp.h:482
MooseMesh wraps a libMesh::Mesh object and enhances its capabilities by caching additional data and s...
Definition: MooseMesh.h:88
void split(int color, int key, Communicator &target) const
FEProblemBase & appProblemBase(unsigned int app)
Get the FEProblemBase for the global app desired.
Definition: MultiApp.C:936
This is a "smart" enum class intended to replace many of the shortcomings in the C++ enum type It sho...
Definition: MooseEnum.h:31
PetscErrorCode PetscInt const PetscInt IS * is
std::vector< unsigned int > _move_apps
The apps to be moved.
Definition: MultiApp.h:577
Executioners are objects that do the actual work of solving your problem.
Definition: Executioner.h:30
MooseApp & _app
The MOOSE application this is associated with.
Definition: MooseBase.h:69
void paramError(const std::string &param, Args... args) const
Emits an error prefixed with the file and line number of the given param (from the input file) along ...
const std::vector< Point > & getPositions(bool initial) const
{ Getters for the positions vector for the desired dimension 1D will be the only one guaranteed to su...
Definition: Positions.C:130
void buildComm()
Create an MPI communicator suitable for each app.
Definition: MultiApp.C:1288
virtual void resetApp(unsigned int global_app, Real time=0.0)
"Reset" the App corresponding to the global App number passed in.
Definition: MultiApp.C:1007
std::string stringify(const T &t)
conversion to string
Definition: Conversion.h:62
static InputParameters validParams()
bool isFirstLocalRank() const
Definition: MultiApp.C:984
Interface for objects interacting with the PerfGraph.
const PostprocessorValue & getPostprocessorValueByName(const PostprocessorName &name, std::size_t t_index=0) const
Get a read-only reference to the value associated with a Postprocessor that exists.
unsigned int _total_num_apps
The total number of apps to simulate.
Definition: MultiApp.h:508
void skipCoordinateCollapsing(bool skip_coordinate_collapsing)
set whether coordinate collapsing operations should be skipped
LocalRankConfig rankConfig(processor_id_type rank, processor_id_type nprocs, dof_id_type napps, processor_id_type min_app_procs, processor_id_type max_app_procs, bool batch_mode)
Returns app partitioning information relevant to the given rank for a multiapp scenario with the give...
Definition: MultiApp.C:1210
Executioner * getExecutioner() const
Retrieve the Executioner for this App.
Definition: MooseApp.C:1482
void broadcast(T &data, const unsigned int root_id=0, const bool identical_sizes=false) const
std::vector< BoundingBox > _bounding_box
This multi-app&#39;s bounding box.
Definition: MultiApp.h:544
Point getOutputPosition() const
Get the output position.
Definition: MooseApp.h:277
unsigned int _my_num_apps
The number of apps this object is involved in simulating.
Definition: MultiApp.h:511
static std::string getMultiAppName(const std::string &base_name, dof_id_type index, dof_id_type total)
Helper for constructing the name of the multiapp.
Definition: MultiApp.C:1375
std::string _node_name
Node Name.
Definition: MultiApp.h:532
static AppFactory & instance()
Get the instance of the AppFactory.
Definition: AppFactory.C:17
bool _has_an_app
Whether or not this processor as an App at all
Definition: MultiApp.h:586
libMesh::Parallel::Communicator _my_communicator
The communicator object that holds the MPI_Comm that we&#39;re going to use.
Definition: MultiApp.h:520
void createApp(unsigned int i, Real start_time)
Helper function for creating an App instance.
Definition: MultiApp.C:1055
bool isParamSetByUser(const std::string &name) const
Method returns true if the parameter was by the user.
static void transformBoundingBox(BoundingBox &box, const MultiAppCoordTransform &transform)
Transform a bounding box according to the transformations in the provided coordinate transformation o...
Definition: MultiApp.C:822
bool usingPositions() const
Whether or not this MultiApp is using positions or its own way for constructing sub-apps.
Definition: MultiApp.h:361
const PerfID _init_timer
Definition: MultiApp.h:614
unsigned int _first_local_app
The number of the first app on this processor.
Definition: MultiApp.h:514
std::string hitCLIFilter(std::string appname, const std::vector< std::string > &argv)
Marks MOOSE hit syntax from supplied command-line arguments.
Definition: Builder.C:311
void addAssociatedTransfer(MultiAppTransfer &transfer)
Add a transfer that is associated with this multiapp.
Definition: MultiApp.C:1354
DIE A HORRIBLE DEATH HERE typedef LIBMESH_DEFAULT_SCALAR_TYPE Real
virtual std::shared_ptr< const DisplacedProblem > getDisplacedProblem() const
MooseApp * localApp(unsigned int local_app)
Get the local MooseApp object.
Definition: MultiApp.C:1000
virtual void initialSetup() override
Method to be called in main-app initial setup for create sub-apps if using positions is false...
Definition: MultiApp.C:413
virtual bool needsRestoration()
Whether or not this MultiApp should be restored at the beginning of each Picard iteration.
Definition: MultiApp.h:210
Base class for all MultiAppTransfer objects.
bool is_first_local_rank
This is true if this rank is the primary/zero rank for a (sub)app slot.
Definition: MultiApp.h:71
FEProblem & appProblem(unsigned int app)
Get the FEProblem for the global app is part of.
Definition: MultiApp.C:947
IntRange< T > make_range(T beg, T end)
SubAppBackups & _sub_app_backups
The cached subapp backups (passed from the parent app)
Definition: MultiApp.h:610
virtual MooseMesh & mesh() override
virtual void postExecute()
Method called at the end of the simulation (after finalize).
Definition: MultiApp.C:710
bool _output_in_position
Whether or not to move the output of the MultiApp into position.
Definition: MultiApp.h:559
void mooseError(Args &&... args) const
Emits an error prefixed with object name and type.
const InputParameters & _pars
Parameters of this object, references the InputParameters stored in the InputParametersWarehouse.
std::vector< Point > _positions
The positions of all of the apps, using input constant vectors (to be deprecated) ...
Definition: MultiApp.h:485
const InputParameters & parameters() const
Get the parameters of the object.
virtual void finalize()
Method called towards the end of the simulation to execute on final.
Definition: MultiApp.C:697
void readCommandLineArguments()
Fill command line arguments for sub apps.
Definition: MultiApp.C:422
int _my_rank
The mpi "rank" of this processor in the sub communicator.
Definition: MultiApp.h:535
void addParam(const std::string &name, const S &value, const std::string &doc_string)
These methods add an option parameter and a documentation string to the InputParameters object...
Moose::CoordinateSystemType getCoordSystem(SubdomainID sid) const
Definition: SubProblem.C:1223
dof_id_type num_local_apps
The number of (sub)apps that should/will be run locally on this rank.
Definition: MultiApp.h:58
virtual void createLocalApp(const unsigned int i)
Create the i-th local app.
Definition: MultiApp.C:406
void addRangeCheckedParam(const std::string &name, const T &value, const std::string &parsed_function, const std::string &doc_string)
std::vector< FileName > _input_files
The input file for each app&#39;s simulation.
Definition: MultiApp.h:496
Holds app partitioning information relevant to the a particular rank for a multiapp scenario...
Definition: MultiApp.h:46
virtual void preRunInputFile()
call back executed right before app->runInputFile()
Definition: MultiApp.C:1349
const ConsoleStream _console
An instance of helper class to write streams to the Console objects.
InputParameters getValidParams(const std::string &name)
Get valid parameters for the object.
Definition: AppFactory.C:30
const UserObject & getUserObjectBase(const std::string &name, const THREAD_ID tid=0) const
Get the user object by its name.
Point _bounding_box_padding
Additional padding added to the bounding box, useful for 1D meshes.
Definition: MultiApp.h:550
const Real _global_time_offset
The offset time so the MultiApp local time relative to the global time.
Definition: MultiApp.h:562
void init(unsigned int num_apps, bool batch_mode=false)
Build communicators and reserve backups.
Definition: MultiApp.C:314
static InputParameters validParams()
Definition: MooseObject.C:24
const MPI_Comm & _orig_comm
The original comm handle.
Definition: MultiApp.h:517
static InputParameters validParams()
Definition: MultiApp.C:50
std::vector< std::unique_ptr< NumericVector< Real > > > _end_solutions
The solution from the end of the previous solve, this is cloned from the Nonlinear solution during re...
Definition: MultiApp.h:598
A MultiApp represents one or more MOOSE applications that are running simultaneously.
Definition: MultiApp.h:112
processor_id_type processor_id() const
void setAppOutputFileBase()
Sets all the app&#39;s output file bases.
Definition: MultiApp.C:1360
bool isRecovering() const
Whether or not this is a "recover" calculation.
Definition: MooseApp.C:1167
auto min(const L &left, const R &right)
MPI_Comm & _my_comm
The MPI communicator this object is going to use.
Definition: MultiApp.h:523
const ExecFlagType EXEC_FINAL
Definition: Moose.C:38
void declareControllable(const std::string &name, std::set< ExecFlagType > execute_flags={})
Declare the given parameters as controllable.
const bool _run_in_position
Whether to run the child apps with their meshes transformed with the coordinate transforms.
Definition: MultiApp.h:607
virtual void moveApp(unsigned int global_app, Point p)
Move the global_app to Point p.
Definition: MultiApp.C:1028
void ErrorVector unsigned int
virtual Executioner * getExecutioner(unsigned int app)
Definition: MultiApp.C:688
auto index_range(const T &sizable)
Base class for user-specific data.
Definition: UserObject.h:39
const Point & position(unsigned int app) const
The physical position of a global App number.
Definition: MultiApp.C:1384
OutputWarehouse & getOutputWarehouse()
Get the OutputWarehouse objects.
Definition: MooseApp.C:1772
std::vector< std::string > _cli_args_from_file
CommandLine arguments from files.
Definition: MultiApp.h:592
const UserObject & appUserObjectBase(unsigned int app, const std::string &name)
Get a UserObject base for a specific global app.
Definition: MultiApp.C:960
uint8_t dof_id_type
unsigned int globalAppToLocal(unsigned int global_app)
Map a global App number to the local number.
Definition: MultiApp.C:1330
This class contains transformation information that only exists in a context in which there are multi...
void dynamicAppRegistration(const std::string &app_name, std::string library_path, const std::string &library_name, bool lib_load_deps)
Definition: MooseApp.C:1938
void addParamNamesToGroup(const std::string &space_delim_names, const std::string group_name)
This method takes a space delimited list of parameter names and adds them to the specified group name...
std::vector< bool > _reset_happened
Whether or not apps have been reset at each time.
Definition: MultiApp.h:571
bool isParamValid(const std::string &name) const
This method returns parameters that have been initialized in one fashion or another, i.e.
const ExecFlagType EXEC_INITIAL
Definition: Moose.C:28
std::vector< bool > _has_bounding_box
Flag if this multi-app computed its bounding box (valid only for non-displaced meshes) ...
Definition: MultiApp.h:541