www.mooseframework.org
TransientMultiApp.C
Go to the documentation of this file.
1 /****************************************************************/
2 /* DO NOT MODIFY THIS HEADER */
3 /* MOOSE - Multiphysics Object Oriented Simulation Environment */
4 /* */
5 /* (c) 2010 Battelle Energy Alliance, LLC */
6 /* ALL RIGHTS RESERVED */
7 /* */
8 /* Prepared by Battelle Energy Alliance, LLC */
9 /* Under Contract No. DE-AC07-05ID14517 */
10 /* With the U. S. Department of Energy */
11 /* */
12 /* See COPYRIGHT for full restrictions */
13 /****************************************************************/
14 
15 // MOOSE includes
16 #include "TransientMultiApp.h"
17 
19 #include "AuxiliarySystem.h"
20 #include "Console.h"
21 #include "LayeredSideFluxAverage.h"
22 #include "MooseMesh.h"
23 #include "Output.h"
24 #include "TimeStepper.h"
25 #include "Transient.h"
26 
27 #include "libmesh/mesh_tools.h"
28 
29 template <>
32 {
35 
36  params.addParam<bool>("sub_cycling",
37  false,
38  "Set to true to allow this MultiApp to take smaller "
39  "timesteps than the rest of the simulation. More "
40  "than one timestep will be performed for each "
41  "'master' timestep");
42 
43  params.addParam<bool>("interpolate_transfers",
44  false,
45  "Only valid when sub_cycling. This allows "
46  "transferred values to be interpolated "
47  "over the time frame the MultiApp is "
48  "executing over when sub_cycling");
49 
50  params.addParam<bool>("detect_steady_state",
51  false,
52  "If true then while sub_cycling a steady state check will be "
53  "done. In this mode output will only be done once the "
54  "MultiApp reaches the target time or steady state is reached");
55 
56  params.addParam<Real>("steady_state_tol",
57  1e-8,
58  "The relative difference between the new "
59  "solution and the old solution that will be "
60  "considered to be at steady state");
61 
62  params.addParam<bool>("output_sub_cycles", false, "If true then every sub-cycle will be output.");
63  params.addParam<bool>(
64  "print_sub_cycles", true, "Toggle the display of sub-cycles on the screen.");
65 
66  params.addParam<unsigned int>(
67  "max_failures", 0, "Maximum number of solve failures tolerated while sub_cycling.");
68 
69  params.addParam<bool>("tolerate_failure",
70  false,
71  "If true this MultiApp won't participate in dt "
72  "decisions and will always be fast-forwarded to "
73  "the current time.");
74 
75  params.addParam<bool>(
76  "catch_up",
77  false,
78  "If true this will allow failed solves to attempt to 'catch up' using smaller timesteps.");
79 
80  params.addParam<Real>("max_catch_up_steps",
81  2,
82  "Maximum number of steps to allow an app to take "
83  "when trying to catch back up after a failed "
84  "solve.");
85 
86  return params;
87 }
88 
90  : MultiApp(parameters),
91  _sub_cycling(getParam<bool>("sub_cycling")),
92  _interpolate_transfers(getParam<bool>("interpolate_transfers")),
93  _detect_steady_state(getParam<bool>("detect_steady_state")),
94  _steady_state_tol(getParam<Real>("steady_state_tol")),
95  _output_sub_cycles(getParam<bool>("output_sub_cycles")),
96  _max_failures(getParam<unsigned int>("max_failures")),
97  _tolerate_failure(getParam<bool>("tolerate_failure")),
98  _failures(0),
99  _catch_up(getParam<bool>("catch_up")),
100  _max_catch_up_steps(getParam<Real>("max_catch_up_steps")),
101  _first(declareRecoverableData<bool>("first", true)),
102  _auto_advance(false),
103  _print_sub_cycles(getParam<bool>("print_sub_cycles"))
104 {
105  // Transfer interpolation only makes sense for sub-cycling solves
107  mooseError("MultiApp ",
108  name(),
109  " is set to interpolate_transfers but is not sub_cycling! That is not valid!");
110 
111  // Subcycling overrides catch up, we don't want to confuse users by allowing them to set both.
112  if (_sub_cycling && _catch_up)
113  mooseError("MultiApp ",
114  name(),
115  " sub_cycling and catch_up cannot both be set to true simultaneously.");
116 }
117 
118 NumericVector<Number> &
119 TransientMultiApp::appTransferVector(unsigned int app, std::string var_name)
120 {
121  if (std::find(_transferred_vars.begin(), _transferred_vars.end(), var_name) ==
122  _transferred_vars.end())
123  _transferred_vars.push_back(var_name);
124 
126  return appProblemBase(app).getAuxiliarySystem().system().get_vector("transfer");
127 
129 }
130 
131 void
133 {
135 
136  if (!_has_an_app)
137  return;
138 
140 
141  if (_has_an_app)
142  {
144  // Grab Transient Executioners from each app
145  for (unsigned int i = 0; i < _my_num_apps; i++)
146  setupApp(i);
147  }
148 }
149 
150 bool
151 TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
152 {
153  if (!_has_an_app)
154  return true;
155 
156  _auto_advance = auto_advance;
157 
158  _console << "Solving MultiApp " << name() << std::endl;
159 
160  // "target_time" must always be in global time
161  target_time += _app.getGlobalTimeOffset();
162 
164  bool return_value = true;
165 
166  // Make sure we swap back the communicator regardless of how this routine is exited
167  try
168  {
169  int rank;
170  int ierr;
171  ierr = MPI_Comm_rank(_orig_comm, &rank);
172  mooseCheckMPIErr(ierr);
173 
174  for (unsigned int i = 0; i < _my_num_apps; i++)
175  {
176 
178 
180 
181  // The App might have a different local time from the rest of the problem
182  Real app_time_offset = _apps[i]->getGlobalTimeOffset();
183 
184  if ((ex->getTime() + app_time_offset) + 2e-14 >=
185  target_time) // Maybe this MultiApp was already solved
186  continue;
187 
188  if (_sub_cycling)
189  {
190  Real time_old = ex->getTime() + app_time_offset;
191 
193  {
194  AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
195  System & libmesh_aux_system = aux_system.system();
196 
197  NumericVector<Number> & solution = *libmesh_aux_system.solution;
198  NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
199 
200  solution.close();
201 
202  // Save off the current auxiliary solution
203  transfer_old = solution;
204 
205  transfer_old.close();
206 
207  // Snag all of the local dof indices for all of these variables
208  AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
209  ConstElemRange & elem_range = *problem.mesh().getActiveLocalElementRange();
210  Threads::parallel_reduce(elem_range, aldit);
211 
213  }
214 
215  // Disable/enable output for sub cycling
216  problem.allowOutput(_output_sub_cycles); // disables all outputs, including console
217  problem.allowOutput<Console>(_print_sub_cycles); // re-enables Console to print, if desired
218 
219  ex->setTargetTime(target_time - app_time_offset);
220 
221  // unsigned int failures = 0;
222 
223  bool at_steady = false;
224 
225  if (_first && !_app.isRecovering())
226  problem.advanceState();
227 
228  bool local_first = _first;
229 
230  // Now do all of the solves we need
231  while ((!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time) ||
232  !ex->lastSolveConverged())
233  {
234  if (local_first != true)
235  ex->incrementStepOrReject();
236 
237  local_first = false;
238 
239  ex->preStep();
240  ex->computeDT();
241 
243  {
244  // See what time this executioner is going to go to.
245  Real future_time = ex->getTime() + app_time_offset + ex->getDT();
246 
247  // How far along we are towards the target time:
248  Real step_percent = (future_time - time_old) / (target_time - time_old);
249 
250  Real one_minus_step_percent = 1.0 - step_percent;
251 
252  // Do the interpolation for each variable that was transferred to
254  AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
255  System & libmesh_aux_system = aux_system.system();
256 
257  NumericVector<Number> & solution = *libmesh_aux_system.solution;
258  NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
259  NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
260 
261  solution.close(); // Just to be sure
262  transfer.close();
263  transfer_old.close();
264 
265  for (const auto & dof : _transferred_dofs)
266  {
267  solution.set(dof,
268  (transfer_old(dof) * one_minus_step_percent) +
269  (transfer(dof) * step_percent));
270  // solution.set(dof, transfer_old(dof));
271  // solution.set(dof, transfer(dof));
272  // solution.set(dof, 1);
273  }
274 
275  solution.close();
276  }
277 
278  ex->takeStep();
279 
280  bool converged = ex->lastSolveConverged();
281 
282  if (!converged)
283  {
284  mooseWarning(
285  "While sub_cycling ", name(), _first_local_app + i, " failed to converge!\n");
286 
287  _failures++;
288 
289  if (_failures > _max_failures)
290  {
291  std::stringstream oss;
292  oss << "While sub_cycling " << name() << _first_local_app << i << " REALLY failed!";
293  throw MultiAppSolveFailure(oss.str());
294  }
295  }
296 
297  Real solution_change_norm = ex->getSolutionChangeNorm();
298 
300  _console << "Solution change norm: " << solution_change_norm << std::endl;
301 
302  if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
303  {
304  _console << "Detected Steady State! Fast-forwarding to " << target_time << std::endl;
305 
306  at_steady = true;
307 
308  // Indicate that the next output call (occurs in ex->endStep()) should output,
309  // regardless of intervals etc...
310  problem.forceOutput();
311 
312  // Clean up the end
313  ex->endStep(target_time - app_time_offset);
314  ex->postStep();
315  }
316  else
317  {
318  ex->endStep();
319  ex->postStep();
320  }
321  }
322 
323  // If we were looking for a steady state, but didn't reach one, we still need to output one
324  // more time, regardless of interval
325  if (!at_steady)
326  problem.outputStep(EXEC_FORCED);
327 
328  } // sub_cycling
329  else if (_tolerate_failure)
330  {
331  ex->takeStep(dt);
332  ex->endStep(target_time - app_time_offset);
333  ex->postStep();
334  }
335  else
336  {
337  _console << "Solving Normal Step!" << std::endl;
338 
339  if (_first && !_app.isRecovering())
340  problem.advanceState();
341 
342  if (auto_advance)
343  if (_first != true)
344  ex->incrementStepOrReject();
345 
346  if (auto_advance)
347  problem.allowOutput(true);
348 
349  ex->takeStep(dt);
350 
351  if (auto_advance)
352  {
353  ex->endStep();
354  ex->postStep();
355 
356  if (!ex->lastSolveConverged())
357  {
358  mooseWarning(name(), _first_local_app + i, " failed to converge!\n");
359 
360  if (_catch_up)
361  {
362  _console << "Starting Catch Up!" << std::endl;
363 
364  bool caught_up = false;
365 
366  unsigned int catch_up_step = 0;
367 
368  Real catch_up_dt = dt / 2;
369 
370  while (!caught_up && catch_up_step < _max_catch_up_steps)
371  {
372  Moose::err << "Solving " << name() << "catch up step " << catch_up_step
373  << std::endl;
374  ex->incrementStepOrReject();
375 
376  ex->computeDT();
377  ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
378 
379  if (ex->lastSolveConverged())
380  {
381  if (ex->getTime() + app_time_offset +
382  ex->timestepTol() * std::abs(ex->getTime()) >=
383  target_time)
384  {
385  problem.outputStep(EXEC_FORCED);
386  caught_up = true;
387  }
388  }
389  else
390  catch_up_dt /= 2.0;
391 
392  ex->endStep();
393  ex->postStep();
394 
395  catch_up_step++;
396  }
397 
398  if (!caught_up)
399  throw MultiAppSolveFailure(name() + " Failed to catch up!\n");
400  }
401  }
402  }
403  else if (!ex->lastSolveConverged())
404  throw MultiAppSolveFailure(name() + " failed to converge");
405  }
406 
407  // Re-enable all output (it may of been disabled by sub-cycling)
408  problem.allowOutput(true);
409  }
410 
411  _first = false;
412 
413  _console << "Successfully Solved MultiApp " << name() << "." << std::endl;
414  }
415  catch (MultiAppSolveFailure & e)
416  {
417  mooseWarning(e.what());
418  _console << "Failed to Solve MultiApp " << name() << ", attempting to recover." << std::endl;
419  return_value = false;
420  }
421 
422  _transferred_vars.clear();
423 
424  return return_value;
425 }
426 
427 void
429 {
430  if (!_auto_advance && !_sub_cycling)
431  {
432  for (unsigned int i = 0; i < _my_num_apps; i++)
433  {
434  /*FEProblemBase * problem =*/appProblemBase(_first_local_app + i);
436 
437  ex->endStep();
438  ex->postStep();
439  ex->incrementStepOrReject();
440  }
441  }
442 }
443 
444 bool
446 {
448 }
449 
450 Real
452 {
453  if (_sub_cycling) // Bow out of the timestep selection dance
454  return std::numeric_limits<Real>::max();
455 
456  Real smallest_dt = std::numeric_limits<Real>::max();
457 
458  if (_has_an_app)
459  {
461 
462  for (unsigned int i = 0; i < _my_num_apps; i++)
463  {
465  ex->computeDT();
466  Real dt = ex->getDT();
467 
468  smallest_dt = std::min(dt, smallest_dt);
469  }
470  }
471 
472  if (_tolerate_failure) // Bow out of the timestep selection dance, we do this down here because we
473  // need to call computeConstrainedDT at least once for these
474  // executioners...
475  return std::numeric_limits<Real>::max();
476 
477  _communicator.min(smallest_dt);
478  return smallest_dt;
479 }
480 
482  unsigned int global_app,
483  Real /*time*/) // FIXME: Note that we are passing in time but also grabbing it below
484 {
485  if (hasLocalApp(global_app))
486  {
487  unsigned int local_app = globalAppToLocal(global_app);
488 
489  // Grab the current time the App is at so we can start the new one at the same place
490  Real time =
491  _transient_executioners[local_app]->getTime() + _apps[local_app]->getGlobalTimeOffset();
492 
493  // Reset the Multiapp
494  MultiApp::resetApp(global_app, time);
495 
497 
498  // Setup the app, disable the output so that the initial condition does not output
499  // When an app is reset the initial condition was effectively already output before reset
500  FEProblemBase & problem = appProblemBase(local_app);
501  problem.allowOutput(false);
502  setupApp(local_app, time);
503  problem.allowOutput(true);
504  }
505 }
506 
507 void TransientMultiApp::setupApp(unsigned int i, Real /*time*/) // FIXME: Should we be passing time?
508 {
509  auto & app = _apps[i];
510  Transient * ex = dynamic_cast<Transient *>(app->getExecutioner());
511  if (!ex)
512  mooseError("MultiApp ", name(), " is not using a Transient Executioner!");
513 
514  // Get the FEProblemBase for the current MultiApp
516 
517  // Update the file numbers for the outputs from the parent application
518  app->getOutputWarehouse().setFileNumbers(_app.getOutputFileNumbers());
519 
520  // Call initialization method of Executioner (Note, this preforms the output of the initial time
521  // step, if desired)
522  ex->init();
523 
525  {
526  AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
527  System & libmesh_aux_system = aux_system.system();
528 
529  // We'll store a copy of the auxiliary system's solution at the old time in here
530  libmesh_aux_system.add_vector("transfer_old", false);
531 
532  // This will be where we'll transfer the value to for the "target" time
533  libmesh_aux_system.add_vector("transfer", false);
534  }
535 
536  ex->preExecute();
537  if (!_app.isRecovering())
538  problem.advanceState();
539  _transient_executioners[i] = ex;
540 }
const std::string & name() const
Get the name of the object.
Definition: MooseObject.h:47
bool hasLocalApp(unsigned int global_app)
Whether or not the given global app number is on this processor.
Definition: MultiApp.C:455
virtual NumericVector< Number > & appTransferVector(unsigned int app, std::string var_name) override
Get the vector to transfer to for this MultiApp.
Utility class for catching solve failure errors so that MOOSE can recover state before continuing...
virtual Real getDT()
Definition: Transient.C:694
ConstElemRange * getActiveLocalElementRange()
Return pointers to range objects for various types of ranges (local nodes, boundary elems...
Definition: MooseMesh.C:685
Transient executioners usually loop through a number of timesteps...
Definition: Transient.h:36
void mooseWarning(Args &&...args) const
Definition: MooseObject.h:89
bool _print_sub_cycles
Flag for toggling console output on sub cycles.
virtual bool lastSolveConverged() override
Whether or not the last solve converged.
Definition: Transient.C:745
virtual void advanceStep() override
Actually advances time and causes output.
bool isRecovering() const
Whether or not this is a "recover" calculation.
Definition: MooseApp.C:608
virtual void setTargetTime(Real target_time)
Can be used to set the next "target time" which is a time to nail perfectly.
Definition: Transient.C:763
Real getGlobalTimeOffset()
Each App has it&#39;s own local time.
Definition: MooseApp.h:207
virtual void computeDT()
Definition: Transient.C:347
An output object for writing to the console (screen)
Definition: Console.h:30
unsigned int _max_failures
virtual void initialSetup() override
Gets called at the beginning of the simulation before this object is asked to do its job...
std::vector< std::shared_ptr< MooseApp > > _apps
Pointers to each of the Apps.
Definition: MultiApp.h:350
virtual void init() override
Initialize the executioner.
Definition: Transient.C:236
void setupApp(unsigned int i, Real time=0.0)
Setup the executioner for the local app.
virtual void endStep(Real input_time=-1.0)
Definition: Transient.C:586
The main MOOSE class responsible for handling user-defined parameters in almost every MOOSE system...
virtual void resetApp(unsigned int global_app, Real time) override
"Reset" the App corresponding to the global App number passed in.
InputParameters validParams< TransientMultiApp >()
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
virtual Real getTime()
Get the current time.
Definition: Transient.h:103
Grab all the local dof indices for the variables passed in, in the system passed in.
virtual void advanceState()
Advance all of the state holding vectors / datastructures so that we can move to the next timestep...
MPI_Comm _orig_comm
The comm that was passed to us specifying our pool of processors.
Definition: MultiApp.h:332
Forces execution to occur (output only)
Definition: MooseTypes.h:106
virtual void takeStep(Real input_dt=-1.0)
Do whatever is necessary to advance one step.
Definition: Transient.C:392
std::set< dof_id_type > _all_dof_indices
TransientMultiApp(const InputParameters &parameters)
InputParameters validParams< MultiApp >()
Definition: MultiApp.C:46
virtual bool needsRestoration() override
Whether or not this MultiApp should be restored at the beginning of each Picard iteration.
std::set< dof_id_type > _transferred_dofs
The DoFs associated with all of the currently transferred variables.
FEProblemBase & appProblemBase(unsigned int app)
Get the FEProblemBase for the global app is part of.
Definition: MultiApp.C:407
void forceOutput()
Indicates that the next call to outputStep should be forced.
bool & _first
Is it our first time through the execution loop?
virtual void incrementStepOrReject()
This is where the solve step is actually incremented.
Definition: Transient.C:353
virtual void resetApp(unsigned int global_app, Real time=0.0)
"Reset" the App corresponding to the global App number passed in.
Definition: MultiApp.C:472
std::vector< std::string > _transferred_vars
The variables that have been transferred to. Used when doing transfer interpolation. This will be cleared after each solve.
AuxiliarySystem & getAuxiliarySystem()
std::vector< Transient * > _transient_executioners
unsigned int _my_num_apps
The number of apps this object is involved in simulating.
Definition: MultiApp.h:326
bool _has_an_app
Whether or not this processor as an App at all
Definition: MultiApp.h:383
virtual NumericVector< Number > & solution() override
Real computeDT()
Finds the smallest dt from among any of the apps.
virtual void preExecute() override
Override this for actions that should take place before execution.
Definition: Transient.C:751
std::map< std::string, unsigned int > & getOutputFileNumbers()
Store a map of outputter names and file numbers The MultiApp system requires this to get the file num...
Definition: MooseApp.h:361
unsigned int _first_local_app
The number of the first app on this processor.
Definition: MultiApp.h:329
virtual void initialSetup() override
Gets called at the beginning of the simulation before this object is asked to do its job...
Definition: MultiApp.C:191
Real & timestepTol()
Get the timestep tolerance.
Definition: Transient.h:178
unsigned int _failures
virtual System & system() override
Get the reference to the libMesh system.
MooseApp & _app
The MooseApp this object is associated with.
Definition: MooseObject.h:108
virtual MooseMesh & mesh() override
virtual void preStep()
Definition: Transient.C:286
void addParam(const std::string &name, const S &value, const std::string &doc_string)
These methods add an option parameter and a documentation string to the InputParameters object...
void mooseError(Args &&...args) const
Definition: MooseObject.h:80
ierr
const ConsoleStream _console
An instance of helper class to write streams to the Console objects.
virtual void postStep()
Definition: Transient.C:292
A MultiApp represents one or more MOOSE applications that are running simultaneously.
Definition: MultiApp.h:59
MPI_Comm _my_comm
The MPI communicator this object is going to use.
Definition: MultiApp.h:335
A system that holds auxiliary variables.
InputParameters validParams< TransientInterface >()
void allowOutput(bool state)
Ability to enable/disable all output calls.
virtual void outputStep(ExecFlagType type)
Output the current step.
Real getSolutionChangeNorm()
Get the Relative L2 norm of the change in the solution.
Definition: Transient.C:769
virtual bool solveStep(Real dt, Real target_time, bool auto_advance=true) override
Re-solve all of the Apps.
unsigned int globalAppToLocal(unsigned int global_app)
Map a global App number to the local number.
Definition: MultiApp.C:680