libMesh
parallel.h
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2017 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 #ifndef LIBMESH_PARALLEL_H
20 #define LIBMESH_PARALLEL_H
21 
22 // libMesh Includes
23 #include "libmesh/libmesh_common.h" // libmesh_assert, cast_int
24 #include "libmesh/libmesh_logging.h"
25 #include "libmesh/auto_ptr.h"
26 
27 // C++ includes
28 #include <cstddef>
29 #include <climits>
30 #include <iterator>
31 #include <limits>
32 #include <map>
33 #include <set>
34 #include <string>
35 #include <vector>
36 
37 namespace libMesh
38 {
39 
40 
41 // Macro to identify and debug functions which should only be called in
42 // parallel on every processor at once
43 
44 #ifdef LIBMESH_ENABLE_DEPRECATED
45 #undef parallel_only
46 #ifndef NDEBUG
47 #define parallel_only() do { \
48  libmesh_deprecated(); \
49  libmesh_assert(CommWorld.verify(std::string(__FILE__).size())); \
50  libmesh_assert(CommWorld.verify(std::string(__FILE__))); \
51  libmesh_assert(CommWorld.verify(__LINE__)); } while (0)
52 #else
53 #define parallel_only() ((void) 0)
54 #endif
55 #endif
56 
57 #undef libmesh_parallel_only
58 #ifndef NDEBUG
59 #define libmesh_parallel_only(comm_obj) do { \
60  libmesh_assert((comm_obj).verify(std::string(__FILE__).size())); \
61  libmesh_assert((comm_obj).verify(std::string(__FILE__))); \
62  libmesh_assert((comm_obj).verify(__LINE__)); } while (0)
63 #else
64 #define libmesh_parallel_only(comm_obj) ((void) 0)
65 #endif
66 
67 // Macro to identify and debug functions which should only be called in
68 // parallel on every processor at once
69 
70 #ifdef LIBMESH_ENABLE_DEPRECATED
71 #undef parallel_only_on
72 #ifndef NDEBUG
73 #define parallel_only_on(comm_arg) do { \
74  libmesh_deprecated(); \
75  libmesh_assert(CommWorld.verify(std::string(__FILE__).size(), comm_arg)); \
76  libmesh_assert(CommWorld.verify(std::string(__FILE__), comm_arg)); \
77  libmesh_assert(CommWorld.verify(__LINE__), comm_arg); } while (0)
78 #else
79 #define parallel_only_on(comm_arg) ((void) 0)
80 #endif
81 #endif
82 
83 #undef libmesh_parallel_only_on
84 #ifndef NDEBUG
85 #define libmesh_parallel_only_on(comm_obj,comm_arg) do { \
86  libmesh_assert(comm_obj.verify(std::string(__FILE__).size(), comm_arg)); \
87  libmesh_assert(comm_obj.verify(std::string(__FILE__), comm_arg)); \
88  libmesh_assert(comm_obj.verify(__LINE__), comm_arg); } while (0)
89 #else
90 #define libmesh_parallel_only_on(comm_obj,comm_arg) ((void) 0)
91 #endif
92 
100 namespace Parallel
101 {
102 //-------------------------------------------------------------------
106 class Communicator;
107 class DataType;
108 class Request;
109 class Status;
110 
111 #ifdef LIBMESH_HAVE_MPI
112 
116 #ifndef NDEBUG
117 #define libmesh_assert_mpi_success(error_code) \
118  do \
119  { \
120  if (error_code != MPI_SUCCESS) \
121  { \
122  char libmesh_mpi_error_string[MPI_MAX_ERROR_STRING+1]; \
123  int libmesh_mpi_error_string_len; \
124  MPI_Error_string(error_code, libmesh_mpi_error_string, \
125  &libmesh_mpi_error_string_len); \
126  libmesh_assert_equal_to_msg(error_code, MPI_SUCCESS, \
127  libmesh_mpi_error_string); \
128  } \
129  } \
130  while (0)
131 
132 #else
133 
134 #define libmesh_assert_mpi_success(error_code) ((void) 0)
135 
136 #endif
137 
138 
139 
140 // Only catch MPI return values when asserts are active.
141 #ifndef NDEBUG
142 #define libmesh_call_mpi(mpi_call) \
143  do \
144  { \
145  unsigned int libmesh_mpi_error_code = mpi_call; \
146  libmesh_assert_mpi_success (libmesh_mpi_error_code); \
147  } \
148  while (0)
149 
150 #else
151 
152 #define libmesh_call_mpi(mpi_call) \
153  do \
154  { \
155  mpi_call; \
156  } \
157  while (0)
158 #endif
159 
160 
161 
162 //-------------------------------------------------------------------
166 typedef MPI_Datatype data_type;
167 
171 typedef MPI_Request request;
172 
176 typedef MPI_Status status;
177 
181 typedef MPI_Comm communicator;
182 
187 template <typename T>
188 inline data_type dataplusint_type();
189 
193 template <typename T>
195 {
196 public:
197  T val;
198  int rank;
199 };
200 
204 const unsigned int any_source =
205  static_cast<unsigned int>(MPI_ANY_SOURCE);
206 
207 
208 #else
209 
210 // These shouldn't actually be needed, but must be
211 // unique types for function overloading to work
212 // properly.
213 struct data_type { /* unsigned int t; */ };
214 struct request { /* unsigned int r; */ };
215 struct status { /* unsigned int s; */ };
216 typedef int communicator; // Must match petsc-nompi definition
217 
218 const unsigned int any_source=0;
219 #endif // LIBMESH_HAVE_MPI
220 
221 
222 
223 //-------------------------------------------------------------------
228 {
229 public:
230 
234  static const int invalid_tag = INT_MIN;
235 
240  explicit MessageTag(int tagvalue = invalid_tag)
241  : _tagvalue(tagvalue), _comm(libmesh_nullptr) {}
242 
247  MessageTag(const MessageTag & other);
248 
253  ~MessageTag();
254 
255  int value() const {
256  return _tagvalue;
257  }
258 
259 private:
262 
263  // Constructor for reference-counted unique tags
264  MessageTag(int tagvalue, const Communicator * comm)
265  : _tagvalue(tagvalue), _comm(comm) {}
266 
267  // Let Communicator handle the reference counting
268  friend class Communicator;
269 };
270 
271 
272 //-------------------------------------------------------------------
276 #ifdef LIBMESH_HAVE_MPI
277 const MessageTag any_tag = MessageTag(MPI_ANY_TAG);
278 #else
279 const MessageTag any_tag = MessageTag(-1);
280 #endif
281 
283 
284 
285 //-------------------------------------------------------------------
289 class DataType
290 {
291 public:
292  DataType () : _datatype() {}
293 
294  DataType (const DataType & other) :
295  _datatype(other._datatype)
296  {}
297 
298  DataType (const data_type & type) :
299  _datatype(type)
300  {}
301 
302 #ifdef LIBMESH_HAVE_MPI
303  DataType (const DataType & other, unsigned int count)
304  {
305  // FIXME - if we nest an inner type here will we run into bug
306  // https://github.com/libMesh/libmesh/issues/631 again?
307  MPI_Type_contiguous(count, other._datatype, &_datatype);
308  this->commit();
309  }
310 #else
311  DataType (const DataType &, unsigned int)
312  {
313  }
314 #endif
315 
316  DataType & operator = (const DataType & other)
317  { _datatype = other._datatype; return *this; }
318 
319  DataType & operator = (const data_type & type)
320  { _datatype = type; return *this; }
321 
322  operator const data_type & () const
323  { return _datatype; }
324 
325  operator data_type & ()
326  { return _datatype; }
327 
328  // operator data_type const * () const
329  // { return &_datatype; }
330 
331  // operator data_type * ()
332  // { return &_datatype; }
333 
334  void commit ()
335  {
336 #ifdef LIBMESH_HAVE_MPI
337  MPI_Type_commit (&_datatype);
338 #endif
339  }
340 
341  void free ()
342  {
343 #ifdef LIBMESH_HAVE_MPI
344  MPI_Type_free (&_datatype);
345 #endif
346  }
347 
348 protected:
349 
350  data_type _datatype;
351 };
352 
353 
354 //-------------------------------------------------------------------
355 
356 #ifdef LIBMESH_HAVE_CXX11
357 // A C++03-compatible replacement for std::false_type
359 {
360  static const bool value = false;
361  typedef bool value_type;
362  typedef false_type type;
363  operator value_type() const { return value; }
364 };
365 
366 // Templated helper class to be used with static_assert.
367 template<typename T>
369 {};
370 #endif
371 
379 template <typename T>
380 class StandardType : public DataType
381 {
382 #ifdef LIBMESH_HAVE_CXX11
383  // Get a slightly better compiler diagnostic if we have C++11
384  static_assert(dependent_false<T>::value,
385  "Only specializations of StandardType may be used, did you forget to include a header file (e.g. parallel_algebra.h)?");
386 #endif
387 
388  /*
389  * The unspecialized class is useless, so we make its constructor
390  * private to catch mistakes at compile-time rather than link-time.
391  * Specializations should have a public constructor of the same
392  * form.
393  */
394 private:
395  StandardType(const T * example = libmesh_nullptr);
396 };
397 
405 template <typename T>
407 {
408 #ifdef LIBMESH_HAVE_CXX11
409  // Get a slightly better compiler diagnostic if we have C++11
410  static_assert(dependent_false<T>::value,
411  "Only specializations of OpFunction may be used, did you forget to include a header file (e.g. parallel_algebra.h)?");
412 #endif
413 
414  /*
415  * The unspecialized class defines none of these functions;
416  * specializations will need to define any functions that need to be
417  * usable.
418  *
419  * Most specializations will just return MPI_MIN, etc, but we'll use
420  * a whitelist rather than a default implementation, so that any
421  * attempt to perform a reduction on an unspecialized type will be a
422  * compile-time rather than a run-time failure.
423  */
424 /*
425  static MPI_Op max();
426  static MPI_Op min();
427  static MPI_Op sum();
428  static MPI_Op product();
429  static MPI_Op logical_and();
430  static MPI_Op bitwise_and();
431  static MPI_Op logical_or();
432  static MPI_Op bitwise_or();
433  static MPI_Op logical_xor();
434  static MPI_Op bitwise_xor();
435  static MPI_Op max_loc();
436  static MPI_Op min_loc();
437  */
438 };
439 
440 /*
441  * The unspecialized class gives default, lowest-common-denominator
442  * attributes, for values which can't be used with Parallel min/max.
443  * Specialized classes can set this to true, and should define
444  * the lowest and highest values possible for the type.
445  */
446 template<typename T>
448 {
449  static const bool has_min_max = false;
450  static void set_lowest(T &) {}
451  static void set_highest(T &) {}
452 };
453 
454 
455 
456 //-------------------------------------------------------------------
461 class Status
462 {
463 public:
464  Status ();
465 
466  explicit Status (const data_type & type);
467 
468  explicit Status (const status & status);
469 
470  Status (const status & status,
471  const data_type & type);
472 
473  Status (const Status & status);
474 
475  Status (const Status & status,
476  const data_type & type);
477 
478  status * get() { return &_status; }
479 
480  status const * get() const { return &_status; }
481 
482  int source () const;
483 
484  int tag () const;
485 
486  data_type & datatype () { return _datatype; }
487 
488  const data_type & datatype () const { return _datatype; }
489 
490  unsigned int size (const data_type & type) const;
491 
492  unsigned int size () const;
493 
494 private:
495 
496  status _status;
497  data_type _datatype;
498 };
499 
500 
501 //-------------------------------------------------------------------
506 struct PostWaitWork {
507  virtual ~PostWaitWork() {}
508 
509  virtual void run() {}
510 };
511 
512 
513 //-------------------------------------------------------------------
517 class Request
518 {
519 public:
520  Request ();
521 
522  Request (const request & r);
523 
524  Request (const Request & other);
525 
526  void cleanup();
527 
528  Request & operator = (const Request & other);
529 
530  Request & operator = (const request & r);
531 
532  ~Request ();
533 
534  request * get() { return &_request; }
535 
536  const request * get() const { return &_request; }
537 
538  Status wait ();
539 
540  bool test ();
541 
542  bool test (status & status);
543 
544  void add_prior_request(const Request & req);
545 
546  void add_post_wait_work(PostWaitWork * work);
547 
548 private:
549  request _request;
550 
551  // Breaking non-blocking sends into multiple requests can require chaining
552  // multiple requests into a single Request
554 
555  // post_wait_work->first is a vector of work to do after a wait
556  // finishes; post_wait_work->second is a reference count so that
557  // Request objects will behave roughly like a shared_ptr and be
558  // usable in STL containers
559  std::pair<std::vector <PostWaitWork * >, unsigned int> * post_wait_work;
560 };
561 
565 inline Status wait (Request & r) { return r.wait(); }
566 
570 inline void wait (std::vector<Request> & r)
571 { for (std::size_t i=0; i<r.size(); i++) r[i].wait(); }
572 
573 
581 template <typename T>
582 class Packing {
583 public:
584  // Should be an MPI sendable type in specializations, e.g.
585  // typedef char buffer_type;
586  // typedef unsigned int buffer_type;
587 
588  // Should copy an encoding of the provided object into the provided
589  // output iterator (which is of type buffer_type)
590  template <typename OutputIter, typename Context>
591  static void pack(const T & object,
592  OutputIter data_out,
593  const Context * context);
594 
595  // Should return the number of array entries (of type buffer_type)
596  // required to encode the provided object
597  template <typename Context>
598  static unsigned int packable_size(const T & object,
599  const Context * context);
600 
601  // Should return the number of array entries which were used to
602  // encode the provided serialization of an object which begins at
603  // \p iter
604  template <typename BufferIter>
605  static unsigned int packed_size(BufferIter iter);
606 
607  // Decode a potentially-variable-size object from a subsequence of a
608  // data array, returning a heap-allocated pointer to the result.
609  template <typename BufferIter, typename Context>
610  static T unpack(BufferIter in, Context * ctx);
611 };
612 
613 
618 template <typename Context, typename buffertype,
619  typename OutputIter, typename T>
620 inline void unpack_range (const typename std::vector<buffertype> & buffer,
621  Context * context,
622  OutputIter out,
623  const T * output_type /* used only to infer T */);
624 
633 template <typename Context, typename buffertype, typename Iter>
634 inline Iter pack_range (const Context * context,
635  Iter range_begin,
636  const Iter range_end,
637  typename std::vector<buffertype> & buffer,
638  std::size_t approx_buffer_size = 1000000);
639 
644 template <typename Context, typename Iter>
645 inline std::size_t packed_range_size (const Context * context,
646  Iter range_begin,
647  const Iter range_end);
648 
649 //-------------------------------------------------------------------
658 {
659  // Basic operations:
660 public:
661 
665  Communicator ();
666 
667  /*
668  * Constructor from MPI_Comm
669  */
670  explicit Communicator (const communicator & comm);
671 
672  /*
673  * NON-VIRTUAL destructor
674  */
675  ~Communicator ();
676 
677  /*
678  * Create a new communicator between some subset of \p this
679  */
680  void split(int color, int key, Communicator & target) const;
681 
682  /*
683  * Create a new duplicate of \p this communicator
684  */
685  void duplicate(const Communicator & comm);
686 
687  /*
688  * Create a new duplicate of an MPI communicator
689  */
690  void duplicate(const communicator & comm);
691 
692  communicator & get() { return _communicator; }
693 
694  const communicator & get() const { return _communicator; }
695 
703  MessageTag get_unique_tag(int tagvalue) const;
704 
709  void reference_unique_tag(int tagvalue) const;
710 
715  void dereference_unique_tag(int tagvalue) const;
716 
720  void clear();
721 
722  Communicator & operator= (const communicator & comm);
723 
724  unsigned int rank() const { return _rank; }
725 
726  unsigned int size() const { return _size; }
727 
731  enum SendMode { DEFAULT=0, SYNCHRONOUS };
732 
733 private:
734 
735  // Don't use the copy constructor, just copy by reference or
736  // pointer - it's too hard to keep a common used_tag_values if
737  // each communicator is shared by more than one Communicator
738  explicit Communicator (const Communicator &);
739 
744  void assign(const communicator & comm);
745 
746  communicator _communicator;
747  unsigned int _rank, _size;
749 
750  // mutable used_tag_values - not thread-safe, but then Parallel::
751  // isn't thread-safe in general.
752  mutable std::map<int, unsigned int> used_tag_values;
754 
755  // Communication operations:
756 public:
757 
761  void send_mode (const SendMode sm) { _send_mode = sm; }
762 
766  SendMode send_mode() const { return _send_mode; }
767 
771  void barrier () const;
772 
777  template <typename T>
778  bool verify(const T & r) const;
779 
785  template <typename T>
786  bool semiverify(const T * r) const;
787 
792  template <typename T>
793  void min(T & r) const;
794 
800  template <typename T>
801  void minloc(T & r,
802  unsigned int & min_id) const;
803 
809  template <typename T>
810  void minloc(std::vector<T> & r,
811  std::vector<unsigned int> & min_id) const;
812 
817  template <typename T>
818  void max(T & r) const;
819 
825  template <typename T>
826  void maxloc(T & r,
827  unsigned int & max_id) const;
828 
834  template <typename T>
835  void maxloc(std::vector<T> & r,
836  std::vector<unsigned int> & max_id) const;
837 
842  template <typename T>
843  void sum(T & r) const;
844 
850  template <typename T>
851  void set_union(T & data, const unsigned int root_id) const;
852 
857  template <typename T>
858  void set_union(T & data) const;
859 
864  status probe (const unsigned int src_processor_id,
865  const MessageTag & tag=any_tag) const;
866 
879  template <typename T>
880  Status packed_range_probe (const unsigned int src_processor_id,
881  const MessageTag & tag,
882  bool & flag) const;
883 
887  template <typename T>
888  void send (const unsigned int dest_processor_id,
889  const T & buf,
890  const MessageTag & tag=no_tag) const;
891 
895  template <typename T>
896  void send (const unsigned int dest_processor_id,
897  const T & buf,
898  Request & req,
899  const MessageTag & tag=no_tag) const;
900 
904  template <typename T>
905  void send (const unsigned int dest_processor_id,
906  const T & buf,
907  const DataType & type,
908  const MessageTag & tag=no_tag) const;
909 
913  template <typename T>
914  void send (const unsigned int dest_processor_id,
915  const T & buf,
916  const DataType & type,
917  Request & req,
918  const MessageTag & tag=no_tag) const;
919 
923  template <typename T>
924  Status receive (const unsigned int dest_processor_id,
925  T & buf,
926  const MessageTag & tag=any_tag) const;
927 
931  template <typename T>
932  void receive (const unsigned int dest_processor_id,
933  T & buf,
934  Request & req,
935  const MessageTag & tag=any_tag) const;
936 
940  template <typename T>
941  Status receive (const unsigned int dest_processor_id,
942  T & buf,
943  const DataType & type,
944  const MessageTag & tag=any_tag) const;
945 
949  template <typename T>
950  void receive (const unsigned int dest_processor_id,
951  T & buf,
952  const DataType & type,
953  Request & req,
954  const MessageTag & tag=any_tag) const;
955 
969  template <typename Context, typename Iter>
970  void send_packed_range (const unsigned int dest_processor_id,
971  const Context * context,
972  Iter range_begin,
973  const Iter range_end,
974  const MessageTag & tag=no_tag) const;
975 
989  template <typename Context, typename Iter>
990  void send_packed_range (const unsigned int dest_processor_id,
991  const Context * context,
992  Iter range_begin,
993  const Iter range_end,
994  Request & req,
995  const MessageTag & tag=no_tag) const;
996 
1005  template <typename Context, typename Iter>
1006  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
1007  const Context * context,
1008  Iter range_begin,
1009  const Iter range_end,
1010  Request & req,
1011  const MessageTag & tag=no_tag) const;
1012 
1039  template <typename Context, typename OutputIter, typename T>
1040  void receive_packed_range (const unsigned int dest_processor_id,
1041  Context * context,
1042  OutputIter out,
1043  const T * output_type, // used only to infer T
1044  const MessageTag & tag=any_tag) const;
1045 
1058  template <typename Context, typename OutputIter, typename T>
1059  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
1060  Context * context,
1061  OutputIter out,
1062  const T * output_type,
1063  Request & req,
1064  Status & stat,
1065  const MessageTag & tag=any_tag) const;
1066 
1071  template <typename T1, typename T2>
1072  void send_receive(const unsigned int dest_processor_id,
1073  const T1 & send,
1074  const unsigned int source_processor_id,
1075  T2 & recv,
1076  const MessageTag & send_tag = no_tag,
1077  const MessageTag & recv_tag = any_tag) const;
1078 
1115  template <typename Context1, typename RangeIter, typename Context2,
1116  typename OutputIter, typename T>
1117  void send_receive_packed_range(const unsigned int dest_processor_id,
1118  const Context1 * context1,
1119  RangeIter send_begin,
1120  const RangeIter send_end,
1121  const unsigned int source_processor_id,
1122  Context2 * context2,
1123  OutputIter out,
1124  const T * output_type, // used only to infer T
1125  const MessageTag & send_tag = no_tag,
1126  const MessageTag & recv_tag = any_tag) const;
1127 
1133  template <typename T1, typename T2>
1134  void send_receive(const unsigned int dest_processor_id,
1135  const T1 & send,
1136  const DataType & type1,
1137  const unsigned int source_processor_id,
1138  T2 & recv,
1139  const DataType & type2,
1140  const MessageTag & send_tag = no_tag,
1141  const MessageTag & recv_tag = any_tag) const;
1142 
1147  template <typename T>
1148  inline void gather(const unsigned int root_id,
1149  const T & send,
1150  std::vector<T> & recv) const;
1151 
1155  template <typename T>
1156  inline void gather(const unsigned int root_id,
1157  const std::basic_string<T> & send,
1158  std::vector<std::basic_string<T>> & recv,
1159  const bool identical_buffer_sizes=false) const;
1160 
1183  template <typename T>
1184  inline void gather(const unsigned int root_id,
1185  std::vector<T> & r) const;
1186 
1191  template <typename T>
1192  inline void allgather(const T & send,
1193  std::vector<T> & recv) const;
1194 
1198  template <typename T>
1199  inline void allgather(const std::basic_string<T> & send,
1200  std::vector<std::basic_string<T>> & recv,
1201  const bool identical_buffer_sizes=false) const;
1202 
1227  template <typename T>
1228  inline void allgather(std::vector<T> & r,
1229  const bool identical_buffer_sizes = false) const;
1230 
1234  template <typename T>
1235  inline void allgather(std::vector<std::basic_string<T>> & r,
1236  const bool identical_buffer_sizes = false) const;
1237 
1238  //-------------------------------------------------------------------
1243  template <typename T>
1244  inline void scatter(const std::vector<T> & data,
1245  T & recv,
1246  const unsigned int root_id=0) const;
1247 
1254  template <typename T>
1255  inline void scatter(const std::vector<T> & data,
1256  std::vector<T> & recv,
1257  const unsigned int root_id=0) const;
1258 
1265  template <typename T>
1266  inline void scatter(const std::vector<T> & data,
1267  const std::vector<int> counts,
1268  std::vector<T> & recv,
1269  const unsigned int root_id=0) const;
1270 
1276  template <typename T>
1277  inline void scatter(const std::vector<std::vector<T>> & data,
1278  std::vector<T> & recv,
1279  const unsigned int root_id=0,
1280  const bool identical_buffer_sizes=false) const;
1281 
1282  //-------------------------------------------------------------------
1287  template <typename Context, typename Iter, typename OutputIter>
1288  inline void gather_packed_range (const unsigned int root_id,
1289  Context * context,
1290  Iter range_begin,
1291  const Iter range_end,
1292  OutputIter out) const;
1293 
1298  template <typename Context, typename Iter, typename OutputIter>
1299  inline void allgather_packed_range (Context * context,
1300  Iter range_begin,
1301  const Iter range_end,
1302  OutputIter out) const;
1303 
1309  template <typename T>
1310  inline void alltoall(std::vector<T> & r) const;
1311 
1320  template <typename T>
1321  inline void broadcast(T & data, const unsigned int root_id=0) const;
1322 
1340  template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
1341  inline void broadcast_packed_range (const Context * context1,
1342  Iter range_begin,
1343  const Iter range_end,
1344  OutputContext * context2,
1345  OutputIter out,
1346  const unsigned int root_id = 0) const;
1347 
1355 #include "libmesh/parallel_communicator_specializations"
1356 
1357 }; // class Communicator
1358 
1359 // FakeCommunicator for debugging inappropriate CommWorld uses
1361 {
1362  operator Communicator & ()
1363  {
1364  libmesh_not_implemented();
1365  static Communicator temp;
1366  return temp;
1367  }
1368 };
1369 
1370 // PostWaitWork specialization for copying from temporary to
1371 // output containers
1372 template <typename Container, typename OutputIter>
1374  PostWaitCopyBuffer(const Container & buffer, const OutputIter out)
1375  : _buf(buffer), _out(out) {}
1376 
1377  virtual void run() libmesh_override { std::copy(_buf.begin(), _buf.end(), _out); }
1378 
1379 private:
1380  const Container & _buf;
1381  OutputIter _out;
1382 };
1383 
1384 // PostWaitWork specialization for unpacking received buffers.
1385 template <typename Container, typename Context, typename OutputIter,
1386  typename T>
1388  PostWaitUnpackBuffer(const Container & buffer, Context * context, OutputIter out) :
1389  _buf(buffer), _context(context), _out(out) {}
1390 
1391  virtual void run() libmesh_override {
1392 
1393  Parallel::unpack_range(_buf, _context, _out, (T*)libmesh_nullptr);
1394  }
1395 
1396 private:
1397  const Container & _buf;
1398  Context * _context;
1399  OutputIter _out;
1400 };
1401 
1402 
1403 // PostWaitWork specialization for freeing no-longer-needed buffers.
1404 template <typename Container>
1406  PostWaitDeleteBuffer(Container * buffer) : _buf(buffer) {}
1407 
1408  virtual void run() libmesh_override { delete _buf; }
1409 
1410 private:
1411  Container * _buf;
1412 };
1413 
1414 } // namespace Parallel
1415 
1429 #ifdef LIBMESH_DISABLE_COMMWORLD
1431 #define LIBMESH_CAN_DEFAULT_TO_COMMWORLD
1432 #else
1434 #define LIBMESH_CAN_DEFAULT_TO_COMMWORLD = libMesh::CommWorld
1435 #endif
1436 
1437 } // namespace libMesh
1438 
1439 // Define all the implementations separately; users might want to look
1440 // through this file for APIs, and it's long enough already.
1441 
1442 #include "libmesh/parallel_implementation.h"
1443 
1444 #endif // LIBMESH_PARALLEL_H
void alltoall(std::vector< T > &r, const Communicator &comm=Communicator_World)
virtual void run() libmesh_override
Definition: parallel.h:1377
SendMode
Whether to use default or synchronous sends?
Definition: parallel.h:731
void receive_packed_range(const unsigned int src_processor_id, Context *context, OutputIter out_iter, const T *output_type, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
MessageTag(int tagvalue=invalid_tag)
Explicit constructor, to discourage using "magic numbers" as tags.
Definition: parallel.h:240
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Status wait(Request &r)
Wait for a non-blocking send or receive to finish.
Definition: parallel.h:565
Encapsulates the MPI_Comm object.
Definition: parallel.h:657
void send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag, const Communicator &comm=Communicator_World)
void send(const unsigned int dest_processor_id, const T &data, const MessageTag &tag=no_tag, const Communicator &comm=Communicator_World)
const unsigned int any_source
Accept from any source.
Definition: parallel.h:204
Encapsulates the MPI_Status struct.
Definition: parallel.h:461
unsigned int size() const
Definition: parallel.h:726
static void set_lowest(T &)
Definition: parallel.h:450
MPI_Comm communicator
Communicator object for talking with subsets of processors.
Definition: parallel.h:181
void allgather_packed_range(Context *context, Iter range_begin, const Iter range_end, OutputIter out_iter, const Communicator &comm=Communicator_World)
MPI_Request request
Request object for non-blocking I/O.
Definition: parallel.h:171
const class libmesh_nullptr_t libmesh_nullptr
MPI_Datatype data_type
Data types for communication.
Definition: parallel.h:166
void send_mode(const SendMode sm)
Explicitly sets the SendMode type used for send operations.
Definition: parallel.h:761
Status receive(const unsigned int src_processor_id, T &buf, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
std::pair< std::vector< PostWaitWork * >, unsigned int > * post_wait_work
Definition: parallel.h:559
The libMesh namespace provides an interface to certain functionality in the library.
void sum(T &r, const Communicator &comm=Communicator_World)
void gather_packed_range(const unsigned int root_id, Context *context, Iter range_begin, const Iter range_end, OutputIter out_iter, const Communicator &comm=Communicator_World)
Define data types and (un)serialization functions for use when encoding a potentially-variable-size o...
Definition: parallel.h:582
void broadcast_packed_range(const Context *context1, Iter range_begin, const Iter range_end, OutputContext *context2, OutputIter out_iter, const unsigned int root_id=0, const Communicator &comm=Communicator_World)
PostWaitDeleteBuffer(Container *buffer)
Definition: parallel.h:1406
std::unique_ptr< T > UniquePtr
Definition: auto_ptr.h:46
std::size_t packed_range_size(const Context *context, Iter range_begin, const Iter range_end)
Return the total buffer size needed to encode a range of potentially-variable-size objects to a data ...
virtual void run() libmesh_override
Definition: parallel.h:1391
void barrier(const Communicator &comm=Communicator_World)
PostWaitCopyBuffer(const Container &buffer, const OutputIter out)
Definition: parallel.h:1374
PostWaitUnpackBuffer(const Container &buffer, Context *context, OutputIter out)
Definition: parallel.h:1388
void broadcast(T &data, const unsigned int root_id=0, const Communicator &comm=Communicator_World)
UniquePtr< Request > _prior_request
Definition: parallel.h:553
void allgather(T send, std::vector< T > &recv, const Communicator &comm=Communicator_World)
MessageTag(int tagvalue, const Communicator *comm)
Definition: parallel.h:264
Encapsulates the MPI tag integers.
Definition: parallel.h:227
DataType(const data_type &type)
Definition: parallel.h:298
const data_type & datatype() const
Definition: parallel.h:488
Templated class to provide the appropriate MPI datatype for use with built-in C types or simple C++ c...
Definition: parallel.h:380
void gather(const unsigned int root_id, T send, std::vector< T > &recv, const Communicator &comm=Communicator_World)
PetscErrorCode Vec Mat Mat void * ctx
MPI_Status status
Status object for querying messages.
Definition: parallel.h:176
DataType(const DataType &other)
Definition: parallel.h:294
virtual void run() libmesh_override
Definition: parallel.h:1408
data_type dataplusint_type()
Templated function to return the appropriate MPI datatype for use with built-in C types when combined...
Encapsulates the MPI_Datatype.
Definition: parallel.h:289
const MessageTag no_tag
Definition: parallel.h:282
tbb::split split
Dummy "splitting object" used to distinguish splitting constructors from copy constructors.
Definition: threads_tbb.h:79
Parallel::FakeCommunicator CommWorld
The default libMesh communicator.
Definition: parallel.h:1433
bool verify(const T &r, const Communicator &comm=Communicator_World)
DataType(const DataType &, unsigned int)
Definition: parallel.h:311
Encapsulates the MPI_Request.
Definition: parallel.h:517
OStreamProxy out
void set_union(T &data, const unsigned int root_id, const Communicator &comm=Communicator_World)
void max(T &r, const Communicator &comm=Communicator_World)
static const bool value
Definition: xdr_io.C:108
data_type & datatype()
Definition: parallel.h:486
Iterator & operator=(const Iterator &rhs)
Assignment operator.
const MessageTag any_tag
Default message tag ids.
Definition: parallel.h:277
DataType(const DataType &other, unsigned int count)
Definition: parallel.h:303
unsigned int rank() const
Definition: parallel.h:724
Types combined with an int.
Definition: parallel.h:194
Templated class to provide the appropriate MPI reduction operations for use with built-in C types or ...
Definition: parallel.h:406
std::map< int, unsigned int > used_tag_values
Definition: parallel.h:752
IterBase * data
Ideally this private member data should have protected access.
static void set_highest(T &)
Definition: parallel.h:451
A class that can be subclassed to allow other code to perform work after a MPI_Wait succeeds...
Definition: parallel.h:506
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
Decode a range of potentially-variable-size objects from a data array.
void maxloc(T &r, U &max_id, const Communicator &comm=Communicator_World)
void min(T &r, const Communicator &comm=Communicator_World)
void send_receive_packed_range(const unsigned int dest_processor_id, const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int source_processor_id, Context2 *context2, OutputIter out_iter, const T *output_type, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag, const Communicator &comm=Communicator_World)
status probe(const unsigned int src_processor_id, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
void minloc(T &r, U &min_id, const Communicator &comm=Communicator_World)
const Communicator * _comm
Definition: parallel.h:261
SendMode send_mode() const
Gets the user-requested SendMode.
Definition: parallel.h:766
void send_receive(const unsigned int dest_processor_id, T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag, const Communicator &comm=Communicator_World)