queso-0.57.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Pages
MpiComm.C
Go to the documentation of this file.
1 //-----------------------------------------------------------------------bl-
2 //--------------------------------------------------------------------------
3 //
4 // QUESO - a library to support the Quantification of Uncertainty
5 // for Estimation, Simulation and Optimization
6 //
7 // Copyright (C) 2008-2017 The PECOS Development Team
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the Version 2.1 GNU Lesser General
11 // Public License as published by the Free Software Foundation.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc. 51 Franklin Street, Fifth Floor,
21 // Boston, MA 02110-1301 USA
22 //
23 //-----------------------------------------------------------------------el-
24 
25 #include <unistd.h>
26 #include <cstring>
27 #include <queso/MpiComm.h>
28 #include <queso/Environment.h>
29 
30 #ifdef QUESO_HAS_TRILINOS
31 #include <Epetra_MpiComm.h>
32 #include <Epetra_SerialComm.h>
33 #endif
34 
35 namespace QUESO {
36 
37 // QUESO MpiComm MPI Constructor ------------------
38 MpiComm::MpiComm(const BaseEnvironment& env, RawType_MPI_Comm inputRawComm)
39  :
40  m_env (env),
41 #ifdef QUESO_HAS_TRILINOS
42  m_epetraComm( new Epetra_MpiComm(inputRawComm) ),
43 #endif
44  m_rawComm (inputRawComm),
45  m_worldRank (-1),
46  m_myPid (-1),
47  m_numProc (-1)
48 {
49 #ifdef QUESO_HAS_MPI
50  int mpiRC = MPI_Comm_rank(inputRawComm,&m_worldRank);
51  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, "failed MPI_Comm_rank() on full rank");
52 
53  mpiRC = MPI_Comm_rank(inputRawComm,&m_myPid);
54  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, "failed MPI_Comm_rank() on inputRawComm");
55 
56  mpiRC = MPI_Comm_size(inputRawComm,&m_numProc);
57  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, "failed MPI_Comm_size() on inputRawComm");
58 #else
59  m_worldRank = 0;
60  m_myPid = 0;
61  m_numProc = 1;
62 #endif
63 }
64 
65 MpiComm::MpiComm(const BaseEnvironment& env)
66  :
67  m_env (env),
68 #ifdef QUESO_HAS_TRILINOS
69  m_epetraComm( new Epetra_SerialComm() ),
70 #endif
71  m_rawComm (RawValue_MPI_COMM_SELF),
72  m_worldRank (0),
73  m_myPid (0),
74  m_numProc (1)
75 {
76 }
77 
78 // Copy constructor ---------------------------------
79 MpiComm::MpiComm(const MpiComm& src)
80  :
81  m_env (src.m_env)
82 #ifdef QUESO_HAS_TRILINOS
83  ,
84  m_epetraComm(NULL)
85 #endif
86 {
87  this->copy(src);
88 }
89 
90 // Destructor ---------------------------------------
91 MpiComm::~MpiComm()
92 {
93 #ifdef QUESO_HAS_TRILINOS
94  delete m_epetraComm;
95  m_epetraComm = NULL;
96 #endif
97 }
98 
99 // --------------------------------------------------
100 // Set methodos -------------------------------------
101 MpiComm&
102 MpiComm::operator=(const MpiComm& rhs)
103 {
104  this->copy(rhs);
105  return *this;
106 }
107 
108 // Attribute access methods -------------------------
109 #ifdef QUESO_HAS_MPI
110 RawType_MPI_Comm
111 MpiComm::Comm() const
112 {
113 #ifdef QUESO_HAS_TRILINOS
114 #ifdef QUESO_HAS_MPI
115  return dynamic_cast<Epetra_MpiComm *>(m_epetraComm)->Comm();
116 #endif
117 #endif
118  return m_rawComm;
119 }
120 #endif // QUESO_HAS_MPI
121 
122 // --------------------------------------------------
123 int
124 MpiComm::MyPID() const
125 {
126 #ifdef QUESO_HAS_TRILINOS
127  return m_epetraComm->MyPID();
128 #endif
129  return m_myPid;
130 }
131 // --------------------------------------------------
132 int
133 MpiComm::NumProc() const
134 {
135 #ifdef QUESO_HAS_TRILINOS
136  return m_epetraComm->NumProc();
137 #endif
138  return m_numProc;
139 }
140 // Methods overridden from Comm ---------------------
141 
142 void
143 MpiComm::Allreduce(void* sendbuf, void* recvbuf, int count, RawType_MPI_Datatype datatype, RawType_MPI_Op op, const char* whereMsg, const char* whatMsg) const
144 {
145  queso_deprecated();
146  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
147 #ifdef QUESO_HAS_MPI
148  int mpiRC = MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, m_rawComm);
149  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
150 #endif
151  }
152 }
153 
154 template <typename T>
155 void
156 MpiComm::Allreduce(const T* sendbuf, T* recvbuf, int count, RawType_MPI_Op op,
157  const char* whereMsg, const char* whatMsg) const
158 {
159  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
160 #ifdef QUESO_HAS_MPI
161  T * sendbuf_noconst = const_cast<T *>(sendbuf);
162  int mpiRC = MPI_Allreduce(sendbuf_noconst, recvbuf, count, StandardType<T>(sendbuf), op, m_rawComm);
163  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
164 #endif
165  }
166  else {
167  size_t dataTypeSize = sizeof(T);
168  size_t dataTotal = dataTypeSize*count;
169  std::memcpy(recvbuf, sendbuf, dataTotal);
170  }
171 }
172 //--------------------------------------------------
173 void
174 MpiComm::Barrier() const // const char* whereMsg, const char* whatMsg) const
175 {
176 #ifdef QUESO_HAS_TRILINOS
177  return m_epetraComm->Barrier();
178 #endif
179 
180  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
181 #ifdef QUESO_HAS_MPI
182  int mpiRC = MPI_Barrier(m_rawComm);
183  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, "mpiRC indicates failure"); // whatMsg);
184 #endif
185  }
186 
187  return;
188 }
189 //--------------------------------------------------
190 void
191 MpiComm::Bcast(void* buffer, int count, RawType_MPI_Datatype datatype, int root, const char* whereMsg, const char* whatMsg) const
192 {
193  if (NumProc() > 1) { // Necesarrily true if QUESO_HAS_MPI
194 #ifdef QUESO_HAS_MPI
195  int mpiRC = MPI_Bcast(buffer, count, datatype, root, m_rawComm);
196  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
197 #endif
198  }
199 }
200 //--------------------------------------------------
201 void
202 MpiComm::Gather(
203  void* sendbuf, int sendcnt, RawType_MPI_Datatype sendtype,
204  void* recvbuf, int recvcount, RawType_MPI_Datatype recvtype,
205  int root,
206  const char* whereMsg, const char* whatMsg) const
207 {
208  queso_deprecated();
209  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
210 #ifdef QUESO_HAS_MPI
211  //int MPI_Gather (void *sendbuf, int sendcnt, MPI_Datatype sendtype,
212  // void *recvbuf, int recvcount, MPI_Datatype recvtype,
213  // int root, MPI_Comm comm )
214  int mpiRC = MPI_Gather(sendbuf, sendcnt, sendtype,
215  recvbuf, recvcount, recvtype,
216  root, m_rawComm);
217  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
218 #endif
219  }
220 }
221 
222 template <typename T>
223 void
224 MpiComm::Gather(const T * sendbuf, int sendcnt, T * recvbuf, int recvcount,
225  int root, const char* whereMsg, const char* whatMsg) const
226 {
227  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
228 #ifdef QUESO_HAS_MPI
229  //int MPI_Gather (void *sendbuf, int sendcnt, MPI_Datatype sendtype,
230  // void *recvbuf, int recvcount, MPI_Datatype recvtype,
231  // int root, MPI_Comm comm )
232  T * sendbuf_noconst = const_cast<T *>(sendbuf);
233  int mpiRC = MPI_Gather(sendbuf_noconst, sendcnt, StandardType<T>(sendbuf),
234  recvbuf, recvcount, StandardType<T>(sendbuf),
235  root, m_rawComm);
236  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
237 #endif
238  }
239  else {
240  size_t dataTypeSize = sizeof(T);
241  size_t sendTotal = dataTypeSize*sendcnt;
242  size_t recvTotal = dataTypeSize*recvcount;
243  if (sendTotal != recvTotal) {
244  std::cerr << "MpiCommClass::Gather()"
245  << ": sendTotal != recvTotal"
246  << std::endl;
247  }
248  queso_require_equal_to_msg(sendTotal, recvTotal, whatMsg);
249  std::memcpy(recvbuf, sendbuf, sendTotal);
250  }
251 }
252 //--------------------------------------------------
253 void
254 MpiComm::Gatherv(
255  void* sendbuf, int sendcnt, RawType_MPI_Datatype sendtype,
256  void* recvbuf, int* recvcnts, int* displs, RawType_MPI_Datatype recvtype,
257  int root,
258  const char* whereMsg, const char* whatMsg) const
259 {
260  queso_deprecated();
261  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
262 #ifdef QUESO_HAS_MPI
263  //int MPI_Gatherv(void *sendbuf, int sendcnt, MPI_Datatype sendtype,
264  // void *recvbuf, int *recvcnts, int *displs, MPI_Datatype recvtype,
265  // int root, MPI_Comm comm )
266  int mpiRC = MPI_Gatherv(sendbuf, sendcnt, sendtype,
267  recvbuf, recvcnts, displs, recvtype,
268  root, m_rawComm);
269  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
270 #endif
271  }
272 }
273 
274 template <typename T>
275 void
276 MpiComm::Gatherv(const T * sendbuf, int sendcnt, T * recvbuf, int * recvcnts,
277  int * displs, int root, const char * whereMsg,
278  const char * whatMsg) const
279 {
280  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
281 #ifdef QUESO_HAS_MPI
282  //int MPI_Gatherv(void *sendbuf, int sendcnt, MPI_Datatype sendtype,
283  // void *recvbuf, int *recvcnts, int *displs, MPI_Datatype recvtype,
284  // int root, MPI_Comm comm )
285  T * sendbuf_noconst = const_cast<T *>(sendbuf);
286  int mpiRC = MPI_Gatherv(sendbuf_noconst, sendcnt, StandardType<T>(sendbuf),
287  recvbuf, recvcnts, displs,
288  StandardType<T>(recvbuf), root, m_rawComm);
289  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
290 #endif
291  }
292  else {
293  size_t dataTypeSize = sizeof(T);
294  size_t sendTotal = dataTypeSize*sendcnt;
295  size_t recvTotal = dataTypeSize*recvcnts[0];
296  if (sendTotal != recvTotal) {
297  std::cerr << "MpiCommClass::Gatherv()"
298  << ": sendTotal != recvTotal"
299  << std::endl;
300  }
301  queso_require_equal_to_msg(sendTotal, recvTotal, whatMsg);
302  std::memcpy(recvbuf, sendbuf, sendTotal);
303  }
304 }
305 //--------------------------------------------------
306 void
307 MpiComm::Recv(
308  void* buf, int count, RawType_MPI_Datatype datatype, int source, int tag, RawType_MPI_Status* status,
309  const char* whereMsg, const char* whatMsg) const
310 {
311  if (NumProc() > 1) { // Necesarrily true if QUESO_HAS_MPI
312 #ifdef QUESO_HAS_MPI
313  int mpiRC = MPI_Recv(buf, count, datatype, source, tag, m_rawComm, status);
314  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
315 #endif
316  }
317 }
318 //--------------------------------------------------
319 void
320 MpiComm::Send(
321  void* buf, int count, RawType_MPI_Datatype datatype, int dest, int tag,
322  const char* whereMsg, const char* whatMsg) const
323 {
324  if (NumProc() > 1) { // Necesarrily true if QUESO_HAS_MPI
325 #ifdef QUESO_HAS_MPI
326  int mpiRC = MPI_Send(buf, count, datatype, dest, tag, m_rawComm);
327  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
328 #endif
329  }
330 }
331 // Misc methods ------------------------------------
332 void
333 MpiComm::syncPrintDebugMsg(const char* msg, unsigned int msgVerbosity, unsigned int numUSecs) const
334 {
335  if (m_env.syncVerbosity() >= msgVerbosity) {
336  this->Barrier();
337  for (int i = 0; i < this->NumProc(); ++i) {
338  if (i == this->MyPID()) {
339  std::cout << msg
340  << ": fullRank " << m_env.fullRank()
341  << ", subEnvironment " << m_env.subId()
342  << ", subRank " << m_env.subRank()
343  << ", inter0Rank " << m_env.inter0Rank()
344  << std::endl;
345  }
346  usleep(numUSecs);
347  this->Barrier();
348  }
349  //if (this->fullRank() == 0) std::cout << "Sleeping " << numUSecs << " microseconds..."
350  // << std::endl;
351  //usleep(numUSecs);
352  this->Barrier();
353  }
354 
355  return;
356 }
357 // -------------------------------------------------
358 #ifdef QUESO_HAS_TRILINOS
359 const Epetra_Comm&
360 MpiComm::epetraMpiComm() const
361 {
362  return *m_epetraComm;
363 }
364 #endif
365 
366 // Private methods----------------------------------
367 void
368 MpiComm::copy(const MpiComm& src)
369 {
370 #ifdef QUESO_HAS_TRILINOS
371  delete m_epetraComm;
372  m_epetraComm = src.m_epetraComm->Clone();
373 #endif
374  m_rawComm = src.m_rawComm;
375  m_worldRank = src.m_worldRank;
376  m_myPid = src.m_myPid;
377  m_numProc = src.m_numProc;
378 
379  return;
380 }
381 // -------------------------------------------------
382 
383 // Explicit template function instantiations
384 template void MpiComm::Allreduce<int>(const int *,
385  int *,
386  int,
387  RawType_MPI_Op,
388  const char *,
389  const char*) const;
390 template void MpiComm::Allreduce<char>(const char *,
391  char *,
392  int,
393  RawType_MPI_Op,
394  const char *,
395  const char*) const;
396 template void MpiComm::Allreduce<unsigned int>(const unsigned int *,
397  unsigned int *,
398  int,
399  RawType_MPI_Op,
400  const char *,
401  const char *) const;
402 template void MpiComm::Allreduce<double>(const double *,
403  double *,
404  int,
405  RawType_MPI_Op,
406  const char *,
407  const char *) const;
408 
409 template void MpiComm::Gather<int>(const int * sendbuf,
410  int sendcnt,
411  int * recvbuf,
412  int recvcount,
413  int root,
414  const char * whereMsg,
415  const char * whatMsg) const;
416 
417 template void MpiComm::Gather<char>(const char * sendbuf,
418  int sendcnt,
419  char * recvbuf,
420  int recvcount,
421  int root,
422  const char * whereMsg,
423  const char * whatMsg) const;
424 
425 template void MpiComm::Gather<unsigned int>(const unsigned int * sendbuf,
426  int sendcnt,
427  unsigned int * recvbuf,
428  int recvcount,
429  int root,
430  const char * whereMsg,
431  const char * whatMsg) const;
432 
433 template void MpiComm::Gather<double>(const double * sendbuf,
434  int sendcnt,
435  double * recvbuf,
436  int recvcount,
437  int root,
438  const char * whereMsg,
439  const char * whatMsg) const;
440 
441 template void MpiComm::Gatherv<int>(const int * sendbuf,
442  int sendcnt,
443  int * recvbuf,
444  int * recvcnts,
445  int * displs,
446  int root,
447  const char * whereMsg,
448  const char * whatMsg) const;
449 
450 template void MpiComm::Gatherv<char>(const char * sendbuf,
451  int sendcnt,
452  char * recvbuf,
453  int * recvcnts,
454  int * displs,
455  int root,
456  const char * whereMsg,
457  const char * whatMsg) const;
458 
459 template void MpiComm::Gatherv<unsigned int>(const unsigned int * sendbuf,
460  int sendcnt,
461  unsigned int * recvbuf,
462  int * recvcnts,
463  int * displs,
464  int root,
465  const char * whereMsg,
466  const char * whatMsg) const;
467 
468 template void MpiComm::Gatherv<double>(const double * sendbuf,
469  int sendcnt,
470  double * recvbuf,
471  int * recvcnts,
472  int * displs,
473  int root,
474  const char * whereMsg,
475  const char * whatMsg) const;
476 
477 } // End namespace QUESO
QUESO::RawType_MPI_Comm
MPI_Comm RawType_MPI_Comm
Definition: MpiComm.h:41
QUESO::MpiComm::m_rawComm
RawType_MPI_Comm m_rawComm
Embedded wrapped opaque MPI_Comm object.
Definition: MpiComm.h:403
QUESO::BaseEnvironment::subId
unsigned int subId() const
Access function to the number of each sub-environment Id: m_subId.
Definition: Environment.C:342
QUESO::MpiComm::Barrier
void Barrier() const
Pause every process in *this communicator until all the processes reach this point.
Definition: MpiComm.C:174
QUESO::MpiComm::operator=
MpiComm & operator=(const MpiComm &rhs)
Assignment operator.
Definition: MpiComm.C:102
QUESO::BaseEnvironment
This (virtual) class sets up the environment underlying the use of the QUESO library by an executable...
Definition: Environment.h:198
QUESO::MpiComm::m_epetraComm
Epetra_Comm * m_epetraComm
Definition: MpiComm.h:399
QUESO::MpiComm::m_env
const BaseEnvironment & m_env
Definition: MpiComm.h:395
QUESO::MpiComm::m_myPid
int m_myPid
Process ID of this process.
Definition: MpiComm.h:409
QUESO::MpiComm::syncPrintDebugMsg
void syncPrintDebugMsg(const char *msg, unsigned int msgVerbosity, unsigned int numUSecs) const
Synchronizes all the processes and print debug message.
Definition: MpiComm.C:333
QUESO::BaseEnvironment::subRank
int subRank() const
Returns the rank of the MPI process in the sub-communicator subComm()
Definition: Environment.C:287
QUESO::MpiComm::MyPID
int MyPID() const
Return my process ID.
Definition: MpiComm.C:124
QUESO::MpiComm::m_worldRank
int m_worldRank
World rank.
Definition: MpiComm.h:406
QUESO::MpiComm::Gather
void Gather(void *sendbuf, int sendcnt, RawType_MPI_Datatype sendtype, void *recvbuf, int recvcount, RawType_MPI_Datatype recvtype, int root, const char *whereMsg, const char *whatMsg) const
Gather values from each process to collect on all processes.
Definition: MpiComm.C:202
QUESO::BaseEnvironment::fullRank
int fullRank() const
Returns the rank of the MPI process in QUESO&#39;s full communicator.
Definition: Environment.C:268
QUESO::MpiComm::copy
void copy(const MpiComm &src)
Copies from an existing MpiComm instance.
Definition: MpiComm.C:368
QUESO::MpiComm::Recv
void Recv(void *buf, int count, RawType_MPI_Datatype datatype, int source, int tag, RawType_MPI_Status *status, const char *whereMsg, const char *whatMsg) const
Blocking receive of data from this process to another process.
Definition: MpiComm.C:307
QUESO::BaseEnvironment::syncVerbosity
unsigned int syncVerbosity() const
Access function to private attribute m_syncVerbosity.
Definition: Environment.C:457
QUESO::MpiComm::Gatherv
void Gatherv(void *sendbuf, int sendcnt, RawType_MPI_Datatype sendtype, void *recvbuf, int *recvcnts, int *displs, RawType_MPI_Datatype recvtype, int root, const char *whereMsg, const char *whatMsg) const
Gathers into specified locations from all processes in a group.
Definition: MpiComm.C:254
QUESO::MpiComm::Allreduce
void Allreduce(void *sendbuf, void *recvbuf, int count, RawType_MPI_Datatype datatype, RawType_MPI_Op op, const char *whereMsg, const char *whatMsg) const
Combines values from all processes and distributes the result back to all processes.
Definition: MpiComm.C:143
QUESO::RawType_MPI_Op
MPI_Op RawType_MPI_Op
Definition: MpiComm.h:45
QUESO::RawType_MPI_Datatype
MPI_Datatype RawType_MPI_Datatype
Definition: MpiComm.h:43
QUESO::MpiComm::Send
void Send(void *buf, int count, RawType_MPI_Datatype datatype, int dest, int tag, const char *whereMsg, const char *whatMsg) const
Possibly blocking send of data from this process to another process.
Definition: MpiComm.C:320
QUESO::queso_require_equal_to_msg
MonteCarloSGOptions::MonteCarloSGOptions(const BaseEnvironment &env, const char *prefix, const McOptionsValues &alternativeOptionsValues queso_require_equal_to_msg)(m_env.optionsInputFileName(), std::string(""), std::string("this constructor is incompatible with the existence of an options input file"))
Definition: MonteCarloSGOptions.C:460
QUESO::MpiComm::~MpiComm
~MpiComm()
Destructor.
QUESO::MpiComm::MpiComm
MpiComm()
Default Constructor.
Definition: MpiComm.C:91
QUESO::MpiComm::epetraMpiComm
const Epetra_Comm & epetraMpiComm() const
Extract MPI Communicator from a Epetra_MpiComm object.
Definition: MpiComm.C:360
QUESO::MpiComm
The QUESO MPI Communicator Class.
Definition: MpiComm.h:203
QUESO::MpiComm::NumProc
int NumProc() const
Returns total number of processes.
Definition: MpiComm.C:133
QUESO::RawType_MPI_Status
MPI_Status RawType_MPI_Status
Definition: MpiComm.h:46
QUESO::BaseEnvironment::inter0Rank
int inter0Rank() const
Returns the process inter0 rank.
Definition: Environment.C:307
QUESO::MpiComm::Comm
RawType_MPI_Comm Comm() const
Extract MPI Communicator from a MpiComm object.
Definition: MpiComm.C:111
QUESO::MpiComm::Bcast
void Bcast(void *buffer, int count, RawType_MPI_Datatype datatype, int root, const char *whereMsg, const char *whatMsg) const
Broadcast values from the root process to the slave processes.
Definition: MpiComm.C:191
Generated on Tue Jun 5 2018 19:48:55 for queso-0.57.1 by  doxygen 1.8.5