queso-0.56.0
MpiComm.C
Go to the documentation of this file.
1 //-----------------------------------------------------------------------bl-
2 //--------------------------------------------------------------------------
3 //
4 // QUESO - a library to support the Quantification of Uncertainty
5 // for Estimation, Simulation and Optimization
6 //
7 // Copyright (C) 2008-2015 The PECOS Development Team
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the Version 2.1 GNU Lesser General
11 // Public License as published by the Free Software Foundation.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc. 51 Franklin Street, Fifth Floor,
21 // Boston, MA 02110-1301 USA
22 //
23 //-----------------------------------------------------------------------el-
24 
25 #include <unistd.h>
26 #include <cstring>
27 #include <queso/MpiComm.h>
28 #include <queso/Environment.h>
29 
30 namespace QUESO {
31 
32 // QUESO MpiComm MPI Constructor ------------------
33 MpiComm::MpiComm(const BaseEnvironment& env, RawType_MPI_Comm inputRawComm)
34  :
35  m_env (env),
36 #ifdef QUESO_HAS_TRILINOS
37  m_epetraMpiComm( new Epetra_MpiComm(inputRawComm) ),
38 #endif
39  m_rawComm (inputRawComm),
40  m_worldRank (-1),
41  m_myPid (-1),
42  m_numProc (-1)
43 {
44 #ifdef QUESO_HAS_MPI
45  int mpiRC = MPI_Comm_rank(inputRawComm,&m_worldRank);
46  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, "failed MPI_Comm_rank() on full rank");
47 
48  mpiRC = MPI_Comm_rank(inputRawComm,&m_myPid);
49  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, "failed MPI_Comm_rank() on inputRawComm");
50 
51  mpiRC = MPI_Comm_size(inputRawComm,&m_numProc);
52  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, "failed MPI_Comm_size() on inputRawComm");
53 #else
54  m_worldRank = 0;
55  m_myPid = 0;
56  m_numProc = 1;
57 #endif
58 }
59 
60 MpiComm::MpiComm(const BaseEnvironment& env)
61  :
62  m_env (env),
63 #ifdef QUESO_HAS_TRILINOS
64  m_epetraMpiComm( new Epetra_MpiComm(MPI_COMM_SELF) ),
65 #endif
66  m_rawComm (RawValue_MPI_COMM_SELF),
67  m_worldRank (0),
68  m_myPid (0),
69  m_numProc (1)
70 {
71 }
72 
73 // Copy constructor ---------------------------------
74 MpiComm::MpiComm(const MpiComm& src)
75  :
76  m_env (src.m_env)
77 #ifdef QUESO_HAS_TRILINOS
78  ,
79  m_epetraMpiComm(NULL)
80 #endif
81 {
82  this->copy(src);
83 }
84 
85 // Destructor ---------------------------------------
86 MpiComm::~MpiComm()
87 {
88 #ifdef QUESO_HAS_TRILINOS
89  delete m_epetraMpiComm;
90  m_epetraMpiComm = NULL;
91 #endif
92 }
93 
94 // --------------------------------------------------
95 // Set methodos -------------------------------------
96 MpiComm&
97 MpiComm::operator=(const MpiComm& rhs)
98 {
99  this->copy(rhs);
100  return *this;
101 }
102 
103 // Attribute access methods -------------------------
104 RawType_MPI_Comm
105 MpiComm::Comm() const
106 {
107 #ifdef QUESO_HAS_TRILINOS
108  return m_epetraMpiComm->Comm();
109 #endif
110  return m_rawComm;
111 }
112 // --------------------------------------------------
113 int
114 MpiComm::MyPID() const
115 {
116 #ifdef QUESO_HAS_TRILINOS
117  return m_epetraMpiComm->MyPID();
118 #endif
119  return m_myPid;
120 }
121 // --------------------------------------------------
122 int
123 MpiComm::NumProc() const
124 {
125 #ifdef QUESO_HAS_TRILINOS
126  return m_epetraMpiComm->NumProc();
127 #endif
128  return m_numProc;
129 }
130 // Methods overridden from Comm ---------------------
131 
132 void
133 MpiComm::Allreduce(void* sendbuf, void* recvbuf, int count, RawType_MPI_Datatype datatype, RawType_MPI_Op op, const char* whereMsg, const char* whatMsg) const
134 {
135  queso_deprecated();
136  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
137 #ifdef QUESO_HAS_MPI
138  int mpiRC = MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, m_rawComm);
139  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
140 #endif
141  }
142 }
143 
144 template <typename T>
145 void
146 MpiComm::Allreduce(const T* sendbuf, T* recvbuf, int count, RawType_MPI_Op op,
147  const char* whereMsg, const char* whatMsg) const
148 {
149  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
150 #ifdef QUESO_HAS_MPI
151  T * sendbuf_noconst = const_cast<T *>(sendbuf);
152  int mpiRC = MPI_Allreduce(sendbuf_noconst, recvbuf, count, StandardType<T>(sendbuf), op, m_rawComm);
153  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
154 #endif
155  }
156  else {
157  size_t dataTypeSize = sizeof(T);
158  size_t dataTotal = dataTypeSize*count;
159  std::memcpy(recvbuf, sendbuf, dataTotal);
160  }
161 }
162 //--------------------------------------------------
163 void
164 MpiComm::Barrier() const // const char* whereMsg, const char* whatMsg) const
165 {
166 #ifdef QUESO_HAS_TRILINOS
167  return m_epetraMpiComm->Barrier();
168 #endif
169 
170  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
171 #ifdef QUESO_HAS_MPI
172  int mpiRC = MPI_Barrier(m_rawComm);
173  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, "mpiRC indicates failure"); // whatMsg);
174 #endif
175  }
176 
177  return;
178 }
179 //--------------------------------------------------
180 void
181 MpiComm::Bcast(void* buffer, int count, RawType_MPI_Datatype datatype, int root, const char* whereMsg, const char* whatMsg) const
182 {
183  if (NumProc() > 1) { // Necesarrily true if QUESO_HAS_MPI
184 #ifdef QUESO_HAS_MPI
185  int mpiRC = MPI_Bcast(buffer, count, datatype, root, m_rawComm);
186  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
187 #endif
188  }
189 }
190 //--------------------------------------------------
191 void
192 MpiComm::Gather(
193  void* sendbuf, int sendcnt, RawType_MPI_Datatype sendtype,
194  void* recvbuf, int recvcount, RawType_MPI_Datatype recvtype,
195  int root,
196  const char* whereMsg, const char* whatMsg) const
197 {
198  queso_deprecated();
199  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
200 #ifdef QUESO_HAS_MPI
201  //int MPI_Gather (void *sendbuf, int sendcnt, MPI_Datatype sendtype,
202  // void *recvbuf, int recvcount, MPI_Datatype recvtype,
203  // int root, MPI_Comm comm )
204  int mpiRC = MPI_Gather(sendbuf, sendcnt, sendtype,
205  recvbuf, recvcount, recvtype,
206  root, m_rawComm);
207  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
208 #endif
209  }
210 }
211 
212 template <typename T>
213 void
214 MpiComm::Gather(const T * sendbuf, int sendcnt, T * recvbuf, int recvcount,
215  int root, const char* whereMsg, const char* whatMsg) const
216 {
217  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
218 #ifdef QUESO_HAS_MPI
219  //int MPI_Gather (void *sendbuf, int sendcnt, MPI_Datatype sendtype,
220  // void *recvbuf, int recvcount, MPI_Datatype recvtype,
221  // int root, MPI_Comm comm )
222  T * sendbuf_noconst = const_cast<T *>(sendbuf);
223  int mpiRC = MPI_Gather(sendbuf_noconst, sendcnt, StandardType<T>(sendbuf),
224  recvbuf, recvcount, StandardType<T>(sendbuf),
225  root, m_rawComm);
226  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
227 #endif
228  }
229  else {
230  size_t dataTypeSize = sizeof(T);
231  size_t sendTotal = dataTypeSize*sendcnt;
232  size_t recvTotal = dataTypeSize*recvcount;
233  if (sendTotal != recvTotal) {
234  std::cerr << "MpiCommClass::Gather()"
235  << ": sendTotal != recvTotal"
236  << std::endl;
237  }
238  queso_require_equal_to_msg(sendTotal, recvTotal, whatMsg);
239  std::memcpy(recvbuf, sendbuf, sendTotal);
240  }
241 }
242 //--------------------------------------------------
243 void
244 MpiComm::Gatherv(
245  void* sendbuf, int sendcnt, RawType_MPI_Datatype sendtype,
246  void* recvbuf, int* recvcnts, int* displs, RawType_MPI_Datatype recvtype,
247  int root,
248  const char* whereMsg, const char* whatMsg) const
249 {
250  queso_deprecated();
251  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
252 #ifdef QUESO_HAS_MPI
253  //int MPI_Gatherv(void *sendbuf, int sendcnt, MPI_Datatype sendtype,
254  // void *recvbuf, int *recvcnts, int *displs, MPI_Datatype recvtype,
255  // int root, MPI_Comm comm )
256  int mpiRC = MPI_Gatherv(sendbuf, sendcnt, sendtype,
257  recvbuf, recvcnts, displs, recvtype,
258  root, m_rawComm);
259  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
260 #endif
261  }
262 }
263 
264 template <typename T>
265 void
266 MpiComm::Gatherv(const T * sendbuf, int sendcnt, T * recvbuf, int * recvcnts,
267  int * displs, int root, const char * whereMsg,
268  const char * whatMsg) const
269 {
270  if (NumProc() > 1) { // Necessarily true if QUESO_HAS_MPI
271 #ifdef QUESO_HAS_MPI
272  //int MPI_Gatherv(void *sendbuf, int sendcnt, MPI_Datatype sendtype,
273  // void *recvbuf, int *recvcnts, int *displs, MPI_Datatype recvtype,
274  // int root, MPI_Comm comm )
275  T * sendbuf_noconst = const_cast<T *>(sendbuf);
276  int mpiRC = MPI_Gatherv(sendbuf_noconst, sendcnt, StandardType<T>(sendbuf),
277  recvbuf, recvcnts, displs,
278  StandardType<T>(recvbuf), root, m_rawComm);
279  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
280 #endif
281  }
282  else {
283  size_t dataTypeSize = sizeof(T);
284  size_t sendTotal = dataTypeSize*sendcnt;
285  size_t recvTotal = dataTypeSize*recvcnts[0];
286  if (sendTotal != recvTotal) {
287  std::cerr << "MpiCommClass::Gatherv()"
288  << ": sendTotal != recvTotal"
289  << std::endl;
290  }
291  queso_require_equal_to_msg(sendTotal, recvTotal, whatMsg);
292  std::memcpy(recvbuf, sendbuf, sendTotal);
293  }
294 }
295 //--------------------------------------------------
296 void
297 MpiComm::Recv(
298  void* buf, int count, RawType_MPI_Datatype datatype, int source, int tag, RawType_MPI_Status* status,
299  const char* whereMsg, const char* whatMsg) const
300 {
301  if (NumProc() > 1) { // Necesarrily true if QUESO_HAS_MPI
302 #ifdef QUESO_HAS_MPI
303  int mpiRC = MPI_Recv(buf, count, datatype, source, tag, m_rawComm, status);
304  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
305 #endif
306  }
307 }
308 //--------------------------------------------------
309 void
310 MpiComm::Send(
311  void* buf, int count, RawType_MPI_Datatype datatype, int dest, int tag,
312  const char* whereMsg, const char* whatMsg) const
313 {
314  if (NumProc() > 1) { // Necesarrily true if QUESO_HAS_MPI
315 #ifdef QUESO_HAS_MPI
316  int mpiRC = MPI_Send(buf, count, datatype, dest, tag, m_rawComm);
317  queso_require_equal_to_msg(mpiRC, MPI_SUCCESS, whatMsg);
318 #endif
319  }
320 }
321 // Misc methods ------------------------------------
322 void
323 MpiComm::syncPrintDebugMsg(const char* msg, unsigned int msgVerbosity, unsigned int numUSecs) const
324 {
325  if (m_env.syncVerbosity() >= msgVerbosity) {
326  this->Barrier();
327  for (int i = 0; i < this->NumProc(); ++i) {
328  if (i == this->MyPID()) {
329  std::cout << msg
330  << ": fullRank " << m_env.fullRank()
331  << ", subEnvironment " << m_env.subId()
332  << ", subRank " << m_env.subRank()
333  << ", inter0Rank " << m_env.inter0Rank()
334  << std::endl;
335  }
336  usleep(numUSecs);
337  this->Barrier();
338  }
339  //if (this->fullRank() == 0) std::cout << "Sleeping " << numUSecs << " microseconds..."
340  // << std::endl;
341  //usleep(numUSecs);
342  this->Barrier();
343  }
344 
345  return;
346 }
347 // -------------------------------------------------
348 #ifdef QUESO_HAS_TRILINOS
349 const Epetra_MpiComm&
350 MpiComm::epetraMpiComm() const
351 {
352  return *m_epetraMpiComm;
353 }
354 #endif
355 
356 // Private methods----------------------------------
357 void
358 MpiComm::copy(const MpiComm& src)
359 {
360 #ifdef QUESO_HAS_TRILINOS
361  delete m_epetraMpiComm;
362  m_epetraMpiComm = new Epetra_MpiComm(*src.m_epetraMpiComm);
363 #endif
364  m_rawComm = src.m_rawComm;
365  m_worldRank = src.m_worldRank;
366  m_myPid = src.m_myPid;
367  m_numProc = src.m_numProc;
368 
369  return;
370 }
371 // -------------------------------------------------
372 
373 // Explicit template function instantiations
374 template void MpiComm::Allreduce<int>(const int *,
375  int *,
376  int,
377  RawType_MPI_Op,
378  const char *,
379  const char*) const;
380 template void MpiComm::Allreduce<char>(const char *,
381  char *,
382  int,
383  RawType_MPI_Op,
384  const char *,
385  const char*) const;
386 template void MpiComm::Allreduce<unsigned int>(const unsigned int *,
387  unsigned int *,
388  int,
389  RawType_MPI_Op,
390  const char *,
391  const char *) const;
392 template void MpiComm::Allreduce<double>(const double *,
393  double *,
394  int,
395  RawType_MPI_Op,
396  const char *,
397  const char *) const;
398 
399 template void MpiComm::Gather<int>(const int * sendbuf,
400  int sendcnt,
401  int * recvbuf,
402  int recvcount,
403  int root,
404  const char * whereMsg,
405  const char * whatMsg) const;
406 
407 template void MpiComm::Gather<char>(const char * sendbuf,
408  int sendcnt,
409  char * recvbuf,
410  int recvcount,
411  int root,
412  const char * whereMsg,
413  const char * whatMsg) const;
414 
415 template void MpiComm::Gather<unsigned int>(const unsigned int * sendbuf,
416  int sendcnt,
417  unsigned int * recvbuf,
418  int recvcount,
419  int root,
420  const char * whereMsg,
421  const char * whatMsg) const;
422 
423 template void MpiComm::Gather<double>(const double * sendbuf,
424  int sendcnt,
425  double * recvbuf,
426  int recvcount,
427  int root,
428  const char * whereMsg,
429  const char * whatMsg) const;
430 
431 template void MpiComm::Gatherv<int>(const int * sendbuf,
432  int sendcnt,
433  int * recvbuf,
434  int * recvcnts,
435  int * displs,
436  int root,
437  const char * whereMsg,
438  const char * whatMsg) const;
439 
440 template void MpiComm::Gatherv<char>(const char * sendbuf,
441  int sendcnt,
442  char * recvbuf,
443  int * recvcnts,
444  int * displs,
445  int root,
446  const char * whereMsg,
447  const char * whatMsg) const;
448 
449 template void MpiComm::Gatherv<unsigned int>(const unsigned int * sendbuf,
450  int sendcnt,
451  unsigned int * recvbuf,
452  int * recvcnts,
453  int * displs,
454  int root,
455  const char * whereMsg,
456  const char * whatMsg) const;
457 
458 template void MpiComm::Gatherv<double>(const double * sendbuf,
459  int sendcnt,
460  double * recvbuf,
461  int * recvcnts,
462  int * displs,
463  int root,
464  const char * whereMsg,
465  const char * whatMsg) const;
466 
467 } // End namespace QUESO
QUESO::MpiComm::NumProc
int NumProc() const
Returns total number of processes.
Definition: MpiComm.C:123
QUESO::MpiComm::Comm
RawType_MPI_Comm Comm() const
Extract MPI Communicator from a MpiComm object.
Definition: MpiComm.C:105
QUESO::MpiComm::Barrier
void Barrier() const
Pause every process in *this communicator until all the processes reach this point.
Definition: MpiComm.C:164
QUESO::RawType_MPI_Status
int RawType_MPI_Status
Definition: MpiComm.h:62
QUESO::MpiComm::~MpiComm
~MpiComm()
Destructor.
Definition: MpiComm.C:86
QUESO::BaseEnvironment::subRank
int subRank() const
Access function for sub-rank.
Definition: Environment.C:287
QUESO::RawType_MPI_Comm
int RawType_MPI_Comm
Definition: MpiComm.h:57
QUESO::BaseEnvironment
This (virtual) class sets up the environment underlying the use of the QUESO library by an executable...
Definition: Environment.h:197
QUESO::MpiComm::MyPID
int MyPID() const
Return my process ID.
Definition: MpiComm.C:114
QUESO::BaseEnvironment::inter0Rank
int inter0Rank() const
Returns the process inter0 rank.
Definition: Environment.C:307
QUESO::MpiComm::Send
void Send(void *buf, int count, RawType_MPI_Datatype datatype, int dest, int tag, const char *whereMsg, const char *whatMsg) const
Possibly blocking send of data from this process to another process.
Definition: MpiComm.C:310
RawValue_MPI_COMM_SELF
#define RawValue_MPI_COMM_SELF
Definition: MpiComm.h:63
queso_require_equal_to_msg
#define queso_require_equal_to_msg(expr1, expr2, msg)
Definition: asserts.h:73
QUESO::MpiComm::Gatherv
void Gatherv(void *sendbuf, int sendcnt, RawType_MPI_Datatype sendtype, void *recvbuf, int *recvcnts, int *displs, RawType_MPI_Datatype recvtype, int root, const char *whereMsg, const char *whatMsg) const
Gathers into specified locations from all processes in a group.
Definition: MpiComm.C:244
QUESO::BaseEnvironment::subId
unsigned int subId() const
Access function to the number of each sub-environment Id: m_subId.
Definition: Environment.C:341
QUESO::BaseEnvironment::syncVerbosity
unsigned int syncVerbosity() const
Access function to private attribute m_syncVerbosity.
Definition: Environment.C:456
queso_deprecated
#define queso_deprecated()
Definition: Defines.h:134
QUESO::RawType_MPI_Op
int RawType_MPI_Op
Definition: MpiComm.h:61
QUESO::BaseEnvironment::fullRank
int fullRank() const
Returns the process full rank.
Definition: Environment.C:268
QUESO::MpiComm::MpiComm
MpiComm()
Default Constructor.
QUESO::MpiComm::m_worldRank
int m_worldRank
World rank.
Definition: MpiComm.h:403
QUESO::MpiComm::m_myPid
int m_myPid
Process ID of this process.
Definition: MpiComm.h:406
QUESO::MpiComm::m_rawComm
RawType_MPI_Comm m_rawComm
Embedded wrapped opaque MPI_Comm object.
Definition: MpiComm.h:400
QUESO::MpiComm::syncPrintDebugMsg
void syncPrintDebugMsg(const char *msg, unsigned int msgVerbosity, unsigned int numUSecs) const
Synchronizes all the processes and print debug message.
Definition: MpiComm.C:323
QUESO::MpiComm
The QUESO MPI Communicator Class.
Definition: MpiComm.h:203
QUESO::MpiComm::Recv
void Recv(void *buf, int count, RawType_MPI_Datatype datatype, int source, int tag, RawType_MPI_Status *status, const char *whereMsg, const char *whatMsg) const
Blocking receive of data from this process to another process.
Definition: MpiComm.C:297
QUESO::MpiComm::operator=
MpiComm & operator=(const MpiComm &rhs)
Assignment operator.
Definition: MpiComm.C:97
QUESO::MpiComm::Gather
void Gather(void *sendbuf, int sendcnt, RawType_MPI_Datatype sendtype, void *recvbuf, int recvcount, RawType_MPI_Datatype recvtype, int root, const char *whereMsg, const char *whatMsg) const
Gather values from each process to collect on all processes.
Definition: MpiComm.C:192
QUESO::MpiComm::Allreduce
void Allreduce(void *sendbuf, void *recvbuf, int count, RawType_MPI_Datatype datatype, RawType_MPI_Op op, const char *whereMsg, const char *whatMsg) const
Combines values from all processes and distributes the result back to all processes.
Definition: MpiComm.C:133
QUESO::MpiComm::copy
void copy(const MpiComm &src)
Copies from an existing MpiComm instance.
Definition: MpiComm.C:358
QUESO::MpiComm::m_env
const BaseEnvironment & m_env
Definition: MpiComm.h:393
QUESO::RawType_MPI_Datatype
int RawType_MPI_Datatype
Definition: MpiComm.h:59
QUESO::MpiComm::Bcast
void Bcast(void *buffer, int count, RawType_MPI_Datatype datatype, int root, const char *whereMsg, const char *whatMsg) const
Broadcast values from the root process to the slave processes.
Definition: MpiComm.C:181
Generated on Tue Nov 29 2016 10:53:10 for queso-0.56.0 by  doxygen 1.8.5