MpiComm.h

Go to the documentation of this file.

//-----------------------------------------------------------------------bl-
//--------------------------------------------------------------------------
//
// QUESO - a library to support the Quantification of Uncertainty
// for Estimation, Simulation and Optimization
//
// Copyright (C) 2008-2015 The PECOS Development Team
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the Version 2.1 GNU Lesser General
// Public License as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc. 51 Franklin Street, Fifth Floor,
// Boston, MA  02110-1301  USA
//
//-----------------------------------------------------------------------el-

#ifndef UQ_MPI_COMM_H
#define UQ_MPI_COMM_H

#include <queso/Defines.h>

#ifdef QUESO_HAS_TRILINOS
#include <Epetra_MpiComm.h>
#endif

#ifdef QUESO_HAS_MPI
#include <mpi.h>
#endif

namespace QUESO {

#ifdef QUESO_HAS_MPI
typedef MPI_Comm     RawType_MPI_Comm ;
typedef MPI_Group    RawType_MPI_Group ;
typedef MPI_Datatype RawType_MPI_Datatype ;
typedef MPI_Datatype data_type ;
typedef MPI_Op       RawType_MPI_Op ;
typedef MPI_Status   RawType_MPI_Status ;
#define RawValue_MPI_COMM_SELF  MPI_COMM_SELF
#define RawValue_MPI_ANY_SOURCE MPI_ANY_SOURCE
#define RawValue_MPI_CHAR       MPI_CHAR
#define RawValue_MPI_INT        MPI_INT
#define RawValue_MPI_DOUBLE     MPI_DOUBLE
#define RawValue_MPI_UNSIGNED   MPI_UNSIGNED
#define RawValue_MPI_MIN        MPI_MIN
#define RawValue_MPI_MAX        MPI_MAX
#define RawValue_MPI_SUM        MPI_SUM
#else
typedef int RawType_MPI_Comm;
typedef int RawType_MPI_Group;
typedef int RawType_MPI_Datatype;
struct data_type { };
typedef int RawType_MPI_Op;
typedef int RawType_MPI_Status;
#define RawValue_MPI_COMM_SELF   0
#define RawValue_MPI_ANY_SOURCE -1
#define RawValue_MPI_CHAR        0
#define RawValue_MPI_INT         1
#define RawValue_MPI_DOUBLE      2
#define RawValue_MPI_UNSIGNED    3
#define RawValue_MPI_MIN         0
#define RawValue_MPI_MAX         1
#define RawValue_MPI_SUM         2
#endif

class DataType
{
public:
  DataType () : _datatype() {}

  DataType (const DataType &other) :
    _datatype(other._datatype)
  {}

  DataType (const RawType_MPI_Datatype &type) :
    _datatype(type)
  {}

#ifdef QUESO_HAS_MPI
  DataType (const DataType &other, unsigned int count)
  {
    // FIXME - if we nest an inner type here will we run into bug
    // https://github.com/libMesh/libmesh/issues/631 again?
    MPI_Type_contiguous(count, other._datatype, &_datatype);
    this->commit();
  }
#else
  DataType (const DataType &, unsigned int)
  {
  }
#endif

  DataType & operator = (const DataType &other)
  { _datatype = other._datatype; return *this; }

  DataType & operator = (const RawType_MPI_Datatype &type)
  { _datatype = type; return *this; }

  operator const RawType_MPI_Datatype & () const
  { return _datatype; }

  operator RawType_MPI_Datatype & ()
  { return _datatype; }

  void commit ()
  {
#ifdef QUESO_HAS_MPI
    MPI_Type_commit (&_datatype);
#endif
  }

  void free ()
  {
#ifdef QUESO_HAS_MPI
    MPI_Type_free (&_datatype);
#endif
  }

protected:
  RawType_MPI_Datatype _datatype;
};

template <typename T>
class StandardType : public DataType
{
#ifdef QUESO_HAS_CXX11  // This macro isn't defined (yet)
    // Get a slightly better compiler diagnostic if we have C++11
  static_assert(dependent_false<T>::value,
                "Only specializations of StandardType may be used, did you forget to include a header file (e.g. parallel_algebra.h)?");
#endif

 /*
  * The unspecialized class is useless, so we make its constructor
  * private to catch mistakes at compile-time rather than link-time.
  * Specializations should have a public constructor of the same
  * form.
  */
private:
  StandardType(const T* example = NULL);
};

#ifdef QUESO_HAS_MPI

#define QUESO_STANDARD_TYPE(cxxtype,mpitype)                                  \
  template<>                                                            \
  class StandardType<cxxtype> : public DataType                         \
  {                                                                     \
  public:                                                               \
    explicit                                                            \
      StandardType(const cxxtype* = NULL) : DataType(mpitype) {}        \
  }

#else

#define QUESO_STANDARD_TYPE(cxxtype,mpitype)                          \
  template<>                                                    \
  class StandardType<cxxtype> : public DataType                 \
  {                                                             \
  public:                                                       \
    explicit                                                    \
      StandardType(const cxxtype* = NULL) : DataType() {}       \
  }

#endif

QUESO_STANDARD_TYPE(char,MPI_CHAR);
QUESO_STANDARD_TYPE(int,MPI_INT);
QUESO_STANDARD_TYPE(unsigned int,MPI_UNSIGNED);
QUESO_STANDARD_TYPE(double,MPI_DOUBLE);

class BaseEnvironment;

class MpiComm
{
public:



  MpiComm(const BaseEnvironment& env, RawType_MPI_Comm inputRawComm);


  MpiComm(const BaseEnvironment& env);


  MpiComm(const MpiComm& src);

 ~MpiComm();


  MpiComm& operator= (const MpiComm& rhs);



  RawType_MPI_Comm Comm     () const;

  int                MyPID    () const;

  int                NumProc  () const;



  void               Allreduce(void* sendbuf, void* recvbuf, int count, RawType_MPI_Datatype datatype,
                               RawType_MPI_Op op, const char* whereMsg, const char* whatMsg) const;


  template <typename T>
  void Allreduce(const T * sendbuf, T * recvbuf, int count, RawType_MPI_Op op,
                 const char* whereMsg, const char* whatMsg) const;


  void               Barrier  () const; // const char* whereMsg, const char* whatMsg) const;


  void               Bcast    (void* buffer, int count, RawType_MPI_Datatype datatype, int root,
                               const char* whereMsg, const char* whatMsg) const;


  void               Gather   (void *sendbuf, int sendcnt, RawType_MPI_Datatype sendtype,
                               void *recvbuf, int recvcount, RawType_MPI_Datatype recvtype,
                               int root,
                               const char* whereMsg, const char* whatMsg) const;


  template <typename T>
  void Gather(const T * sendbuf, int sendcnt, T * recvbuf, int recvcount, int root,
              const char * whereMsg, const char * whatMsg) const;


  void               Gatherv  (void *sendbuf, int sendcnt, RawType_MPI_Datatype sendtype,
                               void *recvbuf, int *recvcnts, int *displs, RawType_MPI_Datatype recvtype,
                               int root,
                               const char* whereMsg, const char* whatMsg) const;


  template <typename T>
  void Gatherv(const T * sendbuf, int sendcnt, T * recvbuf, int * recvcnts,
               int * displs, int root, const char * whereMsg,
               const char * whatMsg) const;


  void               Recv     (void *buf, int count, RawType_MPI_Datatype datatype, int source, int tag, RawType_MPI_Status *status,
                               const char* whereMsg, const char* whatMsg) const;


  void               Send     (void *buf, int count, RawType_MPI_Datatype datatype, int dest, int tag,
                               const char* whereMsg, const char* whatMsg) const;


  void               syncPrintDebugMsg(const char* msg, unsigned int msgVerbosity, unsigned int numUSecs) const;

#ifdef QUESO_HAS_TRILINOS
  const Epetra_MpiComm& epetraMpiComm() const;
#endif

private:

  MpiComm();

  void               copy          (const MpiComm& src);

  // QUESO environment
  const BaseEnvironment& m_env;
#ifdef QUESO_HAS_TRILINOS

  // Epetra MPI communicator
  Epetra_MpiComm*               m_epetraMpiComm;
#endif
  RawType_MPI_Comm            m_rawComm;

  int                           m_worldRank;

  int                           m_myPid;

  // Total number of processes
  int                           m_numProc;
};

}  // End namespace QUESO

#endif // UQ_MPI_COMM_H