Environment.C

Go to the documentation of this file.

//-----------------------------------------------------------------------bl-
//--------------------------------------------------------------------------
//
// QUESO - a library to support the Quantification of Uncertainty
// for Estimation, Simulation and Optimization
//
// Copyright (C) 2008-2015 The PECOS Development Team
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the Version 2.1 GNU Lesser General
// Public License as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc. 51 Franklin Street, Fifth Floor,
// Boston, MA  02110-1301  USA
//
//-----------------------------------------------------------------------el-

#include <queso/queso.h>
#include <queso/Environment.h>
#include <queso/EnvironmentOptions.h>
#include <queso/RngGsl.h>
#include <queso/RngBoost.h>
#include <queso/BasicPdfsGsl.h>
#include <queso/BasicPdfsBoost.h>
#include <queso/Miscellaneous.h>
#include <sys/time.h>
#ifdef HAVE_GRVY
#include <grvy.h>
#endif

// queso error handling
#include <queso/asserts.h>

//-----------------------------
// Library versioning routines
//-----------------------------

namespace QUESO {

  void QUESO_version_print(std::ostream &os)
  {
    {
      os << "------------------------------------------------------------------------------------------" ;
      os << "--------------------" << std::endl;
      os << "QUESO Library: Version = " << QUESO_LIB_VERSION;
      os << " (" << QUESO_get_numeric_version() << ")" << std::endl << std::endl;

      os << QUESO_LIB_RELEASE << std::endl << std::endl;

      os << "Build Date   = " << QUESO_BUILD_DATE     << std::endl;
      os << "Build Host   = " << QUESO_BUILD_HOST     << std::endl;
      os << "Build User   = " << QUESO_BUILD_USER     << std::endl;
      os << "Build Arch   = " << QUESO_BUILD_ARCH     << std::endl;
      os << "Build Rev    = " << QUESO_BUILD_VERSION  << std::endl     << std::endl;

      os << "C++ Config   = " << QUESO_CXX << " " << QUESO_CXXFLAGS    << std::endl;
      os << std::endl;
      os << "Trilinos DIR = " << QUESO_TRILINOS_DIR << std::endl;
      os << "GSL Libs     = " << QUESO_GSL_DIR  << std::endl;
      os << "GRVY DIR     = " << QUESO_GRVY_DIR << std::endl;
      os << "GLPK DIR     = " << QUESO_GLPK_DIR << std::endl;
      os << "HDF5 DIR     = " << QUESO_HDF5_DIR << std::endl;
      os << "------------------------------------------------------------------------------------------" ;
      os << "--------------------" << std::endl;
    }

    return;
  }

  int QUESO_get_numeric_version()
  {
    // Note: return format follows the versioning convention xx.yy.z where
    //
    // xx = major version number
    // yy = minor version number
    // zz = micro version number
    //
    // For example:
    // v.    0.23  -> 002300 = 2300
    // v.   0.23.1 -> 002301 = 2301
    // v.  10.23.2 -> 102302

    int major_version = 0;
    int minor_version = 0;
    int micro_version = 0;

#ifdef QUESO_MAJOR_VERSION
    major_version = QUESO_MAJOR_VERSION;
#endif

#ifdef QUESO_MINOR_VERSION
    minor_version = QUESO_MINOR_VERSION;
#endif

#ifdef QUESO_MICRO_VERSION
    micro_version = QUESO_MICRO_VERSION;
#endif

    return(major_version*10000 + minor_version*100 + micro_version);
  }

FilePtrSetStruct::FilePtrSetStruct()
  :
  ofsVar(NULL),
  ifsVar(NULL)
{
}

FilePtrSetStruct::~FilePtrSetStruct()
{
}

  //
  // queso terminate handler will be invoked for unhandled exceptions
  // needs to be invoked in FullEnvironment constructor
  //
  std::terminate_handler old_terminate_handler;

//*****************************************************
// Base class
//*****************************************************
// Default constructor --------------------------------
BaseEnvironment::BaseEnvironment(
  const char*                    passedOptionsInputFileName,
  const EnvOptionsValues* alternativeOptionsValues)
  :
  m_fullEnvIsReady             (false),
  m_worldRank                  (-1),
  m_fullComm                   (NULL),
  m_fullRank                   (-1),
  m_fullCommSize               (1),
  m_optionsInputFileName       (""),
  m_optionsInputFileAccessState(true),
  m_allOptionsDesc             (NULL),
  m_allOptionsMap              (NULL),
  m_subComm                    (NULL),
  m_subRank                    (-1),
  m_subCommSize                (1),
  m_selfComm                   (NULL),
  m_inter0Comm                 (NULL),
  m_inter0Rank                 (-1),
  m_inter0CommSize             (1),
  m_subDisplayFile             (NULL),
  m_rngObject                  (NULL),
  m_basicPdfs                  (NULL),
  m_exceptionalCircumstance    (false),
  m_alternativeOptionsValues   (),
  m_optionsObj                 (NULL)
{
  if (passedOptionsInputFileName) m_optionsInputFileName     = passedOptionsInputFileName;
  if (alternativeOptionsValues  ) m_alternativeOptionsValues = *alternativeOptionsValues;
}
// Copy constructor -------------------------------------
BaseEnvironment::BaseEnvironment(const BaseEnvironment& obj)
{
  UQ_FATAL_TEST_MACRO(UQ_INVALID_INTERNAL_STATE_RC,
                      obj.worldRank(),
                      "BaseEnvironment::constructor(), copy",
                      "code should not execute through here");
}
// Destructor -------------------------------------------
BaseEnvironment::~BaseEnvironment()
{
  //if (m_subDisplayFile) {
  //  *m_subDisplayFile << "Entering BaseEnvironment::destructor()"
  //                          << std::endl;
  //}

  struct timeval timevalNow;
  /*int iRC = 0;*/
  /*iRC = */gettimeofday(&timevalNow, NULL);

  if( this->displayVerbosity() > 0 )
    {
     if (m_subDisplayFile) {
        *m_subDisplayFile << "Ending run at "    << ctime(&timevalNow.tv_sec)
                          << "Total run time = " << timevalNow.tv_sec - m_timevalBegin.tv_sec
                          << " seconds"
                          << std::endl;
      }

    if (m_fullRank == 0) {
        std::cout << "Ending run at "    << ctime(&timevalNow.tv_sec)
                  << "Total run time = " << timevalNow.tv_sec - m_timevalBegin.tv_sec
                  << " seconds"
                  << std::endl;
      }
    }

  if (m_optionsObj) delete m_optionsObj;
  if (m_allOptionsMap) {
    delete m_allOptionsMap;
    delete m_allOptionsDesc;
  }

  if (m_basicPdfs) delete m_basicPdfs;
  if (m_rngObject) delete m_rngObject;

  //if (m_subDisplayFile) {
  //  *m_subDisplayFile << "Leaving BaseEnvironment::destructor()"
  //                          << std::endl;
  //}

  if (m_subDisplayFile) delete m_subDisplayFile;
  if (m_inter0Comm    ) delete m_inter0Comm;
  if (m_selfComm      ) delete m_selfComm;
  if (m_subComm       ) delete m_subComm;
  if (m_fullComm      ) delete m_fullComm;
}
// Set methods ------------------------------------------
BaseEnvironment&
BaseEnvironment::operator= (const BaseEnvironment& rhs)
{
  UQ_FATAL_TEST_MACRO(UQ_INVALID_INTERNAL_STATE_RC,
                      rhs.worldRank(),
                      "BaseEnvironment::operator=()",
                      "code should not execute through here");
  return *this;
}
// Environment, Communicator and Options Input File methods
bool
BaseEnvironment::fullEnvIsReady() const
{
  return m_fullEnvIsReady;
}
//-------------------------------------------------------
int
BaseEnvironment::worldRank() const
{
  return m_worldRank;
}
//-------------------------------------------------------
int
BaseEnvironment::fullRank() const
{
  return m_fullRank;
}
//-------------------------------------------------------
const MpiComm&
BaseEnvironment::fullComm() const
{
  UQ_FATAL_TEST_MACRO(m_fullComm == NULL,
                      m_worldRank,
                      "BaseEnvironment::fullComm()",
                      "m_fullComm variable is NULL");
  return *m_fullComm;
}
//-------------------------------------------------------
RawType_MPI_Group
BaseEnvironment::subGroup() const
{
  return m_subGroup;
}
//-------------------------------------------------------
int
BaseEnvironment::subRank() const
{
  return m_subRank;
}
//-------------------------------------------------------
const MpiComm&
BaseEnvironment::subComm() const
{
  UQ_FATAL_TEST_MACRO(m_subComm == NULL,
                      m_worldRank,
                      "BaseEnvironment::subComm()",
                      "m_subComm variable is NULL");
  return *m_subComm;
}
//-------------------------------------------------------
const MpiComm&
BaseEnvironment::selfComm() const
{
  UQ_FATAL_TEST_MACRO(m_selfComm == NULL,
                      m_worldRank,
                      "BaseEnvironment::selfComm()",
                      "m_selfComm variable is NULL");
  return *m_selfComm;
}
//-------------------------------------------------------
int
BaseEnvironment::inter0Rank() const
{
  return m_inter0Rank;
}
//-------------------------------------------------------
const MpiComm&
BaseEnvironment::inter0Comm() const
{
  UQ_FATAL_TEST_MACRO(m_inter0Comm == NULL,
                      m_worldRank,
                      "BaseEnvironment::inter0Comm()",
                      "m_inter0Comm variable is NULL");
  return *m_inter0Comm;
}
//-------------------------------------------------------
std::ofstream*
BaseEnvironment::subDisplayFile() const
{
  return m_subDisplayFile;
}
//-------------------------------------------------------
std::string
BaseEnvironment::subDisplayFileName() const
{
  if (m_optionsObj == NULL) return ".";

  return m_optionsObj->m_ov.m_subDisplayFileName;
}
//-------------------------------------------------------
unsigned int
BaseEnvironment::numSubEnvironments() const
{
  UQ_FATAL_TEST_MACRO(m_optionsObj == NULL,
                      m_worldRank,
                      "BaseEnvironment::numSubEnvironments()",
                      "m_optionsObj variable is NULL");
  return m_optionsObj->m_ov.m_numSubEnvironments;
}
//-------------------------------------------------------
unsigned int
BaseEnvironment::subId() const
{
  return m_subId;
}
//-------------------------------------------------------
const std::string&
BaseEnvironment::subIdString() const
{
  return m_subIdString;
}
//-------------------------------------------------------
std::string
BaseEnvironment::optionsInputFileName() const
{
  if (m_optionsInputFileAccessState) {
    return m_optionsInputFileName;
  }
  else {
    return "";
  }
}
//-------------------------------------------------------
void
BaseEnvironment::setOptionsInputFileAccessState(bool newState) const
{
  m_optionsInputFileAccessState = newState;

  return;
}
//-------------------------------------------------------
#ifdef UQ_USES_COMMAND_LINE_OPTIONS
const po::options_description&
BaseEnvironment::allOptionsDesc() const
{
  return *m_allOptionsDesc;
}
#endif
//-------------------------------------------------------
po::variables_map&
BaseEnvironment::allOptionsMap() const
{
  UQ_FATAL_TEST_MACRO(m_allOptionsMap == NULL,
                      m_worldRank,
                      "BaseEnvironment::allOptionsMap()",
                      "m_allOptionsMap variable is NULL");
  return *m_allOptionsMap;
}
//-------------------------------------------------------
void
BaseEnvironment::scanInputFileForMyOptions(const po::options_description& optionsDesc) const
{
#ifdef UQ_USES_COMMAND_LINE_OPTIONS
  // If you want to use command line options, the following line does *not* work outside 'main.C',
  // e.g., in the constructor of FullEnvironment:
  // Line: po::store(po::parse_command_line(argc, argv, *m_allOptionsDesc), *m_allOptionsMap);
  //
  // Instead, put the following three lines *immediately after* instantianting the UQ environment
  // variable "FullEnvironment* env":
  // Line 1: po::store(po::parse_command_line(argc, argv, env->allOptionsDesc()), env->allOptionsMap());
  // Line 2: po::notify(env->allOptionsMap());
  // Line 3: env->getMyOptionValues();
#endif

#ifdef QUESO_MEMORY_DEBUGGING
  std::cout << "Entering BaseEnv::scanInputFileForMyOptions()" << std::endl;
#endif

  UQ_FATAL_TEST_MACRO(m_allOptionsDesc == NULL,
                      m_worldRank,
                      "BaseEnvironment::scanInputFileForMyOptions()",
                      "m_allOptionsDesc variable is NULL");
  m_allOptionsDesc->add(optionsDesc);
  //if (m_subDisplayFile) {
  //  *m_subDisplayFile << *m_allOptionsDesc
  //                    << std::endl;
  //}

  UQ_FATAL_TEST_MACRO(m_optionsInputFileName == "",
                      m_worldRank,
                      "BaseEnvironment::scanInputFileForMyOptions()",
                      "m_optionsInputFileName is 'nothing'");
  //std::ifstream ifs(m_optionsInputFileName.c_str());
  std::ifstream* ifs = new std::ifstream(m_optionsInputFileName.c_str());
#ifdef QUESO_MEMORY_DEBUGGING
  std::cout << "in BaseEnv::scanInputFileForMyOptions(), before store(a)" << std::endl;
#endif

  UQ_FATAL_TEST_MACRO(m_allOptionsMap == NULL,
                      m_worldRank,
                      "BaseEnvironment::scanInputFileForMyOptions()",
                      "m_allOptionsMap variable is NULL");
  po::store(po::parse_config_file(*ifs, *m_allOptionsDesc, true), *m_allOptionsMap);
#ifdef QUESO_MEMORY_DEBUGGING
  std::cout << "in BaseEnv::scanInputFileForMyOptions(), after store(a)" << std::endl;
#endif
  po::notify(*m_allOptionsMap);

  //ifs.close();
  delete ifs;
#ifdef QUESO_MEMORY_DEBUGGING
  std::cout << "Leaving BaseEnv::scanInputFileForMyOptions()" << std::endl;
#endif

  return;
}
//-----------------------------------------------------
unsigned int
BaseEnvironment::displayVerbosity() const
{
  UQ_FATAL_TEST_MACRO(m_optionsObj == NULL,
                      m_worldRank,
                      "BaseEnvironment::displayVerbosity()",
                      "m_optionsObj variable is NULL");
  return m_optionsObj->m_ov.m_displayVerbosity;
}
//-------------------------------------------------------
unsigned int
BaseEnvironment::syncVerbosity() const
{
  UQ_FATAL_TEST_MACRO(m_optionsObj == NULL,
                      m_worldRank,
                      "BaseEnvironment::displayVerbosity()",
                      "m_optionsObj variable is NULL");
  return m_optionsObj->m_ov.m_syncVerbosity;
}
//-------------------------------------------------------
unsigned int
BaseEnvironment::checkingLevel() const
{
  UQ_FATAL_TEST_MACRO(m_optionsObj == NULL,
                      m_worldRank,
                      "BaseEnvironment::checkingLevel()",
                      "m_optionsObj variable is NULL");
  return m_optionsObj->m_ov.m_checkingLevel;
}
//-------------------------------------------------------
const RngBase*
BaseEnvironment::rngObject() const
{
  return m_rngObject;
}
//-------------------------------------------------------
int
BaseEnvironment::seed() const
{
  return m_rngObject->seed();
}
//-------------------------------------------------------
void
BaseEnvironment::resetSeed(int newSeedOption)
{
  m_rngObject->resetSeed(newSeedOption);
  return;
}
//-------------------------------------------------------
const BasicPdfsBase*
BaseEnvironment::basicPdfs() const
{
  return m_basicPdfs;
}
//-------------------------------------------------------
std::string
BaseEnvironment::platformName() const
{
  return m_optionsObj->m_ov.m_platformName;
}
//-------------------------------------------------------
std::string
BaseEnvironment::identifyingString() const
{
  return m_optionsObj->m_ov.m_identifyingString;
}
//-------------------------------------------------------
void
BaseEnvironment::resetIdentifyingString(const std::string& newString) const // Yes, const
{
  m_optionsObj->m_ov.m_identifyingString = newString;
  return;
}
//-------------------------------------------------------
struct timeval
BaseEnvironment::timevalBegin() const
{
  return m_timevalBegin;
}
//-------------------------------------------------------
bool
BaseEnvironment::openOutputFile(
  const std::string&            baseFileName,
  const std::string&            inputFileType,
  const std::set<unsigned int>& allowedSubEnvIds,
        bool                    writeOver,
        FilePtrSetStruct&     filePtrSet) const
{
  std::string fileType(inputFileType);
#ifdef QUESO_HAS_HDF5
  // Do nothing
#else
  if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
    if (m_subDisplayFile) {
      *this->subDisplayFile() << "WARNING in BaseEnvironment::openOutputFile()"
                              << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                              << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' instead..."
                              << std::endl;
    }
    if (this->subRank() == 0) {
      std::cerr << "WARNING in BaseEnvironment::openOutputFile()"
                << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' instead..."
                << std::endl;
    }
    fileType = UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT;
  }
#endif

  bool returnValue = true;
  filePtrSet.ofsVar = NULL;
  if ((baseFileName                         == UQ_ENV_FILENAME_FOR_NO_OUTPUT_FILE) ||
      (allowedSubEnvIds.find(this->subId()) == allowedSubEnvIds.end()            )) {
    if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
      *this->subDisplayFile() << "In BaseEnvironment::openOutputFile()"
                              << ", subId = "     << this->subId()
                              << ": no output file opened with base name '" << baseFileName << "." << fileType
                              << "'"
                              << ", writeOver = " << writeOver
                              << std::endl;
    }
    returnValue = false;
  }
  else {
    // Open file
    if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
      *this->subDisplayFile() << "In BaseEnvironment::openOutputFile()"
                              << ", subId = "     << this->subId()
                              << ": opening output file with base name '" << baseFileName << "." << fileType
                              << "'"
                              << ", writeOver = " << writeOver
                              << std::endl;
    }

    if (this->subRank() == 0) {
#if 0
      std::cout << "In BaseEnvironment::openOutputFile()"
                << ": opening output file with base name '" << baseFileName << "." << fileType
                << "'"
                << ", writeOver = " << writeOver
                << std::endl;
#endif

      // Verify parent directory exists (for cases when a user
      // specifies a relative path for the desired output file).
      // std::cout << "checking " << baseFileName+"_sub"+this->subIdString()+"."+fileType << std::endl;
      if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
        *this->subDisplayFile() << "In BaseEnvironment::openOutputFile()"
                                << ", subId = "     << this->subId()
                                << ", trying to open output file with base name '" << baseFileName << "." << fileType
                                << "'"
                                << ", writeOver = " << writeOver
                                << ": calling CheckFilePath()..."
                                << std::endl;
      }
      int irtrn = CheckFilePath((baseFileName+"_sub"+this->subIdString()+"."+fileType).c_str());
      if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
        *this->subDisplayFile() << "In BaseEnvironment::openOutputFile()"
                                << ", subId = "     << this->subId()
                                << ", trying to open output file with base name '" << baseFileName << "." << fileType
                                << "'"
                                << ", writeOver = " << writeOver
                                << ": returned from CheckFilePath() with irtrn = " << irtrn
                                << std::endl;
      }
      UQ_FATAL_TEST_MACRO(irtrn < 0,
                          m_worldRank,
                          "BaseEnvironment::openOutputFile()",
                          "unable to verify output path");

      if (writeOver) {
        // Write over an eventual pre-existing file
        if (fileType == UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT) {
          filePtrSet.ofsVar = new std::ofstream((baseFileName+"_sub"+this->subIdString()+"."+fileType).c_str(),
                                                std::ofstream::out | std::ofstream::trunc);
        }
        else if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
          UQ_FATAL_TEST_MACRO(true,
                              m_worldRank,
                              "BaseEnvironment::openOutputFile(), writeOver=true",
                              "hdf file type not supported yet");
        }
        else {
          UQ_FATAL_TEST_MACRO(true,
                              m_worldRank,
                              "BaseEnvironment::openOutputFile(), writeOver=true",
                              "invalid file type");
        }
        if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
          *this->subDisplayFile() << "In BaseEnvironment::openOutputFile()"
                                  << ": just opened output file with base name '" << baseFileName << "." << fileType
                                  << "'"
                                  << ", writeOver = " << writeOver
                                  << ", options 'out|trunc'"
                                  << ", osfvar = " << filePtrSet.ofsVar
                                  << std::endl;
        }
      }
      else {
        // Write at the end of an eventual pre-existing file
        if (fileType == UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT) {
#if 0
          filePtrSet.ofsVar = new std::ofstream((baseFileName+"_sub"+this->subIdString()+"."+fileType).c_str(),
                                                std::ofstream::in);
          std::cout << "filePtrSet.ofsVar(1) = " << filePtrSet.ofsVar << std::endl;
          if (filePtrSet.ofsVar) std::cout << "filePtrSet.ofsVar(1)->is_open() = " << filePtrSet.ofsVar->is_open() << std::endl;
          if (filePtrSet.ofsVar) delete filePtrSet.ofsVar;
#endif
          // 'm' and Ranger nodes behave differently on ofstream constructor... prudenci 2010/03/05
          filePtrSet.ofsVar = new std::ofstream((baseFileName+"_sub"+this->subIdString()+"."+fileType).c_str(),
                                                std::ofstream::out /*| std::ofstream::in*/ | std::ofstream::app);
        }
        else if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
          UQ_FATAL_TEST_MACRO(true,
                              m_worldRank,
                              "BaseEnvironment::openOutputFile(), writeOver=false",
                              "hdf file type not supported yet");
        }
        else {
          UQ_FATAL_TEST_MACRO(true,
                              m_worldRank,
                              "BaseEnvironment::openOutputFile(), writeOver=false",
                              "invalid file type");
        }
        if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
          *this->subDisplayFile() << "In BaseEnvironment::openOutputFile()"
                                  << ": just opened output file with base name '" << baseFileName << "." << fileType
                                  << "'"
                                  << ", writeOver = " << writeOver
                                  << ", options 'out|in|app'"
                                  << ", osfvar = " << filePtrSet.ofsVar
                                  << std::endl;
        }

        if (fileType == UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT) {
          //std::cout << "filePtrSet.ofsVar(2) = " << filePtrSet.ofsVar << std::endl;
          //if (filePtrSet.ofsVar) std::cout << "filePtrSet.ofsVar(2)->is_open() = " << filePtrSet.ofsVar->is_open() << std::endl;
          if ((filePtrSet.ofsVar            == NULL ) ||
              (filePtrSet.ofsVar->is_open() == false)) {
            //std::cout << "Retrying 1..." << std::endl;
            delete filePtrSet.ofsVar;
            filePtrSet.ofsVar = new std::ofstream((baseFileName+"_sub"+this->subIdString()+"."+fileType).c_str(),
                                                  std::ofstream::out | std::ofstream::trunc);
            if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
              *this->subDisplayFile() << "In BaseEnvironment::openOutputFile()"
                                      << ": just opened output file with base name '" << baseFileName << "." << fileType
                                      << "'"
                                      << ", writeOver = " << writeOver
                                      << ", options 'out|trunc'"
                                      << ", osfvar = " << filePtrSet.ofsVar
                                      << std::endl;
            }
          }
        } // only for matlab formats
      }
      if (filePtrSet.ofsVar != NULL) {
        if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
          *this->subDisplayFile() << "In BaseEnvironment::openOutputFile()"
                                  << ", subId = "     << this->subId()
                                  << ": succeeded on opening output file with base name '" << baseFileName << "." << fileType
                                  << "'"
                                  << ", writeOver = " << writeOver
                                  << std::endl;
        }
      }
      else {
        std::cerr << "In BaseEnvironment::openOutputFile()"
                  << ": failed to open output file with base name '" << baseFileName << "." << fileType
                  << "'"
                  << std::endl;
      }
      UQ_FATAL_TEST_MACRO((filePtrSet.ofsVar && filePtrSet.ofsVar->is_open()) == false,
                          this->worldRank(),
                          "openOutputFile()",
                          "failed to open output file");
    }
    else {
      returnValue = false;
    }
    //this->subComm().Barrier(); // prudenci-2011-01-17
  }

  return returnValue;
}
//-------------------------------------------------------
bool
BaseEnvironment::openUnifiedOutputFile(
  const std::string&        baseFileName,
  const std::string&        inputFileType,
        bool                writeOver,
        FilePtrSetStruct& filePtrSet) const
{
  std::string fileType(inputFileType);
#ifdef QUESO_HAS_HDF5
  // Do nothing
#else
  if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
    if (m_subDisplayFile) {
      *this->subDisplayFile() << "WARNING in BaseEnvironment::openUnifiedOutputFile()"
                              << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                              << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' instead..."
                              << std::endl;
    }
    if (this->subRank() == 0) {
      std::cerr << "WARNING in BaseEnvironment::openUnifiedOutputFile()"
                << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' instead..."
                << std::endl;
    }
    fileType = UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT;
  }
#endif

  bool returnValue = true;
  filePtrSet.ofsVar = NULL;
  if (baseFileName == ".") {
    if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
      *this->subDisplayFile() << "In BaseEnvironment::openUnifiedOutputFile()"
                              << ": no unified output file opened with base name '" << baseFileName << "." << fileType
                              << "'"
                              << ", writeOver = " << writeOver
                              << std::endl;
    }
    returnValue = false;
  }
  else {
    // Open file
    if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
      *this->subDisplayFile() << "In BaseEnvironment::openUnifiedOutputFile()"
                              << ": opening unified output file with base name '" << baseFileName << "." << fileType
                              << "'"
                              << ", writeOver = " << writeOver
                              << std::endl;
    }


    //if ((this->subRank   () == 0) &&
    //    (this->inter0Rank() == 0)) {
#if 0
      std::cout << "In BaseEnvironment::openUnifiedOutputFile()"
                << ": opening output file with base name '" << baseFileName << "." << fileType
                << "'"
                << ", writeOver = " << writeOver
                << std::endl;
#endif
      // Verify parent directory exists (for cases when a user
      // specifies a relative path for the desired output file). prudenci 2010/06/26
      // std::cout << "checking " << baseFileName+"."+fileType << std::endl;
      int irtrn = CheckFilePath((baseFileName+"."+fileType).c_str());
      UQ_FATAL_TEST_MACRO(irtrn < 0,
                          m_worldRank,
                          "BaseEnvironment::openUnifiedOutputFile()",
                          "unable to verify output path");

      if (writeOver) {
        // Write over an eventual pre-existing file
        if (fileType == UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT) {
          filePtrSet.ofsVar = new std::ofstream((baseFileName+"."+fileType).c_str(),
                                                std::ofstream::out | std::ofstream::trunc);
        }
#ifdef QUESO_HAS_HDF5
        else if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
          filePtrSet.h5Var = H5Fcreate((baseFileName+"."+fileType).c_str(),
                                       H5F_ACC_TRUNC,
                                       H5P_DEFAULT,
                                       H5P_DEFAULT);
        }
#endif
        else {
          UQ_FATAL_TEST_MACRO(true,
                              m_worldRank,
                              "BaseEnvironment::openUnifiedOutputFile(), writeOver=true",
                              "invalid file type");
        }
        if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
          *this->subDisplayFile() << "In BaseEnvironment::openUnifiedOutputFile()"
                                  << ": just opened output file with base name '" << baseFileName << "." << fileType
                                  << "'"
                                  << ", writeOver = " << writeOver
                                  << ", options 'out|trunc'"
                                  << ", osfvar = " << filePtrSet.ofsVar
                                  << std::endl;
        }
      }
      else {
        // Write at the end of an eventual pre-existing file
        // 'm' and Ranger nodes behave differently on ofstream constructor... prudenci 2010/03/05
        if (fileType == UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT) {
          filePtrSet.ofsVar = new std::ofstream((baseFileName+"."+fileType).c_str(),
                                                std::ofstream::out /*| std::ofstream::in*/ | std::ofstream::app);
        }
#ifdef QUESO_HAS_HDF5
        else if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
          filePtrSet.h5Var = H5Fcreate((baseFileName+"."+fileType).c_str(), // TEMPORARY - FIX ME
                                       H5F_ACC_TRUNC,
                                       H5P_DEFAULT,
                                       H5P_DEFAULT);
          //UQ_FATAL_TEST_MACRO(true,
          //                    m_worldRank,
          //                    "BaseEnvironment::openUnifiedOutputFile(), writeOver=false",
          //                    "hdf file type not supported yet");
        }
#endif
        else {
          UQ_FATAL_TEST_MACRO(true,
                              m_worldRank,
                              "BaseEnvironment::openUnifiedOutputFile(), writeOver=false",
                              "invalid file type");
        }
        if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
          *this->subDisplayFile() << "In BaseEnvironment::openUnifiedOutputFile()"
                                  << ": just opened output file with base name '" << baseFileName << "." << fileType
                                  << "'"
                                  << ", writeOver = " << writeOver
                                  << ", options 'out|in|app'"
                                  << ", osfvar = " << filePtrSet.ofsVar
                                  << std::endl;
        }
        if ((filePtrSet.ofsVar            == NULL ) ||
            (filePtrSet.ofsVar->is_open() == false)) {
#if 0
          std::cout << "Retrying 2..." << std::endl;
#endif
          delete filePtrSet.ofsVar;
          filePtrSet.ofsVar = new std::ofstream((baseFileName+"."+fileType).c_str(),
                                                std::ofstream::out | std::ofstream::trunc);
          if ((m_subDisplayFile) && (this->displayVerbosity() > 10)) { // output debug
            *this->subDisplayFile() << "In BaseEnvironment::openUnifiedOutputFile()"
                                    << ": just opened output file with base name '" << baseFileName << "." << fileType
                                    << "'"
                                    << ", writeOver = " << writeOver
                                    << ", options 'out|trunc'"
                                    << ", osfvar = " << filePtrSet.ofsVar
                                    << std::endl;
          }
        }
      }
      if (filePtrSet.ofsVar == NULL) {
        std::cerr << "In BaseEnvironment::openUnifiedOutputFile()"
                  << ": failed to open unified output file with base name '" << baseFileName << "." << fileType
                  << "'"
                  << std::endl;
      }
      UQ_FATAL_TEST_MACRO((filePtrSet.ofsVar && filePtrSet.ofsVar->is_open()) == false,
                          this->worldRank(),
                          "openUnifiedOutputFile()",
                          "failed to open output file");
    //}
  }

  return returnValue;
}
//-------------------------------------------------------
bool
BaseEnvironment::openInputFile(
  const std::string&            baseFileName,
  const std::string&            inputFileType,
  const std::set<unsigned int>& allowedSubEnvIds,
        FilePtrSetStruct&     filePtrSet) const
{
  std::string fileType(inputFileType);
#ifdef QUESO_HAS_HDF5
  // Do nothing
#else
  if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
    if (m_subDisplayFile) {
      *this->subDisplayFile() << "WARNING in BaseEnvironment::openInputFile()"
                              << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                              << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' instead..."
                              << std::endl;
    }
    if (this->subRank() == 0) {
      std::cerr << "WARNING in BaseEnvironment::openInputFile()"
                << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' instead..."
                << std::endl;
    }
    fileType = UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT;
  }
#endif

  bool returnValue = true;
  filePtrSet.ifsVar = NULL;
  if ((baseFileName                         == UQ_ENV_FILENAME_FOR_NO_INPUT_FILE) ||
      (allowedSubEnvIds.find(this->subId()) == allowedSubEnvIds.end()           )) {
    if ((m_subDisplayFile) && (this->displayVerbosity() >= 10)) {
      *this->subDisplayFile() << "In BaseEnvironment::openInputFile()"
                              << ": no input file opened with base name '" << baseFileName << "." << fileType
                              << "'"
                              << std::endl;
    }
    returnValue = false;
  }
  else {
    // Open file
    if ((m_subDisplayFile) && (this->displayVerbosity() >= 10)) {
      *this->subDisplayFile() << "In BaseEnvironment::openInputFile()"
                              << ": opening input file with base name '" << baseFileName << "." << fileType
                              << "'"
                              << std::endl;
    }
    if (this->subRank() == 0) {
      // Verify parent directory exists (for cases when a user
      // specifies a relative path for the desired output file). prudenci 2010/06/26
      // std::cout << "checking " << baseFileName+"."+fileType << std::endl;
      int irtrn = CheckFilePath((baseFileName+"."+fileType).c_str());
      UQ_FATAL_TEST_MACRO(irtrn < 0,
                          m_worldRank,
                          "BaseEnvironment::openInputFile()",
                          "unable to verify input path");

      if (fileType == UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT) {
        filePtrSet.ifsVar = new std::ifstream((baseFileName+"."+fileType).c_str(),
                                              std::ofstream::in);
        if ((filePtrSet.ifsVar == NULL) || (filePtrSet.ifsVar->is_open() == false)) {
          std::cerr << "In BaseEnvironment::openInputFile()"
                    << ": failed to open input file with base name '" << baseFileName << "." << fileType
                    << "'"
                    << std::endl;
        }
        UQ_FATAL_TEST_MACRO((filePtrSet.ifsVar == NULL) || (filePtrSet.ifsVar->is_open() == false),
                            this->worldRank(),
                            "BaseEnvironment::openInputFile()",
                            "file with fileName could not be found");
      }
#ifdef QUESO_HAS_HDF5
      else if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
        filePtrSet.h5Var = H5Fopen((baseFileName+"."+fileType).c_str(),
                                   H5F_ACC_RDONLY,
                                   H5P_DEFAULT);
      }
#endif
      else {
        UQ_FATAL_TEST_MACRO(true,
                            m_worldRank,
                            "BaseEnvironment::openInputFile()",
                            "invalid file type");
      }
    }
    else {
      returnValue = false;
    }
    //this->subComm().Barrier(); // prudenci-2011-01-17
  }

  return returnValue;
}
//-------------------------------------------------------
bool
BaseEnvironment::openUnifiedInputFile(
  const std::string&        baseFileName,
  const std::string&        inputFileType,
        FilePtrSetStruct& filePtrSet) const
{
  std::string fileType(inputFileType);
#ifdef QUESO_HAS_HDF5
  // Do nothing
#else
  if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
    if (m_subDisplayFile) {
      *this->subDisplayFile() << "WARNING in BaseEnvironment::openUnifiedInputFile()"
                              << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                              << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' instead..."
                              << std::endl;
    }
    if (this->subRank() == 0) {
      std::cerr << "WARNING in BaseEnvironment::openUnifiedInputFile()"
                << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' instead..."
                << std::endl;
    }
    fileType = UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT;
  }
#endif

  bool returnValue = true;
  filePtrSet.ifsVar = NULL;
  if (baseFileName == UQ_ENV_FILENAME_FOR_NO_INPUT_FILE) {
    if ((m_subDisplayFile) && (this->displayVerbosity() >= 10)) {
      *this->subDisplayFile() << "In BaseEnvironment::openUnifiedInputFile()"
                              << ": no input file opened with base name '" << baseFileName << "." << fileType
                              << "'"
                              << std::endl;
    }
    returnValue = false;
  }
  else {
    // Open file
    if ((m_subDisplayFile) && (this->displayVerbosity() >= 10)) {
      *this->subDisplayFile() << "In BaseEnvironment::openUnifiedInputFile()"
                              << ": opening input file with base name '" << baseFileName << "." << fileType
                              << "'"
                              << std::endl;
    }
    if (this->subRank() == 0) { // Needed ???????? prudenci 2010-11-11
      // Verify parent directory exists (for cases when a user
      // specifies a relative path for the desired output file). prudenci 2010/06/26
      // std::cout << "checking " << baseFileName+"."+fileType << std::endl;
      int irtrn = CheckFilePath((baseFileName+"."+fileType).c_str());
      UQ_FATAL_TEST_MACRO(irtrn < 0,
                          m_worldRank,
                          "BaseEnvironment::openUnifiedInputFile()",
                          "unable to verify input path");

      if (fileType == UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT) {
        filePtrSet.ifsVar = new std::ifstream((baseFileName+"."+fileType).c_str(),
                                              std::ofstream::in);
        if ((filePtrSet.ifsVar == NULL) || (filePtrSet.ifsVar->is_open() == false)) {
          std::cerr << "In BaseEnvironment::openUnifiedInputFile()"
                    << ": failed to open input file with base name '" << baseFileName << "." << fileType
                    << "'"
                    << std::endl;
        }
        UQ_FATAL_TEST_MACRO((filePtrSet.ifsVar == NULL) || (filePtrSet.ifsVar->is_open() == false),
                            this->worldRank(),
                            "BaseEnvironment::openUnifiedInputFile()",
                            "file with fileName could not be found");
      }
#ifdef QUESO_HAS_HDF5
      else if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
        filePtrSet.h5Var = H5Fopen((baseFileName+"."+fileType).c_str(),
                                   H5F_ACC_RDONLY,
                                   H5P_DEFAULT);
      }
#endif
      else {
        UQ_FATAL_TEST_MACRO(true,
                            m_worldRank,
                            "BaseEnvironment::openUnifiedInputFile()",
                            "invalid file type");
      }
    }
    //else {
    //  returnValue = false;
    //}
    //this->subComm().Barrier();
  }

  return returnValue;
}
//-------------------------------------------------------
void
BaseEnvironment::closeFile(
  FilePtrSetStruct& filePtrSet,
  const std::string&  inputFileType) const
{
  std::string fileType(inputFileType);
#ifdef QUESO_HAS_HDF5
  // Do nothing
#else
  if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
    if (m_subDisplayFile) {
      *this->subDisplayFile() << "WARNING in BaseEnvironment::closeFile()"
                              << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                              << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                              << "' instead..."
                              << std::endl;
    }
    if (this->subRank() == 0) {
      std::cerr << "WARNING in BaseEnvironment::closeFile()"
                << ": file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' has been requested, but this QUESO library has not been built with 'hdf5'"
                << ". Code will therefore process the file format '" << UQ_FILE_EXTENSION_FOR_HDF_FORMAT
                << "' instead..."
                << std::endl;
    }
    fileType = UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT;
  }
#endif

  if (fileType == UQ_FILE_EXTENSION_FOR_MATLAB_FORMAT) {
    //filePtrSet.ofsVar->close(); // close() crashes on Mac; need to use delete(); why? prudenci 2010/June
    delete filePtrSet.ofsVar;
    filePtrSet.ofsVar = NULL;

    //filePtrSet.ifsVar->close();
    delete filePtrSet.ifsVar;
    filePtrSet.ifsVar = NULL;
  }
#ifdef QUESO_HAS_HDF5
  else if (fileType == UQ_FILE_EXTENSION_FOR_HDF_FORMAT) {
    H5Fclose(filePtrSet.h5Var);
  }
#endif
  else {
    UQ_FATAL_TEST_MACRO(true,
                        m_worldRank,
                        "BaseEnvironment::closeFile()",
                        "invalid file type");
  }

  return;
}
//-------------------------------------------------------
void
BaseEnvironment::setExceptionalCircumstance(bool value) const
{
  m_exceptionalCircumstance = value;
  return;
}
//-------------------------------------------------------
bool
BaseEnvironment::exceptionalCircumstance() const
{
  return m_exceptionalCircumstance;
}


//*****************************************************
// Empty Environment
//*****************************************************
EmptyEnvironment::EmptyEnvironment()
  :
  BaseEnvironment("",NULL)
{
}
//-------------------------------------------------------
EmptyEnvironment::~EmptyEnvironment()
{
}
//-------------------------------------------------------
void
EmptyEnvironment::print(std::ostream& os) const
{
  os.flush(); // just to avoid icpc warnings
  return;
}

//*****************************************************
// Full Environment
//*****************************************************
FullEnvironment::FullEnvironment(
  RawType_MPI_Comm             inputComm,
  const char*                    passedOptionsInputFileName,
  const char*                    prefix,
  const EnvOptionsValues* alternativeOptionsValues)
  :
  BaseEnvironment(passedOptionsInputFileName,alternativeOptionsValues)
{
#ifdef QUESO_MEMORY_DEBUGGING
  std::cout << "Entering FullEnv" << std::endl;
#endif

  // Initialize "full" communicator -- Not necessarily MPI_COMM_WORLD
  int mpiRC = MPI_Comm_rank(inputComm,&m_worldRank);
  UQ_FATAL_TEST_MACRO(mpiRC != MPI_SUCCESS,
                      UQ_UNAVAILABLE_RANK,
                      "FullEnvironment::commonConstructor()",
                      "failed to get world fullRank()");

  m_fullComm = new MpiComm(*this,inputComm);
  m_fullRank     = m_fullComm->MyPID();
  m_fullCommSize = m_fullComm->NumProc();

  mpiRC = MPI_Comm_group(m_fullComm->Comm(), &m_fullGroup);
  UQ_FATAL_TEST_MACRO(mpiRC != MPI_SUCCESS,
                      m_worldRank,
                      "FullEnvironment::commonConstructor()",
                      "failed MPI_Comm_group()");

  // saving old uncaught exception handler, invoking queso_terminate
  old_terminate_handler = std::set_terminate(queso_terminate_handler);

#ifdef QUESO_MEMORY_DEBUGGING
  std::cout << "In FullEnv, finished dealing with MPI initially" << std::endl;
#endif

  // Read options
  if (m_optionsInputFileName == "") {
    m_optionsObj = new EnvironmentOptions(*this,prefix,m_alternativeOptionsValues);
  }
  else {
    m_allOptionsMap  = new po::variables_map();
    m_allOptionsDesc = new po::options_description("Allowed options");
    m_optionsObj = new EnvironmentOptions(*this,prefix);

    readOptionsInputFile();

    m_optionsObj->scanOptionsValues();
  }

#ifdef QUESO_MEMORY_DEBUGGING
  std::cout << "In FullEnv, finished scanning options" << std::endl;
#endif

  // Only display these messages if the user wants them
  // NOTE: This got moved below the Read Options section
  // because we need the options to be read to know what
  // the verbosity level is.
  if (this->displayVerbosity() > 0) {
    // Display main initial messages
    // 'std::cout' is for: main trace messages + synchronized trace messages + error messages prior to 'exit()' or 'abort()'
    /*int iRC = 0;*/
    /*iRC = */gettimeofday(&m_timevalBegin, NULL);

    if (m_fullRank == 0) {
      QUESO_version_print(std::cout);
    }

    if (m_fullRank == 0) {
      std::cout << "Beginning run at " << ctime(&m_timevalBegin.tv_sec)
                << std::endl;
    }
  }

  // Deal with multiple subEnvironments: create the sub communicators, one for each subEnvironment
  unsigned int numRanksPerSubEnvironment = m_fullCommSize/m_optionsObj->m_ov.m_numSubEnvironments;

  m_subId = m_fullRank/numRanksPerSubEnvironment;
  char tmpSubId[16];
  sprintf(tmpSubId,"%u",m_subId);
  m_subIdString = tmpSubId;

  if (m_optionsObj->m_ov.m_subDisplayAllowAll) {
    m_optionsObj->m_ov.m_subDisplayAllowedSet.insert((unsigned int) m_subId);
  }

  std::vector<int> fullRanksOfMySubEnvironment(numRanksPerSubEnvironment,0);
  for (unsigned int i = 0; i < numRanksPerSubEnvironment; ++i) {
    fullRanksOfMySubEnvironment[i] = m_subId * numRanksPerSubEnvironment + i;
  }

  mpiRC = MPI_Group_incl(m_fullGroup, (int) numRanksPerSubEnvironment, &fullRanksOfMySubEnvironment[0], &m_subGroup);
  UQ_FATAL_TEST_MACRO(mpiRC != MPI_SUCCESS,
                      m_worldRank,
                      "FullEnvironment::commonConstructor()",
                      "failed MPI_Group_incl() for a subEnvironment");

  RawType_MPI_Comm subRawComm;
  mpiRC = MPI_Comm_create(m_fullComm->Comm(), m_subGroup, &subRawComm);
  UQ_FATAL_TEST_MACRO(mpiRC != MPI_SUCCESS,
                      m_worldRank,
                      "FullEnvironment::commonConstructor()",
                      "failed MPI_Comm_group() for a subEnvironment");
  m_subComm = new MpiComm(*this,subRawComm);
  m_subRank     = m_subComm->MyPID();
  m_subCommSize = m_subComm->NumProc();

  // Deal with multiple subEnvironments: create the self communicator
  m_selfComm = new MpiComm(*this,RawValue_MPI_COMM_SELF);

  // Deal with multiple subEnvironments: create the inter0 communicator
  std::vector<int> fullRanksOfInter0(m_optionsObj->m_ov.m_numSubEnvironments,0);
  for (unsigned int i = 0; i < m_optionsObj->m_ov.m_numSubEnvironments; ++i) {
    fullRanksOfInter0[i] = i * numRanksPerSubEnvironment;
  }
  mpiRC = MPI_Group_incl(m_fullGroup, (int) m_optionsObj->m_ov.m_numSubEnvironments, &fullRanksOfInter0[0], &m_inter0Group);
  UQ_FATAL_TEST_MACRO(mpiRC != MPI_SUCCESS,
                      m_worldRank,
                      "FullEnvironment::commonConstructor()",
                      "failed MPI_Group_incl() for inter0");
  RawType_MPI_Comm inter0RawComm;
  mpiRC = MPI_Comm_create(m_fullComm->Comm(), m_inter0Group, &inter0RawComm);
  UQ_FATAL_TEST_MACRO(mpiRC != MPI_SUCCESS,
                      m_worldRank,
                      "FullEnvironment::commonConstructor()",
                      "failed MPI_Comm_group() for inter0");
  if (m_fullRank%numRanksPerSubEnvironment == 0) {
    m_inter0Comm = new MpiComm(*this,inter0RawComm);
    m_inter0Rank     = m_inter0Comm->MyPID();
    m_inter0CommSize = m_inter0Comm->NumProc();
  }

  if (m_optionsObj->m_ov.m_subDisplayAllowAll) {
    // This situation has been already taken care of above
  }
  else if (m_optionsObj->m_ov.m_subDisplayAllowInter0) {
    if (m_inter0Rank >= 0) {
      m_optionsObj->m_ov.m_subDisplayAllowedSet.insert((unsigned int) m_subId);
    }
  }


  // Open "screen" file
  bool openFile = false;
  if ((m_subRank                                               == 0                                              ) &&
      (m_optionsObj->m_ov.m_subDisplayFileName                 != UQ_ENV_FILENAME_FOR_NO_OUTPUT_FILE             ) &&
      (m_optionsObj->m_ov.m_subDisplayAllowedSet.find(m_subId) != m_optionsObj->m_ov.m_subDisplayAllowedSet.end())) {
    openFile = true;
  }

  if (openFile && m_worldRank == 0) {
    // Verify parent directory exists (for cases when a user
    // specifies a relative path for the desired output file).
    int irtrn = CheckFilePath((m_optionsObj->m_ov.m_subDisplayFileName+"_sub"+m_subIdString+".txt").c_str());
    UQ_FATAL_TEST_MACRO(irtrn < 0,
                        m_worldRank,
                        "Environment::constructor()",
                        "unable to verify output path");
  }

  // Ensure that rank 0 has created path, if necessary, before other tasks use it
  m_fullComm->Barrier();

  if (openFile) {
    // Always write over an eventual pre-existing file
    m_subDisplayFile = new std::ofstream((m_optionsObj->m_ov.m_subDisplayFileName+"_sub"+m_subIdString+".txt").c_str(),
                                         std::ofstream::out | std::ofstream::trunc);
    UQ_FATAL_TEST_MACRO((m_subDisplayFile && m_subDisplayFile->is_open()) == false,
                        m_worldRank,
                        "Environment::constructor()",
                        "failed to open sub screen file");

    QUESO_version_print(*m_subDisplayFile);

    *m_subDisplayFile << "Beginning run at " << ctime(&m_timevalBegin.tv_sec)
                      << std::endl;
  }

  // Debug message related to subEnvironments
  if (this->displayVerbosity() >= 2) {
    for (int i = 0; i < m_fullCommSize; ++i) {
      if (i == m_fullRank) {
        //std::cout << "In FullEnvironment::commonConstructor()"
        std::cout << "MPI node of worldRank "             << m_worldRank
                  << " has fullRank "                     << m_fullRank
                  << ", belongs to subEnvironment of id " << m_subId
                  << ", and has subRank "                 << m_subRank
                  << std::endl;

        std::cout << "MPI node of worldRank " << m_worldRank
                  << " belongs to sub communicator with full ranks";
        for (unsigned int j = 0; j < fullRanksOfMySubEnvironment.size(); ++j) {
          std::cout << " " << fullRanksOfMySubEnvironment[j];
        }
        std::cout << "\n";

        if (m_inter0Comm) {
          std::cout << "MPI node of worldRank " << m_worldRank
                    << " also belongs to inter0 communicator with full ranks";
          for (unsigned int j = 0; j < fullRanksOfInter0.size(); ++j) {
            std::cout << " " << fullRanksOfInter0[j];
          }
          std::cout << ", and has inter0Rank " << m_inter0Rank;
        }
        std::cout << "\n";

        std::cout << std::endl;
      }
      m_fullComm->Barrier();
    }
    //if (this->fullRank() == 0) std::cout << "Sleeping 3 seconds..."
    //                                     << std::endl;
    //sleep(3);
  }

  // Deal with seed
  if (m_optionsObj->m_ov.m_rngType == "gsl") {
    m_rngObject = new RngGsl(m_optionsObj->m_ov.m_seed,m_worldRank);
    m_basicPdfs = new BasicPdfsGsl(m_worldRank);
  }
  else if (m_optionsObj->m_ov.m_rngType == "boost") {
    m_rngObject = new RngBoost(m_optionsObj->m_ov.m_seed,m_worldRank);
    m_basicPdfs = new BasicPdfsBoost(m_worldRank);
  }
  else {
    std::cerr << "In Environment::constructor()"
              << ": rngType = " << m_optionsObj->m_ov.m_rngType
              << std::endl;
    UQ_FATAL_TEST_MACRO(true,
                        m_worldRank,
                        "Environment::constructor()",
                        "the requested 'rngType' is not supported yet");
  }

  // Leave commonConstructor()
  m_fullComm->Barrier();
  m_fullEnvIsReady = true;

  if ((m_subDisplayFile) && (this->displayVerbosity() >= 5)) {
    *m_subDisplayFile << "Done with initializations at FullEnvironment::commonConstructor()"
                      << std::endl;
  }

  return;
}
//-------------------------------------------------------
FullEnvironment::~FullEnvironment()
{
}
//-------------------------------------------------------
void
FullEnvironment::print(std::ostream& os) const
{
  os.flush(); // just to avoid icpc warnings
  return;
}

void queso_terminate_handler()
{
  int mpi_initialized;
  MPI_Initialized (&mpi_initialized);

  if (mpi_initialized)
    {
      //MPI_Abort(m_fullComm->Comm(), 1);
      MPI_Abort(MPI_COMM_WORLD, 1);
    }
  else
    {
      // The system terminate_handler may do useful things like printing
      // uncaught exception information, or the user may have created
      // their own terminate handler that we want to call.
      old_terminate_handler();
    }
  exit(1);
}


//-------------------------------------------------------
void
FullEnvironment::readOptionsInputFile()
{
  std::ifstream* ifs = new std::ifstream(m_optionsInputFileName.c_str());
  if (ifs->is_open()) {
    //ifs->close();
    delete ifs;
  }
  else {
    if (m_fullRank == 0) std::cout << "An invalid input file has been passed to the 'environment' class constructor!"
                                   << ": name of file is '" << m_optionsInputFileName.c_str() << "'"
                                   << std::endl;

    if (m_fullRank == 0) std::cout << "\nThis is a help message of the QUESO library."
                                   << "\nAn application using the QUESO library shall be executed by typing"
                                   << "\n  '<eventual mpi commands and options> <Application> <InputFile>'"
                                   << "\nin the command line."
                                   << "\n"
                                   << std::endl;
    queso_error();
  }

  return;
}

}  // End namespace QUESO