ParallelComm.hpp

Go to the documentation of this file.

/**
 * MOAB, a Mesh-Oriented datABase, is a software component for creating,
 * storing and accessing finite element mesh data.
 *
 * Copyright 2004 Sandia Corporation. Under the terms of Contract
 * DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government
 * retains certain rights in this software.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 */
 
#ifndef MOAB_PARALLEL_COMM_HPP
#define MOAB_PARALLEL_COMM_HPP
 
#include "moab/Forward.hpp"
#include "moab/Interface.hpp"
#include "moab/Range.hpp"
#include "moab/ProcConfig.hpp"
#include <map>
#include <set>
#include <vector>
#include <iostream>
#include <fstream>
#include <cassert>
#include <cstdlib>
#include <cmath>
#include "moab/TupleList.hpp"
 
namespace moab
{
 
class SequenceManager;
class Error;
template < typename KeyType, typename ValType, ValType NullVal >
class RangeMap;
typedef RangeMap< EntityHandle, EntityHandle, 0 > HandleMap;
class ParallelMergeMesh;
class DebugOutput;
class SharedSetData;
 
#define MAX_SHARING_PROCS 64
 
/**
 * \brief Parallel communications in MOAB
 * \author Tim Tautges
 *
 * This class implements methods to communicate mesh between processors
 *
 */
class ParallelComm
{
 public:
 friend class ParallelMergeMesh;
 
 // ==================================
 // \section CONSTRUCTORS/DESTRUCTORS/PCOMM MANAGEMENT
 // ==================================
 
 //! constructor
 ParallelComm( Interface* impl, MPI_Comm comm, int* pcomm_id_out = 0 );
 
 //! constructor taking packed buffer, for testing
 ParallelComm( Interface* impl, std::vector< unsigned char >& tmp_buff, MPI_Comm comm, int* pcomm_id_out = 0 );
 
 //! Get ID used to reference this PCOMM instance
 int get_id() const
 {
 return pcommID;
 }
 
 //! get the indexed pcomm object from the interface
 static ParallelComm* get_pcomm( Interface* impl, const int index );
 
 //! Get ParallelComm instance associated with partition handle
 //! Will create ParallelComm instance if a) one does not already
 //! exist and b) a valid value for MPI_Comm is passed.
 static ParallelComm* get_pcomm( Interface* impl, EntityHandle partitioning, const MPI_Comm* comm = 0 );
 
 static ErrorCode get_all_pcomm( Interface* impl, std::vector< ParallelComm* >& list );
 
 //! destructor
 ~ParallelComm();
 
 static unsigned char PROC_SHARED, PROC_OWNER;
 
 // ==================================
 // \section GLOBAL IDS
 // ==================================
 
 //! assign a global id space, for largest-dimension or all entities (and
 //! in either case for vertices too)
 //!\param owned_only If true, do not get global IDs for non-owned entities
 //! from remote processors.
 ErrorCode assign_global_ids( EntityHandle this_set,
 const int dimension,
 const int start_id = 1,
 const bool largest_dim_only = true,
 const bool parallel = true,
 const bool owned_only = false );
 
 //! assign a global id space, for largest-dimension or all entities (and
 //! in either case for vertices too)
 ErrorCode assign_global_ids( Range entities[],
 const int dimension,
 const int start_id,
 const bool parallel,
 const bool owned_only );
 
 //! check for global ids; based only on tag handle being there or not;
 //! if it's not there, create them for the specified dimensions
 //!\param owned_only If true, do not get global IDs for non-owned entities
 //! from remote processors.
 ErrorCode check_global_ids( EntityHandle this_set,
 const int dimension,
 const int start_id = 1,
 const bool largest_dim_only = true,
 const bool parallel = true,
 const bool owned_only = false );
 
 // ==================================
 // \section HIGH-LEVEL COMMUNICATION (send/recv/bcast/scatter ents, exchange tags)
 // ==================================
 
 /** \brief send entities to another processor, optionally waiting until it's done
 *
 * Send entities to another processor, with adjs, sets, and tags.
 * If store_remote_handles is true, this call receives back handles assigned to
 * entities sent to destination processor and stores them in sharedh_tag or
 * sharedhs_tag.
 * \param to_proc Destination processor
 * \param orig_ents Entities requested to send
 * \param adjs If true, send adjacencies for equiv entities (currently unsupported)
 * \param tags If true, send tag values for all tags assigned to entities
 * \param store_remote_handles If true, also recv message with handles on destination processor
 * (currently unsupported) \param final_ents Range containing all entities sent \param incoming
 * keep track if any messages are coming to this processor (newly added) \param wait_all If
 * true, wait until all messages received/sent complete
 */
 ErrorCode send_entities( const int to_proc,
 Range& orig_ents,
 const bool adjs,
 const bool tags,
 const bool store_remote_handles,
 const bool is_iface,
 Range& final_ents,
 int& incoming1,
 int& incoming2, // newly added
 TupleList& entprocs, // newly added
 std::vector< MPI_Request >& recv_remoteh_reqs, // newly added
 bool wait_all = true );
 
 ErrorCode send_entities( std::vector< unsigned int >& send_procs,
 std::vector< Range* >& send_ents,
 int& incoming1,
 int& incoming2,
 const bool store_remote_handles );
 
 /** \brief Receive entities from another processor, optionally waiting until it's done
 *
 * Receive entities from another processor, with adjs, sets, and tags.
 * If store_remote_handles is true, this call sends back handles assigned to
 * the entities received.
 * \param from_proc Source processor
 * \param store_remote_handles If true, send message with new entity handles to source processor
 * (currently unsupported) \param final_ents Range containing all entities received \param
 * incoming keep track if any messages are coming to this processor (newly added) \param
 * wait_all If true, wait until all messages received/sent complete
 */
 ErrorCode recv_entities( const int from_proc,
 const bool store_remote_handles,
 const bool is_iface,
 Range& final_ents,
 int& incomming1,
 int& incoming2,
 std::vector< std::vector< EntityHandle > >& L1hloc,
 std::vector< std::vector< EntityHandle > >& L1hrem,
 std::vector< std::vector< int > >& L1p,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p,
 std::vector< MPI_Request >& recv_remoteh_reqs,
 bool wait_all = true );
 
 ErrorCode recv_entities( std::set< unsigned int >& recv_procs,
 int incoming1,
 int incoming2,
 const bool store_remote_handles,
 const bool migrate = false );
 
 /** \brief Receive messages from another processor in while loop
 *
 * Receive messages from another processor.
 * \param from_proc Source processor
 * \param store_remote_handles If true, send message with new entity handles to source processor
 * (currently unsupported) \param final_ents Range containing all entities received \param
 * incoming keep track if any messages are coming to this processor (newly added)
 */
 ErrorCode recv_messages( const int from_proc,
 const bool store_remote_handles,
 const bool is_iface,
 Range& final_ents,
 int& incoming1,
 int& incoming2,
 std::vector< std::vector< EntityHandle > >& L1hloc,
 std::vector< std::vector< EntityHandle > >& L1hrem,
 std::vector< std::vector< int > >& L1p,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p,
 std::vector< MPI_Request >& recv_remoteh_reqs );
 
 ErrorCode recv_remote_handle_messages( const int from_proc,
 int& incoming2,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p,
 std::vector< MPI_Request >& recv_remoteh_reqs );
 
 /** \brief Exchange ghost cells with neighboring procs
 * Neighboring processors are those sharing an interface
 * with this processor. All entities of dimension ghost_dim
 * within num_layers of interface, measured going through bridge_dim,
 * are exchanged. See MeshTopoUtil::get_bridge_adjacencies for description
 * of bridge adjacencies. If wait_all is false and store_remote_handles
 * is true, MPI_Request objects are available in the sendReqs[2*MAX_SHARING_PROCS]
 * member array, with inactive requests marked as MPI_REQUEST_NULL. If
 * store_remote_handles or wait_all is false, this function returns after
 * all entities have been received and processed.
 * \param ghost_dim Dimension of ghost entities to be exchanged
 * \param bridge_dim Dimension of entities used to measure layers from interface
 * \param num_layers Number of layers of ghosts requested
 * \param addl_ents Dimension of additional adjacent entities to exchange with ghosts, 0 if none
 * \param store_remote_handles If true, send message with new entity handles to source processor
 * \param wait_all If true, function does not return until all send buffers
 * are cleared.
 */
 
 ErrorCode exchange_ghost_cells( int ghost_dim,
 int bridge_dim,
 int num_layers,
 int addl_ents,
 bool store_remote_handles,
 bool wait_all = true,
 EntityHandle* file_set = NULL );
 
 /** \brief Static version of exchange_ghost_cells, exchanging info through
 * buffers rather than messages
 */
 static ErrorCode exchange_ghost_cells( ParallelComm** pc,
 unsigned int num_procs,
 int ghost_dim,
 int bridge_dim,
 int num_layers,
 int addl_ents,
 bool store_remote_handles,
 EntityHandle* file_sets = NULL );
 
 /** \brief Post "MPI_Irecv" before meshing
 * \param exchange_procs processor vector exchanged
 */
 ErrorCode post_irecv( std::vector< unsigned int >& exchange_procs );
 
 ErrorCode post_irecv( std::vector< unsigned int >& shared_procs, std::set< unsigned int >& recv_procs );
 
 /** \brief Exchange owned mesh for input mesh entities and sets
 * This function should be called collectively over the communicator for this ParallelComm.
 * If this version is called, all shared exchanged entities should have a value for this
 * tag (or the tag should have a default value).
 * \param exchange_procs processor vector exchanged
 * \param exchange_ents exchanged entities for each processors
 * \param migrate if the owner if entities are changed or not
 */
 ErrorCode exchange_owned_meshs( std::vector< unsigned int >& exchange_procs,
 std::vector< Range* >& exchange_ents,
 std::vector< MPI_Request >& recv_ent_reqs,
 std::vector< MPI_Request >& recv_remoteh_reqs,
 bool store_remote_handles,
 bool wait_all = true,
 bool migrate = false,
 int dim = 0 );
 
 /** \brief Exchange owned mesh for input mesh entities and sets
 * This function is called twice by exchange_owned_meshs to exchange entities before sets
 * \param migrate if the owner if entities are changed or not
 */
 ErrorCode exchange_owned_mesh( std::vector< unsigned int >& exchange_procs,
 std::vector< Range* >& exchange_ents,
 std::vector< MPI_Request >& recv_ent_reqs,
 std::vector< MPI_Request >& recv_remoteh_reqs,
 const bool recv_posted,
 bool store_remote_handles,
 bool wait_all,
 bool migrate = false );
 
 /** \brief Exchange tags for all shared and ghosted entities
 * This function should be called collectively over the communicator for this ParallelComm.
 * If this version is called, all ghosted/shared entities should have a value for this
 * tag (or the tag should have a default value). If the entities vector is empty, all shared
 * entities participate in the exchange. If a proc has no owned entities this function must
 * still be called since it is collective. \param src_tags Vector of tag handles to be exchanged
 * \param dst_tags Tag handles to store the tags on the non-owning procs
 * \param entities Entities for which tags are exchanged
 */
 ErrorCode exchange_tags( const std::vector< Tag >& src_tags,
 const std::vector< Tag >& dst_tags,
 const Range& entities );
 
 /** \brief Exchange tags for all shared and ghosted entities
 * This function should be called collectively over the communicator for this ParallelComm.
 * If the entities vector is empty, all shared entities
 * participate in the exchange. If a proc has no owned entities this function must still be
 * called since it is collective. \param tag_name Name of tag to be exchanged \param entities
 * Entities for which tags are exchanged
 */
 ErrorCode exchange_tags( const char* tag_name, const Range& entities );
 
 /** \brief Exchange tags for all shared and ghosted entities
 * This function should be called collectively over the communicator for this ParallelComm.
 * If the entities vector is empty, all shared entities
 * participate in the exchange. If a proc has no owned entities this function must still be
 * called since it is collective. \param tagh Handle of tag to be exchanged \param entities
 * Entities for which tags are exchanged
 */
 ErrorCode exchange_tags( Tag tagh, const Range& entities );
 
 /** \brief Perform data reduction operation for all shared and ghosted entities
 * This function should be called collectively over the communicator for this ParallelComm.
 * If this version is called, all ghosted/shared entities should have a value for this
 * tag (or the tag should have a default value). Operation is any MPI_Op, with result stored
 * in destination tag.
 * \param src_tags Vector of tag handles to be reduced
 * \param dst_tags Vector of tag handles in which the answer will be stored
 * \param mpi_op Operation type
 * \param entities Entities on which reduction will be made; if empty, operates on all shared
 * entities
 */
 ErrorCode reduce_tags( const std::vector< Tag >& src_tags,
 const std::vector< Tag >& dst_tags,
 const MPI_Op mpi_op,
 const Range& entities );
 
 /** \brief Perform data reduction operation for all shared and ghosted entities
 * Same as std::vector variant except for one tag specified by name
 * \param tag_name Name of tag to be reduced
 * \param mpi_op Operation type
 * \param entities Entities on which reduction will be made; if empty, operates on all shared
 * entities
 */
 ErrorCode reduce_tags( const char* tag_name, const MPI_Op mpi_op, const Range& entities );
 
 /** \brief Perform data reduction operation for all shared and ghosted entities
 * Same as std::vector variant except for one tag specified by handle
 * \param tag_name Name of tag to be reduced
 * \param mpi_op Operation type
 * \param entities Entities on which reduction will be made; if empty, operates on all shared
 * entities
 */
 ErrorCode reduce_tags( Tag tag_handle, const MPI_Op mpi_op, const Range& entities );
 
 /** \brief Broadcast all entities resident on from_proc to other processors
 * This function assumes remote handles are *not* being stored, since (usually)
 * every processor will know about the whole mesh.
 * \param from_proc Processor having the mesh to be broadcast
 * \param entities On return, the entities sent or received in this call
 * \param adjacencies If true, adjacencies are sent for equiv entities (currently unsupported)
 * \param tags If true, all non-default-valued tags are sent for sent entities
 */
 ErrorCode broadcast_entities( const int from_proc,
 Range& entities,
 const bool adjacencies = false,
 const bool tags = true );
 
 /** \brief Scatter entities on from_proc to other processors
 * This function assumes remote handles are *not* being stored, since (usually)
 * every processor will know about the whole mesh.
 * \param from_proc Processor having the mesh to be broadcast
 * \param entities On return, the entities sent or received in this call
 * \param adjacencies If true, adjacencies are sent for equiv entities (currently unsupported)
 * \param tags If true, all non-default-valued tags are sent for sent entities
 */
 ErrorCode scatter_entities( const int from_proc,
 std::vector< Range >& entities,
 const bool adjacencies = false,
 const bool tags = true );
 
 /////////////////////////////////////////////////////////////////////////////////
 // Send and Receive routines for a sequence of entities: use case UMR
 /////////////////////////////////////////////////////////////////////////////////
 
 /** \brief Send and receives data from a set of processors
 */
 ErrorCode send_recv_entities( std::vector< int >& send_procs,
 std::vector< std::vector< int > >& msgsizes,
 std::vector< std::vector< EntityHandle > >& senddata,
 std::vector< std::vector< EntityHandle > >& recvdata );
 
 ErrorCode update_remote_data( EntityHandle entity,
 std::vector< int >& procs,
 std::vector< EntityHandle >& handles );
 
 ErrorCode get_remote_handles( EntityHandle* local_vec, EntityHandle* rem_vec, int num_ents, int to_proc );
 
 /////////////////////////////////////////////////////////////////////////////////
 
 // ==================================
 // \section INITIALIZATION OF PARALLEL DATA (resolve_shared_ents, etc.)
 // ==================================
 
 /** \brief Resolve shared entities between processors
 *
 * Resolve shared entities between processors for entities in proc_ents,
 * by comparing global id tag values on vertices on skin of elements in
 * proc_ents. Shared entities are assigned a tag that's either
 * PARALLEL_SHARED_PROC_TAG_NAME, which is 1 integer in length, or
 * PARALLEL_SHARED_PROCS_TAG_NAME, whose length depends on the maximum
 * number of sharing processors. Values in these tags denote the ranks
 * of sharing processors, and the list ends with the value -1.
 *
 * If shared_dim is input as -1 or not input, a value one less than the
 * maximum dimension of entities in proc_ents is used.
 *
 * \param proc_ents Entities for which to resolve shared entities
 * \param shared_dim Maximum dimension of shared entities to look for
 */
 ErrorCode resolve_shared_ents( EntityHandle this_set,
 Range& proc_ents,
 int resolve_dim = -1,
 int shared_dim = -1,
 Range* skin_ents = NULL,
 const Tag* id_tag = 0 );
 
 /** \brief Resolve shared entities between processors
 *
 * Same as resolve_shared_ents(Range&), except works for
 * all entities in instance of dimension dim.
 *
 * If shared_dim is input as -1 or not input, a value one less than the
 * maximum dimension of entities is used.
 
 * \param dim Dimension of entities in the partition
 * \param shared_dim Maximum dimension of shared entities to look for
 */
 ErrorCode resolve_shared_ents( EntityHandle this_set,
 int resolve_dim = 3,
 int shared_dim = -1,
 const Tag* id_tag = 0 );
 
 static ErrorCode resolve_shared_ents( ParallelComm** pc,
 const unsigned int np,
 EntityHandle this_set,
 const int to_dim );
 
 /** Remove shared sets.
 *
 * Generates list of candidate sets using from those (directly)
 * contained in passed set and passes them to the other version
 * of \c resolve_shared_sets.
 *\param this_set Set directly containing candidate sets (e.g. file set)
 *\param id_tag Tag containing global IDs for entity sets.
 */
 
 ErrorCode resolve_shared_sets( EntityHandle this_set, const Tag* id_tag = 0 );
 
 /** Remove shared sets.
 *
 * Use values of id_tag to match sets across processes and populate
 * sharing data for sets.
 *\param candidate_sets Sets to consider as potentially shared.
 *\param id_tag Tag containing global IDs for entity sets.
 */
 ErrorCode resolve_shared_sets( Range& candidate_sets, Tag id_tag );
 
 /** extend shared sets with ghost entities
 * After ghosting, ghost entities do not have yet information about
 * the material set, partition set, Neumann or Dirichlet set they could
 * belong to
 * This method will assign ghosted entities to the those special entity sets
 * In some case we might even have to create those sets, if they do not exist yet on
 * the local processor
 *
 * The special entity sets all have an unique identifier, in a form of an integer
 * tag to the set.
 * The shared sets data is not used, because we do not use the geometry sets, as they are
 * not uniquely identified
 *
 *
 * \param file_set : file set used per application
 *
 */
 ErrorCode augment_default_sets_with_ghosts( EntityHandle file_set );
 // ==================================
 // \section GET PARALLEL DATA (shared/owned/iface entities, etc.)
 // ==================================
 
 /** \brief Get parallel status of an entity
 * Returns the parallel status of an entity
 *
 * \param entity The entity being queried
 * \param pstatus_val Parallel status of the entity
 */
 ErrorCode get_pstatus( EntityHandle entity, unsigned char& pstatus_val );
 
 /** \brief Get entities with the given pstatus bit(s) set
 * Returns any entities whose pstatus tag value v satisfies (v & pstatus_val)
 *
 * \param dim Dimension of entities to be returned, or -1 if any
 * \param pstatus_val pstatus value of desired entities
 * \param pstatus_ents Entities returned from function
 */
 ErrorCode get_pstatus_entities( int dim, unsigned char pstatus_val, Range& pstatus_ents );
 
 /** \brief Return the rank of the entity owner
 */
 ErrorCode get_owner( EntityHandle entity, int& owner );
 
 /** \brief Return the owner processor and handle of a given entity
 */
 ErrorCode get_owner_handle( EntityHandle entity, int& owner, EntityHandle& handle );
 
 /** \brief Get the shared processors/handles for an entity
 * Get the shared processors/handles for an entity. Arrays must
 * be large enough to receive data for all sharing procs. Does *not* include
 * this proc if only shared with one other proc.
 * \param entity Entity being queried
 * \param ps Pointer to sharing proc data
 * \param hs Pointer to shared proc handle data
 * \param pstat Reference to pstatus data returned from this function
 */
 ErrorCode get_sharing_data( const EntityHandle entity,
 int* ps,
 EntityHandle* hs,
 unsigned char& pstat,
 unsigned int& num_ps );
 
 /** \brief Get the shared processors/handles for an entity
 * Same as other version but with int num_ps
 * \param entity Entity being queried
 * \param ps Pointer to sharing proc data
 * \param hs Pointer to shared proc handle data
 * \param pstat Reference to pstatus data returned from this function
 */
 ErrorCode get_sharing_data( const EntityHandle entity,
 int* ps,
 EntityHandle* hs,
 unsigned char& pstat,
 int& num_ps );
 
 /** \brief Get the intersection or union of all sharing processors
 * Get the intersection or union of all sharing processors. Processor set
 * is cleared as part of this function.
 * \param entities Entity list ptr
 * \param num_entities Number of entities
 * \param procs Processors returned
 * \param op Either Interface::UNION or Interface::INTERSECT
 */
 ErrorCode get_sharing_data( const EntityHandle* entities,
 int num_entities,
 std::set< int >& procs,
 int op = Interface::INTERSECT );
 
 /** \brief Get the intersection or union of all sharing processors
 * Same as previous variant but with range as input
 */
 ErrorCode get_sharing_data( const Range& entities, std::set< int >& procs, int op = Interface::INTERSECT );
 
 /** \brief Get shared entities of specified dimension
 * If other_proc is -1, any shared entities are returned. If dim is -1,
 * entities of all dimensions on interface are returned.
 * \param other_proc Rank of processor for which interface entities are requested
 * \param shared_ents Entities returned from function
 * \param dim Dimension of interface entities requested
 * \param iface If true, return only entities on the interface
 * \param owned_filter If true, return only owned shared entities
 */
 ErrorCode get_shared_entities( int other_proc,
 Range& shared_ents,
 int dim = -1,
 const bool iface = false,
 const bool owned_filter = false );
 /*
 //! return partition sets; if tag_name is input, gets sets with
 //! that tag name, otherwise uses PARALLEL_PARTITION tag
 ErrorCode get_partition_sets(EntityHandle this_set,
 Range &part_sets,
 const char *tag_name = NULL);
 */
 //! get processors with which this processor shares an interface
 ErrorCode get_interface_procs( std::set< unsigned int >& iface_procs, const bool get_buffs = false );
 
 //! get processors with which this processor communicates
 ErrorCode get_comm_procs( std::set< unsigned int >& procs );
 
 // ==================================
 // \section SHARED SETS
 // ==================================
 
 //! Get array of process IDs sharing a set. Returns zero
 //! and passes back NULL if set is not shared.
 ErrorCode get_entityset_procs( EntityHandle entity_set, std::vector< unsigned >& ranks ) const;
 
 //! Get rank of the owner of a shared set.
 //! Returns this proc if set is not shared.
 //! Optionally returns handle on owning process for shared set.
 ErrorCode get_entityset_owner( EntityHandle entity_set,
 unsigned& owner_rank,
 EntityHandle* remote_handle = 0 ) const;
 
 //! Given set owner and handle on owner, find local set handle
 ErrorCode get_entityset_local_handle( unsigned owning_rank,
 EntityHandle remote_handle,
 EntityHandle& local_handle ) const;
 
 //! Get all shared sets
 ErrorCode get_shared_sets( Range& result ) const;
 
 //! Get ranks of all processes that own at least one set that is
 //! shared with this process. Will include the rank of this process
 //! if this process owns any shared set.
 ErrorCode get_entityset_owners( std::vector< unsigned >& ranks ) const;
 
 //! Get shared sets owned by process with specified rank.
 ErrorCode get_owned_sets( unsigned owning_rank, Range& sets_out ) const;
 
 // ==================================
 // \section LOW-LEVEL DATA (tags, sets on interface/partition, etc.)
 // ==================================
 
 //! Get proc config for this communication object
 const ProcConfig& proc_config() const
 {
 return procConfig;
 }
 
 //! Get proc config for this communication object
 ProcConfig& proc_config()
 {
 return procConfig;
 }
 
 unsigned rank() const
 {
 return proc_config().proc_rank();
 }
 unsigned size() const
 {
 return proc_config().proc_size();
 }
 MPI_Comm comm() const
 {
 return proc_config().proc_comm();
 }
 
 //! return the tags used to indicate shared procs and handles
 ErrorCode get_shared_proc_tags( Tag& sharedp_tag,
 Tag& sharedps_tag,
 Tag& sharedh_tag,
 Tag& sharedhs_tag,
 Tag& pstatus_tag );
 
 //! return partition, interface set ranges
 Range& partition_sets()
 {
 return partitionSets;
 }
 const Range& partition_sets() const
 {
 return partitionSets;
 }
 Range& interface_sets()
 {
 return interfaceSets;
 }
 const Range& interface_sets() const
 {
 return interfaceSets;
 }
 
 //! return sharedp tag
 Tag sharedp_tag();
 
 //! return sharedps tag
 Tag sharedps_tag();
 
 //! return sharedh tag
 Tag sharedh_tag();
 
 //! return sharedhs tag
 Tag sharedhs_tag();
 
 //! return pstatus tag
 Tag pstatus_tag();
 
 //! return pcomm tag; static because might not have a pcomm before going
 //! to look for one on the interface
 static Tag pcomm_tag( Interface* impl, bool create_if_missing = true );
 
 //! return partitions set tag
 Tag partition_tag();
 Tag part_tag()
 {
 return partition_tag();
 }
 
 // ==================================
 // \section DEBUGGING AIDS
 // ==================================
 
 //! print contents of pstatus value in human-readable form
 void print_pstatus( unsigned char pstat, std::string& ostr );
 
 //! print contents of pstatus value in human-readable form to std::cut
 void print_pstatus( unsigned char pstat );
 
 // ==================================
 // \section IMESHP-RELATED FUNCTIONS
 // ==================================
 
 //! return all the entities in parts owned locally
 ErrorCode get_part_entities( Range& ents, int dim = -1 );
 
 EntityHandle get_partitioning() const
 {
 return partitioningSet;
 }
 ErrorCode set_partitioning( EntityHandle h );
 ErrorCode get_global_part_count( int& count_out ) const;
 ErrorCode get_part_owner( int part_id, int& owner_out ) const;
 ErrorCode get_part_id( EntityHandle part, int& id_out ) const;
 ErrorCode get_part_handle( int id, EntityHandle& handle_out ) const;
 ErrorCode create_part( EntityHandle& part_out );
 ErrorCode destroy_part( EntityHandle part );
 ErrorCode collective_sync_partition();
 ErrorCode get_part_neighbor_ids( EntityHandle part, int neighbors_out[MAX_SHARING_PROCS], int& num_neighbors_out );
 ErrorCode get_interface_sets( EntityHandle part, Range& iface_sets_out, int* adj_part_id = 0 );
 ErrorCode get_owning_part( EntityHandle entity, int& owning_part_id_out, EntityHandle* owning_handle = 0 );
 ErrorCode get_sharing_parts( EntityHandle entity,
 int part_ids_out[MAX_SHARING_PROCS],
 int& num_part_ids_out,
 EntityHandle remote_handles[MAX_SHARING_PROCS] = 0 );
 
 /** Filter the entities by pstatus tag.
 * op is one of PSTATUS_ AND, OR, NOT; an entity is output if:
 * AND: all bits set in pstatus_val are also set on entity
 * OR: any bits set in pstatus_val also set on entity
 * NOT: any bits set in pstatus_val are not set on entity
 *
 * Results returned in input list, unless result_ents is passed in non-null,
 * in which case results are returned in result_ents.
 *
 * If ents is passed in empty, filter is done on shared entities in this
 * pcomm instance, i.e. contents of sharedEnts.
 *
 *\param ents Input entities to filter
 *\param pstatus_val pstatus value to which entities are compared
 *\param op Bitwise operation performed between pstatus values
 *\param to_proc If non-negative and PSTATUS_SHARED is set on pstatus_val,
 * only entities shared with to_proc are returned
 *\param result_ents If non-null, results of filter are put in the
 * pointed-to range
 */
 ErrorCode filter_pstatus( Range& ents,
 const unsigned char pstatus_val,
 const unsigned char op,
 int to_proc = -1,
 Range* returned_ents = NULL );
 
 /** \brief Get entities on interfaces shared with another proc
 *
 * \param other_proc Other proc sharing the interface
 * \param dim Dimension of entities to return, -1 if all dims
 * \param iface_ents Returned entities
 */
 ErrorCode get_iface_entities( int other_proc, int dim, Range& iface_ents );
 
 Interface* get_moab() const
 {
 return mbImpl;
 }
 
 ErrorCode clean_shared_tags( std::vector< Range* >& exchange_ents );
 
 class Buffer
 {
 public:
 unsigned char* mem_ptr;
 unsigned char* buff_ptr;
 unsigned int alloc_size;
 
 Buffer( unsigned int sz = 0 );
 Buffer( const Buffer& );
 ~Buffer();
 void reset_buffer( size_t buff_pos = 0 )
 {
 reset_ptr( buff_pos );
 reserve( INITIAL_BUFF_SIZE );
 }
 void reset_ptr( size_t buff_pos = 0 )
 {
 assert( ( !mem_ptr && !buff_pos ) || ( alloc_size >= buff_pos ) );
 buff_ptr = mem_ptr + buff_pos;
 }
 inline void reserve( unsigned int new_size );
 void set_stored_size()
 {
 *( (int*)mem_ptr ) = (int)( buff_ptr - mem_ptr );
 }
 int get_stored_size()
 {
 return *( (int*)mem_ptr );
 }
 int get_current_size()
 {
 return (int)( buff_ptr - mem_ptr );
 }
 
 void check_space( unsigned int addl_space );
 };
 
 //! public 'cuz we want to unit test these externally
 ErrorCode pack_buffer( Range& orig_ents,
 const bool adjacencies,
 const bool tags,
 const bool store_remote_handles,
 const int to_proc,
 Buffer* buff,
 TupleList* entprocs = NULL,
 Range* allsent = NULL );
 
 ErrorCode unpack_buffer( unsigned char* buff_ptr,
 const bool store_remote_handles,
 const int from_proc,
 const int ind,
 std::vector< std::vector< EntityHandle > >& L1hloc,
 std::vector< std::vector< EntityHandle > >& L1hrem,
 std::vector< std::vector< int > >& L1p,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p,
 std::vector< EntityHandle >& new_ents,
 const bool created_iface = false );
 
 ErrorCode pack_entities( Range& entities,
 Buffer* buff,
 const bool store_remote_handles,
 const int to_proc,
 const bool is_iface,
 TupleList* entprocs = NULL,
 Range* allsent = NULL );
 
 //! unpack entities in buff_ptr
 ErrorCode unpack_entities( unsigned char*& buff_ptr,
 const bool store_remote_handles,
 const int from_ind,
 const bool is_iface,
 std::vector< std::vector< EntityHandle > >& L1hloc,
 std::vector< std::vector< EntityHandle > >& L1hrem,
 std::vector< std::vector< int > >& L1p,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p,
 std::vector< EntityHandle >& new_ents,
 const bool created_iface = false );
 
 //! Call exchange_all_shared_handles, then compare the results with tag data
 //! on local shared entities.
 ErrorCode check_all_shared_handles( bool print_em = false );
 
 static ErrorCode check_all_shared_handles( ParallelComm** pcs, int num_pcs );
 
 struct SharedEntityData
 {
 EntityHandle local;
 EntityHandle remote;
 EntityID owner;
 };
 
 ErrorCode pack_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data );
 
 // check consistency of sharedEnts against their tags and their
 // vertices' tags
 ErrorCode check_local_shared();
 
 // check contents of communicated shared entity data against tags
 ErrorCode check_my_shared_handles( std::vector< std::vector< SharedEntityData > >& shents,
 const char* prefix = NULL );
 
 //! set rank for this pcomm; USED FOR TESTING ONLY!
 void set_rank( unsigned int r );
 
 //! set rank for this pcomm; USED FOR TESTING ONLY!
 void set_size( unsigned int r );
 
 //! get (and possibly allocate) buffers for messages to/from to_proc; returns
 //! index of to_proc in buffProcs vector; if is_new is non-NULL, sets to
 //! whether new buffer was allocated
 //! PUBLIC ONLY FOR TESTING!
 int get_buffers( int to_proc, bool* is_new = NULL );
 
 //! get buff processor vector
 const std::vector< unsigned int >& buff_procs() const;
 
 /* \brief Unpack message with remote handles
 * PUBLIC ONLY FOR TESTING!
 */
 ErrorCode unpack_remote_handles( unsigned int from_proc,
 unsigned char*& buff_ptr,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p );
 
 /* \brief Pack message with remote handles
 * PUBLIC ONLY FOR TESTING!
 */
 ErrorCode pack_remote_handles( std::vector< EntityHandle >& L1hloc,
 std::vector< EntityHandle >& L1hrem,
 std::vector< int >& procs,
 unsigned int to_proc,
 Buffer* buff );
 
 // each iterate in proc_nvecs contains a set of procs and the entities *possibly*
 // on the interface between those procs; this function makes sets for each,
 // and tags the set with the procs sharing it; interface sets are optionally
 // returned; NOTE: a subsequent step is used to verify entities on the interface
 // and remove them if they're not shared
 ErrorCode create_interface_sets( std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs );
 
 // do the same but working straight from sharedEnts
 ErrorCode create_interface_sets( EntityHandle this_set, int resolve_dim, int shared_dim );
 
 ErrorCode tag_shared_verts( TupleList& shared_ents,
 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
 Range& proc_verts,
 unsigned int i_extra = 1 );
 
 ErrorCode list_entities( const EntityHandle* ents, int num_ents );
 
 ErrorCode list_entities( const Range& ents );
 
 void set_send_request( int n_request ); // set send request array
 
 void set_recv_request( int n_request ); // set recv request array
 
 //! reset message buffers to their initial state
 // changed to public function (HJK)
 void reset_all_buffers();
 
 static const unsigned int INITIAL_BUFF_SIZE;
 
 //! set the verbosity level of output from this pcomm
 void set_debug_verbosity( int verb );
 
 //! get the verbosity level of output from this pcomm
 int get_debug_verbosity();
 
 /* \brief Gather tag value from entities down to a specified root proc
 * This function gathers data from a domain-decomposed mesh onto a global mesh
 * represented on the root processor. On the root, this gather mesh is distinct from
 * the root's domain-decomposed subdomain. Entities are matched by global id, or by
 * another tag if its handle is input. The dimension of all entities in gather_ents should
 * be the same, since this is the dimension of entities in gather_set that are queried for
 * matching global id tags.
 * \param gather_ents (Local) entities from which to gather data
 * \param tag_handle Tag whose values are being gathered
 * \param id_tag Tag to use for matching entities (global id used by default)
 * \param gather_set On root, set containing global mesh onto which to put data
 * \param root_proc_rank Rank of the specified root processor (default rank is 0)
 */
 ErrorCode gather_data( Range& gather_ents,
 Tag& tag_handle,
 Tag id_tag = 0,
 EntityHandle gather_set = 0,
 int root_proc_rank = 0 );
 
 /* \brief communicate extra points positions on boundary
 * This function is called after intersection of 2 meshes, to settle the
 * position of the intersection points on the boundary (interface)
 * The initial mesh distributed on each processor is decomposed after
 * intersection with another mesh, such as that new points are created on the
 * boundary. these points should better match at the interface !
 * we perform an extra caution step, to ensure the robustness of the
 * intersection algorithm; only shared edges extra nodes
 * will be actually needed to be communicated, but we just pass by reference
 * the whole extraNodesVec structure, we do
 * not need to construct another data structure
 * The node positions on edges that are owned will be communicated to other
 * processors
 *
 * \param edges total range of entities
 * \param shared_edges_owned edges for which to communicate data
 * \param extraNodesVec handles of intersection vertices on all edges;
 */
 ErrorCode settle_intersection_points( Range& edges,
 Range& shared_edges_owned,
 std::vector< std::vector< EntityHandle >* >& extraNodesVec,
 double tolerance );
 
 /* \brief delete entities from moab database
 * will check the shared ents array, and clean it if necessary
 *
 */
 ErrorCode delete_entities( Range& to_delete );
 
 /*
 * \brief correct multi-sharing info for thin layers
 *
 * will be used for at least 3 processes, when there are thin ghost layers
 * right now it is public, for allowing users to call it directly
 * eventually, it should become private, and be called automatically
 */
 
 ErrorCode correct_thin_ghost_layers();
 
 private:
 ErrorCode reduce_void( int tag_data_type, const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals );
 
 template < class T >
 ErrorCode reduce( const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals );
 
 void print_debug_isend( int from, int to, unsigned char* buff, int tag, int size );
 
 void print_debug_irecv( int to, int from, unsigned char* buff, int size, int tag, int incoming );
 
 void print_debug_recd( MPI_Status status );
 
 void print_debug_waitany( std::vector< MPI_Request >& reqs, int tag, int proc );
 
 // common initialization code, called from various constructors
 void initialize();
 
 ErrorCode set_sharing_data( EntityHandle ent,
 unsigned char pstatus,
 int old_nump,
 int new_nump,
 int* ps,
 EntityHandle* hs );
 
 ErrorCode check_clean_iface( Range& allsent );
 
 void define_mpe();
 
 ErrorCode get_sent_ents( const bool is_iface,
 const int bridge_dim,
 const int ghost_dim,
 const int num_layers,
 const int addl_ents,
 Range* sent_ents,
 Range& allsent,
 TupleList& entprocs );
 
 /** \brief Set pstatus values on entities
 *
 * \param pstatus_ents Entities to be set
 * \param pstatus_val Pstatus value to be set
 * \param lower_dim_ents If true, lower-dimensional ents (incl. vertices) set too
 * (and created if they don't exist)
 * \param verts_too If true, vertices also set
 * \param operation If UNION, pstatus_val is OR-d with existing value, otherwise
 * existing value is over-written
 */
 ErrorCode set_pstatus_entities( Range& pstatus_ents,
 unsigned char pstatus_val,
 bool lower_dim_ents = false,
 bool verts_too = true,
 int operation = Interface::UNION );
 
 /** \brief Set pstatus values on entities (vector-based function)
 *
 * \param pstatus_ents Entities to be set
 * \param pstatus_val Pstatus value to be set
 * \param lower_dim_ents If true, lower-dimensional ents (incl. vertices) set too
 * (and created if they don't exist)
 * \param verts_too If true, vertices also set
 * \param operation If UNION, pstatus_val is OR-d with existing value, otherwise
 * existing value is over-written
 */
 ErrorCode set_pstatus_entities( EntityHandle* pstatus_ents,
 int num_ents,
 unsigned char pstatus_val,
 bool lower_dim_ents = false,
 bool verts_too = true,
 int operation = Interface::UNION );
 
 //! estimate size required to pack entities
 int estimate_ents_buffer_size( Range& entities, const bool store_remote_handles );
 
 //! estimate size required to pack sets
 int estimate_sets_buffer_size( Range& entities, const bool store_remote_handles );
 
 //! send the indicated buffer, possibly sending size first
 ErrorCode send_buffer( const unsigned int to_proc,
 Buffer* send_buff,
 const int msg_tag,
 MPI_Request& send_req,
 MPI_Request& ack_recv_req,
 int* ack_buff,
 int& this_incoming,
 int next_mesg_tag = -1,
 Buffer* next_recv_buff = NULL,
 MPI_Request* next_recv_req = NULL,
 int* next_incoming = NULL );
 
 //! process incoming message; if longer than the initial size, post
 //! recv for next part then send ack; if ack, send second part; else
 //! indicate that we're done and buffer is ready for processing
 ErrorCode recv_buffer( int mesg_tag_expected,
 const MPI_Status& mpi_status,
 Buffer* recv_buff,
 MPI_Request& recv_2nd_req,
 MPI_Request& ack_req,
 int& this_incoming,
 Buffer* send_buff,
 MPI_Request& send_req,
 MPI_Request& sent_ack_req,
 bool& done,
 Buffer* next_buff = NULL,
 int next_tag = -1,
 MPI_Request* next_req = NULL,
 int* next_incoming = NULL );
 
 //! pack a range of entities with equal # verts per entity, along with
 //! the range on the sending proc
 ErrorCode pack_entity_seq( const int nodes_per_entity,
 const bool store_remote_handles,
 const int to_proc,
 Range& these_ents,
 std::vector< EntityHandle >& entities,
 Buffer* buff );
 
 ErrorCode print_buffer( unsigned char* buff_ptr, int mesg_type, int from_proc, bool sent );
 
 //! for all the entities in the received buffer; for each, save
 //! entities in this instance which match connectivity, or zero if none found
 ErrorCode unpack_iface_entities( unsigned char*& buff_ptr,
 const int from_proc,
 const int ind,
 std::vector< EntityHandle >& recd_ents );
 
 ErrorCode pack_sets( Range& entities, Buffer* buff, const bool store_handles, const int to_proc );
 
 ErrorCode unpack_sets( unsigned char*& buff_ptr,
 std::vector< EntityHandle >& entities,
 const bool store_handles,
 const int to_proc );
 
 ErrorCode pack_adjacencies( Range& entities,
 Range::const_iterator& start_rit,
 Range& whole_range,
 unsigned char*& buff_ptr,
 int& count,
 const bool just_count,
 const bool store_handles,
 const int to_proc );
 
 ErrorCode unpack_adjacencies( unsigned char*& buff_ptr,
 Range& entities,
 const bool store_handles,
 const int from_proc );
 
 /* \brief Unpack message with remote handles (const pointer to buffer)
 */
 ErrorCode unpack_remote_handles( unsigned int from_proc,
 const unsigned char* buff_ptr,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p );
 
 //! given connectivity and type, find an existing entity, if there is one
 ErrorCode find_existing_entity( const bool is_iface,
 const int owner_p,
 const EntityHandle owner_h,
 const int num_ents,
 const EntityHandle* connect,
 const int num_connect,
 const EntityType this_type,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p,
 EntityHandle& new_h );
 
 ErrorCode build_sharedhps_list( const EntityHandle entity,
 const unsigned char pstatus,
 const int sharedp,
 const std::set< unsigned int >& procs,
 unsigned int& num_ents,
 int* tmp_procs,
 EntityHandle* tmp_handles );
 
 /**\brief Get list of tags for which to exchange data
 *
 * Get tags and entities for which to exchange tag data. This function
 * was originally part of 'pack_tags' requested with the
 * 'all_possible_tags' parameter.
 *
 *\param all_entities Input. The set of entities for which data is to
 * be communicated.
 *\param all_tags Output. Populated with the handles of tags to be
 * sent.
 *\param tag_ranges Output. For each corresponding tag in all_tags, the
 * subset of 'all_entities' for which a tag value has
 * been set.
 */
 ErrorCode get_tag_send_list( const Range& all_entities,
 std::vector< Tag >& all_tags,
 std::vector< Range >& tag_ranges );
 
 /**\brief Serialize entity tag data
 *
 * This function operates in two passes. The first phase,
 * specified by 'just_count == true' calculates the necessary
 * buffer size for the serialized data. The second phase
 * writes the actual binary serialized representation of the
 * data to the passed buffer.
 *
 *\NOTE First two arguments are not used. (Legacy interface?)
 *
 *\param entities NOT USED
 *\param start_rit NOT USED
 *\param whole_range Should be the union of the sets of entities for
 * which tag values are to be serialized. Also
 * specifies ordering for indexes for tag values and
 * serves as the superset from which to compose entity
 * lists from individual tags if just_count and
 * all_possible_tags are both true.
 *\param buff_ptr Buffer into which to write binary serialized data
 *\param count Output: The size of the serialized data is added
 * to this parameter. NOTE: Should probably initialize
 * to zero before calling.
 *\param just_count If true, just calculate the buffer size required to
 * hold the serialized data. Will also append to
 * 'all_tags' and 'tag_ranges' if all_possible_tags
 * == true.
 *\param store_handles The data for each tag is preceded by a list of
 * EntityHandles designating the entity each of
 * the subsequent tag values corresponds to. This value
 * may be one of:
 * 1) If store_handles == false:
 * An invalid handle composed of {MBMAXTYPE,idx}, where
 * idx is the position of the entity in "whole_range".
 * 2) If store_hanldes == true and a valid remote
 * handle exists, the remote handle.
 * 3) If store_hanldes == true and no valid remote
 * handle is defined for the entity, the same as 1).
 *\param to_proc If 'store_handles' is true, the processor rank for
 * which to store the corresponding remote entity
 * handles.
 *\param all_tags List of tags to write
 *\param tag_ranges List of entities to serialize tag data, one
 * for each corresponding tag handle in 'all_tags.
 */
 ErrorCode pack_tags( Range& entities,
 const std::vector< Tag >& src_tags,
 const std::vector< Tag >& dst_tags,
 const std::vector< Range >& tag_ranges,
 Buffer* buff,
 const bool store_handles,
 const int to_proc );
 
 /**\brief Calculate buffer size required to pack tag data
 *\param source_tag The tag for which data will be serialized
 *\param entities The entities for which tag values will be serialized
 *\param count_out Output: The required buffer size, in bytes.
 */
 ErrorCode packed_tag_size( Tag source_tag, const Range& entities, int& count_out );
 
 /**\brief Serialize tag data
 *\param source_tag The tag for which data will be serialized
 *\param destination_tag Tag in which to store unpacked tag data. Typically
 * the same as source_tag.
 *\param entities The entities for which tag values will be serialized
 *\param whole_range Calculate entity indices as location in this range
 *\param buff_ptr Input/Output: As input, pointer to the start of the
 * buffer in which to serialize data. As output, the
 * position just passed the serialized data.
 *\param count_out Output: The required buffer size, in bytes.
 *\param store_handles The data for each tag is preceded by a list of
 * EntityHandles designating the entity each of
 * the subsequent tag values corresponds to. This value
 * may be one of:
 * 1) If store_handles == false:
 * An invalid handle composed of {MBMAXTYPE,idx}, where
 * idx is the position of the entity in "whole_range".
 * 2) If store_hanldes == true and a valid remote
 * handle exists, the remote handle.
 * 3) If store_hanldes == true and no valid remote
 * handle is defined for the entity, the same as 1).
 *\param to_proc If 'store_handles' is true, the processor rank for
 * which to store the corresponding remote entity
 * handles.
 */
 ErrorCode pack_tag( Tag source_tag,
 Tag destination_tag,
 const Range& entities,
 const std::vector< EntityHandle >& whole_range,
 Buffer* buff,
 const bool store_remote_handles,
 const int to_proc );
 
 ErrorCode unpack_tags( unsigned char*& buff_ptr,
 std::vector< EntityHandle >& entities,
 const bool store_handles,
 const int to_proc,
 const MPI_Op* const mpi_op = NULL );
 
 ErrorCode tag_shared_verts( TupleList& shared_verts,
 Range* skin_ents,
 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
 Range& proc_verts );
 
 ErrorCode get_proc_nvecs( int resolve_dim,
 int shared_dim,
 Range* skin_ents,
 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs );
 
 // after verifying shared entities, now parent/child links between sets can be established
 ErrorCode create_iface_pc_links();
 
 //! pack a range map with keys in this_range and values a contiguous series
 //! of handles starting at actual_start
 ErrorCode pack_range_map( Range& this_range, EntityHandle actual_start, HandleMap& handle_map );
 
 //! returns true if the set is an interface shared with to_proc
 bool is_iface_proc( EntityHandle this_set, int to_proc );
 
 //! for any remote_handles set to zero, remove corresponding sent_ents from
 //! iface_sets corresponding to from_proc
 ErrorCode update_iface_sets( Range& sent_ents, std::vector< EntityHandle >& remote_handles, int from_proc );
 
 //! for specified bridge/ghost dimension, to_proc, and number
 //! of layers, get the entities to be ghosted, and info on additional procs
 //! needing to communicate with to_proc
 ErrorCode get_ghosted_entities( int bridge_dim,
 int ghost_dim,
 int to_proc,
 int num_layers,
 int addl_ents,
 Range& ghosted_ents );
 
 //! add vertices adjacent to entities in this list
 ErrorCode add_verts( Range& sent_ents );
 
 //! Every processor sends shared entity handle data to every other processor
 //! that it shares entities with. Passed back map is all received data,
 //! indexed by processor ID. This function is intended to be used for
 //! debugging.
 ErrorCode exchange_all_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data,
 std::vector< std::vector< SharedEntityData > >& result );
 
 //! replace handles in from_vec with corresponding handles on
 //! to_proc (by checking shared[p/h]_tag and shared[p/h]s_tag;
 //! if no remote handle and new_ents is non-null, substitute
 //! instead CREATE_HANDLE(MBMAXTYPE, index) where index is handle's
 //! position in new_ents
 ErrorCode get_remote_handles( const bool store_remote_handles,
 EntityHandle* from_vec,
 EntityHandle* to_vec_tmp,
 int num_ents,
 int to_proc,
 const std::vector< EntityHandle >& new_ents );
 
 //! same as other version, except from_range and to_range should be
 //! different here
 ErrorCode get_remote_handles( const bool store_remote_handles,
 const Range& from_range,
 Range& to_range,
 int to_proc,
 const std::vector< EntityHandle >& new_ents );
 
 //! same as other version, except packs range into vector
 ErrorCode get_remote_handles( const bool store_remote_handles,
 const Range& from_range,
 EntityHandle* to_vec,
 int to_proc,
 const std::vector< EntityHandle >& new_ents );
 
 //! goes through from_vec, and for any with type MBMAXTYPE, replaces with
 //! new_ents value at index corresponding to id of entity in from_vec
 ErrorCode get_local_handles( EntityHandle* from_vec, int num_ents, const Range& new_ents );
 
 //! same as above except puts results in range
 ErrorCode get_local_handles( const Range& remote_handles,
 Range& local_handles,
 const std::vector< EntityHandle >& new_ents );
 
 //! same as above except gets new_ents from vector
 ErrorCode get_local_handles( EntityHandle* from_vec, int num_ents, const std::vector< EntityHandle >& new_ents );
 
 ErrorCode update_remote_data( Range& local_range,
 Range& remote_range,
 int other_proc,
 const unsigned char add_pstat );
 
 ErrorCode update_remote_data( const EntityHandle new_h,
 const int* ps,
 const EntityHandle* hs,
 const int num_ps,
 const unsigned char add_pstat );
 
 ErrorCode update_remote_data_old( const EntityHandle new_h,
 const int* ps,
 const EntityHandle* hs,
 const int num_ps,
 const unsigned char add_pstat );
 
 /** \brief Set pstatus tag interface bit on entities in sets passed in
 */
 ErrorCode tag_iface_entities();
 
 //! add a pc to the iface instance tag PARALLEL_COMM
 int add_pcomm( ParallelComm* pc );
 
 //! remove a pc from the iface instance tag PARALLEL_COMM
 void remove_pcomm( ParallelComm* pc );
 
 //! check entities to make sure there are no zero-valued remote handles
 //! where they shouldn't be
 ErrorCode check_sent_ents( Range& allsent );
 
 //! assign entities to the input processor part
 ErrorCode assign_entities_part( std::vector< EntityHandle >& entities, const int proc );
 
 //! remove entities to the input processor part
 ErrorCode remove_entities_part( Range& entities, const int proc );
 
 //! MB interface associated with this writer
 Interface* mbImpl;
 
 //! Proc config object, keeps info on parallel stuff
 ProcConfig procConfig;
 
 //! Sequence manager, to get more efficient access to entities
 SequenceManager* sequenceManager;
 
 //! Error handler
 Error* errorHandler;
 
 //! more data buffers, proc-specific
 std::vector< Buffer* > localOwnedBuffs, remoteOwnedBuffs;
 
 //! reset message buffers to their initial state
 // void reset_all_buffers();
 
 //! delete all buffers, freeing up any memory held by them
 void delete_all_buffers();
 
 //! request objects, may be used if store_remote_handles is used
 std::vector< MPI_Request > sendReqs;
 
 //! receive request objects
 std::vector< MPI_Request > recvReqs, recvRemotehReqs;
 
 //! processor rank for each buffer index
 std::vector< unsigned int > buffProcs;
 
 //! the partition, interface sets for this comm'n instance
 Range partitionSets, interfaceSets;
 
 //! all local entities shared with others, whether ghost or ghosted
 std::set< EntityHandle > sharedEnts;
 
 //! tags used to save sharing procs and handles
 Tag sharedpTag, sharedpsTag, sharedhTag, sharedhsTag, pstatusTag, ifaceSetsTag, partitionTag;
 
 int globalPartCount; //!< Cache of global part count
 
 EntityHandle partitioningSet; //!< entity set containing all parts
 
 std::ofstream myFile;
 
 int pcommID;
 
 int ackbuff;
 
 //! used to set verbosity level and to report output
 DebugOutput* myDebug;
 
 //! Data about shared sets
 SharedSetData* sharedSetData;
};
 
inline ParallelComm::Buffer::Buffer( const Buffer& other_buff )
{
 alloc_size = other_buff.alloc_size;
 mem_ptr = (unsigned char*)malloc( alloc_size );
 memcpy( mem_ptr, other_buff.mem_ptr, alloc_size );
 buff_ptr = mem_ptr + ( other_buff.buff_ptr - other_buff.mem_ptr );
}
 
inline ParallelComm::Buffer::Buffer( unsigned int new_size ) : mem_ptr( NULL ), buff_ptr( NULL ), alloc_size( 0 )
{
 if( new_size ) this->reserve( new_size );
}
 
inline ParallelComm::Buffer::~Buffer()
{
 if( mem_ptr )
 {
 free( mem_ptr );
 mem_ptr = NULL;
 }
}
 
#define DEBUG_BUFFER 0
 
inline void ParallelComm::Buffer::reserve( unsigned int new_size )
{
 
#ifdef DEBUG_BUFFER
 int tmp_pos = 0;
 if( mem_ptr )
 {
 tmp_pos = buff_ptr - mem_ptr;
 }
 buff_ptr = (unsigned char*)malloc( new_size );
 assert( 0 <= tmp_pos && tmp_pos <= (int)alloc_size );
 if( tmp_pos ) memcpy( buff_ptr, mem_ptr, tmp_pos );
 if( mem_ptr ) free( mem_ptr );
 mem_ptr = buff_ptr;
 alloc_size = new_size;
 buff_ptr = mem_ptr + tmp_pos;
#else
 if( mem_ptr && alloc_size < new_size )
 {
 size_t tmp_pos = mem_ptr ? buff_ptr - mem_ptr : 0;
 mem_ptr = (unsigned char*)realloc( mem_ptr, new_size );
 alloc_size = new_size;
 buff_ptr = mem_ptr + tmp_pos;
 }
 else if( !mem_ptr )
 {
 mem_ptr = (unsigned char*)malloc( new_size );
 alloc_size = new_size;
 buff_ptr = mem_ptr;
 }
#endif
}
 
inline void ParallelComm::Buffer::check_space( unsigned int addl_space )
{
 assert( buff_ptr >= mem_ptr && buff_ptr <= mem_ptr + alloc_size );
 unsigned int new_size = buff_ptr - mem_ptr + addl_space;
 if( new_size > alloc_size ) reserve( 3 * new_size / 2 );
}
 
inline void ParallelComm::reset_all_buffers()
{
 std::vector< Buffer* >::iterator vit;
 for( vit = localOwnedBuffs.begin(); vit != localOwnedBuffs.end(); ++vit )
 ( *vit )->reset_buffer();
 for( vit = remoteOwnedBuffs.begin(); vit != remoteOwnedBuffs.end(); ++vit )
 ( *vit )->reset_buffer();
}
 
inline void ParallelComm::delete_all_buffers()
{
 std::vector< Buffer* >::iterator vit;
 for( vit = localOwnedBuffs.begin(); vit != localOwnedBuffs.end(); ++vit )
 delete( *vit );
 localOwnedBuffs.clear();
 
 for( vit = remoteOwnedBuffs.begin(); vit != remoteOwnedBuffs.end(); ++vit )
 delete( *vit );
 remoteOwnedBuffs.clear();
}
 
inline const std::vector< unsigned int >& ParallelComm::buff_procs() const
{
 return buffProcs;
}
 
inline ErrorCode ParallelComm::get_shared_proc_tags( Tag& sharedp,
 Tag& sharedps,
 Tag& sharedh,
 Tag& sharedhs,
 Tag& pstatus )
{
 sharedp = sharedp_tag();
 sharedps = sharedps_tag();
 sharedh = sharedh_tag();
 sharedhs = sharedhs_tag();
 pstatus = pstatus_tag();
 
 return MB_SUCCESS;
}
 
inline ErrorCode ParallelComm::exchange_tags( const char* tag_name, const Range& entities )
{
 // get the tag handle
 std::vector< Tag > tags( 1 );
 ErrorCode result = mbImpl->tag_get_handle( tag_name, 0, MB_TYPE_OPAQUE, tags[0], MB_TAG_ANY );
 if( MB_SUCCESS != result )
 return result;
 else if( !tags[0] )
 return MB_TAG_NOT_FOUND;
 
 return exchange_tags( tags, tags, entities );
}
 
inline ErrorCode ParallelComm::exchange_tags( Tag tagh, const Range& entities )
{
 // get the tag handle
 std::vector< Tag > tags;
 tags.push_back( tagh );
 
 return exchange_tags( tags, tags, entities );
}
 
inline ErrorCode ParallelComm::reduce_tags( const char* tag_name, const MPI_Op mpi_op, const Range& entities )
{
 // get the tag handle
 std::vector< Tag > tags( 1 );
 ErrorCode result = mbImpl->tag_get_handle( tag_name, 0, MB_TYPE_OPAQUE, tags[0], MB_TAG_ANY );
 if( MB_SUCCESS != result )
 return result;
 else if( !tags[0] )
 return MB_TAG_NOT_FOUND;
 
 return reduce_tags( tags, tags, mpi_op, entities );
}
 
inline ErrorCode ParallelComm::reduce_tags( Tag tagh, const MPI_Op mpi_op, const Range& entities )
{
 // get the tag handle
 std::vector< Tag > tags;
 tags.push_back( tagh );
 
 return reduce_tags( tags, tags, mpi_op, entities );
}
 
inline ErrorCode ParallelComm::get_comm_procs( std::set< unsigned int >& procs )
{
 ErrorCode result = get_interface_procs( procs );
 if( MB_SUCCESS != result ) return result;
 
 std::copy( buffProcs.begin(), buffProcs.end(), std::inserter( procs, procs.begin() ) );
 
 return MB_SUCCESS;
}
 
inline ErrorCode ParallelComm::get_owner( EntityHandle entity, int& owner )
{
 EntityHandle tmp_handle;
 return get_owner_handle( entity, owner, tmp_handle );
}
 
/* \brief Unpack message with remote handles (const pointer to buffer)
 */
inline ErrorCode ParallelComm::unpack_remote_handles( unsigned int from_proc,
 const unsigned char* buff_ptr,
 std::vector< EntityHandle >& L2hloc,
 std::vector< EntityHandle >& L2hrem,
 std::vector< unsigned int >& L2p )
{
 // cast away const-ness, we won't be passing back a modified ptr
 unsigned char* tmp_buff = const_cast< unsigned char* >( buff_ptr );
 return unpack_remote_handles( from_proc, tmp_buff, L2hloc, L2hrem, L2p );
}
 
inline void ParallelComm::set_rank( unsigned int r )
{
 procConfig.proc_rank( r );
 if( procConfig.proc_size() < r ) procConfig.proc_size( r + 1 );
}
 
inline void ParallelComm::set_size( unsigned int s )
{
 procConfig.proc_size( s );
}
 
inline ErrorCode ParallelComm::get_sharing_data( const EntityHandle* entities,
 int num_entities,
 std::set< int >& procs,
 int op )
{
 Range dum_range;
 // cast away constness 'cuz the range is passed as const
 EntityHandle* ents_cast = const_cast< EntityHandle* >( entities );
 std::copy( ents_cast, ents_cast + num_entities, range_inserter( dum_range ) );
 return get_sharing_data( dum_range, procs, op );
}
 
inline ErrorCode ParallelComm::get_sharing_data( const EntityHandle entity,
 int* ps,
 EntityHandle* hs,
 unsigned char& pstat,
 int& num_ps )
{
 unsigned int dum_ps;
 ErrorCode result = get_sharing_data( entity, ps, hs, pstat, dum_ps );
 if( MB_SUCCESS == result ) num_ps = dum_ps;
 return result;
}
 
inline void ParallelComm::set_send_request( int n_request )
{
 sendReqs.resize( n_request, MPI_REQUEST_NULL );
}
 
inline void ParallelComm::set_recv_request( int n_request )
{
 recvReqs.resize( n_request, MPI_REQUEST_NULL );
}
} // namespace moab
 
#endif